diff --git a/Gopkg.lock b/Gopkg.lock index 6ce508ad9..922ce2fa8 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -388,7 +388,7 @@ revision = "78d079db6d1f3e32e3ed7578a54baa6257b058a7" [[projects]] - digest = "1:590bfb6f8d5741fa38bb3022f55588fb7e06389f6b7b6692a965e0e5a4363fb1" + digest = "1:2f9680b194232a3f8744fbfb2c353360c497ed59840f7cbee34e903b09cb00b6" name = "github.com/kata-containers/agent" packages = [ "pkg/types", @@ -396,7 +396,7 @@ "protocols/grpc", ] pruneopts = "NUT" - revision = "8e48125fa2a793706e5e660e95bdc9755f1cf474" + revision = "17192be88b2725b0e7f482a8b7616935e1b1ddf4" [[projects]] digest = "1:04054595e5c5a35d1553a7f3464d18577caf597445d643992998643df56d4afd" diff --git a/Gopkg.toml b/Gopkg.toml index f121a71f6..4faefaa0d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -56,7 +56,7 @@ [[constraint]] name = "github.com/kata-containers/agent" - revision = "8e48125fa2a793706e5e660e95bdc9755f1cf474" + revision = "17192be88b2725b0e7f482a8b7616935e1b1ddf4" [[constraint]] name = "github.com/containerd/cri-containerd" diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go index 43dd213ab..c617fa60b 100644 --- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go @@ -55,6 +55,7 @@ AgentDetails GuestDetailsRequest GuestDetailsResponse + MemHotplugByProbeRequest SetGuestDateTimeRequest Storage Device @@ -1353,6 +1354,10 @@ type GuestDetailsRequest struct { // for memory hotplug alignment. Typically the server returns what's in // /sys/devices/system/memory/block_size_bytes. MemBlockSize bool `protobuf:"varint,1,opt,name=mem_block_size,json=memBlockSize,proto3" json:"mem_block_size,omitempty"` + // MemoryHotplugProbe asks server to return whether guest kernel supports memory hotplug + // via probeinterface. Typically the server will check if the path + // /sys/devices/system/memory/probe exists. + MemHotplugProbe bool `protobuf:"varint,2,opt,name=mem_hotplug_probe,json=memHotplugProbe,proto3" json:"mem_hotplug_probe,omitempty"` } func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } @@ -1367,10 +1372,18 @@ func (m *GuestDetailsRequest) GetMemBlockSize() bool { return false } +func (m *GuestDetailsRequest) GetMemHotplugProbe() bool { + if m != nil { + return m.MemHotplugProbe + } + return false +} + type GuestDetailsResponse struct { // MemBlockSizeBytes returns the system memory block size in bytes. - MemBlockSizeBytes uint64 `protobuf:"varint,1,opt,name=mem_block_size_bytes,json=memBlockSizeBytes,proto3" json:"mem_block_size_bytes,omitempty"` - AgentDetails *AgentDetails `protobuf:"bytes,2,opt,name=agent_details,json=agentDetails" json:"agent_details,omitempty"` + MemBlockSizeBytes uint64 `protobuf:"varint,1,opt,name=mem_block_size_bytes,json=memBlockSizeBytes,proto3" json:"mem_block_size_bytes,omitempty"` + AgentDetails *AgentDetails `protobuf:"bytes,2,opt,name=agent_details,json=agentDetails" json:"agent_details,omitempty"` + SupportMemHotplugProbe bool `protobuf:"varint,3,opt,name=support_mem_hotplug_probe,json=supportMemHotplugProbe,proto3" json:"support_mem_hotplug_probe,omitempty"` } func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } @@ -1392,6 +1405,31 @@ func (m *GuestDetailsResponse) GetAgentDetails() *AgentDetails { return nil } +func (m *GuestDetailsResponse) GetSupportMemHotplugProbe() bool { + if m != nil { + return m.SupportMemHotplugProbe + } + return false +} + +type MemHotplugByProbeRequest struct { + // server needs to send the value of memHotplugProbeAddr into file /sys/devices/system/memory/probe, + // in order to notify the guest kernel about hot-add memory event + MemHotplugProbeAddr []uint64 `protobuf:"varint,1,rep,packed,name=memHotplugProbeAddr" json:"memHotplugProbeAddr,omitempty"` +} + +func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } +func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } +func (*MemHotplugByProbeRequest) ProtoMessage() {} +func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } + +func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { + if m != nil { + return m.MemHotplugProbeAddr + } + return nil +} + type SetGuestDateTimeRequest struct { // Sec the second since the Epoch. Sec int64 `protobuf:"varint,1,opt,name=Sec,proto3" json:"Sec,omitempty"` @@ -1402,7 +1440,7 @@ type SetGuestDateTimeRequest struct { func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } func (*SetGuestDateTimeRequest) ProtoMessage() {} -func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } +func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } func (m *SetGuestDateTimeRequest) GetSec() int64 { if m != nil { @@ -1451,7 +1489,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (m *Storage) String() string { return proto.CompactTextString(m) } func (*Storage) ProtoMessage() {} -func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } +func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } func (m *Storage) GetDriver() string { if m != nil { @@ -1534,7 +1572,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (m *Device) String() string { return proto.CompactTextString(m) } func (*Device) ProtoMessage() {} -func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } +func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } func (m *Device) GetId() string { if m != nil { @@ -1580,7 +1618,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (m *StringUser) String() string { return proto.CompactTextString(m) } func (*StringUser) ProtoMessage() {} -func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } +func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } func (m *StringUser) GetUid() string { if m != nil { @@ -1628,7 +1666,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } func (*CopyFileRequest) ProtoMessage() {} -func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } +func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } func (m *CopyFileRequest) GetPath() string { if m != nil { @@ -1732,6 +1770,7 @@ func init() { proto.RegisterType((*AgentDetails)(nil), "grpc.AgentDetails") proto.RegisterType((*GuestDetailsRequest)(nil), "grpc.GuestDetailsRequest") proto.RegisterType((*GuestDetailsResponse)(nil), "grpc.GuestDetailsResponse") + proto.RegisterType((*MemHotplugByProbeRequest)(nil), "grpc.MemHotplugByProbeRequest") proto.RegisterType((*SetGuestDateTimeRequest)(nil), "grpc.SetGuestDateTimeRequest") proto.RegisterType((*Storage)(nil), "grpc.Storage") proto.RegisterType((*Device)(nil), "grpc.Device") @@ -1787,6 +1826,7 @@ type AgentServiceClient interface { OnlineCPUMem(ctx context.Context, in *OnlineCPUMemRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) ReseedRandomDev(ctx context.Context, in *ReseedRandomDevRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) GetGuestDetails(ctx context.Context, in *GuestDetailsRequest, opts ...grpc1.CallOption) (*GuestDetailsResponse, error) + MemHotplugByProbe(ctx context.Context, in *MemHotplugByProbeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) SetGuestDateTime(ctx context.Context, in *SetGuestDateTimeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) } @@ -2042,6 +2082,15 @@ func (c *agentServiceClient) GetGuestDetails(ctx context.Context, in *GuestDetai return out, nil } +func (c *agentServiceClient) MemHotplugByProbe(ctx context.Context, in *MemHotplugByProbeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { + out := new(google_protobuf2.Empty) + err := grpc1.Invoke(ctx, "/grpc.AgentService/MemHotplugByProbe", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *agentServiceClient) SetGuestDateTime(ctx context.Context, in *SetGuestDateTimeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { out := new(google_protobuf2.Empty) err := grpc1.Invoke(ctx, "/grpc.AgentService/SetGuestDateTime", in, out, c.cc, opts...) @@ -2100,6 +2149,7 @@ type AgentServiceServer interface { OnlineCPUMem(context.Context, *OnlineCPUMemRequest) (*google_protobuf2.Empty, error) ReseedRandomDev(context.Context, *ReseedRandomDevRequest) (*google_protobuf2.Empty, error) GetGuestDetails(context.Context, *GuestDetailsRequest) (*GuestDetailsResponse, error) + MemHotplugByProbe(context.Context, *MemHotplugByProbeRequest) (*google_protobuf2.Empty, error) SetGuestDateTime(context.Context, *SetGuestDateTimeRequest) (*google_protobuf2.Empty, error) CopyFile(context.Context, *CopyFileRequest) (*google_protobuf2.Empty, error) } @@ -2594,6 +2644,24 @@ func _AgentService_GetGuestDetails_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _AgentService_MemHotplugByProbe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { + in := new(MemHotplugByProbeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentServiceServer).MemHotplugByProbe(ctx, in) + } + info := &grpc1.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.AgentService/MemHotplugByProbe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentServiceServer).MemHotplugByProbe(ctx, req.(*MemHotplugByProbeRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _AgentService_SetGuestDateTime_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { in := new(SetGuestDateTimeRequest) if err := dec(in); err != nil { @@ -2742,6 +2810,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ MethodName: "GetGuestDetails", Handler: _AgentService_GetGuestDetails_Handler, }, + { + MethodName: "MemHotplugByProbe", + Handler: _AgentService_MemHotplugByProbe_Handler, + }, { MethodName: "SetGuestDateTime", Handler: _AgentService_SetGuestDateTime_Handler, @@ -4424,6 +4496,16 @@ func (m *GuestDetailsRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.MemHotplugProbe { + dAtA[i] = 0x10 + i++ + if m.MemHotplugProbe { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -4457,6 +4539,51 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { } i += n23 } + if m.SupportMemHotplugProbe { + dAtA[i] = 0x18 + i++ + if m.SupportMemHotplugProbe { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *MemHotplugByProbeRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { + dAtA25 := make([]byte, len(m.MemHotplugProbeAddr)*10) + var j24 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { + dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j24++ + } + dAtA25[j24] = uint8(num) + j24++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(j24)) + i += copy(dAtA[i:], dAtA25[:j24]) + } return i, nil } @@ -5435,6 +5562,9 @@ func (m *GuestDetailsRequest) Size() (n int) { if m.MemBlockSize { n += 2 } + if m.MemHotplugProbe { + n += 2 + } return n } @@ -5448,6 +5578,22 @@ func (m *GuestDetailsResponse) Size() (n int) { l = m.AgentDetails.Size() n += 1 + l + sovAgent(uint64(l)) } + if m.SupportMemHotplugProbe { + n += 2 + } + return n +} + +func (m *MemHotplugByProbeRequest) Size() (n int) { + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { + l = 0 + for _, e := range m.MemHotplugProbeAddr { + l += sovAgent(uint64(e)) + } + n += 1 + sovAgent(uint64(l)) + l + } return n } @@ -10845,6 +10991,26 @@ func (m *GuestDetailsRequest) Unmarshal(dAtA []byte) error { } } m.MemBlockSize = bool(v != 0) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MemHotplugProbe", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.MemHotplugProbe = bool(v != 0) default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) @@ -10947,6 +11113,138 @@ func (m *GuestDetailsResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SupportMemHotplugProbe", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.SupportMemHotplugProbe = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *MemHotplugByProbeRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MemHotplugByProbeRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MemHotplugByProbeRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.MemHotplugProbeAddr = append(m.MemHotplugProbeAddr, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.MemHotplugProbeAddr = append(m.MemHotplugProbeAddr, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field MemHotplugProbeAddr", wireType) + } default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) @@ -11944,168 +12242,173 @@ var ( func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } var fileDescriptorAgent = []byte{ - // 2596 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0x24, 0x47, - 0x11, 0xd7, 0xfe, 0xf1, 0x7a, 0xb7, 0x76, 0xd7, 0xeb, 0x6d, 0xfb, 0x7c, 0x9b, 0xbd, 0x24, 0x5c, - 0x26, 0xe1, 0xe2, 0x10, 0xb2, 0x26, 0x97, 0x88, 0x84, 0x9c, 0xc2, 0xe9, 0xce, 0x67, 0xce, 0x26, - 0x39, 0xce, 0x8c, 0x63, 0x05, 0x09, 0xa1, 0xd1, 0x78, 0xa6, 0xbd, 0xee, 0x78, 0x67, 0x7a, 0xd2, - 0xdd, 0xe3, 0xbb, 0x0d, 0x12, 0xe2, 0x89, 0x6f, 0xc1, 0x17, 0x40, 0xbc, 0xf1, 0x0d, 0x10, 0x0f, - 0x11, 0x4f, 0xbc, 0x23, 0x21, 0x94, 0x8f, 0xc0, 0x27, 0x40, 0xfd, 0x6f, 0xfe, 0xec, 0x1f, 0x5f, - 0x70, 0x2c, 0xf1, 0x32, 0x9a, 0xaa, 0xae, 0xfe, 0x75, 0x55, 0x75, 0x77, 0x75, 0x55, 0x41, 0xdb, - 0x1f, 0xe3, 0x58, 0x8c, 0x12, 0x46, 0x05, 0x45, 0xf5, 0x31, 0x4b, 0x82, 0x61, 0x8b, 0x06, 0x44, - 0x33, 0x86, 0x3f, 0x1e, 0x13, 0x71, 0x96, 0x9e, 0x8c, 0x02, 0x1a, 0xed, 0x9c, 0xfb, 0xc2, 0x7f, - 0x27, 0xa0, 0xb1, 0xf0, 0x49, 0x8c, 0x19, 0xdf, 0x51, 0x13, 0x77, 0x92, 0xf3, 0xf1, 0x8e, 0x98, - 0x26, 0x98, 0xeb, 0xaf, 0x99, 0x77, 0x6b, 0x4c, 0xe9, 0x78, 0x82, 0x77, 0x14, 0x75, 0x92, 0x9e, - 0xee, 0xe0, 0x28, 0x11, 0x53, 0x3d, 0xe8, 0xfc, 0xb1, 0x0a, 0x5b, 0xbb, 0x0c, 0xfb, 0x02, 0xef, - 0x5a, 0x34, 0x17, 0x7f, 0x99, 0x62, 0x2e, 0xd0, 0x6b, 0xd0, 0xc9, 0x56, 0xf0, 0x48, 0x38, 0xa8, - 0xdc, 0xae, 0x6c, 0xb7, 0xdc, 0x76, 0xc6, 0x3b, 0x08, 0xd1, 0x4d, 0x58, 0xc5, 0xcf, 0x71, 0x20, - 0x47, 0xab, 0x6a, 0xb4, 0x21, 0xc9, 0x83, 0x10, 0xbd, 0x0b, 0x6d, 0x2e, 0x18, 0x89, 0xc7, 0x5e, - 0xca, 0x31, 0x1b, 0xd4, 0x6e, 0x57, 0xb6, 0xdb, 0x77, 0xd7, 0x47, 0xd2, 0xa4, 0xd1, 0x91, 0x1a, - 0x38, 0xe6, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x07, 0x56, 0x43, 0x7c, 0x41, 0x02, 0xcc, 0x07, - 0xf5, 0xdb, 0xb5, 0xed, 0xf6, 0xdd, 0x8e, 0x16, 0x7f, 0xa4, 0x98, 0xae, 0x1d, 0x44, 0x6f, 0x41, - 0x93, 0x0b, 0xca, 0xfc, 0x31, 0xe6, 0x83, 0x15, 0x25, 0xd8, 0xb5, 0xb8, 0x8a, 0xeb, 0x66, 0xc3, - 0xe8, 0x65, 0xa8, 0x3d, 0xdd, 0x3d, 0x18, 0x34, 0xd4, 0xea, 0x60, 0xa4, 0x12, 0x1c, 0xb8, 0x92, - 0x8d, 0x5e, 0x87, 0x2e, 0xf7, 0xe3, 0xf0, 0x84, 0x3e, 0xf7, 0x12, 0x12, 0xc6, 0x7c, 0xb0, 0x7a, - 0xbb, 0xb2, 0xdd, 0x74, 0x3b, 0x86, 0x79, 0x28, 0x79, 0xce, 0x47, 0x70, 0xe3, 0x48, 0xf8, 0x4c, - 0x5c, 0xc1, 0x3b, 0xce, 0x31, 0x6c, 0xb9, 0x38, 0xa2, 0x17, 0x57, 0x72, 0xed, 0x00, 0x56, 0x05, - 0x89, 0x30, 0x4d, 0x85, 0x72, 0x6d, 0xd7, 0xb5, 0xa4, 0xf3, 0xe7, 0x0a, 0xa0, 0xbd, 0xe7, 0x38, - 0x38, 0x64, 0x34, 0xc0, 0x9c, 0xff, 0x9f, 0xb6, 0xeb, 0x4d, 0x58, 0x4d, 0xb4, 0x02, 0x83, 0xba, - 0x12, 0x37, 0xbb, 0x60, 0xb5, 0xb2, 0xa3, 0xce, 0x17, 0xb0, 0x79, 0x44, 0xc6, 0xb1, 0x3f, 0xb9, - 0x46, 0x7d, 0xb7, 0xa0, 0xc1, 0x15, 0xa6, 0x52, 0xb5, 0xeb, 0x1a, 0xca, 0x39, 0x04, 0xf4, 0xb9, - 0x4f, 0xc4, 0xf5, 0xad, 0xe4, 0xbc, 0x03, 0x1b, 0x25, 0x44, 0x9e, 0xd0, 0x98, 0x63, 0xa5, 0x80, - 0xf0, 0x45, 0xca, 0x15, 0xd8, 0x8a, 0x6b, 0x28, 0x07, 0xc3, 0xe6, 0xa7, 0x84, 0x5b, 0x71, 0xfc, - 0xbf, 0xa8, 0xb0, 0x05, 0x8d, 0x53, 0xca, 0x22, 0x5f, 0x58, 0x0d, 0x34, 0x85, 0x10, 0xd4, 0x7d, - 0x36, 0xe6, 0x83, 0xda, 0xed, 0xda, 0x76, 0xcb, 0x55, 0xff, 0xf2, 0x54, 0xce, 0x2c, 0x63, 0xf4, - 0x7a, 0x0d, 0x3a, 0xc6, 0xef, 0xde, 0x84, 0x70, 0xa1, 0xd6, 0xe9, 0xb8, 0x6d, 0xc3, 0x93, 0x73, - 0x1c, 0x0a, 0x5b, 0xc7, 0x49, 0x78, 0xc5, 0x0b, 0x7f, 0x17, 0x5a, 0x0c, 0x73, 0x9a, 0x32, 0x79, - 0x4d, 0xab, 0x6a, 0xdf, 0x37, 0xf5, 0xbe, 0x7f, 0x4a, 0xe2, 0xf4, 0xb9, 0x6b, 0xc7, 0xdc, 0x5c, - 0xcc, 0x5c, 0x21, 0xc1, 0xaf, 0x72, 0x85, 0x3e, 0x82, 0x1b, 0x87, 0x7e, 0xca, 0xaf, 0xa2, 0xab, - 0x73, 0x4f, 0x5e, 0x3f, 0x9e, 0x46, 0x57, 0x9a, 0xfc, 0xa7, 0x0a, 0x34, 0x77, 0x93, 0xf4, 0x98, - 0xfb, 0x63, 0x8c, 0xbe, 0x07, 0x6d, 0x41, 0x85, 0x3f, 0xf1, 0x52, 0x49, 0x2a, 0xf1, 0xba, 0x0b, - 0x8a, 0xa5, 0x05, 0xa4, 0xdb, 0x31, 0x0b, 0x92, 0xd4, 0x48, 0x54, 0x6f, 0xd7, 0xb6, 0xeb, 0x6e, - 0x5b, 0xf3, 0xb4, 0xc8, 0x08, 0x36, 0xd4, 0x98, 0x47, 0x62, 0xef, 0x1c, 0xb3, 0x18, 0x4f, 0x22, - 0x1a, 0x62, 0x75, 0x7e, 0xeb, 0x6e, 0x5f, 0x0d, 0x1d, 0xc4, 0x9f, 0x64, 0x03, 0xe8, 0x07, 0xd0, - 0xcf, 0xe4, 0xe5, 0xa5, 0x54, 0xd2, 0x75, 0x25, 0xdd, 0x33, 0xd2, 0xc7, 0x86, 0xed, 0xfc, 0x0e, - 0xd6, 0x3e, 0x3b, 0x63, 0x54, 0x88, 0x09, 0x89, 0xc7, 0x8f, 0x7c, 0xe1, 0xcb, 0xe8, 0x91, 0x60, - 0x46, 0x68, 0xc8, 0x8d, 0xb6, 0x96, 0x44, 0x6f, 0x43, 0x5f, 0x68, 0x59, 0x1c, 0x7a, 0x56, 0xa6, - 0xaa, 0x64, 0xd6, 0xb3, 0x81, 0x43, 0x23, 0xfc, 0x7d, 0x58, 0xcb, 0x85, 0x65, 0xfc, 0x31, 0xfa, - 0x76, 0x33, 0xee, 0x67, 0x24, 0xc2, 0xce, 0x85, 0xf2, 0x95, 0xda, 0x64, 0xf4, 0x36, 0xb4, 0x72, - 0x3f, 0x54, 0xd4, 0x09, 0x59, 0xd3, 0x27, 0xc4, 0xba, 0xd3, 0x6d, 0x66, 0x4e, 0xf9, 0x18, 0x7a, - 0x22, 0x53, 0xdc, 0x0b, 0x7d, 0xe1, 0x97, 0x0f, 0x55, 0xd9, 0x2a, 0x77, 0x4d, 0x94, 0x68, 0xe7, - 0x1e, 0xb4, 0x0e, 0x49, 0xc8, 0xf5, 0xc2, 0x03, 0x58, 0x0d, 0x52, 0xc6, 0x70, 0x2c, 0xac, 0xc9, - 0x86, 0x44, 0x9b, 0xb0, 0x32, 0x21, 0x11, 0x11, 0xc6, 0x4c, 0x4d, 0x38, 0x14, 0xe0, 0x09, 0x8e, - 0x28, 0x9b, 0x2a, 0x87, 0x6d, 0xc2, 0x4a, 0x71, 0x73, 0x35, 0x81, 0x6e, 0x41, 0x2b, 0xf2, 0x9f, - 0x67, 0x9b, 0x2a, 0x47, 0x9a, 0x91, 0xff, 0x5c, 0x2b, 0x3f, 0x80, 0xd5, 0x53, 0x9f, 0x4c, 0x82, - 0x58, 0x18, 0xaf, 0x58, 0x32, 0x5f, 0xb0, 0x5e, 0x5c, 0xf0, 0x6f, 0x55, 0x68, 0xeb, 0x15, 0xb5, - 0xc2, 0x9b, 0xb0, 0x12, 0xf8, 0xc1, 0x59, 0xb6, 0xa4, 0x22, 0xd0, 0x1d, 0xab, 0x48, 0xb5, 0x18, - 0x84, 0x73, 0x4d, 0xad, 0x6a, 0x3b, 0x00, 0xfc, 0x99, 0x9f, 0x18, 0xdd, 0x6a, 0x4b, 0x84, 0x5b, - 0x52, 0x46, 0xab, 0xfb, 0x1e, 0x74, 0xf4, 0xb9, 0x33, 0x53, 0xea, 0x4b, 0xa6, 0xb4, 0xb5, 0x94, - 0x9e, 0xf4, 0x3a, 0x74, 0x53, 0x8e, 0xbd, 0x33, 0x82, 0x99, 0xcf, 0x82, 0xb3, 0xe9, 0x60, 0x45, - 0xbf, 0x91, 0x29, 0xc7, 0xfb, 0x96, 0x87, 0xee, 0xc2, 0x8a, 0x0c, 0x7f, 0x7c, 0xd0, 0x50, 0xcf, - 0xf1, 0xcb, 0x45, 0x48, 0x65, 0xea, 0x48, 0x7d, 0xf7, 0x62, 0xc1, 0xa6, 0xae, 0x16, 0x1d, 0x7e, - 0x08, 0x90, 0x33, 0xd1, 0x3a, 0xd4, 0xce, 0xf1, 0xd4, 0xdc, 0x43, 0xf9, 0x2b, 0x9d, 0x73, 0xe1, - 0x4f, 0x52, 0xeb, 0x75, 0x4d, 0x7c, 0x54, 0xfd, 0xb0, 0xe2, 0x04, 0xd0, 0x7b, 0x38, 0x39, 0x27, - 0xb4, 0x30, 0x7d, 0x13, 0x56, 0x22, 0xff, 0x0b, 0xca, 0xac, 0x27, 0x15, 0xa1, 0xb8, 0x24, 0xa6, - 0xcc, 0x42, 0x28, 0x02, 0xad, 0x41, 0x95, 0x26, 0xca, 0x5f, 0x2d, 0xb7, 0x4a, 0x93, 0x7c, 0xa1, - 0x7a, 0x61, 0x21, 0xe7, 0x5f, 0x75, 0x80, 0x7c, 0x15, 0xe4, 0xc2, 0x90, 0x50, 0x8f, 0x63, 0x26, - 0x53, 0x10, 0xef, 0x64, 0x2a, 0x30, 0xf7, 0x18, 0x0e, 0x52, 0xc6, 0xc9, 0x85, 0xdc, 0x3f, 0x69, - 0xf6, 0x0d, 0x6d, 0xf6, 0x8c, 0x6e, 0xee, 0x4d, 0x42, 0x8f, 0xf4, 0xbc, 0x87, 0x72, 0x9a, 0x6b, - 0x67, 0xa1, 0x03, 0xb8, 0x91, 0x63, 0x86, 0x05, 0xb8, 0xea, 0x65, 0x70, 0x1b, 0x19, 0x5c, 0x98, - 0x43, 0xed, 0xc1, 0x06, 0xa1, 0xde, 0x97, 0x29, 0x4e, 0x4b, 0x40, 0xb5, 0xcb, 0x80, 0xfa, 0x84, - 0xfe, 0x52, 0x4d, 0xc8, 0x61, 0x0e, 0xe1, 0xa5, 0x82, 0x95, 0xf2, 0xba, 0x17, 0xc0, 0xea, 0x97, - 0x81, 0x6d, 0x65, 0x5a, 0xc9, 0x78, 0x90, 0x23, 0xfe, 0x1c, 0xb6, 0x08, 0xf5, 0x9e, 0xf9, 0x44, - 0xcc, 0xc2, 0xad, 0xbc, 0xc0, 0x48, 0xf9, 0xe8, 0x96, 0xb1, 0xb4, 0x91, 0x11, 0x66, 0xe3, 0x92, - 0x91, 0x8d, 0x17, 0x18, 0xf9, 0x44, 0x4d, 0xc8, 0x61, 0x1e, 0x40, 0x9f, 0xd0, 0x59, 0x6d, 0x56, - 0x2f, 0x03, 0xe9, 0x11, 0x5a, 0xd6, 0xe4, 0x21, 0xf4, 0x39, 0x0e, 0x04, 0x65, 0xc5, 0x43, 0xd0, - 0xbc, 0x0c, 0x62, 0xdd, 0xc8, 0x67, 0x18, 0xce, 0xaf, 0xa1, 0xb3, 0x9f, 0x8e, 0xb1, 0x98, 0x9c, - 0x64, 0xc1, 0xe0, 0xda, 0xe2, 0x8f, 0xf3, 0x9f, 0x2a, 0xb4, 0x77, 0xc7, 0x8c, 0xa6, 0x49, 0x29, - 0x26, 0xeb, 0x4b, 0x3a, 0x1b, 0x93, 0x95, 0x88, 0x8a, 0xc9, 0x5a, 0xf8, 0x7d, 0xe8, 0x44, 0xea, - 0xea, 0x1a, 0x79, 0x1d, 0x87, 0xfa, 0x73, 0x97, 0xda, 0x6d, 0x47, 0x85, 0x60, 0x36, 0x02, 0x48, - 0x48, 0xc8, 0xcd, 0x1c, 0x1d, 0x8e, 0x7a, 0x26, 0x23, 0xb4, 0x21, 0xda, 0x6d, 0x25, 0x59, 0xb4, - 0x7e, 0x17, 0xda, 0x27, 0xd2, 0x49, 0x66, 0x42, 0x29, 0x18, 0xe5, 0xde, 0x73, 0xe1, 0x24, 0xbf, - 0x84, 0xfb, 0xd0, 0x3d, 0xd3, 0x2e, 0x33, 0x93, 0xf4, 0x19, 0x7a, 0xdd, 0x58, 0x92, 0xdb, 0x3b, - 0x2a, 0x7a, 0x56, 0x6f, 0x40, 0xe7, 0xac, 0xc0, 0x1a, 0x1e, 0x41, 0x7f, 0x4e, 0x64, 0x41, 0x0c, - 0xda, 0x2e, 0xc6, 0xa0, 0xf6, 0x5d, 0xa4, 0x17, 0x2a, 0xce, 0x2c, 0xc6, 0xa5, 0x5f, 0xc0, 0xd6, - 0x6c, 0x9a, 0x63, 0x92, 0xb2, 0xf7, 0xa1, 0x13, 0x28, 0xed, 0x4a, 0x3b, 0xd0, 0x9f, 0xd3, 0xdb, - 0x6d, 0x07, 0x39, 0xe1, 0x84, 0x80, 0x3e, 0x67, 0x44, 0xe0, 0x23, 0xc1, 0xb0, 0x1f, 0x5d, 0x47, - 0xd6, 0x8c, 0xa0, 0xae, 0x9e, 0xd8, 0x9a, 0x4a, 0x0a, 0xd5, 0xbf, 0xf3, 0x26, 0x6c, 0x94, 0x56, - 0x31, 0x2a, 0xaf, 0x43, 0x6d, 0x82, 0x63, 0x85, 0xde, 0x75, 0xe5, 0xaf, 0xe3, 0x43, 0xdf, 0xc5, - 0x7e, 0x78, 0x7d, 0xda, 0x98, 0x25, 0x6a, 0xf9, 0x12, 0xdb, 0x80, 0x8a, 0x4b, 0x18, 0x55, 0xac, - 0xd6, 0x95, 0x82, 0xd6, 0x4f, 0xa1, 0xbf, 0x3b, 0xa1, 0x1c, 0x1f, 0x89, 0x90, 0xc4, 0xd7, 0x91, - 0xe6, 0xff, 0x16, 0x36, 0x3e, 0x13, 0xd3, 0xcf, 0x25, 0x18, 0x27, 0x5f, 0xe1, 0x6b, 0xb2, 0x8f, - 0xd1, 0x67, 0xd6, 0x3e, 0x46, 0x9f, 0xc9, 0x0c, 0x3f, 0xa0, 0x93, 0x34, 0x8a, 0xd5, 0x71, 0xef, - 0xba, 0x86, 0x72, 0xfe, 0x59, 0x81, 0x4d, 0x5d, 0x83, 0x1f, 0xe9, 0xd2, 0xd3, 0x2e, 0x3f, 0x84, - 0xe6, 0x19, 0xe5, 0x22, 0xf6, 0x23, 0x6c, 0x96, 0xce, 0x68, 0x09, 0x2f, 0x6b, 0xd6, 0xaa, 0xaa, - 0x0a, 0xe4, 0x6f, 0xa9, 0x30, 0xae, 0x5d, 0x5e, 0x18, 0xcf, 0x95, 0xbe, 0xf5, 0xf9, 0xd2, 0x17, - 0xbd, 0x02, 0x60, 0x85, 0x48, 0xa8, 0x1e, 0xfe, 0x96, 0xdb, 0x32, 0x9c, 0x83, 0x10, 0xdd, 0x81, - 0xde, 0x58, 0x6a, 0xe9, 0x9d, 0x51, 0x7a, 0xee, 0x25, 0xbe, 0x38, 0x53, 0x85, 0x76, 0xcb, 0xed, - 0x2a, 0xf6, 0x3e, 0xa5, 0xe7, 0x87, 0xbe, 0x38, 0x73, 0x6e, 0xc2, 0x8d, 0x47, 0x98, 0x0b, 0x46, - 0xa7, 0x65, 0xeb, 0x9c, 0x9f, 0x02, 0x1c, 0xc4, 0x02, 0xb3, 0x53, 0x5f, 0x96, 0xf5, 0x3f, 0x2a, - 0x52, 0xe6, 0x49, 0x5d, 0x1f, 0xe9, 0x3e, 0x46, 0x36, 0xe0, 0x16, 0x64, 0x9c, 0x11, 0x34, 0x5c, - 0x9a, 0x0a, 0xcc, 0xd1, 0x1b, 0xf6, 0xcf, 0xcc, 0xeb, 0x98, 0x79, 0x8a, 0xe9, 0x9a, 0x31, 0x67, - 0xdf, 0x16, 0x3e, 0x39, 0x9c, 0xf1, 0xf3, 0x08, 0x5a, 0xc4, 0xf2, 0xcc, 0xed, 0x9c, 0x5f, 0x3a, - 0x17, 0x71, 0xf6, 0x60, 0xe3, 0x41, 0x18, 0x7e, 0x67, 0x98, 0x7d, 0xdb, 0x1f, 0xf8, 0xce, 0x48, - 0xf7, 0x60, 0x43, 0x9b, 0xa6, 0x4d, 0xb5, 0x30, 0x6f, 0x40, 0x83, 0x59, 0xbf, 0x54, 0xf2, 0x8e, - 0x8a, 0x11, 0x32, 0x63, 0x72, 0x83, 0x64, 0x61, 0x98, 0x7b, 0xd6, 0x6e, 0xd0, 0x06, 0xf4, 0xe5, - 0x40, 0x09, 0xd3, 0xf9, 0x0d, 0x6c, 0x3c, 0x8d, 0x27, 0x24, 0xc6, 0xbb, 0x87, 0xc7, 0x4f, 0x70, - 0x16, 0x09, 0x10, 0xd4, 0xe5, 0x33, 0xaf, 0x16, 0x6a, 0xba, 0xea, 0x5f, 0x5e, 0x8d, 0xf8, 0xc4, - 0x0b, 0x92, 0x94, 0x9b, 0x16, 0x46, 0x23, 0x3e, 0xd9, 0x4d, 0x52, 0x8e, 0x5e, 0x02, 0xf9, 0xdc, - 0x78, 0x34, 0x9e, 0x4c, 0xd5, 0xfd, 0x68, 0xba, 0xab, 0x41, 0x92, 0x3e, 0x8d, 0x27, 0x53, 0xe7, - 0x87, 0xaa, 0x68, 0xc3, 0x38, 0x74, 0xfd, 0x38, 0xa4, 0xd1, 0x23, 0x7c, 0x51, 0x58, 0x21, 0x2b, - 0x10, 0x6c, 0x1c, 0xf8, 0xba, 0x02, 0x9d, 0x07, 0x63, 0x1c, 0x8b, 0x47, 0x58, 0xf8, 0x64, 0xa2, - 0x8a, 0x80, 0x0b, 0xcc, 0x38, 0xa1, 0xb1, 0xb9, 0x30, 0x96, 0x94, 0x35, 0x1c, 0x89, 0x89, 0xf0, - 0x42, 0x1f, 0x47, 0x34, 0x56, 0x28, 0x4d, 0x17, 0x24, 0xeb, 0x91, 0xe2, 0xa0, 0x37, 0xa1, 0xa7, - 0x5b, 0x4c, 0xde, 0x99, 0x1f, 0x87, 0x13, 0xcc, 0x6c, 0xc9, 0xbd, 0xa6, 0xd9, 0xfb, 0x86, 0x8b, - 0xde, 0x82, 0x75, 0x73, 0x91, 0x72, 0xc9, 0xba, 0x92, 0xec, 0x19, 0x7e, 0x49, 0x34, 0x4d, 0x12, - 0xca, 0x04, 0xf7, 0x38, 0x0e, 0x02, 0x1a, 0x25, 0x26, 0x83, 0xee, 0x59, 0xfe, 0x91, 0x66, 0xcb, - 0x2d, 0x7c, 0x2c, 0xed, 0x34, 0x96, 0xe4, 0x5b, 0xb8, 0x16, 0xe1, 0xc8, 0x3b, 0x99, 0xd0, 0xe0, - 0xdc, 0x93, 0xa1, 0xc9, 0x78, 0x58, 0xbe, 0xd1, 0x0f, 0x25, 0xf3, 0x88, 0x7c, 0x85, 0x9d, 0xdf, - 0x57, 0x60, 0xb3, 0x3c, 0xdb, 0x04, 0xcf, 0x1d, 0xd8, 0x2c, 0x4f, 0xd7, 0xc9, 0xab, 0xc9, 0x32, - 0xfa, 0x45, 0x10, 0x95, 0x9e, 0xa2, 0x0f, 0xa0, 0xab, 0x7a, 0x89, 0x5e, 0xa8, 0x91, 0xca, 0x6f, - 0x5f, 0xd1, 0xd7, 0x6e, 0xc7, 0x2f, 0x50, 0xce, 0x7d, 0xb8, 0x79, 0x84, 0x85, 0x56, 0xc2, 0x17, - 0x26, 0x0f, 0xd4, 0x36, 0xac, 0x43, 0xed, 0x08, 0x07, 0x6a, 0xcd, 0x9a, 0x2b, 0x7f, 0xe5, 0x5e, - 0x1e, 0x73, 0x1c, 0x28, 0xf0, 0x9a, 0xab, 0xfe, 0x9d, 0xbf, 0x54, 0x60, 0xd5, 0x44, 0x2a, 0x19, - 0x29, 0x43, 0x46, 0x2e, 0x30, 0x33, 0xbb, 0x68, 0x28, 0x59, 0x8f, 0xea, 0x3f, 0x8f, 0x26, 0x82, - 0xd0, 0x2c, 0xfe, 0x75, 0x35, 0xf7, 0xa9, 0x66, 0xaa, 0xee, 0x8c, 0x6a, 0x3e, 0x98, 0x3c, 0xdf, - 0x50, 0xaa, 0xc5, 0xc2, 0xe5, 0x35, 0x52, 0xf1, 0xae, 0xe5, 0x1a, 0x4a, 0x9e, 0x1a, 0x8b, 0xb7, - 0xa2, 0xf0, 0x2c, 0x29, 0x4f, 0x4d, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0x01, 0x0e, - 0x14, 0xeb, 0x50, 0x72, 0x9c, 0x3f, 0x54, 0xa0, 0xa1, 0x3b, 0x94, 0xb2, 0xb2, 0xc8, 0x9e, 0x88, - 0x2a, 0x51, 0xcf, 0xad, 0x5a, 0x4b, 0x3f, 0x0b, 0xea, 0x5f, 0x5e, 0x89, 0x8b, 0x48, 0x07, 0x4b, - 0xa3, 0xda, 0x45, 0x24, 0xa3, 0xa4, 0xb4, 0x2c, 0x7f, 0x69, 0xd4, 0xb8, 0x56, 0xb1, 0x9b, 0x71, - 0x95, 0xd8, 0x52, 0x4d, 0x9d, 0x5f, 0xc9, 0x82, 0x2a, 0xeb, 0xce, 0xad, 0x43, 0x2d, 0xcd, 0x94, - 0x91, 0xbf, 0x92, 0x33, 0xce, 0xde, 0x28, 0xf9, 0x8b, 0xee, 0xc0, 0x9a, 0x1f, 0x86, 0x44, 0x4e, - 0xf7, 0x27, 0x8f, 0x49, 0x98, 0x9d, 0xf7, 0x32, 0xd7, 0xf9, 0x7b, 0x05, 0x7a, 0xbb, 0x34, 0x99, - 0xfe, 0x8c, 0x4c, 0x70, 0xe1, 0x32, 0x2a, 0x25, 0xf5, 0x02, 0xea, 0x5f, 0x26, 0xab, 0xa7, 0x64, - 0x82, 0xf5, 0x29, 0xd5, 0x3b, 0xdb, 0x94, 0x0c, 0x79, 0xb8, 0xb2, 0xc1, 0xac, 0xe9, 0xd1, 0xd5, - 0x83, 0x4f, 0x68, 0x88, 0x65, 0x3c, 0x08, 0x09, 0xf3, 0xb2, 0x16, 0x47, 0xd7, 0x5d, 0x0d, 0x09, - 0x53, 0x43, 0xc6, 0x90, 0x15, 0xd5, 0x65, 0x2b, 0x1a, 0xd2, 0xd0, 0x1c, 0x69, 0xc8, 0x16, 0x34, - 0xe8, 0xe9, 0x29, 0xc7, 0x42, 0x75, 0x70, 0x6b, 0xae, 0xa1, 0xb2, 0x88, 0xd1, 0xcc, 0x23, 0xc6, - 0xdd, 0xbf, 0xf6, 0x4c, 0xc4, 0x30, 0xf5, 0x0a, 0x7a, 0x0c, 0xbd, 0x99, 0xfe, 0x37, 0x32, 0x05, - 0xec, 0xe2, 0xb6, 0xf8, 0x70, 0x6b, 0xa4, 0xfb, 0xe9, 0x23, 0xdb, 0x4f, 0x1f, 0xed, 0x45, 0x89, - 0x98, 0xa2, 0x3d, 0x58, 0x2b, 0x77, 0x8a, 0xd1, 0x2d, 0xfb, 0xfc, 0x2e, 0xe8, 0x1f, 0x2f, 0x85, - 0x79, 0x0c, 0xbd, 0x99, 0xa6, 0xb1, 0xd5, 0x67, 0x71, 0x2f, 0x79, 0x29, 0xd0, 0x7d, 0x68, 0x17, - 0xba, 0xc4, 0x68, 0xa0, 0x41, 0xe6, 0x1b, 0xc7, 0x4b, 0x01, 0x76, 0xa1, 0x5b, 0x6a, 0xdc, 0xa2, - 0xa1, 0xb1, 0x67, 0x41, 0x37, 0x77, 0x29, 0xc8, 0x43, 0x68, 0x17, 0xfa, 0xa7, 0x56, 0x8b, 0xf9, - 0x26, 0xed, 0xf0, 0xa5, 0x05, 0x23, 0x26, 0x88, 0xed, 0x43, 0xb7, 0xd4, 0xed, 0xb4, 0x8a, 0x2c, - 0xea, 0xb4, 0x0e, 0x6f, 0x2d, 0x1c, 0x33, 0x48, 0x8f, 0xa1, 0x37, 0xd3, 0xfb, 0xb4, 0xce, 0x5d, - 0xdc, 0x12, 0x5d, 0x6a, 0xd6, 0x27, 0x6a, 0xb3, 0x0b, 0xc9, 0x7e, 0x61, 0xb3, 0xe7, 0x3b, 0x9d, - 0xc3, 0x97, 0x17, 0x0f, 0x1a, 0xad, 0xf6, 0x60, 0xad, 0xdc, 0xe4, 0xb4, 0x60, 0x0b, 0x5b, 0x9f, - 0x97, 0x9f, 0x9c, 0x52, 0xbf, 0x33, 0x3f, 0x39, 0x8b, 0xda, 0xa0, 0x4b, 0x81, 0x1e, 0x00, 0x98, - 0x9a, 0x20, 0x24, 0x71, 0xb6, 0x65, 0x73, 0xb5, 0x48, 0xb6, 0x65, 0x0b, 0xea, 0x87, 0xfb, 0x00, - 0x3a, 0x95, 0x0f, 0x69, 0x2a, 0xd0, 0x4d, 0xab, 0xc6, 0x4c, 0xfd, 0x30, 0x1c, 0xcc, 0x0f, 0xcc, - 0x01, 0x60, 0xc6, 0xae, 0x02, 0xf0, 0x31, 0x40, 0x5e, 0x22, 0x58, 0x80, 0xb9, 0xa2, 0xe1, 0x12, - 0x1f, 0x74, 0x8a, 0x05, 0x01, 0x32, 0xb6, 0x2e, 0x28, 0x12, 0x96, 0x42, 0xdc, 0x83, 0x4e, 0x31, - 0x4b, 0xb4, 0x10, 0x0b, 0x32, 0xc7, 0xe1, 0x5c, 0x72, 0x87, 0x1e, 0xd8, 0x93, 0x9a, 0xb3, 0x4a, - 0x27, 0xf5, 0xdb, 0x41, 0xcc, 0xa4, 0x97, 0xe5, 0x48, 0xf2, 0x2d, 0x20, 0x3e, 0x80, 0x4e, 0x31, - 0xaf, 0xb4, 0x26, 0x2c, 0xc8, 0x35, 0x87, 0xa5, 0xdc, 0x12, 0xdd, 0x87, 0xb5, 0x72, 0x4e, 0x89, - 0x0a, 0xf7, 0x72, 0x2e, 0xd3, 0x1c, 0x9a, 0xc2, 0xbf, 0x20, 0xfe, 0x1e, 0x40, 0x9e, 0x7b, 0xda, - 0xed, 0x9b, 0xcb, 0x46, 0x67, 0x56, 0xdd, 0x85, 0x6e, 0xa9, 0x8e, 0xb2, 0x81, 0x62, 0x51, 0x71, - 0x75, 0x59, 0x1c, 0x2f, 0xd7, 0x2b, 0x56, 0xf5, 0x85, 0x55, 0xcc, 0x65, 0x07, 0xa8, 0x98, 0x27, - 0x5b, 0xd7, 0x2d, 0xc8, 0x9d, 0x5f, 0x70, 0xa1, 0x8b, 0xb9, 0x70, 0xe1, 0x42, 0x2f, 0x48, 0x91, - 0x97, 0x02, 0xed, 0x43, 0xef, 0xb1, 0xcd, 0xcd, 0x4c, 0xa2, 0x6c, 0xd4, 0x59, 0x90, 0x72, 0x0e, - 0x87, 0x8b, 0x86, 0xcc, 0xad, 0x3a, 0x80, 0xf5, 0xd9, 0x2c, 0x0f, 0xbd, 0x62, 0x82, 0xdb, 0xe2, - 0xec, 0x6f, 0xa9, 0x52, 0x3f, 0x81, 0xa6, 0xcd, 0x2a, 0x90, 0x69, 0x9b, 0xcd, 0x64, 0x19, 0xcb, - 0xa6, 0x3e, 0xec, 0x7c, 0xfd, 0xcd, 0xab, 0x95, 0x7f, 0x7c, 0xf3, 0x6a, 0xe5, 0xdf, 0xdf, 0xbc, - 0x5a, 0x39, 0x69, 0xa8, 0xd1, 0xf7, 0xfe, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x44, 0x3b, 0xc9, 0x9d, - 0x3e, 0x1f, 0x00, 0x00, + // 2675 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x19, 0xdb, 0x6e, 0x1b, 0xc7, + 0x15, 0xbc, 0x88, 0x22, 0x0f, 0x49, 0x51, 0x1c, 0xc9, 0x32, 0x4d, 0x27, 0xae, 0xb3, 0x49, 0x1d, + 0xa5, 0x69, 0xa8, 0xc4, 0x09, 0x9a, 0x8b, 0x91, 0x1a, 0x96, 0xac, 0x5a, 0x6a, 0xe2, 0x5a, 0x5d, + 0x45, 0x48, 0x81, 0xa2, 0x58, 0xac, 0x76, 0x47, 0xe4, 0x44, 0xdc, 0x9d, 0xcd, 0xce, 0xac, 0x6c, + 0xa6, 0x40, 0x1f, 0xfb, 0xda, 0x2f, 0xe8, 0x0f, 0x14, 0x7d, 0xeb, 0x63, 0x5f, 0xfb, 0x10, 0xf4, + 0xa9, 0xef, 0x05, 0x8a, 0x22, 0x9f, 0xd0, 0x2f, 0x28, 0xe6, 0xb6, 0x17, 0x72, 0xa9, 0xa4, 0x8a, + 0x80, 0xbe, 0x10, 0x73, 0x2e, 0x73, 0x6e, 0x33, 0x73, 0xf6, 0x9c, 0x43, 0x68, 0xbb, 0x63, 0x1c, + 0xf2, 0x51, 0x14, 0x53, 0x4e, 0x51, 0x7d, 0x1c, 0x47, 0xde, 0xb0, 0x45, 0x3d, 0xa2, 0x10, 0xc3, + 0x9f, 0x8c, 0x09, 0x9f, 0x24, 0xa7, 0x23, 0x8f, 0x06, 0x3b, 0xe7, 0x2e, 0x77, 0xdf, 0xf2, 0x68, + 0xc8, 0x5d, 0x12, 0xe2, 0x98, 0xed, 0xc8, 0x8d, 0x3b, 0xd1, 0xf9, 0x78, 0x87, 0xcf, 0x22, 0xcc, + 0xd4, 0xaf, 0xde, 0x77, 0x7b, 0x4c, 0xe9, 0x78, 0x8a, 0x77, 0x24, 0x74, 0x9a, 0x9c, 0xed, 0xe0, + 0x20, 0xe2, 0x33, 0x45, 0xb4, 0xfe, 0x58, 0x85, 0xad, 0xbd, 0x18, 0xbb, 0x1c, 0xef, 0x19, 0x69, + 0x36, 0xfe, 0x32, 0xc1, 0x8c, 0xa3, 0x57, 0xa0, 0x93, 0x6a, 0x70, 0x88, 0x3f, 0xa8, 0xdc, 0xad, + 0x6c, 0xb7, 0xec, 0x76, 0x8a, 0x3b, 0xf4, 0xd1, 0x4d, 0x58, 0xc5, 0x2f, 0xb0, 0x27, 0xa8, 0x55, + 0x49, 0x6d, 0x08, 0xf0, 0xd0, 0x47, 0xef, 0x40, 0x9b, 0xf1, 0x98, 0x84, 0x63, 0x27, 0x61, 0x38, + 0x1e, 0xd4, 0xee, 0x56, 0xb6, 0xdb, 0xf7, 0xd7, 0x47, 0xc2, 0xa5, 0xd1, 0xb1, 0x24, 0x9c, 0x30, + 0x1c, 0xdb, 0xc0, 0xd2, 0x35, 0xba, 0x07, 0xab, 0x3e, 0xbe, 0x20, 0x1e, 0x66, 0x83, 0xfa, 0xdd, + 0xda, 0x76, 0xfb, 0x7e, 0x47, 0xb1, 0x3f, 0x96, 0x48, 0xdb, 0x10, 0xd1, 0x1b, 0xd0, 0x64, 0x9c, + 0xc6, 0xee, 0x18, 0xb3, 0xc1, 0x8a, 0x64, 0xec, 0x1a, 0xb9, 0x12, 0x6b, 0xa7, 0x64, 0xf4, 0x12, + 0xd4, 0x9e, 0xed, 0x1d, 0x0e, 0x1a, 0x52, 0x3b, 0x68, 0xae, 0x08, 0x7b, 0xb6, 0x40, 0xa3, 0x57, + 0xa1, 0xcb, 0xdc, 0xd0, 0x3f, 0xa5, 0x2f, 0x9c, 0x88, 0xf8, 0x21, 0x1b, 0xac, 0xde, 0xad, 0x6c, + 0x37, 0xed, 0x8e, 0x46, 0x1e, 0x09, 0x9c, 0xf5, 0x11, 0xdc, 0x38, 0xe6, 0x6e, 0xcc, 0xaf, 0x10, + 0x1d, 0xeb, 0x04, 0xb6, 0x6c, 0x1c, 0xd0, 0x8b, 0x2b, 0x85, 0x76, 0x00, 0xab, 0x9c, 0x04, 0x98, + 0x26, 0x5c, 0x86, 0xb6, 0x6b, 0x1b, 0xd0, 0xfa, 0x73, 0x05, 0xd0, 0xfe, 0x0b, 0xec, 0x1d, 0xc5, + 0xd4, 0xc3, 0x8c, 0xfd, 0x9f, 0x8e, 0xeb, 0x75, 0x58, 0x8d, 0x94, 0x01, 0x83, 0xba, 0x64, 0xd7, + 0xa7, 0x60, 0xac, 0x32, 0x54, 0xeb, 0x0b, 0xd8, 0x3c, 0x26, 0xe3, 0xd0, 0x9d, 0x5e, 0xa3, 0xbd, + 0x5b, 0xd0, 0x60, 0x52, 0xa6, 0x34, 0xb5, 0x6b, 0x6b, 0xc8, 0x3a, 0x02, 0xf4, 0xb9, 0x4b, 0xf8, + 0xf5, 0x69, 0xb2, 0xde, 0x82, 0x8d, 0x82, 0x44, 0x16, 0xd1, 0x90, 0x61, 0x69, 0x00, 0x77, 0x79, + 0xc2, 0xa4, 0xb0, 0x15, 0x5b, 0x43, 0x16, 0x86, 0xcd, 0x4f, 0x09, 0x33, 0xec, 0xf8, 0x7f, 0x31, + 0x61, 0x0b, 0x1a, 0x67, 0x34, 0x0e, 0x5c, 0x6e, 0x2c, 0x50, 0x10, 0x42, 0x50, 0x77, 0xe3, 0x31, + 0x1b, 0xd4, 0xee, 0xd6, 0xb6, 0x5b, 0xb6, 0x5c, 0x8b, 0x5b, 0x39, 0xa7, 0x46, 0xdb, 0xf5, 0x0a, + 0x74, 0x74, 0xdc, 0x9d, 0x29, 0x61, 0x5c, 0xea, 0xe9, 0xd8, 0x6d, 0x8d, 0x13, 0x7b, 0x2c, 0x0a, + 0x5b, 0x27, 0x91, 0x7f, 0xc5, 0x07, 0x7f, 0x1f, 0x5a, 0x31, 0x66, 0x34, 0x89, 0xc5, 0x33, 0xad, + 0xca, 0x73, 0xdf, 0x54, 0xe7, 0xfe, 0x29, 0x09, 0x93, 0x17, 0xb6, 0xa1, 0xd9, 0x19, 0x9b, 0x7e, + 0x42, 0x9c, 0x5d, 0xe5, 0x09, 0x7d, 0x04, 0x37, 0x8e, 0xdc, 0x84, 0x5d, 0xc5, 0x56, 0xeb, 0x81, + 0x78, 0x7e, 0x2c, 0x09, 0xae, 0xb4, 0xf9, 0x4f, 0x15, 0x68, 0xee, 0x45, 0xc9, 0x09, 0x73, 0xc7, + 0x18, 0xfd, 0x00, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, 0xec, 0x75, 0x1b, 0x24, 0x4a, + 0x31, 0x88, 0xb0, 0xe3, 0xd8, 0x8b, 0x12, 0xcd, 0x51, 0xbd, 0x5b, 0xdb, 0xae, 0xdb, 0x6d, 0x85, + 0x53, 0x2c, 0x23, 0xd8, 0x90, 0x34, 0x87, 0x84, 0xce, 0x39, 0x8e, 0x43, 0x3c, 0x0d, 0xa8, 0x8f, + 0xe5, 0xfd, 0xad, 0xdb, 0x7d, 0x49, 0x3a, 0x0c, 0x3f, 0x49, 0x09, 0xe8, 0x47, 0xd0, 0x4f, 0xf9, + 0xc5, 0xa3, 0x94, 0xdc, 0x75, 0xc9, 0xdd, 0xd3, 0xdc, 0x27, 0x1a, 0x6d, 0xfd, 0x0e, 0xd6, 0x3e, + 0x9b, 0xc4, 0x94, 0xf3, 0x29, 0x09, 0xc7, 0x8f, 0x5d, 0xee, 0x8a, 0xec, 0x11, 0xe1, 0x98, 0x50, + 0x9f, 0x69, 0x6b, 0x0d, 0x88, 0xde, 0x84, 0x3e, 0x57, 0xbc, 0xd8, 0x77, 0x0c, 0x4f, 0x55, 0xf2, + 0xac, 0xa7, 0x84, 0x23, 0xcd, 0xfc, 0x43, 0x58, 0xcb, 0x98, 0x45, 0xfe, 0xd1, 0xf6, 0x76, 0x53, + 0xec, 0x67, 0x24, 0xc0, 0xd6, 0x85, 0x8c, 0x95, 0x3c, 0x64, 0xf4, 0x26, 0xb4, 0xb2, 0x38, 0x54, + 0xe4, 0x0d, 0x59, 0x53, 0x37, 0xc4, 0x84, 0xd3, 0x6e, 0xa6, 0x41, 0xf9, 0x18, 0x7a, 0x3c, 0x35, + 0xdc, 0xf1, 0x5d, 0xee, 0x16, 0x2f, 0x55, 0xd1, 0x2b, 0x7b, 0x8d, 0x17, 0x60, 0xeb, 0x01, 0xb4, + 0x8e, 0x88, 0xcf, 0x94, 0xe2, 0x01, 0xac, 0x7a, 0x49, 0x1c, 0xe3, 0x90, 0x1b, 0x97, 0x35, 0x88, + 0x36, 0x61, 0x65, 0x4a, 0x02, 0xc2, 0xb5, 0x9b, 0x0a, 0xb0, 0x28, 0xc0, 0x53, 0x1c, 0xd0, 0x78, + 0x26, 0x03, 0xb6, 0x09, 0x2b, 0xf9, 0xc3, 0x55, 0x00, 0xba, 0x0d, 0xad, 0xc0, 0x7d, 0x91, 0x1e, + 0xaa, 0xa0, 0x34, 0x03, 0xf7, 0x85, 0x32, 0x7e, 0x00, 0xab, 0x67, 0x2e, 0x99, 0x7a, 0x21, 0xd7, + 0x51, 0x31, 0x60, 0xa6, 0xb0, 0x9e, 0x57, 0xf8, 0xb7, 0x2a, 0xb4, 0x95, 0x46, 0x65, 0xf0, 0x26, + 0xac, 0x78, 0xae, 0x37, 0x49, 0x55, 0x4a, 0x00, 0xdd, 0x33, 0x86, 0x54, 0xf3, 0x49, 0x38, 0xb3, + 0xd4, 0x98, 0xb6, 0x03, 0xc0, 0x9e, 0xbb, 0x91, 0xb6, 0xad, 0xb6, 0x84, 0xb9, 0x25, 0x78, 0x94, + 0xb9, 0xef, 0x42, 0x47, 0xdd, 0x3b, 0xbd, 0xa5, 0xbe, 0x64, 0x4b, 0x5b, 0x71, 0xa9, 0x4d, 0xaf, + 0x42, 0x37, 0x61, 0xd8, 0x99, 0x10, 0x1c, 0xbb, 0xb1, 0x37, 0x99, 0x0d, 0x56, 0xd4, 0x37, 0x32, + 0x61, 0xf8, 0xc0, 0xe0, 0xd0, 0x7d, 0x58, 0x11, 0xe9, 0x8f, 0x0d, 0x1a, 0xf2, 0x73, 0xfc, 0x52, + 0x5e, 0xa4, 0x74, 0x75, 0x24, 0x7f, 0xf7, 0x43, 0x1e, 0xcf, 0x6c, 0xc5, 0x3a, 0xfc, 0x00, 0x20, + 0x43, 0xa2, 0x75, 0xa8, 0x9d, 0xe3, 0x99, 0x7e, 0x87, 0x62, 0x29, 0x82, 0x73, 0xe1, 0x4e, 0x13, + 0x13, 0x75, 0x05, 0x7c, 0x54, 0xfd, 0xa0, 0x62, 0x79, 0xd0, 0xdb, 0x9d, 0x9e, 0x13, 0x9a, 0xdb, + 0xbe, 0x09, 0x2b, 0x81, 0xfb, 0x05, 0x8d, 0x4d, 0x24, 0x25, 0x20, 0xb1, 0x24, 0xa4, 0xb1, 0x11, + 0x21, 0x01, 0xb4, 0x06, 0x55, 0x1a, 0xc9, 0x78, 0xb5, 0xec, 0x2a, 0x8d, 0x32, 0x45, 0xf5, 0x9c, + 0x22, 0xeb, 0x5f, 0x75, 0x80, 0x4c, 0x0b, 0xb2, 0x61, 0x48, 0xa8, 0xc3, 0x70, 0x2c, 0x4a, 0x10, + 0xe7, 0x74, 0xc6, 0x31, 0x73, 0x62, 0xec, 0x25, 0x31, 0x23, 0x17, 0xe2, 0xfc, 0x84, 0xdb, 0x37, + 0x94, 0xdb, 0x73, 0xb6, 0xd9, 0x37, 0x09, 0x3d, 0x56, 0xfb, 0x76, 0xc5, 0x36, 0xdb, 0xec, 0x42, + 0x87, 0x70, 0x23, 0x93, 0xe9, 0xe7, 0xc4, 0x55, 0x2f, 0x13, 0xb7, 0x91, 0x8a, 0xf3, 0x33, 0x51, + 0xfb, 0xb0, 0x41, 0xa8, 0xf3, 0x65, 0x82, 0x93, 0x82, 0xa0, 0xda, 0x65, 0x82, 0xfa, 0x84, 0xfe, + 0x52, 0x6e, 0xc8, 0xc4, 0x1c, 0xc1, 0xad, 0x9c, 0x97, 0xe2, 0xb9, 0xe7, 0x84, 0xd5, 0x2f, 0x13, + 0xb6, 0x95, 0x5a, 0x25, 0xf2, 0x41, 0x26, 0xf1, 0xe7, 0xb0, 0x45, 0xa8, 0xf3, 0xdc, 0x25, 0x7c, + 0x5e, 0xdc, 0xca, 0xb7, 0x38, 0x29, 0x3e, 0xba, 0x45, 0x59, 0xca, 0xc9, 0x00, 0xc7, 0xe3, 0x82, + 0x93, 0x8d, 0x6f, 0x71, 0xf2, 0xa9, 0xdc, 0x90, 0x89, 0x79, 0x04, 0x7d, 0x42, 0xe7, 0xad, 0x59, + 0xbd, 0x4c, 0x48, 0x8f, 0xd0, 0xa2, 0x25, 0xbb, 0xd0, 0x67, 0xd8, 0xe3, 0x34, 0xce, 0x5f, 0x82, + 0xe6, 0x65, 0x22, 0xd6, 0x35, 0x7f, 0x2a, 0xc3, 0xfa, 0x35, 0x74, 0x0e, 0x92, 0x31, 0xe6, 0xd3, + 0xd3, 0x34, 0x19, 0x5c, 0x5b, 0xfe, 0xb1, 0xfe, 0x53, 0x85, 0xf6, 0xde, 0x38, 0xa6, 0x49, 0x54, + 0xc8, 0xc9, 0xea, 0x91, 0xce, 0xe7, 0x64, 0xc9, 0x22, 0x73, 0xb2, 0x62, 0x7e, 0x0f, 0x3a, 0x81, + 0x7c, 0xba, 0x9a, 0x5f, 0xe5, 0xa1, 0xfe, 0xc2, 0xa3, 0xb6, 0xdb, 0x41, 0x2e, 0x99, 0x8d, 0x00, + 0x22, 0xe2, 0x33, 0xbd, 0x47, 0xa5, 0xa3, 0x9e, 0xae, 0x08, 0x4d, 0x8a, 0xb6, 0x5b, 0x51, 0x9a, + 0xad, 0xdf, 0x81, 0xf6, 0xa9, 0x08, 0x92, 0xde, 0x50, 0x48, 0x46, 0x59, 0xf4, 0x6c, 0x38, 0xcd, + 0x1e, 0xe1, 0x01, 0x74, 0x27, 0x2a, 0x64, 0x7a, 0x93, 0xba, 0x43, 0xaf, 0x6a, 0x4f, 0x32, 0x7f, + 0x47, 0xf9, 0xc8, 0xaa, 0x03, 0xe8, 0x4c, 0x72, 0xa8, 0xe1, 0x31, 0xf4, 0x17, 0x58, 0x4a, 0x72, + 0xd0, 0x76, 0x3e, 0x07, 0xb5, 0xef, 0x23, 0xa5, 0x28, 0xbf, 0x33, 0x9f, 0x97, 0x7e, 0x01, 0x5b, + 0xf3, 0x65, 0x8e, 0x2e, 0xca, 0xde, 0x83, 0x8e, 0x27, 0xad, 0x2b, 0x9c, 0x40, 0x7f, 0xc1, 0x6e, + 0xbb, 0xed, 0x65, 0x80, 0xe5, 0x03, 0xfa, 0x3c, 0x26, 0x1c, 0x1f, 0xf3, 0x18, 0xbb, 0xc1, 0x75, + 0x54, 0xcd, 0x08, 0xea, 0xf2, 0x13, 0x5b, 0x93, 0x45, 0xa1, 0x5c, 0x5b, 0xaf, 0xc3, 0x46, 0x41, + 0x8b, 0x36, 0x79, 0x1d, 0x6a, 0x53, 0x1c, 0x4a, 0xe9, 0x5d, 0x5b, 0x2c, 0x2d, 0x17, 0xfa, 0x36, + 0x76, 0xfd, 0xeb, 0xb3, 0x46, 0xab, 0xa8, 0x65, 0x2a, 0xb6, 0x01, 0xe5, 0x55, 0x68, 0x53, 0x8c, + 0xd5, 0x95, 0x9c, 0xd5, 0xcf, 0xa0, 0xbf, 0x37, 0xa5, 0x0c, 0x1f, 0x73, 0x9f, 0x84, 0xd7, 0x51, + 0xe6, 0xff, 0x16, 0x36, 0x3e, 0xe3, 0xb3, 0xcf, 0x85, 0x30, 0x46, 0xbe, 0xc2, 0xd7, 0xe4, 0x5f, + 0x4c, 0x9f, 0x1b, 0xff, 0x62, 0xfa, 0x5c, 0x54, 0xf8, 0x1e, 0x9d, 0x26, 0x41, 0x28, 0xaf, 0x7b, + 0xd7, 0xd6, 0x90, 0xf5, 0xcf, 0x0a, 0x6c, 0xaa, 0x1e, 0xfc, 0x58, 0xb5, 0x9e, 0x46, 0xfd, 0x10, + 0x9a, 0x13, 0xca, 0x78, 0xe8, 0x06, 0x58, 0xab, 0x4e, 0x61, 0x21, 0x5e, 0xf4, 0xac, 0x55, 0xd9, + 0x15, 0x88, 0x65, 0xa1, 0x31, 0xae, 0x5d, 0xde, 0x18, 0x2f, 0xb4, 0xbe, 0xf5, 0xc5, 0xd6, 0x17, + 0xbd, 0x0c, 0x60, 0x98, 0x88, 0x2f, 0x3f, 0xfc, 0x2d, 0xbb, 0xa5, 0x31, 0x87, 0x3e, 0xba, 0x07, + 0xbd, 0xb1, 0xb0, 0xd2, 0x99, 0x50, 0x7a, 0xee, 0x44, 0x2e, 0x9f, 0xc8, 0x46, 0xbb, 0x65, 0x77, + 0x25, 0xfa, 0x80, 0xd2, 0xf3, 0x23, 0x97, 0x4f, 0xac, 0x9b, 0x70, 0xe3, 0x31, 0x66, 0x3c, 0xa6, + 0xb3, 0xa2, 0x77, 0xd6, 0x4f, 0x01, 0x0e, 0x43, 0x8e, 0xe3, 0x33, 0x57, 0xb4, 0xf5, 0x6f, 0xe7, + 0x21, 0xfd, 0x49, 0x5d, 0x1f, 0xa9, 0x39, 0x46, 0x4a, 0xb0, 0x73, 0x3c, 0xd6, 0x08, 0x1a, 0x36, + 0x4d, 0x38, 0x66, 0xe8, 0x35, 0xb3, 0xd2, 0xfb, 0x3a, 0x7a, 0x9f, 0x44, 0xda, 0x9a, 0x66, 0x1d, + 0x98, 0xc6, 0x27, 0x13, 0xa7, 0xe3, 0x3c, 0x82, 0x16, 0x31, 0x38, 0xfd, 0x3a, 0x17, 0x55, 0x67, + 0x2c, 0xd6, 0x3e, 0x6c, 0x3c, 0xf2, 0xfd, 0xef, 0x2d, 0xe6, 0xc0, 0xcc, 0x07, 0xbe, 0xb7, 0xa4, + 0x07, 0xb0, 0xa1, 0x5c, 0x53, 0xae, 0x1a, 0x31, 0xaf, 0x41, 0x23, 0x36, 0x71, 0xa9, 0x64, 0x13, + 0x15, 0xcd, 0xa4, 0x69, 0xe2, 0x80, 0x44, 0x63, 0x98, 0x45, 0xd6, 0x1c, 0xd0, 0x06, 0xf4, 0x05, + 0xa1, 0x20, 0xd3, 0xfa, 0x0d, 0x6c, 0x3c, 0x0b, 0xa7, 0x24, 0xc4, 0x7b, 0x47, 0x27, 0x4f, 0x71, + 0x9a, 0x09, 0x10, 0xd4, 0xc5, 0x67, 0x5e, 0x2a, 0x6a, 0xda, 0x72, 0x2d, 0x9e, 0x46, 0x78, 0xea, + 0x78, 0x51, 0xc2, 0xf4, 0x08, 0xa3, 0x11, 0x9e, 0xee, 0x45, 0x09, 0x43, 0xb7, 0x40, 0x7c, 0x6e, + 0x1c, 0x1a, 0x4e, 0x67, 0xf2, 0x7d, 0x34, 0xed, 0x55, 0x2f, 0x4a, 0x9e, 0x85, 0xd3, 0x99, 0xf5, + 0x63, 0xd9, 0xb4, 0x61, 0xec, 0xdb, 0x6e, 0xe8, 0xd3, 0xe0, 0x31, 0xbe, 0xc8, 0x69, 0x48, 0x1b, + 0x04, 0x93, 0x07, 0xbe, 0xae, 0x40, 0xe7, 0xd1, 0x18, 0x87, 0xfc, 0x31, 0xe6, 0x2e, 0x99, 0xca, + 0x26, 0xe0, 0x02, 0xc7, 0x8c, 0xd0, 0x50, 0x3f, 0x18, 0x03, 0x8a, 0x1e, 0x8e, 0x84, 0x84, 0x3b, + 0xbe, 0x8b, 0x03, 0x1a, 0x4a, 0x29, 0x4d, 0x1b, 0x04, 0xea, 0xb1, 0xc4, 0xa0, 0xd7, 0xa1, 0xa7, + 0x46, 0x4c, 0xce, 0xc4, 0x0d, 0xfd, 0x29, 0x8e, 0x4d, 0xcb, 0xbd, 0xa6, 0xd0, 0x07, 0x1a, 0x8b, + 0xde, 0x80, 0x75, 0xfd, 0x90, 0x32, 0xce, 0xba, 0xe4, 0xec, 0x69, 0x7c, 0x81, 0x35, 0x89, 0x22, + 0x1a, 0x73, 0xe6, 0x30, 0xec, 0x79, 0x34, 0x88, 0x74, 0x05, 0xdd, 0x33, 0xf8, 0x63, 0x85, 0xb6, + 0xc6, 0xb0, 0xf1, 0x44, 0xf8, 0xa9, 0x3d, 0xc9, 0x8e, 0x70, 0x2d, 0xc0, 0x81, 0x73, 0x3a, 0xa5, + 0xde, 0xb9, 0x23, 0x52, 0x93, 0x8e, 0xb0, 0xf8, 0x46, 0xef, 0x0a, 0xe4, 0x31, 0xf9, 0x4a, 0x36, + 0x8b, 0x82, 0x6b, 0x42, 0x79, 0x34, 0x4d, 0xc6, 0x4e, 0x14, 0xd3, 0x53, 0xac, 0x5d, 0xec, 0x05, + 0x38, 0x38, 0x50, 0xf8, 0x23, 0x81, 0xb6, 0xfe, 0x5a, 0x81, 0xcd, 0xa2, 0x26, 0x9d, 0x68, 0x77, + 0x60, 0xb3, 0xa8, 0x4a, 0x15, 0xba, 0xba, 0x22, 0xe9, 0xe7, 0x15, 0xca, 0x52, 0x16, 0xbd, 0x0f, + 0x5d, 0x39, 0x77, 0x74, 0x7c, 0x25, 0xa9, 0xf8, 0x9d, 0xcc, 0x9f, 0x8b, 0xdd, 0x71, 0xf3, 0xa7, + 0xf4, 0x21, 0xdc, 0xd2, 0xee, 0x3b, 0x8b, 0x66, 0xab, 0x0b, 0xb1, 0xa5, 0x19, 0x9e, 0xce, 0x59, + 0xff, 0x29, 0x0c, 0x32, 0xd4, 0xee, 0x4c, 0x22, 0x4d, 0xac, 0xde, 0x86, 0x8d, 0x39, 0x67, 0x1f, + 0xf9, 0x7e, 0x2c, 0x73, 0x42, 0xdd, 0x2e, 0x23, 0x59, 0x0f, 0xe1, 0xe6, 0x31, 0xe6, 0x2a, 0x1a, + 0x2e, 0xd7, 0xc5, 0xab, 0x12, 0xb6, 0x0e, 0xb5, 0x63, 0xec, 0x49, 0xe7, 0x6b, 0xb6, 0x58, 0x8a, + 0x0b, 0x78, 0xc2, 0xb0, 0x27, 0xbd, 0xac, 0xd9, 0x72, 0x6d, 0xfd, 0xa5, 0x02, 0xab, 0x3a, 0xbd, + 0x8a, 0xf4, 0xee, 0xc7, 0xe4, 0x02, 0xc7, 0xfa, 0xea, 0x69, 0x48, 0x34, 0xd1, 0x6a, 0xe5, 0xd0, + 0x88, 0x13, 0x9a, 0x26, 0xed, 0xae, 0xc2, 0x3e, 0x53, 0x48, 0x39, 0x52, 0x92, 0x13, 0x13, 0xdd, + 0x9c, 0x68, 0x48, 0xce, 0x85, 0x98, 0x78, 0xfb, 0x32, 0x49, 0xb7, 0x6c, 0x0d, 0x89, 0xab, 0x6e, + 0xe4, 0xad, 0x48, 0x79, 0x06, 0x14, 0x57, 0x3d, 0xa0, 0x49, 0xc8, 0x9d, 0x88, 0x92, 0x90, 0xeb, + 0xac, 0x0c, 0x12, 0x75, 0x24, 0x30, 0xd6, 0xef, 0x2b, 0xd0, 0x50, 0x63, 0x55, 0xd1, 0x0e, 0xa5, + 0xdf, 0xb5, 0x2a, 0x91, 0x35, 0x82, 0xd4, 0xa5, 0xbe, 0x65, 0x72, 0x2d, 0xde, 0xf1, 0x45, 0xa0, + 0x32, 0xbc, 0x36, 0xed, 0x22, 0x10, 0xa9, 0x5d, 0x78, 0x96, 0x7d, 0x1e, 0x25, 0x5d, 0x99, 0xd8, + 0x4d, 0xb1, 0x92, 0x6d, 0xa9, 0xa5, 0xd6, 0xaf, 0x44, 0x17, 0x98, 0x8e, 0x14, 0xd7, 0xa1, 0x96, + 0xa4, 0xc6, 0x88, 0xa5, 0xc0, 0x8c, 0xd3, 0x0f, 0xab, 0x58, 0xa2, 0x7b, 0xb0, 0xe6, 0xfa, 0x3e, + 0x11, 0xdb, 0xdd, 0xe9, 0x13, 0xe2, 0xa7, 0x8f, 0xb4, 0x88, 0xb5, 0xfe, 0x5e, 0x81, 0xde, 0x1e, + 0x8d, 0x66, 0x3f, 0x23, 0x53, 0x9c, 0xcb, 0x20, 0xd2, 0x48, 0xa5, 0x40, 0xae, 0x45, 0x85, 0x7d, + 0x46, 0xa6, 0x58, 0x3d, 0x2d, 0x75, 0xb2, 0x4d, 0x81, 0x90, 0xcf, 0xca, 0x10, 0xd3, 0x49, 0x4d, + 0x57, 0x11, 0x9f, 0x52, 0x1f, 0x8b, 0x24, 0xe6, 0x93, 0xd8, 0x49, 0xe7, 0x32, 0x5d, 0x7b, 0xd5, + 0x27, 0xb1, 0x24, 0x69, 0x47, 0x56, 0xe4, 0x68, 0x30, 0xef, 0x48, 0x43, 0x61, 0x84, 0x23, 0x5b, + 0xd0, 0xa0, 0x67, 0x67, 0x0c, 0x73, 0x39, 0x76, 0xae, 0xd9, 0x1a, 0x4a, 0xd3, 0x5c, 0x33, 0x4b, + 0x73, 0xf7, 0xff, 0xb0, 0xae, 0xd3, 0x9c, 0x6e, 0xb2, 0xd0, 0x13, 0xe8, 0xcd, 0x0d, 0xed, 0x91, + 0xee, 0xba, 0xcb, 0x67, 0xf9, 0xc3, 0xad, 0x91, 0xfa, 0x13, 0x60, 0x64, 0xfe, 0x04, 0x18, 0xed, + 0x07, 0x11, 0x9f, 0xa1, 0x7d, 0x58, 0x2b, 0x8e, 0xb7, 0xd1, 0x6d, 0x53, 0x33, 0x94, 0x0c, 0xbd, + 0x97, 0x8a, 0x79, 0x02, 0xbd, 0xb9, 0x49, 0xb7, 0xb1, 0xa7, 0x7c, 0x00, 0xbe, 0x54, 0xd0, 0x43, + 0x68, 0xe7, 0x46, 0xdb, 0x68, 0xa0, 0x84, 0x2c, 0x4e, 0xbb, 0x97, 0x0a, 0xd8, 0x83, 0x6e, 0x61, + 0xda, 0x8c, 0x86, 0xda, 0x9f, 0x92, 0x11, 0xf4, 0x52, 0x21, 0xbb, 0xd0, 0xce, 0x0d, 0x7d, 0x8d, + 0x15, 0x8b, 0x93, 0xe5, 0xe1, 0xad, 0x12, 0x8a, 0xce, 0xa6, 0x07, 0xd0, 0x2d, 0x8c, 0x68, 0x8d, + 0x21, 0x65, 0xe3, 0xe1, 0xe1, 0xed, 0x52, 0x9a, 0x96, 0xf4, 0x04, 0x7a, 0x73, 0x03, 0x5b, 0x13, + 0xdc, 0xf2, 0x39, 0xee, 0x52, 0xb7, 0x3e, 0x91, 0x87, 0x9d, 0xeb, 0x50, 0x72, 0x87, 0xbd, 0x38, + 0x9e, 0x1d, 0xbe, 0x54, 0x4e, 0xd4, 0x56, 0xed, 0xc3, 0x5a, 0x71, 0x32, 0x6b, 0x84, 0x95, 0xce, + 0x6b, 0x2f, 0xbf, 0x39, 0x85, 0x21, 0x6d, 0x76, 0x73, 0xca, 0x66, 0xb7, 0x4b, 0x05, 0x3d, 0x02, + 0xd0, 0x8d, 0x8c, 0x4f, 0xc2, 0xf4, 0xc8, 0x16, 0x1a, 0xa8, 0xf4, 0xc8, 0x4a, 0x9a, 0x9e, 0x87, + 0x00, 0xaa, 0xff, 0xf0, 0x69, 0xc2, 0xd1, 0x4d, 0x63, 0xc6, 0x5c, 0xd3, 0x33, 0x1c, 0x2c, 0x12, + 0x16, 0x04, 0xe0, 0x38, 0xbe, 0x8a, 0x80, 0x8f, 0x01, 0xb2, 0xbe, 0xc6, 0x08, 0x58, 0xe8, 0x74, + 0x2e, 0x89, 0x41, 0x27, 0xdf, 0xc5, 0x20, 0xed, 0x6b, 0x49, 0x67, 0xb3, 0x54, 0xc4, 0x03, 0xe8, + 0xe4, 0x4b, 0x5b, 0x23, 0xa2, 0xa4, 0xdc, 0x1d, 0x2e, 0x54, 0xa4, 0xe8, 0x91, 0xb9, 0xa9, 0x19, + 0xaa, 0x70, 0x53, 0xbf, 0x9b, 0x88, 0xb9, 0x9a, 0xb8, 0x98, 0x49, 0xbe, 0x83, 0x88, 0xf7, 0xa1, + 0x93, 0x2f, 0x86, 0x8d, 0x0b, 0x25, 0x05, 0xf2, 0xb0, 0x50, 0x10, 0xa3, 0x87, 0xb0, 0x56, 0x2c, + 0x84, 0x51, 0xee, 0x5d, 0x2e, 0x94, 0xc7, 0x43, 0x3d, 0xad, 0xc8, 0xb1, 0xbf, 0x0b, 0x90, 0x15, + 0xcc, 0xe6, 0xf8, 0x16, 0x4a, 0xe8, 0x39, 0xad, 0x7b, 0xd0, 0x2d, 0x34, 0x7f, 0x26, 0x51, 0x94, + 0x75, 0x84, 0x97, 0xe5, 0xf1, 0x62, 0x93, 0x65, 0x4c, 0x2f, 0x6d, 0xbd, 0x2e, 0xbb, 0x40, 0xf9, + 0xe2, 0xde, 0x84, 0xae, 0xa4, 0xe0, 0xff, 0x96, 0x07, 0x9d, 0x2f, 0xe0, 0x73, 0x0f, 0xba, 0xa4, + 0xae, 0x5f, 0x2a, 0xe8, 0x00, 0x7a, 0x4f, 0x4c, 0x6d, 0xa6, 0xeb, 0x46, 0x6d, 0x4e, 0x49, 0x9d, + 0x3c, 0x1c, 0x96, 0x91, 0xf4, 0xab, 0xfa, 0x04, 0xfa, 0x0b, 0x35, 0x23, 0xba, 0x93, 0x0e, 0xb4, + 0x4a, 0x8b, 0xc9, 0xa5, 0x66, 0x1d, 0xc2, 0xfa, 0x7c, 0xc9, 0x88, 0x5e, 0xd6, 0x99, 0xb2, 0xbc, + 0x94, 0x5c, 0x2a, 0xea, 0x43, 0x68, 0x9a, 0x12, 0x05, 0xe9, 0xc1, 0xe1, 0x5c, 0xc9, 0xb2, 0x6c, + 0xeb, 0x6e, 0xe7, 0xeb, 0x6f, 0xee, 0x54, 0xfe, 0xf1, 0xcd, 0x9d, 0xca, 0xbf, 0xbf, 0xb9, 0x53, + 0x39, 0x6d, 0x48, 0xea, 0xbb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xbe, 0xbd, 0x6e, 0x1d, 0x40, + 0x20, 0x00, 0x00, } diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index 804f47b78..8b94f3023 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -216,6 +216,12 @@ type agent interface { // cpuOnly specifies that we should online cpu or online memory or both onlineCPUMem(cpus uint32, cpuOnly bool) error + // memHotplugByProbe will notify the guest kernel about memory hotplug event through + // probe interface. + // This function should be called after hot adding Memory and before online memory. + // addr specifies the address of the recently hotplugged or unhotplugged memory device. + memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error + // statsContainer will tell the agent to get stats from a container related to a Sandbox statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 96ac05829..52f3fa2f6 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -683,8 +683,8 @@ func (fc *firecracker) hypervisorConfig() HypervisorConfig { return fc.config } -func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { - return 0, nil +func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index 95790c5bd..08c3b227e 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -891,6 +891,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) { return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message) } +func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + // hyperstart-agent does not support notify memory hotplug event via probe interface + return nil +} + func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error { // hyperstart-agent uses udev to online CPUs automatically return nil diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 1157d4214..5dd008c7b 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -91,6 +91,8 @@ const ( type memoryDevice struct { slot int sizeMB int + addr uint64 + probe bool } // Set sets an hypervisor type based on the input string. @@ -592,7 +594,7 @@ type hypervisor interface { addDevice(devInfo interface{}, devType deviceType) error hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) + resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) resizeVCPUs(vcpus uint32) (uint32, uint32, error) getSandboxConsole(sandboxID string) (string, error) disconnect() diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index f3e8dfad5..56a3caa66 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -1346,6 +1346,30 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error { return err } +func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + if memorySectionSizeMB == uint32(0) { + return fmt.Errorf("memorySectionSizeMB couldn't be zero") + } + // hot-added memory device should be sliced into the size of memory section, which is the basic unit for + // memory hotplug + numSection := uint64(sizeMB / memorySectionSizeMB) + var addrList []uint64 + index := uint64(0) + for index < numSection { + k.Logger().WithFields(logrus.Fields{ + "addr": fmt.Sprintf("0x%x", addr+(index*uint64(memorySectionSizeMB))<<20), + }).Debugf("notify guest kernel the address of memory device") + addrList = append(addrList, addr+(index*uint64(memorySectionSizeMB))<<20) + index++ + } + req := &grpc.MemHotplugByProbeRequest{ + MemHotplugProbeAddr: addrList, + } + + _, err := k.sendReq(req) + return err +} + func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { req := &grpc.OnlineCPUMemRequest{ Wait: false, @@ -1574,6 +1598,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...) } + k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...) + } k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...) } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index bdc869a0e..378c4fc65 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -256,6 +256,10 @@ func (p *gRPCProxy) CopyFile(ctx context.Context, req *pb.CopyFileRequest) (*gpb return &gpb.Empty{}, nil } +func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByProbeRequest) (*gpb.Empty, error) { + return &gpb.Empty{}, nil +} + func gRPCRegister(s *grpc.Server, srv interface{}) { switch g := srv.(type) { case *gRPCProxy: diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 780ae566f..86e3739f7 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -84,8 +84,8 @@ func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) { return "", nil } -func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { - return 0, nil +func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) { return 0, 0, nil diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index db9a3dcbd..cf609128f 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe return nil } +// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing. +func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error { + return nil +} + // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing. func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { return nil diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 9b018b801..157c25f86 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1240,7 +1240,19 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { q.Logger().WithError(err).Error("hotplug memory") return 0, err } - + // if guest kernel only supports memory hotplug via probe interface, we need to get address of hot-add memory device + if memDev.probe { + memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) + if err != nil { + return 0, fmt.Errorf("failed to query memory devices: %v", err) + } + if len(memoryDevices) != 0 { + q.Logger().WithField("addr", fmt.Sprintf("0x%x", memoryDevices[len(memoryDevices)-1].Data.Addr)).Debug("recently hot-add memory device") + memDev.addr = memoryDevices[len(memoryDevices)-1].Data.Addr + } else { + return 0, fmt.Errorf("failed to probe address of recently hot-add memory device, no device exists") + } + } q.state.HotpluggedMemory += memDev.sizeMB return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) } @@ -1370,32 +1382,33 @@ func (q *qemu) disconnect() { // the memory to remove has to be at least the size of one slot. // To return memory back we are resizing the VM memory balloon. // A longer term solution is evaluate solutions like virtio-mem -func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) err := q.qmpSetup() if err != nil { - return 0, err + return 0, memoryDevice{}, err } + var addMemDevice memoryDevice switch { case currentMemory < reqMemMB: //hotplug addMemMB := reqMemMB - currentMemory memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotplugMB), - } - data, err := q.hotplugAddDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugAddDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryAdded, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory added, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory added, got %+v", data) } currentMemory += uint32(memoryAdded) case currentMemory > reqMemMB: @@ -1403,31 +1416,31 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, addMemMB := currentMemory - reqMemMB memHotunplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotunplugMB), - } - data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotunplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugRemoveDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryRemoved, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory removed, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory removed, got %+v", data) } //FIXME: This is to check memory hotplugRemoveDevice reported 0, as this is not supported. // In the future if this is implemented this validation should be removed. if memoryRemoved != 0 { - return currentMemory, fmt.Errorf("memory hot unplug is not supported, something went wrong") + return currentMemory, addMemDevice, fmt.Errorf("memory hot unplug is not supported, something went wrong") } currentMemory -= uint32(memoryRemoved) } // currentMemory is the current memory (updated) of the VM, return to caller to allow verify // the current VM memory state. - return currentMemory, nil + return currentMemory, addMemDevice, nil } // genericAppendBridges appends to devices the given bridges diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 28b4f3cd6..cf073134f 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -393,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) - _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 009273740..58fe1f5ad 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -416,7 +416,8 @@ func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error { func (s *Sandbox) getAndStoreGuestDetails() error { guestDetailRes, err := s.agent.getGuestDetails(&grpc.GuestDetailsRequest{ - MemBlockSize: true, + MemBlockSize: true, + MemHotplugProbe: true, }) if err != nil { return err @@ -427,6 +428,7 @@ func (s *Sandbox) getAndStoreGuestDetails() error { if guestDetailRes.AgentDetails != nil { s.seccompSupported = guestDetailRes.AgentDetails.SupportsSeccomp } + s.state.GuestMemoryHotplugProbe = guestDetailRes.SupportMemHotplugProbe if err = s.store.Store(store.State, s.state); err != nil { return err @@ -1671,11 +1673,18 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) + newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { return err } s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory) + if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 { + //notify the guest kernel about memory hot-add event, before onlining them + s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr) + if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil { + return err + } + } if err := s.agent.onlineCPUMem(0, false); err != nil { return err } diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go index 84dc14f5e..bd1d6b167 100644 --- a/virtcontainers/types/sandbox.go +++ b/virtcontainers/types/sandbox.go @@ -45,6 +45,9 @@ type State struct { // GuestMemoryBlockSizeMB is the size of memory block of guestos GuestMemoryBlockSizeMB uint32 `json:"guestMemoryBlockSize"` + // GuestMemoryHotplugProbe determines whether guest kernel supports memory hotplug probe interface + GuestMemoryHotplugProbe bool `json:"guestMemoryHotplugProbe"` + // CgroupPath is the cgroup hierarchy where sandbox's processes // including the hypervisor are placed. CgroupPath string `json:"cgroupPath,omitempty"` diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index dc9d6f097..b621830bf 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -373,7 +373,7 @@ func (v *VM) AddCPUs(num uint32) error { func (v *VM) AddMemory(numMB uint32) error { if numMB > 0 { v.logger().Infof("hot adding %d MB memory", numMB) - dev := &memoryDevice{1, int(numMB)} + dev := &memoryDevice{1, int(numMB), 0, false} if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil { return err }