From 88e58a4f4b2f347361c2d4be3ed1d11289753aec Mon Sep 17 00:00:00 2001 From: "fupan.lfp" Date: Wed, 31 Mar 2021 21:57:23 +0800 Subject: [PATCH 1/2] agent: fix the issue of missing pass fsGroup For k8s emptyDir volume, a specific fsGroup would be set for it, thus runtime should pass this fsGroup to guest and set it properly on the emptyDir volume in guest. Fixes: #1580 Signed-off-by: fupan.lfp --- src/runtime/virtcontainers/kata_agent.go | 32 ++++++++++++++++--- src/runtime/virtcontainers/kata_agent_test.go | 5 +-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 33001a035..bc1badb5f 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -51,6 +51,9 @@ const ( // containers. KataLocalDevType = "local" + // Allocating an FSGroup that owns the pod's volumes + fsGid = "fsgid" + // path to vfio devices vfioPath = "/dev/vfio/" @@ -1327,7 +1330,11 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co epheStorages := k.handleEphemeralStorage(ociSpec.Mounts) ctrStorages = append(ctrStorages, epheStorages...) - localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) + localStorages, err := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) + if err != nil { + return nil, err + } + ctrStorages = append(ctrStorages, localStorages...) // We replace all OCI mount sources that match our container mount @@ -1426,10 +1433,27 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage // handleLocalStorage handles local storage within the VM // by creating a directory in the VM from the source of the mount point. -func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage { +func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) ([]*grpc.Storage, error) { var localStorages []*grpc.Storage for idx, mnt := range mounts { if mnt.Type == KataLocalDevType { + origin_src := mounts[idx].Source + stat := syscall.Stat_t{} + err := syscall.Stat(origin_src, &stat) + if err != nil { + k.Logger().WithError(err).Errorf("failed to stat %s", origin_src) + return nil, err + } + + dir_options := localDirOptions + + // if volume's gid isn't root group(default group), this means there's + // an specific fsGroup is set on this local volume, then it should pass + // to guest. + if stat.Gid != 0 { + dir_options = append(dir_options, fmt.Sprintf("%s=%d", fsGid, stat.Gid)) + } + // Set the mount source path to a the desired directory point in the VM. // In this case it is located in the sandbox directory. // We rely on the fact that the first container in the VM has the same ID as the sandbox ID. @@ -1444,12 +1468,12 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r Source: KataLocalDevType, Fstype: KataLocalDevType, MountPoint: mounts[idx].Source, - Options: localDirOptions, + Options: dir_options, } localStorages = append(localStorages, localStorage) } } - return localStorages + return localStorages, nil } // handleDeviceBlockVolume handles volume that is block device file diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index cafc1c45a..114a4e3b0 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -205,7 +205,8 @@ func TestHandleEphemeralStorage(t *testing.T) { func TestHandleLocalStorage(t *testing.T) { k := kataAgent{} var ociMounts []specs.Mount - mountSource := "mountPoint" + mountSource := "/tmp/mountPoint" + os.Mkdir(mountSource, 0755) mount := specs.Mount{ Type: KataLocalDevType, @@ -216,7 +217,7 @@ func TestHandleLocalStorage(t *testing.T) { rootfsSuffix := "rootfs" ociMounts = append(ociMounts, mount) - localStorages := k.handleLocalStorage(ociMounts, sandboxID, rootfsSuffix) + localStorages, _ := k.handleLocalStorage(ociMounts, sandboxID, rootfsSuffix) assert.NotNil(t, localStorages) assert.Equal(t, len(localStorages), 1) From 649394256868cf2280011093e39d3722b2ed93e6 Mon Sep 17 00:00:00 2001 From: "fupan.lfp" Date: Wed, 31 Mar 2021 22:01:25 +0800 Subject: [PATCH 2/2] mount: fix the issue of missing set fsGroup For k8s emptyDir volume, a specific fsGroup would be set for it, thus guest should get this fsGroup from runtime and set it properly on the emptyDir volume in guest. Fixes: #1580 Signed-off-by: fupan.lfp --- src/agent/src/mount.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 8cbbb0e06..461330d91 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -17,6 +17,7 @@ use tokio::sync::Mutex; use libc::{c_void, mount}; use nix::mount::{self, MsFlags}; +use nix::unistd::Gid; use regex::Regex; use std::fs::File; @@ -45,6 +46,9 @@ pub const TYPE_ROOTFS: &str = "rootfs"; pub const MOUNT_GUEST_TAG: &str = "kataShared"; +// Allocating an FSGroup that owns the pod's volumes +const FS_GID: &str = "fsgid"; + #[rustfmt::skip] lazy_static! { pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = { @@ -266,11 +270,24 @@ async fn local_storage_handler( let opts_vec: Vec = storage.options.to_vec(); let opts = parse_options(opts_vec); - let mode = opts.get("mode"); - if let Some(mode) = mode { + + let mut need_set_fsgid = false; + if let Some(fsgid) = opts.get(FS_GID) { + let gid = fsgid.parse::()?; + + nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?; + need_set_fsgid = true; + } + + if let Some(mode) = opts.get("mode") { let mut permission = fs::metadata(&storage.mount_point)?.permissions(); - let o_mode = u32::from_str_radix(mode, 8)?; + let mut o_mode = u32::from_str_radix(mode, 8)?; + + if need_set_fsgid { + // set SetGid mode mask. + o_mode |= 0o2000; + } permission.set_mode(o_mode); fs::set_permissions(&storage.mount_point, permission)?;