tracing: Consolidate tracing into a new katatrace package

Removes custom trace functions defined across the repo and creates
a single trace function in a new katatrace package. Also moves
span tag management into this package and provides a function to
dynamically add a tag at runtime, such as a container id, etc.

Fixes #1162

Signed-off-by: Benjamin Porter <bporter816@gmail.com>
This commit is contained in:
Benjamin Porter
2021-07-03 17:37:22 -05:00
parent c8f32936d3
commit b10e3e22b5
18 changed files with 348 additions and 339 deletions

View File

@@ -26,6 +26,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
@@ -43,11 +44,18 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// tracingTags defines tags for the trace span
func (s *Sandbox) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "sandbox",
"sandbox_id": s.id,
}
}
const (
// vmStartTimeout represents the time in seconds a sandbox can wait before
// to consider the VM starting operation failed.
@@ -127,18 +135,6 @@ type SandboxConfig struct {
Cgroups *configs.Cgroup
}
func (s *Sandbox) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
s.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "sandbox"), otelLabel.String("sandbox_id", s.id)))
return span, ctx
}
// valid checks that the sandbox configuration is valid.
func (sandboxConfig *SandboxConfig) valid() bool {
if sandboxConfig.ID == "" {
@@ -393,8 +389,9 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
}
func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
span, _ := trace(ctx, "createAssets")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, _ := katatrace.Trace(ctx, nil, "createAssets", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
for _, name := range types.AssetTypes() {
@@ -444,8 +441,9 @@ func (s *Sandbox) getAndStoreGuestDetails(ctx context.Context) error {
// to physically create that sandbox i.e. starts a VM for that sandbox to eventually
// be started.
func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error) {
span, ctx := trace(ctx, "createSandbox")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, ctx := katatrace.Trace(ctx, nil, "createSandbox", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
if err := createAssets(ctx, &sandboxConfig); err != nil {
@@ -483,8 +481,9 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
}
func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (sb *Sandbox, retErr error) {
span, ctx := trace(ctx, "newSandbox")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, ctx := katatrace.Trace(ctx, nil, "newSandbox", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
if !sandboxConfig.valid() {
@@ -618,7 +617,7 @@ func (s *Sandbox) createCgroupManager() error {
// storeSandbox stores a sandbox config.
func (s *Sandbox) storeSandbox(ctx context.Context) error {
span, _ := s.trace(ctx, "storeSandbox")
span, _ := katatrace.Trace(ctx, s.Logger(), "storeSandbox", s.tracingTags())
defer span.End()
// flush data to storage
@@ -712,8 +711,7 @@ func (s *Sandbox) Delete(ctx context.Context) error {
}
func (s *Sandbox) startNetworkMonitor(ctx context.Context) error {
var span otelTrace.Span
span, ctx = s.trace(ctx, "startNetworkMonitor")
span, ctx := katatrace.Trace(ctx, s.Logger(), "startNetworkMonitor", s.tracingTags())
defer span.End()
binPath, err := os.Executable()
@@ -752,7 +750,7 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
return nil
}
span, ctx := s.trace(ctx, "createNetwork")
span, ctx := katatrace.Trace(ctx, s.Logger(), "createNetwork", s.tracingTags())
defer span.End()
s.networkNS = NetworkNamespace{
@@ -760,8 +758,8 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
NetNsCreated: s.config.NetworkConfig.NetNsCreated,
}
span.SetAttributes(otelLabel.Any("networkNS", s.networkNS))
span.SetAttributes(otelLabel.Any("NetworkConfig", s.config.NetworkConfig))
katatrace.AddTag(span, "networkNS", s.networkNS)
katatrace.AddTag(span, "NetworkConfig", s.config.NetworkConfig)
// In case there is a factory, network interfaces are hotplugged
// after vm is started.
@@ -789,8 +787,7 @@ func (s *Sandbox) postCreatedNetwork(ctx context.Context) error {
}
func (s *Sandbox) removeNetwork(ctx context.Context) error {
var span otelTrace.Span
span, ctx = s.trace(ctx, "removeNetwork")
span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", s.tracingTags())
defer span.End()
if s.config.NetworkConfig.NetmonConfig.Enable {
@@ -997,7 +994,7 @@ func (cw *consoleWatcher) stop() {
// startVM starts the VM.
func (s *Sandbox) startVM(ctx context.Context) (err error) {
span, ctx := s.trace(ctx, "startVM")
span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags())
defer span.End()
s.Logger().Info("Starting VM")
@@ -1078,7 +1075,7 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
// stopVM: stop the sandbox's VM
func (s *Sandbox) stopVM(ctx context.Context) error {
span, ctx := s.trace(ctx, "stopVM")
span, ctx := katatrace.Trace(ctx, s.Logger(), "stopVM", s.tracingTags())
defer span.End()
s.Logger().Info("Stopping sandbox in the VM")
@@ -1437,7 +1434,7 @@ func (s *Sandbox) ResumeContainer(ctx context.Context, containerID string) error
// createContainers registers all containers, create the
// containers in the guest and starts one shim per container.
func (s *Sandbox) createContainers(ctx context.Context) error {
span, ctx := s.trace(ctx, "createContainers")
span, ctx := katatrace.Trace(ctx, s.Logger(), "createContainers", s.tracingTags())
defer span.End()
for i := range s.config.Containers {
@@ -1509,7 +1506,7 @@ func (s *Sandbox) Start(ctx context.Context) error {
// will be destroyed.
// When force is true, ignore guest related stop failures.
func (s *Sandbox) Stop(ctx context.Context, force bool) error {
span, ctx := s.trace(ctx, "Stop")
span, ctx := katatrace.Trace(ctx, s.Logger(), "Stop", s.tracingTags())
defer span.End()
if s.state.State == types.StateStopped {
@@ -1620,7 +1617,7 @@ func (s *Sandbox) unsetSandboxBlockIndex(index int) error {
// HotplugAddDevice is used for add a device to sandbox
// Sandbox implement DeviceReceiver interface from device/api/interface.go
func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devType config.DeviceType) error {
span, ctx := s.trace(ctx, "HotplugAddDevice")
span, ctx := katatrace.Trace(ctx, s.Logger(), "HotplugAddDevice", s.tracingTags())
defer span.End()
if s.config.SandboxCgroupOnly {