runtime: migrate from opentracing to opentelemetry

This commit includes two changes:
- migrate from opentracing to opentelemetry
- add jaeger configuration items

Fixes: #1351

Signed-off-by: bin <bin@hyper.sh>

src/runtime/pkg/katatestutils/utils.go

@@ -41,6 +41,9 @@ type RuntimeConfigOptions struct {
 	AgentDebug           bool
 	AgentTrace           bool
 	EnablePprof          bool
+	JaegerEndpoint       string
+	JaegerUser           string
+	JaegerPassword       string
 }
 
 func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
@@ -85,5 +88,8 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_debug = ` + strconv.FormatBool(config.RuntimeDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.RuntimeTrace) + `
 	disable_new_netns= ` + strconv.FormatBool(config.DisableNewNetNs) + `
-	enable_pprof= ` + strconv.FormatBool(config.EnablePprof)
+	enable_pprof= ` + strconv.FormatBool(config.EnablePprof) + `
+	jaeger_endpoint= "` + config.JaegerEndpoint + `"
+	jaeger_user= "` + config.JaegerUser + `"
+	jaeger_password= "` + config.JaegerPassword + `"`
 }

src/runtime/pkg/katautils/config.go

@@ -138,6 +138,9 @@ type runtime struct {
 	Experimental        []string `toml:"experimental"`
 	InterNetworkModel   string   `toml:"internetworking_model"`
 	EnablePprof         bool     `toml:"enable_pprof"`
+	JaegerEndpoint      string   `toml:"jaeger_endpoint"`
+	JaegerUser          string   `toml:"jaeger_user"`
+	JaegerPassword      string   `toml:"jaeger_password"`
 }
 
 type agent struct {
@@ -1119,6 +1122,9 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
 	config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
 	config.EnablePprof = tomlConf.Runtime.EnablePprof
+	config.JaegerEndpoint = tomlConf.Runtime.JaegerEndpoint
+	config.JaegerUser = tomlConf.Runtime.JaegerUser
+	config.JaegerPassword = tomlConf.Runtime.JaegerPassword
 	for _, f := range tomlConf.Runtime.Experimental {
 		feature := exp.Get(f)
 		if feature == nil {

src/runtime/pkg/katautils/config_test.go

@@ -34,6 +34,9 @@ var (
 	agentDebug      = false
 	agentTrace      = false
 	enablePprof     = true
+	jaegerEndpoint  = "localhost"
+	jaegerUser      = "jaeger_user1"
+	jaegerPassword  = "jaeger_password1"
 )
 
 type testRuntimeConfig struct {
@@ -114,6 +117,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		SharedFS:             sharedFS,
 		VirtioFSDaemon:       virtioFSdaemon,
 		EnablePprof:          enablePprof,
+		JaegerEndpoint:       jaegerEndpoint,
+		JaegerUser:           jaegerUser,
+		JaegerPassword:       jaegerPassword,
 	}
 
 	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)
@@ -193,6 +199,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		NetmonConfig:    netmonConfig,
 		DisableNewNetNs: disableNewNetNs,
 		EnablePprof:     enablePprof,
+		JaegerEndpoint:  jaegerEndpoint,
+		JaegerUser:      jaegerUser,
+		JaegerPassword:  jaegerPassword,
 
 		FactoryConfig: factoryConfig,
 	}

src/runtime/pkg/katautils/create.go

@@ -17,6 +17,7 @@ import (
 	vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"go.opentelemetry.io/otel/label"
 )
 
 // GetKernelParamsFunc use a variable to allow tests to modify its value
@@ -104,7 +105,7 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
 	span, ctx := Trace(ctx, "createSandbox")
-	defer span.Finish()
+	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
 	if err != nil {
@@ -158,7 +159,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	span.SetTag("sandbox", sid)
+	span.SetAttributes(label.Key("sandbox").String(sid))
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -202,7 +203,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 	var c vc.VCContainer
 
 	span, ctx := Trace(ctx, "createContainer")
-	defer span.Finish()
+	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
 
@@ -227,7 +228,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	span.SetTag("sandbox", sandboxID)
+	span.SetAttributes(label.Key("sandbox").String(sandboxID))
 
 	c, err = sandbox.CreateContainer(contConfig)
 	if err != nil {

src/runtime/pkg/katautils/hook.go

@@ -12,13 +12,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"os/exec"
-	"strings"
 	"syscall"
 	"time"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opentracing/opentracing-go/log"
 	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/label"
 )
 
 // Logger returns a logrus logger appropriate for logging hook messages
@@ -28,13 +27,14 @@ func hookLogger() *logrus.Entry {
 
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
 	span, _ := Trace(ctx, "hook")
-	defer span.Finish()
+	defer span.End()
 
-	span.SetTag("subsystem", "runHook")
+	span.SetAttributes(label.Key("subsystem").String("runHook"))
 
-	span.LogFields(
-		log.String("hook-name", hook.Path),
-		log.String("hook-args", strings.Join(hook.Args, " ")))
+	// FIXME
+	// span.LogFields(
+	// 	log.String("hook-name", hook.Path),
+	// 	log.String("hook-args", strings.Join(hook.Args, " ")))
 
 	state := specs.State{
 		Pid:    syscall.Gettid(),
@@ -91,9 +91,9 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
 	span, _ := Trace(ctx, "hooks")
-	defer span.Finish()
+	defer span.End()
 
-	span.SetTag("subsystem", hookType)
+	span.SetAttributes(label.Key("subsystem").String(hookType))
 
 	for _, hook := range hooks {
 		if err := runHook(ctx, hook, cid, bundlePath); err != nil {

src/runtime/pkg/katautils/tracing.go

@@ -7,67 +7,89 @@ package katautils
 
 import (
 	"context"
-	"io"
 
-	opentracing "github.com/opentracing/opentracing-go"
-	"github.com/uber/jaeger-client-go/config"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/trace/jaeger"
+	"go.opentelemetry.io/otel/label"
+	"go.opentelemetry.io/otel/propagation"
+	export "go.opentelemetry.io/otel/sdk/export/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
-// Implements jaeger-client-go.Logger interface
-type traceLogger struct {
+// kataSpanExporter is used to ensure that Jaeger logs each span.
+// This is essential as it is used by:
+//
+// https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
+type kataSpanExporter struct{}
+
+var _ export.SpanExporter = (*kataSpanExporter)(nil)
+
+// ExportSpans exports SpanData to Jaeger.
+func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
+	for _, span := range spans {
+		kataUtilsLogger.Infof("Reporting span %+v", span)
+	}
+	return nil
+}
+
+func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
+	return nil
 }
 
 // tracerCloser contains a copy of the closer returned by createTracer() which
 // is used by stopTracing().
-var tracerCloser io.Closer
-
-func (t traceLogger) Error(msg string) {
-	kataUtilsLogger.Error(msg)
-}
-
-func (t traceLogger) Infof(msg string, args ...interface{}) {
-	kataUtilsLogger.Infof(msg, args...)
-}
+var tracerCloser func()
 
 // CreateTracer create a tracer
-func CreateTracer(name string) (opentracing.Tracer, error) {
-	cfg := &config.Configuration{
-		ServiceName: name,
-
-		// If tracing is disabled, use a NOP trace implementation
-		Disabled: !tracing,
-
-		// Note that span logging reporter option cannot be enabled as
-		// it pollutes the output stream which causes (atleast) the
-		// "state" command to fail under Docker.
-		Sampler: &config.SamplerConfig{
-			Type:  "const",
-			Param: 1,
-		},
-
-		// Ensure that Jaeger logs each span.
-		// This is essential as it is used by:
-		//
-		// https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
-		Reporter: &config.ReporterConfig{
-			LogSpans: tracing,
-		},
+func CreateTracer(name string, config *oci.RuntimeConfig) (func(), error) {
+	if !tracing {
+		otel.SetTracerProvider(trace.NewNoopTracerProvider())
+		return func() {}, nil
 	}
 
-	logger := traceLogger{}
+	// build kata exporter to log reporting span records
+	kataExporter := &kataSpanExporter{}
 
-	tracer, closer, err := cfg.NewTracer(config.Logger(logger))
+	// build jaeger exporter
+	collectorEndpoint := config.JaegerEndpoint
+	if collectorEndpoint == "" {
+		collectorEndpoint = "http://localhost:14268/api/traces"
+	}
+
+	jaegerExporter, err := jaeger.NewRawExporter(
+		jaeger.WithCollectorEndpoint(collectorEndpoint,
+			jaeger.WithUsername(config.JaegerUser),
+			jaeger.WithPassword(config.JaegerPassword),
+		), jaeger.WithProcess(jaeger.Process{
+			ServiceName: name,
+			Tags: []label.KeyValue{
+				label.String("exporter", "jaeger"),
+				label.String("lib", "opentelemetry"),
+			},
+		}))
 	if err != nil {
 		return nil, err
 	}
 
-	// save for stopTracing()'s exclusive use
-	tracerCloser = closer
+	// build tracer provider, that combining both jaeger exporter and kata exporter.
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithConfig(
+			sdktrace.Config{
+				DefaultSampler: sdktrace.AlwaysSample(),
+			},
+		),
+		sdktrace.WithSyncer(kataExporter),
+		sdktrace.WithSyncer(jaegerExporter),
+	)
 
-	// Seems to be essential to ensure non-root spans are logged
-	opentracing.SetGlobalTracer(tracer)
+	tracerCloser = jaegerExporter.Flush
 
-	return tracer, nil
+	otel.SetTracerProvider(tp)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
+	return tracerCloser, nil
 }
 
 // StopTracing ends all tracing, reporting the spans to the collector.
@@ -76,25 +98,24 @@ func StopTracing(ctx context.Context) {
 		return
 	}
 
-	span := opentracing.SpanFromContext(ctx)
-
+	span := otelTrace.SpanFromContext(ctx)
 	if span != nil {
-		span.Finish()
+		span.End()
 	}
 
 	// report all possible spans to the collector
 	if tracerCloser != nil {
-		tracerCloser.Close()
+		tracerCloser()
 	}
 }
 
 // Trace creates a new tracing span based on the specified name and parent
 // context.
-func Trace(parent context.Context, name string) (opentracing.Span, context.Context) {
-	span, ctx := opentracing.StartSpanFromContext(parent, name)
+func Trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
 
-	span.SetTag("source", "runtime")
-	span.SetTag("component", "cli")
+	tracer := otel.Tracer("kata")
+	ctx, span := tracer.Start(parent, name)
+	span.SetAttributes(label.Key("source").String("runtime"))
 
 	// This is slightly confusing: when tracing is disabled, trace spans
 	// are still created - but the tracer used is a NOP. Therefore, only