mirror of
https://github.com/aljazceru/kata-containers.git
synced 2025-12-27 02:54:27 +01:00
Add initial support for opentracing by using the `jaeger` package.
Since opentracing uses the `context` package, add a `context.Context`
as the first parameter to all the functions that we might want to
trace. Trace "spans" (trace points) are then added by extracting the
trace details from the specified context parameter.
Notes:
- Although the tracer is created in `main()`, the "root span"
(aka the first trace point) is not added until `beforeSubcommands()`.
This is by design and is a compromise: by delaying the creation of the
root span, the spans become much more readable since using the web-based
JaegerUI, you will see traces like this:
```
kata-runtime: kata-runtime create
------------ -------------------
^ ^
| |
Trace name First span name
(which clearly shows the CLI command that was run)
```
Creating the span earlier means it is necessary to expand 'n' spans in
the UI before you get to see the name of the CLI command that was run.
In adding support, this became very tedious, hence my design decision to
defer the creation of the root span until after signal handling has been
setup and after CLI options have been parsed, but still very early in
the code path.
- At this stage, the tracing stops at the `virtcontainers` call
boundary.
- Tracing is "always on" as there doesn't appear to be a way to toggle
it. However, its resolves to a "nop" unless the tracer can talk to a
jaeger agent.
Note that this commit required a bit of rework to `beforeSubcommands()`
to reduce the cyclomatic complexity.
Fixes #557.
Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
338 lines
7.9 KiB
Go
338 lines
7.9 KiB
Go
// Copyright (c) 2017 Uber Technologies, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package metrics
|
|
|
|
import (
|
|
"sort"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/codahale/hdrhistogram"
|
|
)
|
|
|
|
// This is intentionally very similar to github.com/codahale/metrics, the
|
|
// main difference being that counters/gauges are scoped to the provider
|
|
// rather than being global (to facilitate testing).
|
|
|
|
// A LocalBackend is a metrics provider which aggregates data in-vm, and
|
|
// allows exporting snapshots to shove the data into a remote collector
|
|
type LocalBackend struct {
|
|
cm sync.Mutex
|
|
gm sync.Mutex
|
|
tm sync.Mutex
|
|
counters map[string]*int64
|
|
gauges map[string]*int64
|
|
timers map[string]*localBackendTimer
|
|
stop chan struct{}
|
|
wg sync.WaitGroup
|
|
TagsSep string
|
|
TagKVSep string
|
|
}
|
|
|
|
// NewLocalBackend returns a new LocalBackend. The collectionInterval is the histogram
|
|
// time window for each timer.
|
|
func NewLocalBackend(collectionInterval time.Duration) *LocalBackend {
|
|
b := &LocalBackend{
|
|
counters: make(map[string]*int64),
|
|
gauges: make(map[string]*int64),
|
|
timers: make(map[string]*localBackendTimer),
|
|
stop: make(chan struct{}),
|
|
TagsSep: "|",
|
|
TagKVSep: "=",
|
|
}
|
|
if collectionInterval == 0 {
|
|
// Use one histogram time window for all timers
|
|
return b
|
|
}
|
|
b.wg.Add(1)
|
|
go b.runLoop(collectionInterval)
|
|
return b
|
|
}
|
|
|
|
// Clear discards accumulated stats
|
|
func (b *LocalBackend) Clear() {
|
|
b.cm.Lock()
|
|
defer b.cm.Unlock()
|
|
b.gm.Lock()
|
|
defer b.gm.Unlock()
|
|
b.tm.Lock()
|
|
defer b.tm.Unlock()
|
|
b.counters = make(map[string]*int64)
|
|
b.gauges = make(map[string]*int64)
|
|
b.timers = make(map[string]*localBackendTimer)
|
|
}
|
|
|
|
func (b *LocalBackend) runLoop(collectionInterval time.Duration) {
|
|
defer b.wg.Done()
|
|
ticker := time.NewTicker(collectionInterval)
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
b.tm.Lock()
|
|
timers := make(map[string]*localBackendTimer, len(b.timers))
|
|
for timerName, timer := range b.timers {
|
|
timers[timerName] = timer
|
|
}
|
|
b.tm.Unlock()
|
|
|
|
for _, t := range timers {
|
|
t.Lock()
|
|
t.hist.Rotate()
|
|
t.Unlock()
|
|
}
|
|
case <-b.stop:
|
|
ticker.Stop()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// IncCounter increments a counter value
|
|
func (b *LocalBackend) IncCounter(name string, tags map[string]string, delta int64) {
|
|
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
|
|
b.cm.Lock()
|
|
defer b.cm.Unlock()
|
|
counter := b.counters[name]
|
|
if counter == nil {
|
|
b.counters[name] = new(int64)
|
|
*b.counters[name] = delta
|
|
return
|
|
}
|
|
atomic.AddInt64(counter, delta)
|
|
}
|
|
|
|
// UpdateGauge updates the value of a gauge
|
|
func (b *LocalBackend) UpdateGauge(name string, tags map[string]string, value int64) {
|
|
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
|
|
b.gm.Lock()
|
|
defer b.gm.Unlock()
|
|
gauge := b.gauges[name]
|
|
if gauge == nil {
|
|
b.gauges[name] = new(int64)
|
|
*b.gauges[name] = value
|
|
return
|
|
}
|
|
atomic.StoreInt64(gauge, value)
|
|
}
|
|
|
|
// RecordTimer records a timing duration
|
|
func (b *LocalBackend) RecordTimer(name string, tags map[string]string, d time.Duration) {
|
|
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
|
|
timer := b.findOrCreateTimer(name)
|
|
timer.Lock()
|
|
timer.hist.Current.RecordValue(int64(d / time.Millisecond))
|
|
timer.Unlock()
|
|
}
|
|
|
|
func (b *LocalBackend) findOrCreateTimer(name string) *localBackendTimer {
|
|
b.tm.Lock()
|
|
defer b.tm.Unlock()
|
|
if t, ok := b.timers[name]; ok {
|
|
return t
|
|
}
|
|
|
|
t := &localBackendTimer{
|
|
hist: hdrhistogram.NewWindowed(5, 0, int64((5*time.Minute)/time.Millisecond), 1),
|
|
}
|
|
b.timers[name] = t
|
|
return t
|
|
}
|
|
|
|
type localBackendTimer struct {
|
|
sync.Mutex
|
|
hist *hdrhistogram.WindowedHistogram
|
|
}
|
|
|
|
var (
|
|
percentiles = map[string]float64{
|
|
"P50": 50,
|
|
"P75": 75,
|
|
"P90": 90,
|
|
"P95": 95,
|
|
"P99": 99,
|
|
"P999": 99.9,
|
|
}
|
|
)
|
|
|
|
// Snapshot captures a snapshot of the current counter and gauge values
|
|
func (b *LocalBackend) Snapshot() (counters, gauges map[string]int64) {
|
|
b.cm.Lock()
|
|
defer b.cm.Unlock()
|
|
|
|
counters = make(map[string]int64, len(b.counters))
|
|
for name, value := range b.counters {
|
|
counters[name] = atomic.LoadInt64(value)
|
|
}
|
|
|
|
b.gm.Lock()
|
|
defer b.gm.Unlock()
|
|
|
|
gauges = make(map[string]int64, len(b.gauges))
|
|
for name, value := range b.gauges {
|
|
gauges[name] = atomic.LoadInt64(value)
|
|
}
|
|
|
|
b.tm.Lock()
|
|
timers := make(map[string]*localBackendTimer)
|
|
for timerName, timer := range b.timers {
|
|
timers[timerName] = timer
|
|
}
|
|
b.tm.Unlock()
|
|
|
|
for timerName, timer := range timers {
|
|
timer.Lock()
|
|
hist := timer.hist.Merge()
|
|
timer.Unlock()
|
|
for name, q := range percentiles {
|
|
gauges[timerName+"."+name] = hist.ValueAtQuantile(q)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// Stop cleanly closes the background goroutine spawned by NewLocalBackend.
|
|
func (b *LocalBackend) Stop() {
|
|
close(b.stop)
|
|
b.wg.Wait()
|
|
}
|
|
|
|
// GetKey converts name+tags into a single string of the form
|
|
// "name|tag1=value1|...|tagN=valueN", where tag names are
|
|
// sorted alphabetically.
|
|
func GetKey(name string, tags map[string]string, tagsSep string, tagKVSep string) string {
|
|
keys := make([]string, 0, len(tags))
|
|
for k := range tags {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
key := name
|
|
for _, k := range keys {
|
|
key = key + tagsSep + k + tagKVSep + tags[k]
|
|
}
|
|
return key
|
|
}
|
|
|
|
type stats struct {
|
|
name string
|
|
tags map[string]string
|
|
localBackend *LocalBackend
|
|
}
|
|
|
|
type localTimer struct {
|
|
stats
|
|
}
|
|
|
|
func (l *localTimer) Record(d time.Duration) {
|
|
l.localBackend.RecordTimer(l.name, l.tags, d)
|
|
}
|
|
|
|
type localCounter struct {
|
|
stats
|
|
}
|
|
|
|
func (l *localCounter) Inc(delta int64) {
|
|
l.localBackend.IncCounter(l.name, l.tags, delta)
|
|
}
|
|
|
|
type localGauge struct {
|
|
stats
|
|
}
|
|
|
|
func (l *localGauge) Update(value int64) {
|
|
l.localBackend.UpdateGauge(l.name, l.tags, value)
|
|
}
|
|
|
|
// LocalFactory stats factory that creates metrics that are stored locally
|
|
type LocalFactory struct {
|
|
*LocalBackend
|
|
namespace string
|
|
tags map[string]string
|
|
}
|
|
|
|
// NewLocalFactory returns a new LocalMetricsFactory
|
|
func NewLocalFactory(collectionInterval time.Duration) *LocalFactory {
|
|
return &LocalFactory{
|
|
LocalBackend: NewLocalBackend(collectionInterval),
|
|
}
|
|
}
|
|
|
|
// appendTags adds the tags to the namespace tags and returns a combined map.
|
|
func (l *LocalFactory) appendTags(tags map[string]string) map[string]string {
|
|
newTags := make(map[string]string)
|
|
for k, v := range l.tags {
|
|
newTags[k] = v
|
|
}
|
|
for k, v := range tags {
|
|
newTags[k] = v
|
|
}
|
|
return newTags
|
|
}
|
|
|
|
func (l *LocalFactory) newNamespace(name string) string {
|
|
if l.namespace == "" {
|
|
return name
|
|
}
|
|
|
|
if name == "" {
|
|
return l.namespace
|
|
}
|
|
|
|
return l.namespace + "." + name
|
|
}
|
|
|
|
// Counter returns a local stats counter
|
|
func (l *LocalFactory) Counter(name string, tags map[string]string) Counter {
|
|
return &localCounter{
|
|
stats{
|
|
name: l.newNamespace(name),
|
|
tags: l.appendTags(tags),
|
|
localBackend: l.LocalBackend,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Timer returns a local stats timer.
|
|
func (l *LocalFactory) Timer(name string, tags map[string]string) Timer {
|
|
return &localTimer{
|
|
stats{
|
|
name: l.newNamespace(name),
|
|
tags: l.appendTags(tags),
|
|
localBackend: l.LocalBackend,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Gauge returns a local stats gauge.
|
|
func (l *LocalFactory) Gauge(name string, tags map[string]string) Gauge {
|
|
return &localGauge{
|
|
stats{
|
|
name: l.newNamespace(name),
|
|
tags: l.appendTags(tags),
|
|
localBackend: l.LocalBackend,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Namespace returns a new namespace.
|
|
func (l *LocalFactory) Namespace(name string, tags map[string]string) Factory {
|
|
return &LocalFactory{
|
|
namespace: l.newNamespace(name),
|
|
tags: l.appendTags(tags),
|
|
LocalBackend: l.LocalBackend,
|
|
}
|
|
}
|