Files
kata-containers/vendor/github.com/uber/jaeger-lib/metrics/local.go
James O. D. Hunt 3a1bbd0271 tracing: Add initial opentracing support
Add initial support for opentracing by using the `jaeger` package.
Since opentracing uses the `context` package, add a `context.Context`
as the first parameter to all the functions that we might want to
trace. Trace "spans" (trace points) are then added by extracting the
trace details from the specified context parameter.

Notes:

- Although the tracer is created in `main()`, the "root span"
  (aka the first trace point) is not added until `beforeSubcommands()`.

  This is by design and is a compromise: by delaying the creation of the
  root span, the spans become much more readable since using the web-based
  JaegerUI, you will see traces like this:

  ```
  kata-runtime: kata-runtime create
  ------------  -------------------
       ^                ^
       |                |
  Trace name        First span name
                    (which clearly shows the CLI command that was run)
  ```

  Creating the span earlier means it is necessary to expand 'n' spans in
  the UI before you get to see the name of the CLI command that was run.
  In adding support, this became very tedious, hence my design decision to
  defer the creation of the root span until after signal handling has been
  setup and after CLI options have been parsed, but still very early in
  the code path.

  - At this stage, the tracing stops at the `virtcontainers` call
  boundary.

- Tracing is "always on" as there doesn't appear to be a way to toggle
  it. However, its resolves to a "nop" unless the tracer can talk to a
  jaeger agent.

Note that this commit required a bit of rework to `beforeSubcommands()`
to reduce the cyclomatic complexity.

Fixes #557.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
2018-08-10 16:13:48 +01:00

338 lines
7.9 KiB
Go

// Copyright (c) 2017 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metrics
import (
"sort"
"sync"
"sync/atomic"
"time"
"github.com/codahale/hdrhistogram"
)
// This is intentionally very similar to github.com/codahale/metrics, the
// main difference being that counters/gauges are scoped to the provider
// rather than being global (to facilitate testing).
// A LocalBackend is a metrics provider which aggregates data in-vm, and
// allows exporting snapshots to shove the data into a remote collector
type LocalBackend struct {
cm sync.Mutex
gm sync.Mutex
tm sync.Mutex
counters map[string]*int64
gauges map[string]*int64
timers map[string]*localBackendTimer
stop chan struct{}
wg sync.WaitGroup
TagsSep string
TagKVSep string
}
// NewLocalBackend returns a new LocalBackend. The collectionInterval is the histogram
// time window for each timer.
func NewLocalBackend(collectionInterval time.Duration) *LocalBackend {
b := &LocalBackend{
counters: make(map[string]*int64),
gauges: make(map[string]*int64),
timers: make(map[string]*localBackendTimer),
stop: make(chan struct{}),
TagsSep: "|",
TagKVSep: "=",
}
if collectionInterval == 0 {
// Use one histogram time window for all timers
return b
}
b.wg.Add(1)
go b.runLoop(collectionInterval)
return b
}
// Clear discards accumulated stats
func (b *LocalBackend) Clear() {
b.cm.Lock()
defer b.cm.Unlock()
b.gm.Lock()
defer b.gm.Unlock()
b.tm.Lock()
defer b.tm.Unlock()
b.counters = make(map[string]*int64)
b.gauges = make(map[string]*int64)
b.timers = make(map[string]*localBackendTimer)
}
func (b *LocalBackend) runLoop(collectionInterval time.Duration) {
defer b.wg.Done()
ticker := time.NewTicker(collectionInterval)
for {
select {
case <-ticker.C:
b.tm.Lock()
timers := make(map[string]*localBackendTimer, len(b.timers))
for timerName, timer := range b.timers {
timers[timerName] = timer
}
b.tm.Unlock()
for _, t := range timers {
t.Lock()
t.hist.Rotate()
t.Unlock()
}
case <-b.stop:
ticker.Stop()
return
}
}
}
// IncCounter increments a counter value
func (b *LocalBackend) IncCounter(name string, tags map[string]string, delta int64) {
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
b.cm.Lock()
defer b.cm.Unlock()
counter := b.counters[name]
if counter == nil {
b.counters[name] = new(int64)
*b.counters[name] = delta
return
}
atomic.AddInt64(counter, delta)
}
// UpdateGauge updates the value of a gauge
func (b *LocalBackend) UpdateGauge(name string, tags map[string]string, value int64) {
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
b.gm.Lock()
defer b.gm.Unlock()
gauge := b.gauges[name]
if gauge == nil {
b.gauges[name] = new(int64)
*b.gauges[name] = value
return
}
atomic.StoreInt64(gauge, value)
}
// RecordTimer records a timing duration
func (b *LocalBackend) RecordTimer(name string, tags map[string]string, d time.Duration) {
name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
timer := b.findOrCreateTimer(name)
timer.Lock()
timer.hist.Current.RecordValue(int64(d / time.Millisecond))
timer.Unlock()
}
func (b *LocalBackend) findOrCreateTimer(name string) *localBackendTimer {
b.tm.Lock()
defer b.tm.Unlock()
if t, ok := b.timers[name]; ok {
return t
}
t := &localBackendTimer{
hist: hdrhistogram.NewWindowed(5, 0, int64((5*time.Minute)/time.Millisecond), 1),
}
b.timers[name] = t
return t
}
type localBackendTimer struct {
sync.Mutex
hist *hdrhistogram.WindowedHistogram
}
var (
percentiles = map[string]float64{
"P50": 50,
"P75": 75,
"P90": 90,
"P95": 95,
"P99": 99,
"P999": 99.9,
}
)
// Snapshot captures a snapshot of the current counter and gauge values
func (b *LocalBackend) Snapshot() (counters, gauges map[string]int64) {
b.cm.Lock()
defer b.cm.Unlock()
counters = make(map[string]int64, len(b.counters))
for name, value := range b.counters {
counters[name] = atomic.LoadInt64(value)
}
b.gm.Lock()
defer b.gm.Unlock()
gauges = make(map[string]int64, len(b.gauges))
for name, value := range b.gauges {
gauges[name] = atomic.LoadInt64(value)
}
b.tm.Lock()
timers := make(map[string]*localBackendTimer)
for timerName, timer := range b.timers {
timers[timerName] = timer
}
b.tm.Unlock()
for timerName, timer := range timers {
timer.Lock()
hist := timer.hist.Merge()
timer.Unlock()
for name, q := range percentiles {
gauges[timerName+"."+name] = hist.ValueAtQuantile(q)
}
}
return
}
// Stop cleanly closes the background goroutine spawned by NewLocalBackend.
func (b *LocalBackend) Stop() {
close(b.stop)
b.wg.Wait()
}
// GetKey converts name+tags into a single string of the form
// "name|tag1=value1|...|tagN=valueN", where tag names are
// sorted alphabetically.
func GetKey(name string, tags map[string]string, tagsSep string, tagKVSep string) string {
keys := make([]string, 0, len(tags))
for k := range tags {
keys = append(keys, k)
}
sort.Strings(keys)
key := name
for _, k := range keys {
key = key + tagsSep + k + tagKVSep + tags[k]
}
return key
}
type stats struct {
name string
tags map[string]string
localBackend *LocalBackend
}
type localTimer struct {
stats
}
func (l *localTimer) Record(d time.Duration) {
l.localBackend.RecordTimer(l.name, l.tags, d)
}
type localCounter struct {
stats
}
func (l *localCounter) Inc(delta int64) {
l.localBackend.IncCounter(l.name, l.tags, delta)
}
type localGauge struct {
stats
}
func (l *localGauge) Update(value int64) {
l.localBackend.UpdateGauge(l.name, l.tags, value)
}
// LocalFactory stats factory that creates metrics that are stored locally
type LocalFactory struct {
*LocalBackend
namespace string
tags map[string]string
}
// NewLocalFactory returns a new LocalMetricsFactory
func NewLocalFactory(collectionInterval time.Duration) *LocalFactory {
return &LocalFactory{
LocalBackend: NewLocalBackend(collectionInterval),
}
}
// appendTags adds the tags to the namespace tags and returns a combined map.
func (l *LocalFactory) appendTags(tags map[string]string) map[string]string {
newTags := make(map[string]string)
for k, v := range l.tags {
newTags[k] = v
}
for k, v := range tags {
newTags[k] = v
}
return newTags
}
func (l *LocalFactory) newNamespace(name string) string {
if l.namespace == "" {
return name
}
if name == "" {
return l.namespace
}
return l.namespace + "." + name
}
// Counter returns a local stats counter
func (l *LocalFactory) Counter(name string, tags map[string]string) Counter {
return &localCounter{
stats{
name: l.newNamespace(name),
tags: l.appendTags(tags),
localBackend: l.LocalBackend,
},
}
}
// Timer returns a local stats timer.
func (l *LocalFactory) Timer(name string, tags map[string]string) Timer {
return &localTimer{
stats{
name: l.newNamespace(name),
tags: l.appendTags(tags),
localBackend: l.LocalBackend,
},
}
}
// Gauge returns a local stats gauge.
func (l *LocalFactory) Gauge(name string, tags map[string]string) Gauge {
return &localGauge{
stats{
name: l.newNamespace(name),
tags: l.appendTags(tags),
localBackend: l.LocalBackend,
},
}
}
// Namespace returns a new namespace.
func (l *LocalFactory) Namespace(name string, tags map[string]string) Factory {
return &LocalFactory{
namespace: l.newNamespace(name),
tags: l.appendTags(tags),
LocalBackend: l.LocalBackend,
}
}