技术
Gin 通过中间件接入 Jaeger 收集 Tracing
前言
Distributed tracing, also called distributed request tracing, is a method used to profile and monitor applications. Distributed tracing helps pinpoint where failures occur and what causes poor performance.Distributed tracing is particularly well-suited to debugging and monitoring modern distributed software architectures, such as microservices.
分布式追踪，也被成为分布式请求跟踪，是一种分析和监控应用程序的手段。分布式追踪有助于查明故障发生的位置以及导致性能低下的原因。此外，分布式跟踪十分合适调试和监控现代分布式架构的软件，尤其是那些使用微服务体系结构构建的应用程序
如上述引用所属，分布式追踪在现代分布式架构的软件中起着举足轻重的作用，尤其是在微服务这类调用链冗长且复杂的软件架构中，分布式追踪的出现使得上述分析和监控应用程序等工作变得更加简单且方便。
OpenTracing
OpenTracing API提供了一个标准的、与供应商无关的框架，这意味着如果开发者想要尝试一种不同的分布式追踪系统，开发者只需要简单地修改Tracer配置即可，而不需要替换整个分布式追踪系统。
OpenTracing 由 API 规范(描述了语言无关的数据模型和 Opentracing API 指南)、实现该规范的框架和库以及项目文档组成，OpenTracing 不是一个标准，OpenTracing API 项目正致力于为分布式跟踪创建更加标准化的 API 和工具。
Jaeger
Jaeger 是由 Uber 开发的且兼容 OpenTracing API 的分布式追踪系统，用于监控和调试基于微服务体系结构构建的应用程序，主要功能包括:
- 分布式上下文传播
- 分布式事务监控
- 根因分析
- 服务依赖分析
- 性能和延迟优化
测试环境
测试环境使用 Docker 进行测试，测试镜像为 Uber 官方提供的 All-In-One 镜像:
version: '3'
services:
jaeger:
container_name: local-jaeger
image: jaegertracing/all-in-one:1.18
ports:
- "5778:5778"
- "6831-6832:6831-6832/udp"
- "14250:14250"
- "14268:14268"
- "16686:16686"
networks:
- local
networks:
local:
初始化 Jaeger 并注册为全局 Tracer
func NewGlobalTestTracer() (opentracing.Tracer, io.Closer, error) {
cfg := jaegercfg.Configuration{
Sampler: &jaegercfg.SamplerConfig{
Type: jaeger.SamplerTypeConst,
Param: 1,
},
Reporter: &jaegercfg.ReporterConfig{
LogSpans: true,
},
}
// 示例 Logger 和 Metric 分别使用 github.com/uber/jaeger-client-go/log 和 github.com/uber/jaeger-lib/metrics
jLogger := jaegerlog.StdLogger
jMetricsFactory := metrics.NullFactory
// 初始化 Tracer 实例
tracer, closer, err := cfg.NewTracer(
"serviceName",
jaegercfg.Logger(jLogger),
jaegercfg.Metrics(jMetricsFactory),
// 设置最大 Tag 长度，根据情况设置
jaegercfg.MaxTagValueLength(65535),
)
if err != nil {
log.Printf("Could not initialize jaeger tracer: %s", err.Error())
panic(err)
}
return tracer, closer, err
}
编写 Gin 中间件
func OpenTracing() gin.HandlerFunc {
return func(c *gin.Context) {
// 使用 opentracing.GlobalTracer() 获取全局 Tracer
wireCtx, _ := opentracing.GlobalTracer().Extract(
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(c.Request.Header),
)
// OpenTracing Span 概念，详情参见 https://opentracing.io/docs/overview/spans/
serverSpan := opentracing.StartSpan(
c.Request.URL.Path,
ext.RPCServerOption(wireCtx),
)
defer serverSpan.Finish()
// 记录请求 Url
ext.HTTPUrl.Set(serverSpan, c.Request.URL.Path)
// Http Method
ext.HTTPMethod.Set(serverSpan, c.Request.Method)
// 记录组件名称
ext.Component.Set(serverSpan, "Gin-Http")
// 自定义 Tag X-Forwarded-For
opentracing.Tag{Key: "http.headers.x-forwarded-for", Value: c.Request.Header.Get("X-Forwarded-For")}.Set(serverSpan)
// 自定义 Tag User-Agent
opentracing.Tag{Key: "http.headers.user-agent", Value: c.Request.Header.Get("User-Agent")}.Set(serverSpan)
// 自定义 Tag Request-Time
opentracing.Tag{Key: "request.time", Value: time.Now().Format(time.RFC3339)}.Set(serverSpan)
// 自定义 Tag Server-Mode
opentracing.Tag{Key: "http.server.mode", Value: gin.Mode()}.Set(serverSpan)
body, err := ioutil.ReadAll(c.Request.Body)
if err == nil {
// 自定义 Tag Request-Body
opentracing.Tag{Key: "http.request_body", Value: string(body)}.Set(serverSpan)
}
c.Request = c.Request.WithContext(opentracing.ContextWithSpan(c.Request.Context(), serverSpan))
c.Next()
if gin.Mode() == gin.DebugMode {
// 自定义 Tag StackTrace
opentracing.Tag{Key: "debug.trace", Value: string(debug.Stack())}.Set(serverSpan)
}
ext.HTTPStatusCode.Set(serverSpan, uint16(c.Writer.Status()))
opentracing.Tag{Key: "request.errors", Value: c.Errors.String()}.Set(serverSpan)
}
}
启动 Gin 服务
package main
import (
"io/ioutil"
"runtime/debug"
"time"
"github.com/gin-gonic/gin"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/gin-gonic/gin"
)
func main() {
t, closer, err := NewGlobalTestTracer()
if err != nil {
panic(err)
}
defer closer.Close()
opentracing.SetGlobalTracer(t)
r := gin.Default()
// 注入先前编写的中间件
r.Use(OpenTracing())
r.GET("/ping", func(c *gin.Context) {
c.JSON(200, gin.H{
"message": "pong",
})
})
r.Run()
}
结果验证
打开 Jaeger UI 进行验证，按照对应的测试环境，Jaeger Web UI 地址为 http://127.0.0.1:16686/search
Tags:
component=Gin-Http
debug.trace=goroutine 60 [running]: runtime/debug.Stack(0xc00017c0f0, 0xc00062c2d0, 0x54a2380) /usr/local/Cellar/go/1.15/libexec/src/runtime/debug/stack.go:24 +0x9f fusion-pc.com/eam/internal/pkg/plugins/jaeger.OpenTracing.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/internal/pkg/plugins/jaeger/jaeger_gin.go:43 +0x81c github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-contrib/zap.RecoveryWithZap.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-contrib/zap/zap.go:109 +0x68 github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-contrib/zap.Ginzap.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-contrib/zap/zap.go:32 +0xce github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-gonic/gin.RecoveryWithWriter.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/recovery.go:83 +0x65 github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc00053a000, 0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/gin.go:409 +0x67a github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc00053a000, 0x5487700, 0xc0006281c0, 0xc000372600) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/gin.go:367 +0x14d net/http.serverHandler.ServeHTTP(0xc0006280e0, 0x5487700, 0xc0006281c0, 0xc000372600) /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:2843 +0xa3 net/http.(*conn).serve(0xc0000e0140, 0x548c680, 0xc0005643c0) /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:1925 +0x8ad created by net/http.(*Server).Serve /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:2969 +0x36c
http.headers.user-agent=PostmanRuntime/7.26.3
http.headers.x-forwarded-for=
http.method=GET
http.request_body=
http.server.mode=debug
http.status_code=200
http.url=/ping
internal.span.format=proto
request.errors=
request.time=2020-09-01T20:55:34+08:00
sampler.param=1
sampler.type=probabilistic
span.kind=server
Process:
client-uuid=8060d5ba910b166
hostname=zhouhaoweis-MacBook-Pro.localip=192.168.31.83
jaeger.version=Go-2.25.0
延伸: 从配置初始化 Jaeger
package jaeger
import (
"fmt"
"strings"
"time"
"github.com/google/wire"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"github.com/spf13/viper"
"github.com/uber/jaeger-client-go"
"github.com/uber/jaeger-client-go/config"
// 使用 zap 作为日志输出
jaegerZap "github.com/uber/jaeger-client-go/log/zap"
"go.uber.org/zap"
)
type Sampler struct {
SamplerType string
Probability float64
Rate int64
}
type Reporter struct {
QueueSize int
LogSpans bool
BufferFlushInterval time.Duration
Username string
Password string
}
type Options struct {
Enable bool
ServiceName string
Host string
Port uint16
Sampler *Sampler
Reporter *Reporter
}
const defaultSamplingLimitRate = 1
var NoDefaultTracingConfigError = errors.New("unable to load tracing config")
func NewOptions(v *viper.Viper, logger *zap.Logger) (opts *Options, err error) {
cfg := v.GetString("tracing.default")
if len(cfg) <= 0 {
return nil, NoDefaultTracingConfigError
}
opts = new(Options)
key := fmt.Sprintf("tracing.%s", cfg)
if err = v.UnmarshalKey(key, opts); err != nil {
return nil, errors.Wrap(err, "unmarshal tracing options error")
}
logger.With(zap.String("type", "tracing")).Info(fmt.Sprintf("load %s tracing options success", cfg))
return
}
func NewTracer(logger *zap.Logger, opts *Options) (opentracing.Tracer, error) {
sampleCfg := &config.SamplerConfig{}
switch strings.ToLower(opts.Sampler.SamplerType) {
case jaeger.SamplerTypeConst:
sampleCfg.Type = jaeger.SamplerTypeConst
case jaeger.SamplerTypeRateLimiting:
sampleCfg.Type = jaeger.SamplerTypeRateLimiting
if opts.Sampler.Rate > 0 {
sampleCfg.Param = float64(opts.Sampler.Rate)
} else {
sampleCfg.Param = defaultSamplingLimitRate
}
case jaeger.SamplerTypeRemote:
sampleCfg.Type = jaeger.SamplerTypeRemote
default:
sampleCfg.Type = jaeger.SamplerTypeProbabilistic
if opts.Sampler.Probability > 0 {
sampleCfg.Param = opts.Sampler.Probability
}
}
reporterCfg := &config.ReporterConfig{
QueueSize: opts.Reporter.QueueSize,
LogSpans: opts.Reporter.LogSpans,
BufferFlushInterval: opts.Reporter.BufferFlushInterval,
}
if len(opts.Reporter.Username) > 0 && len(opts.Reporter.Password) > 0 {
reporterCfg.User = opts.Reporter.Username
reporterCfg.Password = opts.Reporter.Password
}
cfg := config.Configuration{
ServiceName: opts.ServiceName,
Sampler: sampleCfg,
Reporter: reporterCfg,
}
addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
sender, err := jaeger.NewUDPTransport(addr, 0)
if err != nil {
return nil, err
}
reporter := jaeger.NewRemoteReporter(sender)
tracer, _, err := cfg.NewTracer(
config.Reporter(reporter),
config.Logger(jaegerZap.NewLogger(logger.With(zap.String("type", "tracing")))),
config.MaxTagValueLength(65535),
)
opentracing.SetGlobalTracer(tracer)
return tracer, err
}
var ProviderSet = wire.NewSet(NewOptions, NewTracer)
