feat: opentelemetry for tracing and metrics

internal/logz/init.go

@@ -0,0 +1,28 @@
+package logz
+
+import (
+	"context"
+	"log"
+
+	"go.uber.org/multierr"
+)
+
+func Init(ctx context.Context, name string) (context.Context, func() error) {
+	stop := [3]func() error{
+		initLogger(name),
+	}
+	ctx, stop[1] = initMetrics(ctx, name)
+	ctx, stop[2] = initTracing(ctx, name)
+
+	reverse(stop[:])
+
+	return ctx, func() error {
+		log.Println("flushing logs...")
+		errs := make([]error, len(stop))
+		for i, fn := range stop {
+			errs[i] = fn()
+		}
+		log.Println("all stopped.")
+		return multierr.Combine(errs...)
+	}
+}

internal/logz/logger.go

@@ -0,0 +1,111 @@
+package logz
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+	"runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/go-logr/stdr"
+	"github.com/logzio/logzio-go"
+	"go.opentelemetry.io/otel"
+)
+
+type logzwriter struct {
+	name      string
+	pkg       string
+	goversion string
+	hostname  string
+
+	w io.Writer
+}
+
+func (l *logzwriter) Write(b []byte) (int, error) {
+	i := 0
+	for _, sp := range bytes.Split(b, []byte("\n")) {
+		msg := struct {
+			Message   string `json:"message"`
+			Host      string `json:"host"`
+			GoVersion string `json:"go_version"`
+			Package   string `json:"pkg"`
+			App       string `json:"app"`
+		}{
+			Message:   strings.TrimSpace(string(sp)),
+			Host:      l.hostname,
+			GoVersion: l.goversion,
+			Package:   l.pkg,
+			App:       l.name,
+		}
+
+		if msg.Message == "" || strings.HasPrefix(msg.Message, "#") {
+			continue
+		}
+
+		b, err := json.Marshal(msg)
+		if err != nil {
+			return 0, err
+		}
+
+		j, err := l.w.Write(b)
+
+		i += j
+		if err != nil {
+			return i, err
+		}
+	}
+	return i, nil
+}
+
+func initLogger(name string) func() error {
+	log.SetPrefix("[" + name + "] ")
+	log.SetFlags(log.LstdFlags&^(log.Ldate|log.Ltime) | log.Lshortfile)
+
+	token := env("LOGZIO_LOG_TOKEN", "")
+	if token == "" {
+		return nil
+	}
+
+	l, err := logzio.New(
+		token,
+		// logzio.SetDebug(os.Stderr),
+		logzio.SetUrl(env("LOGZIO_LOG_URL", "https://listener.logz.io:8071")),
+		logzio.SetDrainDuration(time.Second*5),
+		logzio.SetTempDirectory(env("LOGZIO_DIR", os.TempDir())),
+		logzio.SetCheckDiskSpace(true),
+		logzio.SetDrainDiskThreshold(70),
+	)
+	if err != nil {
+		return nil
+	}
+
+	w := io.MultiWriter(os.Stderr, lzw(l, name))
+	log.SetOutput(w)
+	otel.SetLogger(stdr.New(log.Default()))
+
+	return func() error {
+		defer log.Println("logger stopped")
+		log.SetOutput(os.Stderr)
+		l.Stop()
+		return nil
+	}
+}
+func lzw(l io.Writer, name string) io.Writer {
+	lz := &logzwriter{
+		name: name,
+		w:    l,
+	}
+
+	if info, ok := debug.ReadBuildInfo(); ok {
+		lz.goversion = info.GoVersion
+		lz.pkg = info.Path
+	}
+	if hostname, err := os.Hostname(); err == nil {
+		lz.hostname = hostname
+	}
+
+	return lz
+}

internal/logz/metric.go

@@ -0,0 +1,92 @@
+package logz
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"os"
+	"runtime/debug"
+	"time"
+
+	// metricsExporter "github.com/logzio/go-metrics-sdk"
+	"go.opentelemetry.io/contrib/instrumentation/runtime"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/global"
+	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
+	"go.opentelemetry.io/otel/sdk/metric/controller/basic"
+	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
+	"go.opentelemetry.io/otel/sdk/metric/export/aggregation"
+	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
+	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+)
+
+var meterKey = contextKey{"meter"}
+var promHTTPKey = contextKey{"promHTTP"}
+
+func Meter(ctx context.Context) metric.Meter {
+	if t := fromContext[contextKey, metric.Meter](ctx, tracerKey); t != nil {
+		return t
+	}
+	return global.Meter("")
+}
+func PromHTTP(ctx context.Context) http.Handler {
+	if t := fromContext[contextKey, *prometheus.Exporter](ctx, promHTTPKey); t != nil {
+		return t
+	}
+	return http.NotFoundHandler()
+}
+
+func initMetrics(ctx context.Context, name string) (context.Context, func() error) {
+	goversion := ""
+	pkg := ""
+	host := ""
+	if info, ok := debug.ReadBuildInfo(); ok {
+		goversion = info.GoVersion
+		pkg = info.Path
+	}
+	if h, err := os.Hostname(); err == nil {
+		host = h
+	}
+
+	config := prometheus.Config{}
+	cont := controller.New(
+		processor.NewFactory(
+			selector.NewWithHistogramDistribution(
+				histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries),
+			),
+			aggregation.CumulativeTemporalitySelector(),
+			processor.WithMemory(true),
+		),
+		basic.WithResource(
+			resource.NewWithAttributes(
+				semconv.SchemaURL,
+				attribute.String("app", name),
+				attribute.String("host", host),
+				attribute.String("go_version", goversion),
+				attribute.String("pkg", pkg),
+			),
+		),
+	)
+	ex, err := prometheus.New(config, cont)
+	if err != nil {
+		return ctx, nil
+	}
+
+	ctx = toContext(ctx, promHTTPKey, ex)
+
+	global.SetMeterProvider(cont)
+	m := cont.Meter(name)
+	ctx = toContext(ctx, meterKey, m)
+	runtime.Start()
+
+	return ctx, func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
+		defer log.Println("metrics stopped")
+		return cont.Stop(ctx)
+	}
+}

internal/logz/tracer.go

@@ -0,0 +1,146 @@
+package logz
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"time"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+type contextKey struct {
+	name string
+}
+
+var tracerKey = contextKey{"tracer"}
+
+func Tracer(ctx context.Context) trace.Tracer {
+	if t := fromContext[contextKey, trace.Tracer](ctx, tracerKey); t != nil {
+		return t
+	}
+	return otel.Tracer("")
+}
+
+func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	var attrs []attribute.KeyValue
+	var name string
+	if pc, file, line, ok := runtime.Caller(1); ok {
+		if fn := runtime.FuncForPC(pc); fn != nil {
+			name = fn.Name()
+		}
+		attrs = append(attrs,
+			attribute.String("pc", fmt.Sprintf("%v", pc)),
+			attribute.String("file", file),
+			attribute.Int("line", line),
+			attribute.String("name", name),
+		)
+	}
+	ctx, span := Tracer(ctx).Start(ctx, name, opts...)
+	span.SetAttributes(attrs...)
+
+	return ctx, span
+}
+
+func initTracing(ctx context.Context, name string) (context.Context, func() error) {
+	res, err := resource.New(ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(name),
+		),
+	)
+	if err != nil {
+		log.Println(wrap(err, "failed to create trace resource"))
+		return ctx, nil
+	}
+
+	traceExporter, err := otlptracehttp.New(ctx,
+		otlptracehttp.WithInsecure(),
+		otlptracehttp.WithEndpoint("localhost:4318"),
+	)
+	if err != nil {
+		log.Println(wrap(err, "failed to create trace exporter"))
+		return ctx, nil
+	}
+	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+
+	var sample sdktrace.TracerProviderOption
+	sampleRate := env("LOGZIO_TRACE_SAMPLE", "always")
+	switch sampleRate {
+	case "always":
+		sample = sdktrace.WithSampler(sdktrace.AlwaysSample())
+	case "never":
+		sample = sdktrace.WithSampler(sdktrace.NeverSample())
+	default:
+		if v, err := strconv.Atoi(sampleRate); err != nil {
+			sample = sdktrace.WithSampler(sdktrace.AlwaysSample())
+		} else {
+			sample = sdktrace.WithSampler(sdktrace.TraceIDRatioBased(float64(v) * 0.01))
+		}
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(
+		sample,
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(bsp),
+	)
+	otel.SetTracerProvider(tracerProvider)
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+
+	ctx = toContext(ctx, tracerKey, otel.Tracer(name))
+
+	return ctx, func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
+		defer log.Println("tracer stopped")
+		return wrap(tracerProvider.Shutdown(ctx), "failed to shutdown TracerProvider")
+	}
+}
+
+func wrap(err error, s string) error {
+	if err != nil {
+		return fmt.Errorf(s, err)
+	}
+	return nil
+}
+func reverse[T any](s []T) {
+	first, last := 0, len(s)-1
+	for first < last {
+		s[first], s[last] = s[last], s[first]
+		first++
+		last--
+	}
+}
+func env(name, defaultValue string) string {
+	if v := os.Getenv(name); v != "" {
+		log.Println("# ", name, " = ", v)
+		return v
+	}
+	return defaultValue
+}
+
+func Htrace(h http.Handler, name string) http.Handler {
+	return otelhttp.NewHandler(h, name)
+}
+
+func toContext[K comparable, V any](ctx context.Context, key K, value V) context.Context {
+	return context.WithValue(ctx, key, value)
+}
+func fromContext[K comparable, V any](ctx context.Context, key K) V {
+	var empty V
+	if v, ok := ctx.Value(key).(V); ok {
+		return v
+	}
+	return empty
+}