OpenTelemetry の Log を使いこなそう

Jagu'e'r オブザーバビリティ分科会 Meetup #3
https://jaguer-o11y-sre.connpass.com/event/359593/

1. 4IPUB
   *XBNJ
   ]
   1MBUGPSN
   &OHJOFFS
   ]
   OFXNP
   *OD OpenTelemetry ͷ Log Λ࢖͍͜ͳͦ͏
 Jagu'e'r ΦϒβʔόϏϦςΟ෼Պձ Meetup #3 1
2. ؠݟজଠ
   
   *XBNJO
   Platform Engineer newmo, Inc. 2
3. ؠݟজଠ
   
   *XBNJO
   Platform Engineer Autonomous Driving Vehicle Team? newmo, Inc. 3
4. Software Design 5݄߸ 4 4 • ͸͡Ίʹ • OpenTelemetryͱ͸ • OpenTelemetryͷجຊΞʔΩςΫνϟ • αϯϓϦϯάઓུͱ࣮૷ • ܭ૷ͷछྨͱબ୒ج४ • खಈ/ϥΠϒϥϦ/ࣗಈܭ૷ • TraceProvider • MetricProvider • LoggerProvider • ·ͱΊ 4 “OpenTelemetryೖ໳” Λدߘ͠·ͨ͠
5. Software Design 5݄߸ 5 “OpenTelemetryೖ໳” Λدߘ͠·ͨ͠ 5 • ͸͡Ίʹ • OpenTelemetryͱ͸ • OpenTelemetryͷجຊΞʔΩςΫνϟ • αϯϓϦϯάઓུͱ࣮૷ • ܭ૷ͷछྨͱબ୒ج४ • खಈ/ϥΠϒϥϦ/ࣗಈܭ૷ • TraceProvider • MetricProvider • LoggerProvider • ·ͱΊ 5
6. LoggerProviderͷར༻ 6
7. Logger Provider 7 7 • Trace, Metricsͱ͸গ͠ੑ࣭͕ҧ͏ • طଘͷϩΪϯάϥΠϒϥϦͱ Otel Λ࿈ܞͤ͞ΔͨΊͷ Bridge ػೳΛఏڙ • ޓ׵ੑΛอͪͳ͕Β Trace ͱඥ෇͚Δ Context ͷ௥Ճ΍ڞ௨ϑΥʔϚοτʹղੳɾม׵͢Δͨ ΊͷπʔϧΩοτΛఏڙ 7
8. Logger Provider 8 8 • Trace, Metricsͱ͸গ͠ੑ࣭͕ҧ͏ • طଘͷϩΪϯάϥΠϒϥϦͱ Otel Λ࿈ܞͤ͞ΔͨΊͷ Bridge ػೳΛఏڙ • ޓ׵ੑΛอͪͳ͕Β Trace ͱඥ෇͚Δ Context ͷ௥Ճ΍ڞ௨ϑΥʔϚοτʹղੳɾม׵͢Δͨ ΊͷπʔϧΩοτΛఏڙ 8
9. Otel log bridge Λ࢖Θͳ͍৔߹ 9
10. 10 func (s *slogJSONHandler) Handle( ctx context.Context, r slog.Record, ) error { newRecord := slog.NewRecord(…) ... sc := trace.SpanContextFromContext(ctx) if sc.HasTraceID() { newRecord.AddAttrs( slog.String("trace_id", sc.TraceID().String()), slog.String("trace_flags", sc.TraceFlags().String()), ) if sc.HasSpanID() { newRecord.AddAttrs(slog.String("span_id", sc.SpanID().String())) } } return s.handler.Handle(ctx, newRecord) } Bridge ౳Λ࢖Θͣʹ trace ͱඥ͚ͮΔ৔߹ • Log Handler ʹcontext ͔Β trace id ౳Λऔಘ • ࣗલͰ attribute ʹ௥Ճ
11. 11 func (s *slogJSONHandler) Handle( ctx context.Context, r slog.Record, ) error { newRecord := slog.NewRecord(…) ... sc := trace.SpanContextFromContext(ctx) if sc.HasTraceID() { newRecord.AddAttrs( slog.String("trace_id", sc.TraceID().String()), slog.String("trace_flags", sc.TraceFlags().String()), ) if sc.HasSpanID() { newRecord.AddAttrs(slog.String("span_id", sc.SpanID().String())) } } return s.handler.Handle(ctx, newRecord) } Bridge ౳Λ࢖Θͣʹ trace ͱඥ͚ͮΔ৔߹ • Log Handler ʹcontext ͔Β trace id ౳Λऔಘ • ࣗલͰ attribute ʹ௥Ճ
12. 12 func (s *slogJSONHandler) Handle( ctx context.Context, r slog.Record, ) error { newRecord := slog.NewRecord(…) ... sc := trace.SpanContextFromContext(ctx) if sc.HasTraceID() { newRecord.AddAttrs( slog.String("trace_id", sc.TraceID().String()), slog.String("trace_flags", sc.TraceFlags().String()), ) if sc.HasSpanID() { newRecord.AddAttrs(slog.String("span_id", sc.SpanID().String())) } } return s.handler.Handle(ctx, newRecord) } Bridge ౳Λ࢖Θͣʹ trace ͱඥ͚ͮΔ৔߹ • Log Handler ʹcontext ͔Β trace id ౳Λऔಘ • ࣗલͰ attribute ʹ௥Ճ
13. Otel log bridge Λ࢖͏৔߹ 13
14. lp := initLoggerProvider(ctx) // slogͱOpenTelemetryͷ࿈ܞ logger := otelslog.NewLogger("my-service", otelslog.WithLoggerProvider(lp)) // τϨʔεͱͷ࿈ܞྫ tracer := otel.Tracer("my-service") ctx, span := tracer.Start(ctx, "operation-name") defer span.End() // contextΛ౉͢͜ͱͰɺtrace৘ใΛ఻ൖ͍ͯ͠Δ logger.InfoContext(ctx, "Start processing", "user_id", "12345", ) OpenTelemetry Logger Provider for Go 14 • ීஈ࢖͍ͬͯΔ slog Λ wrap ͢Δ͚ͩ
15. lp := initLoggerProvider(ctx) // slogͱOpenTelemetryͷ࿈ܞ logger := otelslog.NewLogger("my-service", otelslog.WithLoggerProvider(lp)) // τϨʔεͱͷ࿈ܞྫ tracer := otel.Tracer("my-service") ctx, span := tracer.Start(ctx, "operation-name") defer span.End() // contextΛ౉͢͜ͱͰɺtrace৘ใΛ఻ൖ͍ͯ͠Δ logger.InfoContext(ctx, "Start processing", "user_id", "12345", ) OpenTelemetry Logger Provider for Go 15 • ීஈ࢖͍ͬͯΔ slog Λ wrap func initLoggerProvider(ctx context.Context) (*sdklog.LoggerProvider, error) { // ΤΫεϙʔλʔͷઃఆ exporter, _ := otlploggrpc.New(ctx, otlploggrpc.WithEndpoint("localhost:4317"), otlploggrpc.WithInsecure(), ) // LoggerProviderͷઃఆ loggerProvider := sdklog.NewLoggerProvider( sdklog.WithProcessor( sdklog.NewBatchProcessor( exporter, ), ), ) return loggerProvider, nil }
16. Logger Provider 16 16 16
17. Logger Provider 17 17 17
18. Logger Provider 18 18 18
19. Logger Provider 19 19 19
20. Logger Provider 20 20 20
21. Logger Provider 21 21 21
22. lp := initLoggerProvider(ctx) // slogͱOpenTelemetryͷ࿈ܞ logger := otelslog.NewLogger("my-service", otelslog.WithLoggerProvider(lp)) // τϨʔεͱͷ࿈ܞྫ tracer := otel.Tracer("my-service") ctx, span := tracer.Start(ctx, "operation-name") defer span.End() // contextΛ౉͢͜ͱͰɺtrace৘ใΛ఻ൖ͍ͯ͠Δ logger.InfoContext(ctx, "Start processing", "user_id", "12345", ) OpenTelemetry Logger Provider for Go 22 • ීஈ࢖͍ͬͯΔ slog Λ wrap ͢Δ͚ͩ ͳΜ͔͍͍ײ͡ʹͯ͘͠ΕΔ ͲͷΑ͏ͳ࢓૊Έʁ
23. Logger Provider ͷ ઃܭͱ಺෦࣮૷ 23
24. Logger Provider ͷઃܭͱ಺෦࣮૷ 24 24 24
25. Logger Provider ͷઃܭͱ಺෦࣮૷ 25 25 25
26. 26 // ΞϓϦέʔγϣϯ logger.InfoContext(ctx, "ॲཧ։࢝", "user_id", 123) // otelslog.Handler.Handle() func (h *handler) Handle(ctx context.Context, r slog.Record) error { // slog.Record Λ log.Record ʹม׵ record := convert.LogRecord(r) // ಺෦ͷOTel Loggerʹసૹ h.logger.Emit(ctx, record) return nil } otelslog Bridge
27. 27 // ΞϓϦέʔγϣϯ logger.InfoContext(ctx, "ॲཧ։࢝", "user_id", 123) // otelslog.Handler.Handle() func (h *handler) Handle(ctx context.Context, r slog.Record) error { // slog.Record Λ log.Record ʹม׵ record := convert.LogRecord(r) // ಺෦ͷOTel Loggerʹసૹ h.logger.Emit(ctx, record) return nil } otelslog Bridge slog logܗ͔ࣜΒ otel logܗࣜʹ Convert
28. 28 // ΞϓϦέʔγϣϯ logger.InfoContext(ctx, "ॲཧ։࢝", "user_id", 123) // otelslog.Handler.Handle() func (h *handler) Handle(ctx context.Context, r slog.Record) error { // slog.Record Λ log.Record ʹม׵ record := convert.LogRecord(r) // ಺෦ͷOTel Loggerʹసૹ h.logger.Emit(ctx, record) return nil } otelslog Bridge otel logger ͷ Emit Λ࣮ߦ
29. Logger Provider ͷઃܭͱ಺෦࣮૷ 29 29 29
30. 30 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK
31. 31 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK ৽ن Record ࡞੒
32. 32 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK Context ͔Β Trace ID / Span ID Λ औಘͯ͠ Record ʹຒΊࠐΉ
33. 33 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK Processor Λ ॱ࣮࣍ߦ
34. 34 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK Processor Λ ॱ࣮࣍ߦ
35. 35 func (l *logger) Emit(ctx context.Context, r log.Record) { newRecord := l.newRecord(ctx, r) for _, p := range l.provider.processors { p.OnEmit(ctx, &newRecord) } } func (l *logger) newRecord(ctx context.Context, r log.Record) Record { sc := trace.SpanContextFromContext(ctx) return Record{ body: r.Body(), severity: r.Severity(), timestamp: r.Timestamp(), traceID: sc.TraceID(), spanID: sc.SpanID(), traceFlags: sc.TraceFlags(), resource: l.provider.resource, scope: &l.instrumentationScope, } } Otel SDK Processor Λ ॱ࣮࣍ߦ
36. 36 // SimpleProcessor func (s *SimpleProcessor) OnEmit(ctx context.Context, r *Record) error { records := []*Record{r} return s.exporter.Export(ctx, records) } // BatchProcessor func (b *BatchProcessor) OnEmit(ctx context.Context, r *Record) error { b.queue.Enqueue(r.Clone()) return nil } Processor
37. 37 // SimpleProcessor func (s *SimpleProcessor) OnEmit(ctx context.Context, r *Record) error { records := []*Record{r} return s.exporter.Export(ctx, records) } // BatchProcessor func (b *BatchProcessor) OnEmit(ctx context.Context, r *Record) error { b.queue.Enqueue(r.Clone()) return nil } Processor SimpleProcessor ଈ࠲ʹ Export
38. 38 // SimpleProcessor func (s *SimpleProcessor) OnEmit(ctx context.Context, r *Record) error { records := []*Record{r} return s.exporter.Export(ctx, records) } // BatchProcessor func (b *BatchProcessor) OnEmit(ctx context.Context, r *Record) error { b.queue.Enqueue(r.Clone()) return nil } Processor BatchProcessor όοναΠζʹୡͨ͠Β ผͷ Goroutine Ͱ Export
39. Logger Provider ͷઃܭͱ಺෦࣮૷ 39 39 39
40. Logger Provider ͷઃܭͱ಺෦࣮૷ 40 40 40 local Ͱ͸ otlp Ͱग़ྗͯ͠ o11y πʔϧͰ trace ͱඥ͚ͮͯ֬ೝ dev/stg/prod Ͱ͸ඪ४ग़ྗͰग़ͯ͠ӬଓԽͭͭ͠ϩάΛpipelineͰྲྀͯ֬͠ೝ
41. Logger Provider ͷઃܭͱ಺෦࣮૷ 41 41 41
42. Logger Provider ͷઃܭͱ಺෦࣮૷ 42 42 42 ͍ͭ΋௨Γ
43. Logger Provider ͷઃܭͱ಺෦࣮૷ 43 43 43 ͍ͭ΋௨Γ ࢖ͬͯΔLogͷܗࣜΛ OtelLogʹม׵
44. Logger Provider ͷઃܭͱ಺෦࣮૷ 44 44 44 ͍ͭ΋௨Γ ࢖ͬͯΔLogͷܗࣜΛ OtelLogʹม׵ Trace ৘ใͷநग़ ೚ҙͷ Export
45. Logger Provider ͷઃܭͱ಺෦࣮૷ 45 45 45 ͍ͭ΋௨Γ ࢖ͬͯΔLogͷܗࣜΛ OtelLogʹม׵ Trace ৘ใͷநग़ ೚ҙͷ Export ؀ڥʹԠͨ֬͡ೝ
46. Logger Provider ͷઃܭͱ಺෦࣮૷ 46 46 46 ͍ͭ΋௨Γ ࢖ͬͯΔLogͷܗࣜΛ OtelLogʹม׵ Trace ৘ใͷநग़ ೚ҙͷ Export ؀ڥʹԠͨ֬͡ೝ ͜Ε͚ͩͰ৭ʑ΍ͬͯ͘ΕΔ
47. ·ͱΊ 47
48. • ϩάͱτϨʔεͷඥ෇͚Λউखʹ΍ͬͯ͘ΕΔ • Bridge ʹΑͬͯطଘͷϩάϥΠϒϥϦͷࠩ෼Λٵऩ • Local Ͱ OTLP Ͱు͖ग़͢͜ͱͰɺීஈͷϩʔΧϧ։ൃͰ΋ϩάͱ τϨʔεΛඥ͍ͮͯΈΕΔ • ೚ҙͷม׵ॲཧΛ processor ͱ࣮ͯ͠૷Ͱ͖Δ ·ͱΊ 48 48 ศརػೳ͕੝Γͩ͘͞ΜʂOpenTelemetry Logger Λ࢖͍͜ͳͦ͏ʂ