Mobile crash support (#191)

* Traversing logs * Renaming files * Changing package name * Reorganizing files * Reorganizing files * Adding test case for crash events * Making mobile enricher internal * Moving resource enricher * Passing resource config to resource enrichment * Enriching log resources * Extracting attribute config to an independent package to get reused * Using resource and attribute configs for logs * Testing log resource enrichment * Adding timestamp.us verification * Extracting getting timestamp.us tool * Adding timestamp.us to crash events * Moving attributeconfig to elasticattr * Adding error.id * Improving event tests * Updatig tests * Validating error.id * Adding java stacktrace testdata * Adding groupingkey go files * Adding stacktrace without message to testdata * Creating test for curating stacktraces * Curating stacktraces * Validating curating different types of stacktraces * Updating tests * Creating grouping key for java stacktraces * Updating tests * Adding grouping_key to crash events * Adding error.type crash to crash events * Using camelcase for variable names * Updating tests * Using observerdtimestamp when timestamp is not available * Adding validation for non-crash events * Validating only events are sent to the event enricher * Updating stacktrace test data * Using elasticattr attributes * Adding license headers * Update enrichments/logs/internal/mobile/groupingkey.go Co-authored-by: Felix Barnsteiner <felixbarny@users.noreply.github.com> * Keeping cause exception name * Removing stacktrace line numbers * Updating groupingkey tests * Updating event tests * Replacing sha256 by xxhash * Replace SHA-256 with xxHash and optimize regex patterns in groupingkey.go * Update test expectations in event_test.go for xxHash output * Make error grouping key conditional on Java language * Addressing struct pointer lint issue * Avoiding creating new map instances on each loop iteration * Move Android stacktrace test data to dedicated subfolder * Adding ios crash test files * Add Swift stack trace grouping key function using xxHash for iOS crashes * Adding testdata curated swift stacktraces * Removing unicode chars from ios stacktraces * removing unicode chars * Creating swift stacktrace groupingkey * Add Swift error grouping key support to enrichCrashEvent function * Renaming function * Add test case for Swift error grouping key support * Ensuring swift grouping keys don't contain the thread number * Updating swift curated stacktrace test files * Removing frame and image load addresses from swift stacktraces * Moving mobile enrichments to enricher * Clean up * Moving GetTimestampUs out of elasticattr --------- Co-authored-by: Felix Barnsteiner <felixbarny@users.noreply.github.com>

Author: LikeTheSalad

elasticattr/attributes.go

@@ -41,6 +41,7 @@ const (
 SpanType = "span.type"
 SpanSubtype = "span.subtype"
 EventOutcome = "event.outcome"
+EventKind = "event.kind"
 SuccessCount = "event.success_count"
 ServiceTargetType = "service.target.type"
 ServiceTargetName = "service.target.name"
@@ -55,4 +56,5 @@ const (
 ErrorExceptionHandled = "error.exception.handled"
 ErrorGroupingKey = "error.grouping_key"
 ErrorGroupingName = "error.grouping_name"
+ErrorType = "error.type"
 )

enrichments/enricher.go

@@ -44,7 +44,7 @@ func (e *Enricher) EnrichTraces(pt ptrace.Traces) {
 resSpans := pt.ResourceSpans()
 for i := 0; i < resSpans.Len(); i++ {
 resSpan := resSpans.At(i)
-elastic.EnrichResource(resSpan.Resource(), e.Config)
+elastic.EnrichResource(resSpan.Resource(), e.Config.Resource)
 scopeSpans := resSpan.ScopeSpans()
 for j := 0; j < scopeSpans.Len(); j++ {
 scopeSpan := scopeSpans.At(j)
@@ -63,14 +63,16 @@ func (e *Enricher) EnrichLogs(pl plog.Logs) {
 resLogs := pl.ResourceLogs()
 for i := 0; i < resLogs.Len(); i++ {
 resLog := resLogs.At(i)
-elastic.EnrichResource(resLog.Resource(), e.Config)
+resource := resLog.Resource()
+elastic.EnrichResource(resource, e.Config.Resource)
+resourceAttrs := resource.Attributes().AsRaw()
 scopeLogs := resLog.ScopeLogs()
 for j := 0; j < scopeLogs.Len(); j++ {
 scopeSpan := scopeLogs.At(j)
 elastic.EnrichScope(scopeSpan.Scope(), e.Config)
 logRecords := scopeSpan.LogRecords()
 for k := 0; k < logRecords.Len(); k++ {
-elastic.EnrichLog(logRecords.At(k), e.Config)
+elastic.EnrichLog(resourceAttrs, logRecords.At(k), e.Config)
 }
 }
 }
@@ -83,7 +85,7 @@ func (e *Enricher) EnrichMetrics(pl pmetric.Metrics) {
 for i := 0; i < resMetrics.Len(); i++ {
 resMetric := resMetrics.At(i)
 elastic.EnrichMetric(resMetric, e.Config)
-elastic.EnrichResource(resMetric.Resource(), e.Config)
+elastic.EnrichResource(resMetric.Resource(), e.Config.Resource)
 scopeMetics := resMetric.ScopeMetrics()
 for j := 0; j < scopeMetics.Len(); j++ {
 scopeMetric := scopeMetics.At(j)

enrichments/internal/elastic/log.go

@@ -20,13 +20,35 @@ package elastic
 import (
 "github.com/elastic/opentelemetry-lib/elasticattr"
 "github.com/elastic/opentelemetry-lib/enrichments/config"
+"github.com/elastic/opentelemetry-lib/enrichments/internal/elastic/mobile"
 "go.opentelemetry.io/collector/pdata/plog"
 )
 
-func EnrichLog(log plog.LogRecord, cfg config.Config) {
+func EnrichLog(resourceAttrs map[string]any, log plog.LogRecord, cfg config.Config) {
 if cfg.Log.ProcessorEvent.Enabled {
 if _, exists := log.Attributes().Get(elasticattr.ProcessorEvent); !exists {
 log.Attributes().PutStr(elasticattr.ProcessorEvent, "log")
 }
 }
+eventName, ok := getEventName(log)
+if ok {
+ctx := mobile.EventContext{
+ResourceAttributes: resourceAttrs,
+EventName: eventName,
+}
+mobile.EnrichLogEvent(ctx, log)
+}
+}
+
+// getEventName returns the event name from the log record.
+// If the event name is not set, it returns an empty string.
+func getEventName(logRecord plog.LogRecord) (string, bool) {
+if logRecord.EventName() != "" {
+return logRecord.EventName(), true
+}
+attributeValue, ok := logRecord.Attributes().Get("event.name")
+if ok {
+return attributeValue.AsString(), true
+}
+return "", false
 }

enrichments/internal/elastic/mobile/event.go

@@ -0,0 +1,86 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mobile
+
+import (
+"crypto/rand"
+"encoding/hex"
+"io"
+
+"github.com/elastic/opentelemetry-lib/elasticattr"
+"go.opentelemetry.io/collector/pdata/pcommon"
+"go.opentelemetry.io/collector/pdata/plog"
+)
+
+// EventContext contains contextual information for log event enrichment
+type EventContext struct {
+ResourceAttributes map[string]any
+EventName string
+}
+
+func EnrichLogEvent(ctx EventContext, logRecord plog.LogRecord) {
+logRecord.Attributes().PutStr(elasticattr.EventKind, "event")
+
+if ctx.EventName == "device.crash" {
+enrichCrashEvent(logRecord, ctx.ResourceAttributes)
+}
+}
+
+func enrichCrashEvent(logRecord plog.LogRecord, resourceAttrs map[string]any) {
+timestamp := logRecord.Timestamp()
+if timestamp == 0 {
+timestamp = logRecord.ObservedTimestamp()
+}
+logRecord.Attributes().PutStr(elasticattr.ProcessorEvent, "error")
+logRecord.Attributes().PutInt(elasticattr.TimestampUs, getTimestampUs(timestamp))
+if id, err := newUniqueID(); err == nil {
+logRecord.Attributes().PutStr(elasticattr.ErrorID, id)
+}
+stacktrace, ok := logRecord.Attributes().Get("exception.stacktrace")
+if ok {
+language, hasLanguage := resourceAttrs["telemetry.sdk.language"]
+if hasLanguage {
+switch language {
+case "java":
+logRecord.Attributes().PutStr(elasticattr.ErrorGroupingKey, CreateJavaStacktraceGroupingKey(stacktrace.AsString()))
+case "swift":
+if key, err := CreateSwiftStacktraceGroupingKey(stacktrace.AsString()); err == nil {
+logRecord.Attributes().PutStr(elasticattr.ErrorGroupingKey, key)
+}
+}
+}
+}
+logRecord.Attributes().PutStr(elasticattr.ErrorType, "crash")
+}
+
+func newUniqueID() (string, error) {
+var u [16]byte
+if _, err := io.ReadFull(rand.Reader, u[:]); err != nil {
+return "", err
+}
+
+// convert to string
+buf := make([]byte, 32)
+hex.Encode(buf, u[:])
+
+return string(buf), nil
+}
+
+func getTimestampUs(ts pcommon.Timestamp) int64 {
+return int64(ts) / 1000
+}

enrichments/internal/elastic/mobile/event_test.go

@@ -0,0 +1,183 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mobile
+
+import (
+"testing"
+"time"
+
+"maps"
+
+"github.com/google/go-cmp/cmp"
+"github.com/stretchr/testify/assert"
+"go.opentelemetry.io/collector/pdata/pcommon"
+"go.opentelemetry.io/collector/pdata/plog"
+)
+
+func TestEnrichEvents(t *testing.T) {
+now := time.Unix(3600, 0)
+timestamp := pcommon.NewTimestampFromTime(now)
+javaStacktrace := "Exception in thread \"main\" java.lang.RuntimeException: Test exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at com.example.GenerateTrace.main(GenerateTrace.java:5)"
+javaStacktraceHash := "e25c4196dc720d91"
+
+swiftStacktrace := readSwiftStacktraceFile(t, "thread-8-crash.txt")
+swiftStacktraceHash := "e737b0da1c8f9d5a"
+
+for _, tc := range []struct {
+name string
+eventName string
+input func() plog.LogRecord
+resourceAttrs map[string]any
+expectedAttributes map[string]any
+}{
+{
+name: "crash_event_java",
+eventName: "device.crash",
+resourceAttrs: map[string]any{
+"telemetry.sdk.language": "java",
+},
+input: func() plog.LogRecord {
+logRecord := plog.NewLogRecord()
+logRecord.SetTimestamp(timestamp)
+logRecord.Attributes().PutStr("event.name", "device.crash")
+logRecord.Attributes().PutStr("exception.message", "Exception message")
+logRecord.Attributes().PutStr("exception.type", "java.lang.RuntimeException")
+logRecord.Attributes().PutStr("exception.stacktrace", javaStacktrace)
+return logRecord
+},
+expectedAttributes: map[string]any{
+"processor.event": "error",
+"timestamp.us": timestamp.AsTime().UnixMicro(),
+"error.grouping_key": javaStacktraceHash,
+"error.type": "crash",
+"event.kind": "event",
+},
+},
+{
+name: "crash_event_without_timestamp_java",
+eventName: "device.crash",
+resourceAttrs: map[string]any{
+"telemetry.sdk.language": "java",
+},
+input: func() plog.LogRecord {
+logRecord := plog.NewLogRecord()
+logRecord.SetObservedTimestamp(timestamp)
+logRecord.Attributes().PutStr("event.name", "device.crash")
+logRecord.Attributes().PutStr("exception.message", "Exception message")
+logRecord.Attributes().PutStr("exception.type", "java.lang.RuntimeException")
+logRecord.Attributes().PutStr("exception.stacktrace", javaStacktrace)
+return logRecord
+},
+expectedAttributes: map[string]any{
+"processor.event": "error",
+"timestamp.us": timestamp.AsTime().UnixMicro(),
+"error.grouping_key": javaStacktraceHash,
+"error.type": "crash",
+"event.kind": "event",
+},
+},
+{
+name: "crash_event_non_java",
+eventName: "device.crash",
+resourceAttrs: map[string]any{
+"telemetry.sdk.language": "go",
+},
+input: func() plog.LogRecord {
+logRecord := plog.NewLogRecord()
+logRecord.SetTimestamp(timestamp)
+logRecord.Attributes().PutStr("event.name", "device.crash")
+logRecord.Attributes().PutStr("exception.message", "Exception message")
+logRecord.Attributes().PutStr("exception.type", "go.error")
+logRecord.Attributes().PutStr("exception.stacktrace", javaStacktrace)
+return logRecord
+},
+expectedAttributes: map[string]any{
+"processor.event": "error",
+"timestamp.us": timestamp.AsTime().UnixMicro(),
+"error.type": "crash",
+"event.kind": "event",
+},
+},
+{
+name: "non_crash_event",
+eventName: "othername",
+resourceAttrs: map[string]any{},
+input: func() plog.LogRecord {
+logRecord := plog.NewLogRecord()
+logRecord.Attributes().PutStr("event.name", "othername")
+return logRecord
+},
+expectedAttributes: map[string]any{
+"event.kind": "event",
+},
+},
+{
+name: "crash_event_swift",
+eventName: "device.crash",
+resourceAttrs: map[string]any{
+"telemetry.sdk.language": "swift",
+},
+input: func() plog.LogRecord {
+logRecord := plog.NewLogRecord()
+logRecord.SetTimestamp(timestamp)
+logRecord.Attributes().PutStr("event.name", "device.crash")
+logRecord.Attributes().PutStr("exception.type", "SIGTRAP")
+logRecord.Attributes().PutStr("exception.stacktrace", swiftStacktrace)
+return logRecord
+},
+expectedAttributes: map[string]any{
+"processor.event": "error",
+"timestamp.us": timestamp.AsTime().UnixMicro(),
+"error.grouping_key": swiftStacktraceHash,
+"error.type": "crash",
+"event.kind": "event",
+},
+},
+} {
+t.Run(tc.name, func(t *testing.T) {
+inputLogRecord := tc.input()
+
+maps.Copy(tc.expectedAttributes, inputLogRecord.Attributes().AsRaw())
+
+ctx := EventContext{
+ResourceAttributes: tc.resourceAttrs,
+EventName: tc.eventName,
+}
+EnrichLogEvent(ctx, inputLogRecord)
+
+assert.Empty(t, cmp.Diff(inputLogRecord.Attributes().AsRaw(), tc.expectedAttributes, ignoreMapKey("error.id")))
+errorId, ok := inputLogRecord.Attributes().Get("error.id")
+if ok {
+assert.Equal(t, "device.crash", tc.eventName)
+assert.Equal(t, 32, len(errorId.AsString()))
+} else {
+assert.NotEqual(t, "device.crash", tc.eventName)
+}
+})
+}
+}
+
+func ignoreMapKey(k string) cmp.Option {
+return cmp.FilterPath(func(p cmp.Path) bool {
+mapIndex, ok := p.Last().(cmp.MapIndex)
+if !ok {
+return false
+}
+return mapIndex.Key().String() == k
+}, cmp.Ignore())
+}