fix: Improve telemetry allocations (#2185)

This PR: - moves OTEL counter initialization to a single place - removes startTime and endTime non-counter metrics - introduces duration metric - removes client-level cardinality from the metrics, preventing memory accumulation

Author: murarustefaan

scheduler/metrics/duration.go

@@ -0,0 +1,38 @@
+package metrics
+
+import (
+"sync"
+"time"
+)
+
+type durationMeasurement struct {
+startTime time.Time
+started bool
+duration time.Duration
+sem sync.Mutex
+}
+
+func (dm *durationMeasurement) Start(start time.Time) {
+// If we have already started, don't start again. This can happen for relational tables that are resolved multiple times (per parent resource)
+dm.sem.Lock()
+defer dm.sem.Unlock()
+if dm.started {
+return
+}
+
+dm.started = true
+dm.startTime = start
+}
+
+// End calculates, updates and returns the delta duration for updating OTEL counters.
+func (dm *durationMeasurement) End(end time.Time) time.Duration {
+var delta time.Duration
+newDuration := end.Sub(dm.startTime)
+
+dm.sem.Lock()
+defer dm.sem.Unlock()
+
+delta = newDuration - dm.duration
+dm.duration = newDuration
+return delta
+}