From b453f26cf9ef057d6ade90782e81b93c7665bd2c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 27 Feb 2026 16:15:56 -0500 Subject: [PATCH 1/4] Reducing memory allocation from client-side stats Introduced DDCache-s around UTF8BytesString construction UTF8BytesString are advantageous for serialization, but that only applies to the key instance that is actually serialized Most key instances here are being created to do a look-up into the map. so the UTF8BytesString creation was extra work. This change is intended as a quick fix. I think there's still a lot of room for improvement by restructuring the code further, but that is left for a future PR. As is this changer reduces the impact on application throughput by 5-20% depending on the heap size. --- .../trace/common/metrics/MetricKey.java | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java index e10c081035c..0313f2e0ddd 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java @@ -1,15 +1,25 @@ package datadog.trace.common.metrics; -import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; - +import datadog.trace.api.cache.DDCaches; +import datadog.trace.api.cache.DDCache; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.HashingUtils; + import java.util.Collections; import java.util.List; import java.util.Objects; /** The aggregation key for tracked metrics. */ public final class MetricKey { + static final DDCache RESOURCE_CACHE = DDCaches.newFixedSizeCache(32); + static final DDCache SERVICE_CACHE = DDCaches.newFixedSizeCache(8); + static final DDCache SERVICE_SOURCE_CACHE = DDCaches.newFixedSizeCache(4); + static final DDCache OPERATION_CACHE = DDCaches.newFixedSizeCache(64); + static final DDCache TYPE_CACHE = DDCaches.newFixedSizeCache(8); + static final DDCache KIND_CACHE = DDCaches.newFixedSizeCache(8); + static final DDCache HTTP_METHOD_CACHE = DDCaches.newFixedSizeCache(8); + static final DDCache HTTP_ENDPOINT_CACHE = DDCaches.newFixedSizeCache(32); + private final UTF8BytesString resource; private final UTF8BytesString service; private final UTF8BytesString serviceSource; @@ -37,19 +47,19 @@ public MetricKey( List peerTags, CharSequence httpMethod, CharSequence httpEndpoint) { - this.resource = null == resource ? EMPTY : UTF8BytesString.create(resource); - this.service = null == service ? EMPTY : UTF8BytesString.create(service); - this.serviceSource = null == serviceSource ? null : UTF8BytesString.create(serviceSource); - this.operationName = null == operationName ? EMPTY : UTF8BytesString.create(operationName); - this.type = null == type ? EMPTY : UTF8BytesString.create(type); + this.resource = utf8(RESOURCE_CACHE, resource); + this.service = utf8(SERVICE_CACHE, service); + this.serviceSource = utf8(SERVICE_SOURCE_CACHE, serviceSource); + this.operationName = utf8(OPERATION_CACHE, operationName); + this.type = utf8(TYPE_CACHE, type); this.httpStatusCode = httpStatusCode; this.synthetics = synthetics; this.isTraceRoot = isTraceRoot; - this.spanKind = null == spanKind ? EMPTY : UTF8BytesString.create(spanKind); + this.spanKind = utf8(KIND_CACHE, spanKind); this.peerTags = peerTags == null ? Collections.emptyList() : peerTags; - this.httpMethod = httpMethod == null ? null : UTF8BytesString.create(httpMethod); - this.httpEndpoint = httpEndpoint == null ? null : UTF8BytesString.create(httpEndpoint); - + this.httpMethod = utf8(HTTP_METHOD_CACHE, httpMethod); + this.httpEndpoint = utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); + int tmpHash = 0; tmpHash = HashingUtils.addToHash(tmpHash, this.isTraceRoot); tmpHash = HashingUtils.addToHash(tmpHash, this.spanKind); @@ -65,6 +75,16 @@ public MetricKey( tmpHash = HashingUtils.addToHash(tmpHash, this.httpMethod); this.hash = tmpHash; } + + static UTF8BytesString utf8(DDCache cache, CharSequence charSeq) { + if ( charSeq == null ) { + return UTF8BytesString.EMPTY; + } else if ( charSeq instanceof UTF8BytesString ) { + return (UTF8BytesString)charSeq; + } else { + return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); + } + } public UTF8BytesString getResource() { return resource; From bd6a4188075b1b02c0932ed3d1a55034f031b7bd Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 9 Mar 2026 13:19:46 -0400 Subject: [PATCH 2/4] Addressing review feedback - fix null handling for http method & endpoint - increase service source cache size --- .../main/java/datadog/trace/common/metrics/MetricKey.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java index 0313f2e0ddd..484a72a36a4 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java @@ -13,7 +13,7 @@ public final class MetricKey { static final DDCache RESOURCE_CACHE = DDCaches.newFixedSizeCache(32); static final DDCache SERVICE_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache SERVICE_SOURCE_CACHE = DDCaches.newFixedSizeCache(4); + static final DDCache SERVICE_SOURCE_CACHE = DDCaches.newFixedSizeCache(16); static final DDCache OPERATION_CACHE = DDCaches.newFixedSizeCache(64); static final DDCache TYPE_CACHE = DDCaches.newFixedSizeCache(8); static final DDCache KIND_CACHE = DDCaches.newFixedSizeCache(8); @@ -57,8 +57,8 @@ public MetricKey( this.isTraceRoot = isTraceRoot; this.spanKind = utf8(KIND_CACHE, spanKind); this.peerTags = peerTags == null ? Collections.emptyList() : peerTags; - this.httpMethod = utf8(HTTP_METHOD_CACHE, httpMethod); - this.httpEndpoint = utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); + this.httpMethod = httpMethod == null ? null : utf8(HTTP_METHOD_CACHE, httpMethod); + this.httpEndpoint = httpEndpoint == null ? null : utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); int tmpHash = 0; tmpHash = HashingUtils.addToHash(tmpHash, this.isTraceRoot); From 123ed377351beff96cf69e4154ee4eeb18dc1b5a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 9 Mar 2026 15:11:08 -0400 Subject: [PATCH 3/4] Fixing null handling Since it about a 50/50 split between cases that use EMPTY vs null, when a null is passed I decided to just put the null handling back into the constructor. That makes the choice more explicit, and makes the PR easier to review / compare to the prior logic --- .../datadog/trace/common/metrics/MetricKey.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java index 484a72a36a4..fe6585fc78a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java @@ -1,5 +1,7 @@ package datadog.trace.common.metrics; +import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; + import datadog.trace.api.cache.DDCaches; import datadog.trace.api.cache.DDCache; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -47,15 +49,15 @@ public MetricKey( List peerTags, CharSequence httpMethod, CharSequence httpEndpoint) { - this.resource = utf8(RESOURCE_CACHE, resource); - this.service = utf8(SERVICE_CACHE, service); - this.serviceSource = utf8(SERVICE_SOURCE_CACHE, serviceSource); - this.operationName = utf8(OPERATION_CACHE, operationName); + this.resource = resource == null ? EMPTY : utf8(RESOURCE_CACHE, resource); + this.service = service == null ? EMPTY : utf8(SERVICE_CACHE, service); + this.serviceSource = serviceSource == null ? null : utf8(SERVICE_SOURCE_CACHE, serviceSource); + this.operationName = operationName == null ? EMPTY : utf8(OPERATION_CACHE, operationName); this.type = utf8(TYPE_CACHE, type); this.httpStatusCode = httpStatusCode; this.synthetics = synthetics; this.isTraceRoot = isTraceRoot; - this.spanKind = utf8(KIND_CACHE, spanKind); + this.spanKind = null == spanKind ? EMPTY : utf8(KIND_CACHE, spanKind); this.peerTags = peerTags == null ? Collections.emptyList() : peerTags; this.httpMethod = httpMethod == null ? null : utf8(HTTP_METHOD_CACHE, httpMethod); this.httpEndpoint = httpEndpoint == null ? null : utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); @@ -77,9 +79,7 @@ public MetricKey( } static UTF8BytesString utf8(DDCache cache, CharSequence charSeq) { - if ( charSeq == null ) { - return UTF8BytesString.EMPTY; - } else if ( charSeq instanceof UTF8BytesString ) { + if ( charSeq instanceof UTF8BytesString ) { return (UTF8BytesString)charSeq; } else { return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); From 8f5255de5c1f4345100d99d9d0cd8bf1ce4dd349 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 9 Mar 2026 15:42:52 -0400 Subject: [PATCH 4/4] Cleaning up null handling - reducing diff size --- .../trace/common/metrics/MetricKey.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java index fe6585fc78a..0f8147faefa 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java @@ -2,11 +2,10 @@ import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; -import datadog.trace.api.cache.DDCaches; import datadog.trace.api.cache.DDCache; +import datadog.trace.api.cache.DDCaches; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.HashingUtils; - import java.util.Collections; import java.util.List; import java.util.Objects; @@ -15,13 +14,15 @@ public final class MetricKey { static final DDCache RESOURCE_CACHE = DDCaches.newFixedSizeCache(32); static final DDCache SERVICE_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache SERVICE_SOURCE_CACHE = DDCaches.newFixedSizeCache(16); + static final DDCache SERVICE_SOURCE_CACHE = + DDCaches.newFixedSizeCache(16); static final DDCache OPERATION_CACHE = DDCaches.newFixedSizeCache(64); static final DDCache TYPE_CACHE = DDCaches.newFixedSizeCache(8); static final DDCache KIND_CACHE = DDCaches.newFixedSizeCache(8); static final DDCache HTTP_METHOD_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache HTTP_ENDPOINT_CACHE = DDCaches.newFixedSizeCache(32); - + static final DDCache HTTP_ENDPOINT_CACHE = + DDCaches.newFixedSizeCache(32); + private final UTF8BytesString resource; private final UTF8BytesString service; private final UTF8BytesString serviceSource; @@ -49,11 +50,11 @@ public MetricKey( List peerTags, CharSequence httpMethod, CharSequence httpEndpoint) { - this.resource = resource == null ? EMPTY : utf8(RESOURCE_CACHE, resource); - this.service = service == null ? EMPTY : utf8(SERVICE_CACHE, service); - this.serviceSource = serviceSource == null ? null : utf8(SERVICE_SOURCE_CACHE, serviceSource); - this.operationName = operationName == null ? EMPTY : utf8(OPERATION_CACHE, operationName); - this.type = utf8(TYPE_CACHE, type); + this.resource = null == resource ? EMPTY : utf8(RESOURCE_CACHE, resource); + this.service = null == service ? EMPTY : utf8(SERVICE_CACHE, service); + this.serviceSource = null == serviceSource ? null : utf8(SERVICE_SOURCE_CACHE, serviceSource); + this.operationName = null == operationName ? EMPTY : utf8(OPERATION_CACHE, operationName); + this.type = null == type ? EMPTY : utf8(TYPE_CACHE, type); this.httpStatusCode = httpStatusCode; this.synthetics = synthetics; this.isTraceRoot = isTraceRoot; @@ -61,7 +62,7 @@ public MetricKey( this.peerTags = peerTags == null ? Collections.emptyList() : peerTags; this.httpMethod = httpMethod == null ? null : utf8(HTTP_METHOD_CACHE, httpMethod); this.httpEndpoint = httpEndpoint == null ? null : utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); - + int tmpHash = 0; tmpHash = HashingUtils.addToHash(tmpHash, this.isTraceRoot); tmpHash = HashingUtils.addToHash(tmpHash, this.spanKind); @@ -77,13 +78,13 @@ public MetricKey( tmpHash = HashingUtils.addToHash(tmpHash, this.httpMethod); this.hash = tmpHash; } - + static UTF8BytesString utf8(DDCache cache, CharSequence charSeq) { - if ( charSeq instanceof UTF8BytesString ) { - return (UTF8BytesString)charSeq; - } else { - return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); - } + if (charSeq instanceof UTF8BytesString) { + return (UTF8BytesString) charSeq; + } else { + return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); + } } public UTF8BytesString getResource() {