diff --git a/extensions-contrib/opentelemetry-emitter/pom.xml b/extensions-contrib/opentelemetry-emitter/pom.xml
index 45335d65e4f8..5395aa29de32 100644
--- a/extensions-contrib/opentelemetry-emitter/pom.xml
+++ b/extensions-contrib/opentelemetry-emitter/pom.xml
@@ -39,7 +39,7 @@
33.7.2-jre
- 1.84.2
+ 1.84.3
diff --git a/extensions-contrib/tdigestsketch/pom.xml b/extensions-contrib/tdigestsketch/pom.xml
index e476fd2ffa48..1b5287361598 100644
--- a/extensions-contrib/tdigestsketch/pom.xml
+++ b/extensions-contrib/tdigestsketch/pom.xml
@@ -56,7 +56,7 @@
com.tdunningt-digest
- 3.2
+ 3.3com.google.guava
diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java
index 8d27659dbc10..efb2ad3cff92 100644
--- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java
+++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java
@@ -62,15 +62,19 @@ static byte[] toBytes(MergingDigest tDigest)
* This method computes an estimate of the max intermediate size of a {@link MergingDigest}.
* Since there is no utility available in the T-Digest library to compute this size,
* the below code is inspired by looking at
- * {@link MergingDigest#MergingDigest(double, int, int)}
- * This method is current as of 3.2 version and many need to change in future.
+ * {@link MergingDigest#MergingDigest(double, int, int)} with default buffer and centroid sizes.
+ * This method is current as of t-digest 3.3. TDigestSketchAggregatorFactoryTest checks this estimate
+ * against the arrays the library allocates, so it fails if a future version changes the sizing.
*/
static int getMaxIntermdiateTDigestSize(int compression)
{
+ // MergingDigest raises compression to at least 10 and adds extra centroid room for small compressions
+ final int effectiveCompression = Math.max(compression, 10);
+ final int sizeFudge = effectiveCompression < 30 ? 30 : 10;
+ final int centroidArraySize = 2 * effectiveCompression + sizeFudge;
+ final int tempBufferSize = 5 * centroidArraySize;
int intermediateSize = 0;
- int centroidArraySize = (int) (2 * Math.ceil(compression)) + 10;
intermediateSize += 2 * centroidArraySize * Double.BYTES;
- int tempBufferSize = (int) (5 * Math.ceil(compression));
intermediateSize += 2 * tempBufferSize * Double.BYTES;
intermediateSize += tempBufferSize * Integer.BYTES;
// Adding an extra buffer of 1K for overhead
diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
index 1e618d48ff50..91ad55a179b1 100644
--- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
+++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.query.aggregation.tdigestsketch;
+import com.tdunning.math.stats.MergingDigest;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
@@ -31,6 +32,10 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
public class TDigestSketchAggregatorFactoryTest
{
@Test
@@ -86,4 +91,37 @@ public void testCompressionBelowMaxCompressionUnchanged()
TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("tdigest", "col", 100, config);
Assertions.assertEquals(100, factory.getCompression());
}
+
+ /**
+ * Guards {@link TDigestSketchUtils#getMaxIntermdiateTDigestSize} against changes in the t-digest library.
+ * The estimate is used for on-heap memory accounting during ingestion, so if it is smaller than what a
+ * {@link MergingDigest} really allocates, ingestion persists later than maxBytesInMemory intends.
+ */
+ @Test
+ public void testMaxIntermediateSizeCoversDigestBuffers() throws IllegalAccessException
+ {
+ // Raise the cap so the factory doesn't reduce the larger compressions below
+ final TDigestConfig config = TDigestConfig.builder().maxCompression(1000).build();
+ // 5 is raised to the library minimum of 10; 29 and 30 sit on either side of the extra centroid room below 30
+ for (int compression : new int[]{5, 10, 29, 30, 100, 200, 1000}) {
+ final MergingDigest digest = new MergingDigest(compression);
+ // Sum the sizes of the digest's instance arrays (centroids and temp buffers), which dominate its footprint
+ long arrayBytes = 0;
+ for (Field field : MergingDigest.class.getDeclaredFields()) {
+ if (field.getType().isArray() && !Modifier.isStatic(field.getModifiers())) {
+ field.setAccessible(true);
+ final Object array = field.get(digest);
+ if (array != null) {
+ final int elementBytes = field.getType().getComponentType() == double.class ? Double.BYTES : Integer.BYTES;
+ arrayBytes += (long) Array.getLength(array) * elementBytes;
+ }
+ }
+ }
+ final TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("sketch", "col", compression, config);
+ Assertions.assertTrue(
+ factory.getMaxIntermediateSize() >= arrayBytes,
+ "compression " + compression + ": estimate " + factory.getMaxIntermediateSize() + " < allocated " + arrayBytes
+ );
+ }
+ }
}
diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
index c18d0ad7f434..0b87eb560669 100644
--- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
+++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
@@ -182,7 +182,7 @@ public void testCastedQuantileAndCompressionParamForTDigestQuantileAgg()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
- new Object[]{1.0, 3.5, 6.0}
+ new Object[]{1.0, 4.0, 6.0}
)
);
}
@@ -287,9 +287,9 @@ public void testComputingQuantileOnPreAggregatedSketch()
final List