diff --git a/extensions-contrib/opentelemetry-emitter/pom.xml b/extensions-contrib/opentelemetry-emitter/pom.xml index 45335d65e4f8..5395aa29de32 100644 --- a/extensions-contrib/opentelemetry-emitter/pom.xml +++ b/extensions-contrib/opentelemetry-emitter/pom.xml @@ -39,7 +39,7 @@ 33.7.2-jre - 1.84.2 + 1.84.3 diff --git a/extensions-contrib/tdigestsketch/pom.xml b/extensions-contrib/tdigestsketch/pom.xml index e476fd2ffa48..1b5287361598 100644 --- a/extensions-contrib/tdigestsketch/pom.xml +++ b/extensions-contrib/tdigestsketch/pom.xml @@ -56,7 +56,7 @@ com.tdunning t-digest - 3.2 + 3.3 com.google.guava diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java index 8d27659dbc10..efb2ad3cff92 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchUtils.java @@ -62,15 +62,19 @@ static byte[] toBytes(MergingDigest tDigest) * This method computes an estimate of the max intermediate size of a {@link MergingDigest}. * Since there is no utility available in the T-Digest library to compute this size, * the below code is inspired by looking at - * {@link MergingDigest#MergingDigest(double, int, int)} - * This method is current as of 3.2 version and many need to change in future. + * {@link MergingDigest#MergingDigest(double, int, int)} with default buffer and centroid sizes. + * This method is current as of t-digest 3.3. TDigestSketchAggregatorFactoryTest checks this estimate + * against the arrays the library allocates, so it fails if a future version changes the sizing. */ static int getMaxIntermdiateTDigestSize(int compression) { + // MergingDigest raises compression to at least 10 and adds extra centroid room for small compressions + final int effectiveCompression = Math.max(compression, 10); + final int sizeFudge = effectiveCompression < 30 ? 30 : 10; + final int centroidArraySize = 2 * effectiveCompression + sizeFudge; + final int tempBufferSize = 5 * centroidArraySize; int intermediateSize = 0; - int centroidArraySize = (int) (2 * Math.ceil(compression)) + 10; intermediateSize += 2 * centroidArraySize * Double.BYTES; - int tempBufferSize = (int) (5 * Math.ceil(compression)); intermediateSize += 2 * tempBufferSize * Double.BYTES; intermediateSize += tempBufferSize * Integer.BYTES; // Adding an extra buffer of 1K for overhead diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java index 1e618d48ff50..91ad55a179b1 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.tdigestsketch; +import com.tdunning.math.stats.MergingDigest; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregatorFactory; @@ -31,6 +32,10 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + public class TDigestSketchAggregatorFactoryTest { @Test @@ -86,4 +91,37 @@ public void testCompressionBelowMaxCompressionUnchanged() TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("tdigest", "col", 100, config); Assertions.assertEquals(100, factory.getCompression()); } + + /** + * Guards {@link TDigestSketchUtils#getMaxIntermdiateTDigestSize} against changes in the t-digest library. + * The estimate is used for on-heap memory accounting during ingestion, so if it is smaller than what a + * {@link MergingDigest} really allocates, ingestion persists later than maxBytesInMemory intends. + */ + @Test + public void testMaxIntermediateSizeCoversDigestBuffers() throws IllegalAccessException + { + // Raise the cap so the factory doesn't reduce the larger compressions below + final TDigestConfig config = TDigestConfig.builder().maxCompression(1000).build(); + // 5 is raised to the library minimum of 10; 29 and 30 sit on either side of the extra centroid room below 30 + for (int compression : new int[]{5, 10, 29, 30, 100, 200, 1000}) { + final MergingDigest digest = new MergingDigest(compression); + // Sum the sizes of the digest's instance arrays (centroids and temp buffers), which dominate its footprint + long arrayBytes = 0; + for (Field field : MergingDigest.class.getDeclaredFields()) { + if (field.getType().isArray() && !Modifier.isStatic(field.getModifiers())) { + field.setAccessible(true); + final Object array = field.get(digest); + if (array != null) { + final int elementBytes = field.getType().getComponentType() == double.class ? Double.BYTES : Integer.BYTES; + arrayBytes += (long) Array.getLength(array) * elementBytes; + } + } + } + final TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("sketch", "col", compression, config); + Assertions.assertTrue( + factory.getMaxIntermediateSize() >= arrayBytes, + "compression " + compression + ": estimate " + factory.getMaxIntermediateSize() + " < allocated " + arrayBytes + ); + } + } } diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java index c18d0ad7f434..0b87eb560669 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java @@ -182,7 +182,7 @@ public void testCastedQuantileAndCompressionParamForTDigestQuantileAgg() ), ResultMatchMode.EQUALS_EPS, ImmutableList.of( - new Object[]{1.0, 3.5, 6.0} + new Object[]{1.0, 4.0, 6.0} ) ); } @@ -287,9 +287,9 @@ public void testComputingQuantileOnPreAggregatedSketch() final List expectedResults = ImmutableList.of( new Object[]{ - 1.1, - 2.9, - 5.3, + 1.0, + 3.0, + 5.0, 6.0 } ); @@ -331,7 +331,7 @@ public void testGeneratingSketchAndComputingQuantileOnFly() final List expectedResults = ImmutableList.of( new Object[]{ 1.0, - 3.5, + 4.0, 6.0 } ); @@ -387,7 +387,7 @@ public void testQuantileOnNumericValues() final List expectedResults = ImmutableList.of( new Object[]{ 1.0, - 3.5, + 4.0, 6.0 } ); @@ -451,7 +451,7 @@ public void testCompressionParamForTDigestQuantileAgg() ), ResultMatchMode.EQUALS_EPS, ImmutableList.of( - new Object[]{1.0, 3.5, 6.0} + new Object[]{1.0, 4.0, 6.0} ) ); } diff --git a/licenses.yaml b/licenses.yaml index 1221c6f98f8c..54adf1835d3f 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -446,7 +446,7 @@ name: Caffeine license_category: binary module: java-core license_name: Apache License version 2.0 -version: 3.3.0 +version: 3.3.1 libraries: - com.github.ben-manes.caffeine: caffeine @@ -2577,7 +2577,7 @@ name: Hibernate Validator license_category: binary module: java-core license_name: Apache License version 2.0 -version: 9.1.4.Final +version: 9.1.5.Final libraries: - org.hibernate.validator: hibernate-validator @@ -4051,7 +4051,7 @@ name: XZ license_category: binary module: java-core license_name: Public Domain -version: 1.9 +version: 1.12 libraries: - org.tukaani: xz diff --git a/pom.xml b/pom.xml index b60ffd806d90..10e916a73982 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 1.29.0 1.12.2 1.18.1 - 1.18.14 + 1.18.15 @@ -128,10 +128,10 @@ 1.12.797 2.55.6 2.3.9 - 3.3.0 + 3.3.1 0.8.15 2.0.5 - 9.1.4.Final + 9.1.5.Final 4.5.14 5.5.0 25.0.0-legacy @@ -726,7 +726,7 @@ org.tukaani xz - 1.9 + 1.12 com.github.luben