Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
6f5a113
build(deps): bump the maven-minor-patch group with 14 updates
dependabot[bot] Oct 28, 2026
9b1da70
fix: avoid fastutil grow override conflict
FrankChen021 Oct 28, 2026
e0a48a5
build: align zstd with kafka client
FrankChen021 Oct 28, 2026
f8b681b
build: update dependency license registry
FrankChen021 Oct 28, 2026
cb12ea9
build: declare fastutil in server
FrankChen021 Oct 28, 2026
7cd23fa
build: align indexing service fastutil dependency
FrankChen021 Oct 28, 2026
5456513
build: align sql fastutil dependency
FrankChen021 Oct 28, 2026
df99ece
build: align datasketches fastutil dependency
FrankChen021 Oct 28, 2026
b0d5771
build: align histogram fastutil dependency
FrankChen021 Oct 28, 2026
cf12f96
build: align benchmark fastutil dependency
FrankChen021 Oct 28, 2026
f46e2f5
build: align lookups fastutil dependency
FrankChen021 Oct 28, 2026
da51c09
build: align S3 fastutil dependency
FrankChen021 Oct 28, 2026
b522ec2
fix: handle null dictionary values with fastutil 8.5.19
FrankChen021 Oct 28, 2026
ffa25bb
fix: declare fastutil dependency for tdigestsketch
FrankChen021 Oct 28, 2026
76073e8
fix: declare fastutil dependency for ddsketch
FrankChen021 Oct 28, 2026
512318c
build: replace fastutil-core with fastutil in remaining modules
FrankChen021 Oct 28, 2026
9c5fb44
build: exclude kafka-clients, javacc-maven-plugin and spotbugs-maven-…
FrankChen021 Oct 28, 2026
57b7e2c
test: update t-digest 3.3 expectations
FrankChen021 Oct 28, 2026
38abce5
fix: support null array overlap constants
FrankChen021 Oct 28, 2026
463eb7a
build: move the fastutil upgrade to a separate PR
FrankChen021 Oct 28, 2026
a542235
fix: update t-digest intermediate size estimate for 3.3
FrankChen021 Oct 29, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion extensions-contrib/opentelemetry-emitter/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
<!-- These guava and grpc versions are used only in the opentelemetry-extension.
Look at build section for more details about shading. -->
<shade.guava.version>33.7.2-jre</shade.guava.version>
<shade.grpc.version>1.84.2</shade.grpc.version>
<shade.grpc.version>1.84.3</shade.grpc.version>
</properties>
<dependencyManagement>
<dependencies>
Expand Down
2 changes: 1 addition & 1 deletion extensions-contrib/tdigestsketch/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>t-digest</artifactId>
<version>3.2</version>
<version>3.3</version>
Comment thread
amaechler marked this conversation as resolved.
Resolved
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,19 @@ static byte[] toBytes(MergingDigest tDigest)
* This method computes an estimate of the max intermediate size of a {@link MergingDigest}.
* Since there is no utility available in the T-Digest library to compute this size,
* the below code is inspired by looking at
* {@link MergingDigest#MergingDigest(double, int, int)}
* This method is current as of 3.2 version and many need to change in future.
* {@link MergingDigest#MergingDigest(double, int, int)} with default buffer and centroid sizes.
* This method is current as of t-digest 3.3. TDigestSketchAggregatorFactoryTest checks this estimate
* against the arrays the library allocates, so it fails if a future version changes the sizing.
*/
static int getMaxIntermdiateTDigestSize(int compression)
{
// MergingDigest raises compression to at least 10 and adds extra centroid room for small compressions
final int effectiveCompression = Math.max(compression, 10);
final int sizeFudge = effectiveCompression < 30 ? 30 : 10;
final int centroidArraySize = 2 * effectiveCompression + sizeFudge;
final int tempBufferSize = 5 * centroidArraySize;
int intermediateSize = 0;
int centroidArraySize = (int) (2 * Math.ceil(compression)) + 10;
intermediateSize += 2 * centroidArraySize * Double.BYTES;
int tempBufferSize = (int) (5 * Math.ceil(compression));
intermediateSize += 2 * tempBufferSize * Double.BYTES;
intermediateSize += tempBufferSize * Integer.BYTES;
// Adding an extra buffer of 1K for overhead
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.druid.query.aggregation.tdigestsketch;

import com.tdunning.math.stats.MergingDigest;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
Expand All @@ -31,6 +32,10 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;

public class TDigestSketchAggregatorFactoryTest
{
@Test
Expand Down Expand Up @@ -86,4 +91,37 @@ public void testCompressionBelowMaxCompressionUnchanged()
TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("tdigest", "col", 100, config);
Assertions.assertEquals(100, factory.getCompression());
}

/**
* Guards {@link TDigestSketchUtils#getMaxIntermdiateTDigestSize} against changes in the t-digest library.
* The estimate is used for on-heap memory accounting during ingestion, so if it is smaller than what a
* {@link MergingDigest} really allocates, ingestion persists later than maxBytesInMemory intends.
*/
@Test
public void testMaxIntermediateSizeCoversDigestBuffers() throws IllegalAccessException
{
// Raise the cap so the factory doesn't reduce the larger compressions below
final TDigestConfig config = TDigestConfig.builder().maxCompression(1000).build();
// 5 is raised to the library minimum of 10; 29 and 30 sit on either side of the extra centroid room below 30
for (int compression : new int[]{5, 10, 29, 30, 100, 200, 1000}) {
final MergingDigest digest = new MergingDigest(compression);
// Sum the sizes of the digest's instance arrays (centroids and temp buffers), which dominate its footprint
long arrayBytes = 0;
for (Field field : MergingDigest.class.getDeclaredFields()) {
if (field.getType().isArray() && !Modifier.isStatic(field.getModifiers())) {
field.setAccessible(true);
final Object array = field.get(digest);
if (array != null) {
final int elementBytes = field.getType().getComponentType() == double.class ? Double.BYTES : Integer.BYTES;
arrayBytes += (long) Array.getLength(array) * elementBytes;
}
}
}
final TDigestSketchAggregatorFactory factory = new TDigestSketchAggregatorFactory("sketch", "col", compression, config);
Assertions.assertTrue(
factory.getMaxIntermediateSize() >= arrayBytes,
"compression " + compression + ": estimate " + factory.getMaxIntermediateSize() + " < allocated " + arrayBytes
);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ public void testCastedQuantileAndCompressionParamForTDigestQuantileAgg()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{1.0, 3.5, 6.0}
new Object[]{1.0, 4.0, 6.0}
)
);
}
Expand Down Expand Up @@ -287,9 +287,9 @@ public void testComputingQuantileOnPreAggregatedSketch()

final List<Object[]> expectedResults = ImmutableList.of(
new Object[]{
1.1,
2.9,
5.3,
1.0,
3.0,
5.0,
6.0
}
);
Expand Down Expand Up @@ -331,7 +331,7 @@ public void testGeneratingSketchAndComputingQuantileOnFly()
final List<Object[]> expectedResults = ImmutableList.of(
new Object[]{
1.0,
3.5,
4.0,
6.0
}
);
Expand Down Expand Up @@ -387,7 +387,7 @@ public void testQuantileOnNumericValues()
final List<Object[]> expectedResults = ImmutableList.of(
new Object[]{
1.0,
3.5,
4.0,
6.0
}
);
Expand Down Expand Up @@ -451,7 +451,7 @@ public void testCompressionParamForTDigestQuantileAgg()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{1.0, 3.5, 6.0}
new Object[]{1.0, 4.0, 6.0}
)
);
}
Expand Down
6 changes: 3 additions & 3 deletions licenses.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ name: Caffeine
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 3.3.0
version: 3.3.1
libraries:
- com.github.ben-manes.caffeine: caffeine

Expand Down Expand Up @@ -2577,7 +2577,7 @@ name: Hibernate Validator
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 9.1.4.Final
version: 9.1.5.Final
libraries:
- org.hibernate.validator: hibernate-validator

Expand Down Expand Up @@ -4051,7 +4051,7 @@ name: XZ
license_category: binary
module: java-core
license_name: Public Domain
Comment thread
amaechler marked this conversation as resolved.
Resolved
version: 1.9
version: 1.12
libraries:
- org.tukaani: xz

Expand Down
8 changes: 4 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@
<avatica.version>1.29.0</avatica.version>
<avro.version>1.12.2</avro.version>
<parquet.version>1.18.1</parquet.version>
<bytebuddy.version>1.18.14</bytebuddy.version>
<bytebuddy.version>1.18.15</bytebuddy.version>
<!--
The base calcite parser was copied into the project; when updating Calcite run dev/upgrade-calcite-parser to adopt upstream changes
-->
Expand Down Expand Up @@ -128,10 +128,10 @@
<aws.sdk.v1.version>1.12.797</aws.sdk.v1.version>
<aws.sdk.v2.version>2.55.6</aws.sdk.v2.version>
<aws.msk.iam.auth.version>2.3.9</aws.msk.iam.auth.version>
<caffeine.version>3.3.0</caffeine.version>
<caffeine.version>3.3.1</caffeine.version>
<jacoco.version>0.8.15</jacoco.version>
<testcontainers.version>2.0.5</testcontainers.version>
<hibernate-validator.version>9.1.4.Final</hibernate-validator.version>
<hibernate-validator.version>9.1.5.Final</hibernate-validator.version>
<httpclient.version>4.5.14</httpclient.version>
<okhttp.version>5.5.0</okhttp.version>
<kubernetes.client.version>25.0.0-legacy</kubernetes.client.version>
Expand Down Expand Up @@ -726,7 +726,7 @@
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>1.9</version>
<version>1.12</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
Expand Down
Loading