This is an automated email from the ASF dual-hosted git repository. asdf2014 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push: new cacdc83 Improve error message for integer overflow in compaction task (#7131) cacdc83 is described below commit cacdc83cad98a2707591257705a2dd3f951b3095 Author: Jihoon Son <jihoon...@apache.org> AuthorDate: Wed Feb 27 19:07:37 2019 -0800 Improve error message for integer overflow in compaction task (#7131) * improve error message for integer overflow in compaction task * fix build --- .../org/apache/druid/java/util/common/Numbers.java | 8 ++++++++ .../druid/indexing/common/task/CompactionTask.java | 13 ++++++++++++- .../indexing/common/task/CompactionTaskTest.java | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/Numbers.java b/core/src/main/java/org/apache/druid/java/util/common/Numbers.java index d3ccf26..c9d40d7 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/Numbers.java +++ b/core/src/main/java/org/apache/druid/java/util/common/Numbers.java @@ -92,6 +92,14 @@ public final class Numbers } } + public static int toIntExact(long value, String error) + { + if ((int) value != value) { + throw new ArithmeticException(error); + } + return (int) value; + } + private Numbers() { } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index c897de3..53c3641 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -54,8 +54,10 @@ import org.apache.druid.indexing.firehose.IngestSegmentFirehoseFactory; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.JodaUtils; +import org.apache.druid.java.util.common.Numbers; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; @@ -845,7 +847,16 @@ public class CompactionTask extends AbstractTask } final double avgRowsPerByte = totalNumRows / (double) totalSizeBytes; - final int maxRowsPerSegment = Math.toIntExact(Math.round(avgRowsPerByte * nonNullTargetCompactionSizeBytes)); + final long maxRowsPerSegmentLong = Math.round(avgRowsPerByte * nonNullTargetCompactionSizeBytes); + final int maxRowsPerSegment = Numbers.toIntExact( + maxRowsPerSegmentLong, + StringUtils.format( + "Estimated maxRowsPerSegment[%s] is out of integer value range. " + + "Please consider reducing targetCompactionSizeBytes[%s].", + maxRowsPerSegmentLong, + targetCompactionSizeBytes + ) + ); Preconditions.checkState(maxRowsPerSegment > 0, "Negative maxRowsPerSegment[%s]", maxRowsPerSegment); log.info( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 5117c1a..283dee8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -62,6 +62,7 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; @@ -1135,6 +1136,27 @@ public class CompactionTaskTest .build(); } + @Test + public void testHugeTargetCompactionSize() + { + final PartitionConfigurationManager manager = new PartitionConfigurationManager(Long.MAX_VALUE, TUNING_CONFIG); + final TestIndexIO indexIO = (TestIndexIO) toolbox.getIndexIO(); + final Map<File, QueryableIndex> queryableIndexMap = indexIO.getQueryableIndexMap(); + final List<Pair<QueryableIndex, DataSegment>> segments = new ArrayList<>(); + + for (Entry<DataSegment, File> entry : segmentMap.entrySet()) { + final DataSegment segment = entry.getKey(); + final File file = entry.getValue(); + segments.add(Pair.of(Preconditions.checkNotNull(queryableIndexMap.get(file)), segment)); + } + + expectedException.expect(ArithmeticException.class); + expectedException.expectMessage( + CoreMatchers.startsWith("Estimated maxRowsPerSegment[922337203685477632] is out of integer value range.") + ); + manager.computeTuningConfig(segments); + } + private static List<DimensionsSpec> getExpectedDimensionsSpecForAutoGeneration(boolean keepSegmentGranularity) { if (keepSegmentGranularity) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org