[incubator-druid] branch master updated: Move build badge to https://travis-ci.org/apache/incubator-druid (#6007)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new e206f8c Move build badge to https://travis-ci.org/apache/incubator-druid (#6007) e206f8c is described below commit e206f8ca98c6750083680af187341bdcbda807ed Author: Charles Allen AuthorDate: Tue Jul 17 11:57:33 2018 -0700 Move build badge to https://travis-ci.org/apache/incubator-druid (#6007) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2108e7e..098acf8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[](https://travis-ci.org/druid-io/druid) [](https://teamcity.jetbrains.com/viewType.html?buildTypeId=OpenSourceProjects_Druid_Inspections) [](https://coveralls.io/r/druid-io/druid?br [...] +[](https://travis-ci.org/apache/incubator-druid) [](https://teamcity.jetbrains.com/viewType.html?buildTypeId=OpenSourceProjects_Druid_Inspections) [](https://coveralls.io/r/d [...] ## Druid - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Synchronize scheduled poll() calls in SQLMetadataSegmentManager (#6041)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 5ee7b0c Synchronize scheduled poll() calls in
SQLMetadataSegmentManager (#6041)
5ee7b0c is described below
commit 5ee7b0cada983912083a85f079086cb579488efd
Author: Jihoon Son
AuthorDate: Tue Jul 24 20:57:30 2018 -0700
Synchronize scheduled poll() calls in SQLMetadataSegmentManager (#6041)
Similar issue to https://github.com/apache/incubator-druid/issues/6028.
---
.../io/druid/metadata/SQLMetadataRuleManager.java | 14 ++--
.../druid/metadata/SQLMetadataSegmentManager.java | 94 --
.../metadata/SQLMetadataSegmentManagerTest.java| 5 ++
3 files changed, 81 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java
b/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java
index 6900dff..4bbb0fa 100644
--- a/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java
+++ b/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java
@@ -148,10 +148,12 @@ public class SQLMetadataRuleManager implements
MetadataRuleManager
* This field is used to implement a simple stamp mechanism instead of just
a boolean "started" flag to prevent
* the theoretical situation of two tasks scheduled in {@link #start()}
calling {@link #poll()} concurrently, if
* the sequence of {@link #start()} - {@link #stop()} - {@link #start()}
actions occurs quickly.
+ *
+ * {@link SQLMetadataSegmentManager} also have a similar issue.
*/
private long currentStartOrder = -1;
private ScheduledExecutorService exec = null;
- private long retryStartTime = 0;
+ private long failStartTimeMs = 0;
@Inject
public SQLMetadataRuleManager(
@@ -311,17 +313,17 @@ public class SQLMetadataRuleManager implements
MetadataRuleManager
log.info("Polled and found rules for %,d datasource(s)",
newRules.size());
rules.set(newRules);
- retryStartTime = 0;
+ failStartTimeMs = 0;
}
catch (Exception e) {
- if (retryStartTime == 0) {
-retryStartTime = System.currentTimeMillis();
+ if (failStartTimeMs == 0) {
+failStartTimeMs = System.currentTimeMillis();
}
- if (System.currentTimeMillis() - retryStartTime >
config.getAlertThreshold().toStandardDuration().getMillis()) {
+ if (System.currentTimeMillis() - failStartTimeMs >
config.getAlertThreshold().toStandardDuration().getMillis()) {
log.makeAlert(e, "Exception while polling for rules")
.emit();
-retryStartTime = 0;
+failStartTimeMs = 0;
} else {
log.error(e, "Exception while polling for rules");
}
diff --git
a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java
b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java
index 7d5a7e6..4a83125 100644
--- a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java
+++ b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java
@@ -29,9 +29,6 @@ import com.google.common.collect.Interner;
import com.google.common.collect.Interners;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import io.druid.client.DruidDataSource;
import io.druid.client.ImmutableDruidDataSource;
@@ -73,8 +70,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
/**
@@ -85,10 +85,16 @@ public class SQLMetadataSegmentManager implements
MetadataSegmentManager
private static final Interner DATA_SEGMENT_INTERNER =
Interners.newWeakInterner();
private static final EmittingLogger log = new
EmittingLogger(SQLMetadataSegmentManager.class);
- // Use to synchronize start() and stop(). These methods should be
synchronized to prevent from being called at the
- // same time if two different threads are calling them. This might be
possible if a druid coordinator gets and drops
- // leadership repeatedly in quick succession.
- private final Object lock = new Object();
+ /**
+ * Use to synchronize {@link #start()}, {@link #stop()}, {@link #poll()},
and {@link #isStarted()}. These methods
+ * should be synchronized to prev
[incubator-druid] branch 0.12.2 updated: [Backport] Fix 'auto' encoded longs + compression serializer (#6083)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch 0.12.2
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/0.12.2 by this push:
new be6481c [Backport] Fix 'auto' encoded longs + compression serializer
(#6083)
be6481c is described below
commit be6481cdc0dcdd5c308881eb565b8409cc800a85
Author: Jihoon Son
AuthorDate: Tue Jul 31 15:38:31 2018 -0700
[Backport] Fix 'auto' encoded longs + compression serializer (#6083)
* [Backport] Fix 'auto' encoded longs + compression serializer
* fix license
---
.../data/BlockLayoutColumnarLongsSerializer.java | 1 +
.../data/CompressedLongsAutoEncodingSerdeTest.java | 138 +
2 files changed, 139 insertions(+)
diff --git
a/processing/src/main/java/io/druid/segment/data/BlockLayoutColumnarLongsSerializer.java
b/processing/src/main/java/io/druid/segment/data/BlockLayoutColumnarLongsSerializer.java
index a73c39b..58eeb54 100644
---
a/processing/src/main/java/io/druid/segment/data/BlockLayoutColumnarLongsSerializer.java
+++
b/processing/src/main/java/io/druid/segment/data/BlockLayoutColumnarLongsSerializer.java
@@ -92,6 +92,7 @@ public class BlockLayoutColumnarLongsSerializer implements
ColumnarLongsSerializ
endBuffer.flip();
flattener.write(endBuffer);
endBuffer.clear();
+ writer.setBuffer(endBuffer);
}
writer.write(value);
diff --git
a/processing/src/test/java/io/druid/segment/data/CompressedLongsAutoEncodingSerdeTest.java
b/processing/src/test/java/io/druid/segment/data/CompressedLongsAutoEncodingSerdeTest.java
new file mode 100644
index 000..af770f0
--- /dev/null
+++
b/processing/src/test/java/io/druid/segment/data/CompressedLongsAutoEncodingSerdeTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.data;
+
+import io.druid.java.util.common.StringUtils;
+import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+@RunWith(Parameterized.class)
+public class CompressedLongsAutoEncodingSerdeTest
+{
+ @Parameterized.Parameters(name = "{0} {1} {2}")
+ public static Iterable compressionStrategies()
+ {
+List data = new ArrayList<>();
+for (long bpv : bitsPerValueParameters) {
+ for (CompressionStrategy strategy : CompressionStrategy.values()) {
+data.add(new Object[]{bpv, strategy, ByteOrder.BIG_ENDIAN});
+data.add(new Object[]{bpv, strategy, ByteOrder.LITTLE_ENDIAN});
+ }
+}
+return data;
+ }
+
+ private static final long[] bitsPerValueParameters = new long[]{1, 2, 4, 7,
11, 14, 18, 23, 31, 39, 46, 55, 62};
+
+ protected final CompressionFactory.LongEncodingStrategy encodingStrategy =
CompressionFactory.LongEncodingStrategy.AUTO;
+ protected final CompressionStrategy compressionStrategy;
+ protected final ByteOrder order;
+ protected final long bitsPerValue;
+
+ public CompressedLongsAutoEncodingSerdeTest(
+ long bitsPerValue,
+ CompressionStrategy compressionStrategy,
+ ByteOrder order
+ )
+ {
+this.bitsPerValue = bitsPerValue;
+this.compressionStrategy = compressionStrategy;
+this.order = order;
+ }
+
+ @Test
+ public void testFidelity() throws Exception
+ {
+final long bound = 1L << bitsPerValue;
+// big enough to have at least 2 blocks, and a handful of sizes offset by
1 from each other
+int blockSize = 1 << 16;
+int numBits = (Long.SIZE - Long.numberOfLeadingZeros(1 << (bitsPerValue -
1)));
+double numValuesPerByte = 8.0 / (double) numBits;
+
+int numRows = (int) (blockSize * numValuesPerByte) * 2 +
ThreadLocalRandom.current().nextInt(1, 101);
+long
[incubator-druid] branch master updated: Minor followup to #6220. (#6231)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 157e75a Minor followup to #6220. (#6231)
157e75a is described below
commit 157e75a1fe2213d2dcf51d3f36ece10f43f54df8
Author: Gian Merlino
AuthorDate: Mon Aug 27 10:01:44 2018 -0700
Minor followup to #6220. (#6231)
Adjustments to comments and usage of generics.
---
.../topn/types/StringTopNColumnSelectorStrategy.java | 16
.../topn/types/TopNColumnSelectorStrategyFactory.java| 6 +-
.../java/io/druid/segment/DimensionHandlerUtils.java | 5 +
3 files changed, 14 insertions(+), 13 deletions(-)
diff --git
a/processing/src/main/java/io/druid/query/topn/types/StringTopNColumnSelectorStrategy.java
b/processing/src/main/java/io/druid/query/topn/types/StringTopNColumnSelectorStrategy.java
index ed89bbd..76936dd 100644
---
a/processing/src/main/java/io/druid/query/topn/types/StringTopNColumnSelectorStrategy.java
+++
b/processing/src/main/java/io/druid/query/topn/types/StringTopNColumnSelectorStrategy.java
@@ -36,7 +36,7 @@ import java.util.Map;
import java.util.function.Function;
public class StringTopNColumnSelectorStrategy
-implements TopNColumnSelectorStrategy>
+implements TopNColumnSelectorStrategy, Aggregator[]>>
{
private final Function> dimensionValueConverter;
@@ -73,7 +73,7 @@ public class StringTopNColumnSelectorStrategy
}
@Override
- public Map makeDimExtractionAggregateStore()
+ public Map, Aggregator[]> makeDimExtractionAggregateStore()
{
return new HashMap<>();
}
@@ -84,7 +84,7 @@ public class StringTopNColumnSelectorStrategy
DimensionSelector selector,
Cursor cursor,
Aggregator[][] rowSelector,
- Map aggregatesStore
+ Map, Aggregator[]> aggregatesStore
)
{
if (selector.getValueCardinality() !=
DimensionSelector.CARDINALITY_UNKNOWN) {
@@ -96,11 +96,11 @@ public class StringTopNColumnSelectorStrategy
@Override
public void updateDimExtractionResults(
- final Map aggregatesStore,
+ final Map, Aggregator[]> aggregatesStore,
final TopNResultBuilder resultBuilder
)
{
-for (Map.Entry entry :
aggregatesStore.entrySet()) {
+for (Map.Entry, Aggregator[]> entry :
aggregatesStore.entrySet()) {
Aggregator[] aggs = entry.getValue();
if (aggs != null) {
Object[] vals = new Object[aggs.length];
@@ -108,7 +108,7 @@ public class StringTopNColumnSelectorStrategy
vals[i] = aggs[i].get();
}
-final Comparable key = dimensionValueConverter.apply(entry.getKey());
+final Comparable key =
dimensionValueConverter.apply(entry.getKey());
resultBuilder.addEntry(key, key, vals);
}
}
@@ -119,7 +119,7 @@ public class StringTopNColumnSelectorStrategy
Cursor cursor,
DimensionSelector selector,
Aggregator[][] rowSelector,
- Map aggregatesStore
+ Map, Aggregator[]> aggregatesStore
)
{
long processedRows = 0;
@@ -152,7 +152,7 @@ public class StringTopNColumnSelectorStrategy
TopNQuery query,
Cursor cursor,
DimensionSelector selector,
- Map aggregatesStore
+ Map, Aggregator[]> aggregatesStore
)
{
long processedRows = 0;
diff --git
a/processing/src/main/java/io/druid/query/topn/types/TopNColumnSelectorStrategyFactory.java
b/processing/src/main/java/io/druid/query/topn/types/TopNColumnSelectorStrategyFactory.java
index 08e76ff..5f29874 100644
---
a/processing/src/main/java/io/druid/query/topn/types/TopNColumnSelectorStrategyFactory.java
+++
b/processing/src/main/java/io/druid/query/topn/types/TopNColumnSelectorStrategyFactory.java
@@ -49,9 +49,13 @@ public class TopNColumnSelectorStrategyFactory implements
ColumnSelectorStrategy
case LONG:
case FLOAT:
case DOUBLE:
+// When the selector is numeric, we want to use
NumericTopNColumnSelectorStrategy. It aggregates using
+// a numeric type and then converts to the desired output type after
aggregating. We must be careful not to
+// convert to an output type that cannot represent all possible values
of the input type.
+
if (ValueType.isNumeric(dimensionType)) {
// Return strategy that aggregates using the _output_ type, because
this allows us to collapse values
- // properly (numeric types cannot represent all values of other
numeric types).
+ // properly (numeric types cannot always represent all values of
other numeric types).
return NumericTopNColumnSelectorStrategy.ofType(dimensionType,
dimensionType);
} else {
// Return strategy that aggregates using the _input_ type. Here we
are assuming that the output type can
diff --git
[incubator-druid] branch master updated: Fix dictionary ID race condition in IncrementalIndexStorageAdapter (#6340)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 609da01 Fix dictionary ID race condition in
IncrementalIndexStorageAdapter (#6340)
609da01 is described below
commit 609da018827697b7ae5ce6cc464762041e9a865e
Author: Jonathan Wei
AuthorDate: Mon Sep 17 23:43:29 2018 -0700
Fix dictionary ID race condition in IncrementalIndexStorageAdapter (#6340)
Possibly related to https://github.com/apache/incubator-druid/issues/4937
There is currently a race condition in IncrementalIndexStorageAdapter that
can lead to exceptions like the following, when running queries with filters on
String dimensions that hit realtime tasks:
```
org.apache.druid.java.util.common.ISE: id[5] >= maxId[5]
at
org.apache.druid.segment.StringDimensionIndexer$1IndexerDimensionSelector.lookupName(StringDimensionIndexer.java:591)
at
org.apache.druid.segment.StringDimensionIndexer$1IndexerDimensionSelector$2.matches(StringDimensionIndexer.java:562)
at
org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter$IncrementalIndexCursor.advance(IncrementalIndexStorageAdapter.java:284)
```
When the `filterMatcher` is created in the constructor of
`IncrementalIndexStorageAdapter.IncrementalIndexCursor`,
`StringDimensionIndexer.makeDimensionSelector` gets called eventually, which
calls:
```
final int maxId = getCardinality();
...
@Override
public int getCardinality()
{
return dimLookup.size();
}
```
So `maxId` is set to the size of the dictionary at the time that the
`filterMatcher` is created.
However, the `maxRowIndex` which is meant to prevent the Cursor from
returning rows that were added after the Cursor was created (see
https://github.com/apache/incubator-druid/pull/4049) is set after the
`filterMatcher` is created.
If rows with new dictionary values are added after the `filterMatcher` is
created but before `maxRowIndex` is set, then it is possible for the Cursor to
return rows that contain the new values, which will have `id >= maxId`.
This PR sets `maxRowIndex` before creating the `filterMatcher` to prevent
rows with unknown dictionary IDs from being passed to the `filterMatcher`.
---
The included test triggers the error with a custom Filter +
DruidPredicateFactory.
The DimensionSelector for predicate-based filter matching is created here
in `Filters.makeValueMatcher`:
```
public static ValueMatcher makeValueMatcher(
final ColumnSelectorFactory columnSelectorFactory,
final String columnName,
final DruidPredicateFactory predicateFactory
)
{
final ColumnCapabilities capabilities =
columnSelectorFactory.getColumnCapabilities(columnName);
// This should be folded into the ValueMatcherColumnSelectorStrategy
once that can handle LONG typed columns.
if (capabilities != null && capabilities.getType() == ValueType.LONG) {
return getLongPredicateMatcher(
columnSelectorFactory.makeColumnValueSelector(columnName),
predicateFactory.makeLongPredicate()
);
}
final ColumnSelectorPlus selector =
DimensionHandlerUtils.createColumnSelectorPlus(
ValueMatcherColumnSelectorStrategyFactory.instance(),
DefaultDimensionSpec.of(columnName),
columnSelectorFactory
);
return
selector.getColumnSelectorStrategy().makeValueMatcher(selector.getSelector(),
predicateFactory);
}
```
The test Filter adds a row to the IncrementalIndex in the test when the
predicateFactory creates a new String predicate, after
`DimensionHandlerUtils.createColumnSelectorPlus` is called.
---
.../IncrementalIndexStorageAdapter.java| 3 +-
.../IncrementalIndexStorageAdapterTest.java| 159 +
2 files changed, 161 insertions(+), 1 deletion(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index 53cba0f..94c3cf2 100644
---
a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -236,9 +236,10 @@ public class IncrementalIndexStorageAdapter implements
StorageAdapter
{
currEntry = new IncrementalIndexRowHolder();
columnSelectorFactory = new IncrementalIndexColumnSelectorFactory(index,
virtualColumns, descend
[incubator-druid] branch master updated: Fix issue that the forbidden-apis check do not always work (#6371)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6843cbb Fix issue that the forbidden-apis check do not always work
(#6371)
6843cbb is described below
commit 6843cbba1ddce00c1f81e08477f4ec8ca3742110
Author: QiuMM
AuthorDate: Thu Sep 27 00:39:39 2018 +0800
Fix issue that the forbidden-apis check do not always work (#6371)
* Fix the forbidden apis check do not work issue
* use SuppressForbidden annotation
---
.../java/org/apache/druid/java/util/common/DateTimes.java | 3 +++
pom.xml | 13 +
.../src/main/java/org/apache/druid/cli/DumpSegment.java | 2 ++
services/src/main/java/org/apache/druid/cli/Main.java | 2 ++
.../main/java/org/apache/druid/cli/PullDependencies.java| 2 ++
services/src/main/java/org/apache/druid/cli/Version.java| 2 ++
.../org/apache/druid/cli/validate/DruidJsonValidator.java | 2 ++
7 files changed, 18 insertions(+), 8 deletions(-)
diff --git
a/java-util/src/main/java/org/apache/druid/java/util/common/DateTimes.java
b/java-util/src/main/java/org/apache/druid/java/util/common/DateTimes.java
index 74e221c..f9a708d 100644
--- a/java-util/src/main/java/org/apache/druid/java/util/common/DateTimes.java
+++ b/java-util/src/main/java/org/apache/druid/java/util/common/DateTimes.java
@@ -19,6 +19,7 @@
package org.apache.druid.java.util.common;
+import io.netty.util.SuppressForbidden;
import org.joda.time.Chronology;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -40,6 +41,7 @@ public final class DateTimes
ISODateTimeFormat.dateTimeParser().withOffsetParsed()
);
+ @SuppressForbidden(reason = "DateTimeZone#forID")
public static DateTimeZone inferTzfromString(String tzId)
{
try {
@@ -65,6 +67,7 @@ public final class DateTimes
this.innerFormatter =
innerFormatter.withChronology(ISOChronology.getInstanceUTC());
}
+@SuppressForbidden(reason = "DateTimeFormatter#parseDateTime")
public DateTime parse(final String instant)
{
return innerFormatter.parseDateTime(instant);
diff --git a/pom.xml b/pom.xml
index c576759..8da6770 100644
--- a/pom.xml
+++ b/pom.xml
@@ -931,15 +931,12 @@
${session.executionRootDirectory}/codestyle/joda-time-forbidden-apis.txt
${session.executionRootDirectory}/codestyle/druid-forbidden-apis.txt
-
-
org/apache/druid/java/util/common/DateTimes$UtcFormatter.class
-
org/apache/druid/java/util/common/DateTimes.class
-
+
**.SuppressForbidden
-validate
-validate
+compile
+compile
check
@@ -952,8 +949,8 @@
-testValidate
-validate
+testCompile
+test-compile
testCheck
diff --git a/services/src/main/java/org/apache/druid/cli/DumpSegment.java
b/services/src/main/java/org/apache/druid/cli/DumpSegment.java
index 28bfe0e..abe2512 100644
--- a/services/src/main/java/org/apache/druid/cli/DumpSegment.java
+++ b/services/src/main/java/org/apache/druid/cli/DumpSegment.java
@@ -36,6 +36,7 @@ import com.google.inject.Module;
import com.google.inject.name.Names;
import io.airlift.airline.Command;
import io.airlift.airline.Option;
+import io.netty.util.SuppressForbidden;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ConciseBitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
@@ -417,6 +418,7 @@ public class DumpSegment extends GuiceRunnable
return ImmutableList.copyOf(columnNames);
}
+ @SuppressForbidden(reason = "System#out")
private T withOutputStream(Function f) throws
IOException
{
if (outputFileName == null) {
diff --git a/services/src/main/java/org/apache/druid/cli/Main.java
b/services/src/main/java/org/apache/druid/cli/Main.java
index 0c0b0c6..9f77bb3 100644
--- a/services/src/main/java/org/apache/druid/cli/Main.java
+++ b/services/src/main/java/org/apache/druid/cli/Main.java
@@ -23,6 +23,7 @@ import com.google.inject.Injector;
import io.airlift.airline.Cli;
import io.airlift.airline.Help;
import io.airlift.airline.ParseException;
+import io.netty.u
[incubator-druid] branch master updated: Prohibit jackson ObjectMapper#reader methods which are deprecated (#6386)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0b8085a Prohibit jackson ObjectMapper#reader methods which are
deprecated (#6386)
0b8085a is described below
commit 0b8085aff77d215995da994eb2d86676fdb50911
Author: QiuMM
AuthorDate: Thu Oct 4 04:55:20 2018 +0800
Prohibit jackson ObjectMapper#reader methods which are deprecated (#6386)
* Prohibit jackson ObjectMapper#reader methods which are deprecated
* address comments
---
codestyle/druid-forbidden-apis.txt | 5 -
.../metrics/WhiteListBasedDruidToTimelineEventConverter.java | 2 +-
.../ambari/metrics/AmbariMetricsEmitterConfigTest.java | 12 ++--
.../druid/emitter/graphite/WhiteListBasedConverter.java | 2 +-
.../druid/emitter/graphite/GraphiteEmitterConfigTest.java| 6 +++---
.../apache/druid/emitter/kafka/KafkaEmitterConfigTest.java | 2 +-
.../org/apache/druid/emitter/opentsdb/EventConverter.java| 2 +-
.../druid/emitter/opentsdb/OpentsdbEmitterConfigTest.java| 2 +-
.../org/apache/druid/emitter/opentsdb/OpentsdbEventTest.java | 2 +-
.../org/apache/druid/emitter/statsd/DimensionConverter.java | 2 +-
.../apache/druid/storage/hdfs/HdfsKerberosConfigTest.java| 2 +-
.../apache/druid/server/lookup/LoadingLookupFactoryTest.java | 2 +-
.../druid/server/lookup/PollingLookupSerDeserTest.java | 2 +-
.../druid/server/lookup/cache/loading/LoadingCacheTest.java | 4 ++--
.../apache/druid/server/lookup/jdbc/JdbcDataFetcherTest.java | 2 +-
.../org/apache/druid/indexer/HadoopKerberosConfigTest.java | 2 +-
.../query/extraction/MapLookupExtractionFnSerDeTest.java | 4 ++--
.../org/apache/druid/query/filter/BoundDimFilterTest.java| 2 +-
.../apache/druid/query/filter/InDimFilterSerDesrTest.java| 2 +-
.../org/apache/druid/query/filter/IntervalDimFilterTest.java | 4 ++--
.../java/org/apache/druid/query/lookup/LookupConfigTest.java | 2 +-
.../org/apache/druid/query/lookup/LookupExtractorTest.java | 2 +-
.../druid/query/dimension/LookupDimensionSpecTest.java | 2 +-
.../druid/query/lookup/MapLookupExtractorFactoryTest.java| 2 +-
.../appenderator/DefaultOfflineAppenderatorFactoryTest.java | 2 +-
25 files changed, 38 insertions(+), 35 deletions(-)
diff --git a/codestyle/druid-forbidden-apis.txt
b/codestyle/druid-forbidden-apis.txt
index 5a459f4..49b00fd 100644
--- a/codestyle/druid-forbidden-apis.txt
+++ b/codestyle/druid-forbidden-apis.txt
@@ -8,4 +8,7 @@ org.apache.commons.io.FileUtils#getTempDirectory() @ Use
org.junit.rules.Tempora
java.util.LinkedList @ Use ArrayList or ArrayDeque instead
com.google.common.collect.Lists#newLinkedList() @ Use ArrayList or ArrayDeque
instead
com.google.common.collect.Lists#newLinkedList(java.lang.Iterable) @ Use
ArrayList or ArrayDeque instead
-java.util.Random#() @ Use ThreadLocalRandom.current() or the constructor
with a seed (the latter in tests only!)
\ No newline at end of file
+java.util.Random#() @ Use ThreadLocalRandom.current() or the constructor
with a seed (the latter in tests only!)
+com.fasterxml.jackson.databind.ObjectMapper#reader(com.fasterxml.jackson.databind.JavaType)
@ Use ObjectMapper#readerFor instead
+com.fasterxml.jackson.databind.ObjectMapper#reader(java.lang.Class) @ Use
ObjectMapper#readerFor instead
+com.fasterxml.jackson.databind.ObjectMapper#reader(com.fasterxml.jackson.core.type.TypeReference)
@ Use ObjectMapper#readerFor instead
\ No newline at end of file
diff --git
a/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
b/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
index f204b22..6d77d8e 100644
---
a/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
+++
b/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
@@ -218,7 +218,7 @@ public class WhiteListBasedDruidToTimelineEventConverter
implements DruidToTimel
} else {
fileContent = Files.asCharSource(new File(mapPath),
StandardCharsets.UTF_8).read();
}
- return mapper.reader(new TypeReference>>()
+ return mapper.readerFor(new TypeReference>>()
{
}).readValue(fileContent);
}
diff --git
a/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/ambari/metrics/AmbariMetricsEmitterConfigTest.java
b/extensions-contrib/ambari-metrics-emitter/src/test/java/org/apache/druid/emitter/am
[incubator-druid] branch master updated: HttpPostEmitterMonitor: don't emit maxTime and minTime if no times were recorded (#6418)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new af9efdb HttpPostEmitterMonitor: don't emit maxTime and minTime if no
times were recorded (#6418)
af9efdb is described below
commit af9efdbedfa00410ede691a98daea387c8b79912
Author: Roman Leventov
AuthorDate: Mon Oct 8 17:11:42 2018 -0300
HttpPostEmitterMonitor: don't emit maxTime and minTime if no times were
recorded (#6418)
* HttpPostEmitterMonitor: don't emit maxTime and minTime if no times were
recorded
* Don't emit sum and count if none
* Remove outdated comments
---
.../util/emitter/core/ConcurrentTimeCounter.java | 29 +-
.../java/util/metrics/HttpPostEmitterMonitor.java | 18 +++---
2 files changed, 37 insertions(+), 10 deletions(-)
diff --git
a/java-util/src/main/java/org/apache/druid/java/util/emitter/core/ConcurrentTimeCounter.java
b/java-util/src/main/java/org/apache/druid/java/util/emitter/core/ConcurrentTimeCounter.java
index d4c5435..a71b2fc 100644
---
a/java-util/src/main/java/org/apache/druid/java/util/emitter/core/ConcurrentTimeCounter.java
+++
b/java-util/src/main/java/org/apache/druid/java/util/emitter/core/ConcurrentTimeCounter.java
@@ -21,6 +21,7 @@ package org.apache.druid.java.util.emitter.core;
import com.google.common.primitives.UnsignedInts;
+import javax.annotation.Nullable;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -70,18 +71,34 @@ public class ConcurrentTimeCounter
return timeSumAndCount.getAndSet(0L);
}
- public int getAndResetMaxTime()
+ /**
+ * Returns the max time {@link #add}ed since the previous call to this
method or since the creation of this object,
+ * or null if no times were added.
+ */
+ @Nullable
+ public Integer getAndResetMaxTime()
{
long max = this.max.getAndSet(-1);
-// If max < 0, means no times added yet, then return 0
-return max >= 0 ? (int) max : 0;
+if (max >= 0) {
+ return (int) max;
+} else {
+ return null;
+}
}
- public int getAndResetMinTime()
+ /**
+ * Returns the min time {@link #add}ed since the previous call to this
method or since the creation of this object,
+ * or null if no times were added.
+ */
+ @Nullable
+ public Integer getAndResetMinTime()
{
long min = this.min.getAndSet(-1);
-// If min < 0, means no times added yet, then return 0
-return min >= 0 ? (int) min : 0;
+if (min >= 0) {
+ return (int) min;
+} else {
+ return null;
+}
}
public static int timeSum(long timeSumAndCount)
diff --git
a/java-util/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
b/java-util/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
index de40f0e..a8fd802 100644
---
a/java-util/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
+++
b/java-util/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
@@ -75,10 +75,20 @@ public class HttpPostEmitterMonitor extends
FeedDefiningMonitor
private void emitTimeCounterMetrics(ServiceEmitter emitter,
ConcurrentTimeCounter timeCounter, String metricNameBase)
{
long timeSumAndCount = timeCounter.getTimeSumAndCountAndReset();
-emitter.emit(builder.build(metricNameBase + "timeMsSum",
ConcurrentTimeCounter.timeSum(timeSumAndCount)));
-emitter.emit(builder.build(metricNameBase + "count",
ConcurrentTimeCounter.count(timeSumAndCount)));
-emitter.emit(builder.build(metricNameBase + "maxTimeMs",
timeCounter.getAndResetMaxTime()));
-emitter.emit(builder.build(metricNameBase + "minTimeMs",
timeCounter.getAndResetMinTime()));
+int timeSum = ConcurrentTimeCounter.timeSum(timeSumAndCount);
+int count = ConcurrentTimeCounter.count(timeSumAndCount);
+if (count != 0) {
+ emitter.emit(builder.build(metricNameBase + "timeMsSum", timeSum));
+ emitter.emit(builder.build(metricNameBase + "count", count));
+}
+Integer maxTime = timeCounter.getAndResetMaxTime();
+if (maxTime != null) {
+ emitter.emit(builder.build(metricNameBase + "maxTimeMs", maxTime));
+}
+Integer minTime = timeCounter.getAndResetMinTime();
+if (minTime != null) {
+ emitter.emit(builder.build(metricNameBase + "minTimeMs", minTime));
+}
}
@Override
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Remove Aggregator.clone() methods (#6437)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 09126c0 Remove Aggregator.clone() methods (#6437)
09126c0 is described below
commit 09126c021a786ea13656e8a390732021a8e61b94
Author: Roman Leventov
AuthorDate: Wed Oct 10 10:07:56 2018 -0300
Remove Aggregator.clone() methods (#6437)
* Remove Aggregator.clone() methods
* Remove CardinalityAggregator.name
---
.../query/aggregation/TimestampAggregator.java | 6 --
.../druid/query/aggregation/CountAggregator.java | 6 --
.../query/aggregation/DoubleMaxAggregator.java | 6 --
.../query/aggregation/DoubleMinAggregator.java | 6 --
.../query/aggregation/DoubleSumAggregator.java | 6 --
.../query/aggregation/FloatMaxAggregator.java | 6 --
.../query/aggregation/FloatMinAggregator.java | 6 --
.../query/aggregation/FloatSumAggregator.java | 6 --
.../druid/query/aggregation/LongMaxAggregator.java | 6 --
.../druid/query/aggregation/LongMinAggregator.java | 6 --
.../druid/query/aggregation/LongSumAggregator.java | 6 --
.../cardinality/CardinalityAggregator.java | 22 --
.../cardinality/CardinalityAggregatorFactory.java | 2 +-
.../hyperloglog/HyperUniquesAggregator.java| 6 --
.../cardinality/CardinalityAggregatorTest.java | 14 --
15 files changed, 9 insertions(+), 101 deletions(-)
diff --git
a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregator.java
b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregator.java
index d24741c..c11b50d 100644
---
a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregator.java
+++
b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregator.java
@@ -101,10 +101,4 @@ public class TimestampAggregator implements Aggregator
{
// no resource to cleanup
}
-
- @Override
- public Aggregator clone()
- {
-return new TimestampAggregator(name, selector, timestampSpec, comparator,
initValue);
- }
}
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregator.java
b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregator.java
index da8cdda..3b24ac5 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregator.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregator.java
@@ -69,12 +69,6 @@ public class CountAggregator implements Aggregator
}
@Override
- public Aggregator clone()
- {
-return new CountAggregator();
- }
-
- @Override
public void close()
{
// no resources to cleanup
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregator.java
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregator.java
index 5d9b88a..139c99d 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregator.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregator.java
@@ -71,12 +71,6 @@ public class DoubleMaxAggregator implements Aggregator
}
@Override
- public Aggregator clone()
- {
-return new DoubleMaxAggregator(selector);
- }
-
- @Override
public void close()
{
// no resources to cleanup
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregator.java
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregator.java
index 7956d99..755eb1f 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregator.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregator.java
@@ -71,12 +71,6 @@ public class DoubleMinAggregator implements Aggregator
}
@Override
- public Aggregator clone()
- {
-return new DoubleMinAggregator(selector);
- }
-
- @Override
public void close()
{
// no resources to cleanup
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregator.java
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregator.java
index 67c72b1..3d74b4e 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregator.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregator.java
@@ -79,12 +79,6 @@ public class DoubleSumAggregator implements Aggregator
}
@Override
- public Aggregator clone()
- {
-return new DoubleSumAggregator(selector);
- }
-
- @Override
public void close()
{
// no resources to cleanup
diff --git
a
[incubator-druid] branch master updated: Emit emitter/buffers/allocated/delta and emitter/buffers/failed/delta (#6425)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b57d5df Emit emitter/buffers/allocated/delta and
emitter/buffers/failed/delta (#6425)
b57d5df is described below
commit b57d5df07479174f9a8bbbab7b131040457ce6c6
Author: Roman Leventov
AuthorDate: Mon Oct 15 17:30:29 2018 -0300
Emit emitter/buffers/allocated/delta and emitter/buffers/failed/delta
(#6425)
@gianm gauge `emitter/buffers/failed` and `emitter/buffers/totalAllocated`
metrics proved to be very inconvenient in practice: they are not additive
across a fleet of nodes, are randomly reset when a JVM restarts (that may be
completely independent from metrics emitting issues). Regarding [problem
identification](https://github.com/apache/incubator-druid/pull/4973#issuecomment-338119125),
`emitter/buffers/dropped` serves this well.
Currently in this PR I added `/delta` vs. `/total` metrics, but if there
are no objections I would remove `/total` and so align
`emitter/buffers/allocated` and `/failed` with `/dropped` and `/events/emitted`
which are already delta-only.
---
.../java/util/emitter/core/HttpPostEmitter.java| 4 +-
.../java/util/metrics/HttpPostEmitterMonitor.java | 53 +-
.../emitter/core/HttpPostEmitterStressTest.java| 4 +-
3 files changed, 45 insertions(+), 16 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
index 8f4951e..61dc66b 100644
---
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
+++
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
@@ -882,12 +882,12 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
return approximateBuffersToReuseCount.get();
}
- public int getFailedBuffers()
+ public int getTotalFailedBuffers()
{
return emittingThread.approximateFailedBuffersCount.get();
}
- public int getDroppedBuffers()
+ public int getTotalDroppedBuffers()
{
return droppedBuffers.get();
}
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
b/core/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
index a8fd802..e925fab 100644
---
a/core/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
+++
b/core/src/main/java/org/apache/druid/java/util/metrics/HttpPostEmitterMonitor.java
@@ -31,7 +31,9 @@ public class HttpPostEmitterMonitor extends
FeedDefiningMonitor
private final ImmutableMap extraDimensions;
private final ServiceMetricEvent.Builder builder;
private long lastTotalEmittedEvents = 0;
- private int lastDroppedBuffers = 0;
+ private int lastTotalDroppedBuffers = 0;
+ private int lastTotalAllocatedBuffers = 0;
+ private int lastTotalFailedBuffers = 0;
public HttpPostEmitterMonitor(
String feed,
@@ -48,15 +50,10 @@ public class HttpPostEmitterMonitor extends
FeedDefiningMonitor
@Override
public boolean doMonitor(ServiceEmitter emitter)
{
-long newTotalEmittedEvents = httpPostEmitter.getTotalEmittedEvents();
-long totalEmittedEventsDiff = newTotalEmittedEvents -
lastTotalEmittedEvents;
-emitter.emit(builder.build("emitter/events/emitted",
totalEmittedEventsDiff));
-lastTotalEmittedEvents = newTotalEmittedEvents;
-
-int newDroppedBuffers = httpPostEmitter.getDroppedBuffers();
-int droppedBuffersDiff = newDroppedBuffers - lastDroppedBuffers;
-emitter.emit(builder.build("emitter/buffers/dropped", droppedBuffersDiff));
-lastDroppedBuffers = newDroppedBuffers;
+emitEmittedEvents(emitter);
+emitDroppedBuffers(emitter);
+emitAllocatedBuffers(emitter);
+emitFailedBuffers(emitter);
emitTimeCounterMetrics(emitter,
httpPostEmitter.getBatchFillingTimeCounter(), "emitter/batchFilling/");
emitTimeCounterMetrics(emitter,
httpPostEmitter.getSuccessfulSendingTimeCounter(),
"emitter/successfulSending/");
@@ -64,14 +61,46 @@ public class HttpPostEmitterMonitor extends
FeedDefiningMonitor
emitter.emit(builder.build("emitter/events/emitQueue",
httpPostEmitter.getEventsToEmit()));
emitter.emit(builder.build("emitter/events/large/emitQueue",
httpPostEmitter.getLargeEventsToEmit()));
-emitter.emit(builder.build("emitter/buffers/totalAllocated",
httpPostEmitter.getTotalAllocatedBuffers()));
+
emitter.emit(builder.build("emitter/buffers/emitQueue",
httpPostEmitter.getBuffersToEmit()));
-emitter.emit(builder.build("emitter/buffers/failed",
httpPostEmitter.getFailedBuffers()));
+
emitter.emit(builder.buil
[incubator-druid] branch master updated: QueryCountStatsMonitor: emit query/count (#6473)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new f5f4171 QueryCountStatsMonitor: emit query/count (#6473)
f5f4171 is described below
commit f5f4171a45ad9f79d1b627631643c98a1270ff21
Author: QiuMM
AuthorDate: Fri Oct 19 21:15:02 2018 +0800
QueryCountStatsMonitor: emit query/count (#6473)
Let `QueryCountStatsMonitor` emit `query/count`, then I can monitor QPS of
my services, or I have to count it by myself.
---
docs/content/operations/metrics.md| 3 +++
.../apache/druid/server/metrics/QueryCountStatsMonitor.java | 11 ---
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/docs/content/operations/metrics.md
b/docs/content/operations/metrics.md
index e08aecb..5144b0a 100644
--- a/docs/content/operations/metrics.md
+++ b/docs/content/operations/metrics.md
@@ -35,6 +35,7 @@ Available Metrics
|`query/node/ttfb`|Time to first byte. Milliseconds elapsed until broker
starts receiving the response from individual historical/realtime nodes.|id,
status, server.|< 1s|
|`query/node/backpressure`|Milliseconds that the channel to this node has
spent suspended due to backpressure.|id, status, server.| |
|`query/intervalChunk/time`|Only emitted if interval chunking is enabled.
Milliseconds required to query an interval chunk.|id, status, chunkInterval (if
interval chunking is enabled).|< 1s|
+|`query/count`|number of total queries|This metric is only available if the
QueryCountStatsMonitor module is included.||
|`query/success/count`|number of queries successfully processed|This metric is
only available if the QueryCountStatsMonitor module is included.||
|`query/failed/count`|number of failed queries|This metric is only available
if the QueryCountStatsMonitor module is included.||
|`query/interrupted/count`|number of queries interrupted due to cancellation
or timeout|This metric is only available if the QueryCountStatsMonitor module
is included.||
@@ -49,6 +50,7 @@ Available Metrics
|`segment/scan/pending`|Number of segments in queue waiting to be
scanned.||Close to 0|
|`query/segmentAndCache/time`|Milliseconds taken to query individual segment
or hit the cache (if it is enabled on the historical node).|id,
segment.|several hundred milliseconds|
|`query/cpu/time`|Microseconds of CPU time taken to complete a query|Common:
dataSource, type, interval, hasFilters, duration, context, remoteAddress, id.
Aggregation Queries: numMetrics, numComplexMetrics. GroupBy: numDimensions.
TopN: threshold, dimension.|Varies|
+|`query/count`|number of total queries|This metric is only available if the
QueryCountStatsMonitor module is included.||
|`query/success/count`|number of queries successfully processed|This metric is
only available if the QueryCountStatsMonitor module is included.||
|`query/failed/count`|number of failed queries|This metric is only available
if the QueryCountStatsMonitor module is included.||
|`query/interrupted/count`|number of queries interrupted due to cancellation
or timeout|This metric is only available if the QueryCountStatsMonitor module
is included.||
@@ -60,6 +62,7 @@ Available Metrics
|`query/time`|Milliseconds taken to complete a query.|Common: dataSource,
type, interval, hasFilters, duration, context, remoteAddress, id. Aggregation
Queries: numMetrics, numComplexMetrics. GroupBy: numDimensions. TopN:
threshold, dimension.|< 1s|
|`query/wait/time`|Milliseconds spent waiting for a segment to be scanned.|id,
segment.|several hundred milliseconds|
|`segment/scan/pending`|Number of segments in queue waiting to be
scanned.||Close to 0|
+|`query/count`|number of total queries|This metric is only available if the
QueryCountStatsMonitor module is included.||
|`query/success/count`|number of queries successfully processed|This metric is
only available if the QueryCountStatsMonitor module is included.||
|`query/failed/count`|number of failed queries|This metric is only available
if the QueryCountStatsMonitor module is included.||
|`query/interrupted/count`|number of queries interrupted due to cancellation
or timeout|This metric is only available if the QueryCountStatsMonitor module
is included.||
diff --git
a/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java
b/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java
index 6d88dc6..e217e8d 100644
---
a/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java
+++
b/server/src/main/java/org/apache/druid/server/metrics/QueryCountStatsMonitor.java
@@ -44,11 +44,16 @@ public class QueryCountStatsMonitor extends AbstractMonitor
public boolean doMonitor(ServiceEmitter emitter)
{
final ServiceMetricEvent.Builder builder = new
ServiceMetricEven
[incubator-druid] branch master updated: Add RequestLogEventBuilderFactory (#6477)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2cdce2e Add RequestLogEventBuilderFactory (#6477)
2cdce2e is described below
commit 2cdce2e2a6111acdd8fffd4ef61edc1c89f187bf
Author: Roman Leventov
AuthorDate: Wed Oct 31 22:24:37 2018 +0100
Add RequestLogEventBuilderFactory (#6477)
This PR allows to control the fields in `RequestLogEvent`, emitted in
`EmittingRequestLogger`. In our case, we want to get rid of the `intervals`
fields of the query objects that are a part of `DefaultRequestLogEvent`. They
are enormous (thousands of segments) and not useful.
Related to #5522, FYI @a2l007.
---
.../metrics/DruidToTimelineMetricConverter.java| 1 -
.../druid/emitter/graphite/GraphiteEmitter.java| 4 +-
.../druid/server/log/DefaultRequestLogEvent.java | 104 ++
.../log/DefaultRequestLogEventBuilderFactory.java | 60 +++
.../druid/server/log/EmittingRequestLogger.java| 118 +++--
.../server/log/EmittingRequestLoggerProvider.java | 6 +-
.../apache/druid/server/log/RequestLogEvent.java | 31 ++
.../server/log/RequestLogEventBuilderFactory.java | 38 +++
8 files changed, 252 insertions(+), 110 deletions(-)
diff --git
a/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java
b/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java
index fe0c774..10183e5 100644
---
a/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java
+++
b/extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java
@@ -30,7 +30,6 @@ import
org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
@JsonSubTypes.Type(name = "all", value =
SendAllTimelineEventConverter.class),
@JsonSubTypes.Type(name = "whiteList", value =
WhiteListBasedDruidToTimelineEventConverter.class)
})
-
public interface DruidToTimelineMetricConverter
{
/**
diff --git
a/extensions-contrib/graphite-emitter/src/main/java/org/apache/druid/emitter/graphite/GraphiteEmitter.java
b/extensions-contrib/graphite-emitter/src/main/java/org/apache/druid/emitter/graphite/GraphiteEmitter.java
index fba66e3..e80a249 100644
---
a/extensions-contrib/graphite-emitter/src/main/java/org/apache/druid/emitter/graphite/GraphiteEmitter.java
+++
b/extensions-contrib/graphite-emitter/src/main/java/org/apache/druid/emitter/graphite/GraphiteEmitter.java
@@ -29,7 +29,7 @@ import org.apache.druid.java.util.emitter.core.Emitter;
import org.apache.druid.java.util.emitter.core.Event;
import org.apache.druid.java.util.emitter.service.AlertEvent;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
-import org.apache.druid.server.log.EmittingRequestLogger;
+import org.apache.druid.server.log.RequestLogEvent;
import java.io.IOException;
import java.net.SocketException;
@@ -124,7 +124,7 @@ public class GraphiteEmitter implements Emitter
log.error(e, "got interrupted with message [%s]", e.getMessage());
Thread.currentThread().interrupt();
}
-} else if (event instanceof EmittingRequestLogger.RequestLogEvent) {
+} else if (event instanceof RequestLogEvent) {
for (Emitter emitter : requestLogEmitters) {
emitter.emit(event);
}
diff --git
a/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java
b/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java
new file mode 100644
index 000..83cdbd2
--- /dev/null
+++
b/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.log;
+
+import com.
[incubator-druid] branch master updated: Find duplicate lines with checkstyle; enable some duplicate inspections in IntelliJ (#6558)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 887c645 Find duplicate lines with checkstyle; enable some duplicate
inspections in IntelliJ (#6558)
887c645 is described below
commit 887c6456755bcd8d1bab96b41c9e569f00e9123e
Author: Roman Leventov
AuthorDate: Mon Nov 26 16:55:42 2018 +0100
Find duplicate lines with checkstyle; enable some duplicate inspections in
IntelliJ (#6558)
Not putting this to 0.13 milestone because the found bugs are not critical
(one is a harmless DI config duplicate, and another is in a benchmark.
Change in `DumpSegment` is just an indentation change.
---
.idea/inspectionProfiles/Druid.xml | 8 +++
.../druid/benchmark/query/SearchBenchmark.java | 7 +--
codestyle/checkstyle-suppressions.xml | 7 ++-
codestyle/checkstyle.xml | 8 +++
.../druid/jackson/CommaListJoinDeserializer.java | 3 +-
.../druid/jackson/CommaListJoinSerializer.java | 4 +-
.../apache/druid/indexer/TaskStatusPlusTest.java | 4 +-
.../tuple/ArrayOfDoublesSketchJsonSerializer.java | 3 +-
.../cache/JdbcExtractionNamespaceTest.java | 7 +--
.../org/apache/druid/guice/NullHandlingModule.java | 1 -
.../jackson/DruidDefaultSerializersModule.java | 8 +--
.../java/org/apache/druid/jackson/JodaStuff.java | 6 +-
.../java/org/apache/druid/cli/DumpSegment.java | 71 +++---
13 files changed, 69 insertions(+), 68 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index 97f079f..6bb556d 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -29,6 +29,11 @@
+
+
+
+
+
@@ -71,6 +76,7 @@
+
@@ -80,6 +86,8 @@
+
+
diff --git
a/benchmarks/src/main/java/org/apache/druid/benchmark/query/SearchBenchmark.java
b/benchmarks/src/main/java/org/apache/druid/benchmark/query/SearchBenchmark.java
index 3b2f667..c7008fc 100644
---
a/benchmarks/src/main/java/org/apache/druid/benchmark/query/SearchBenchmark.java
+++
b/benchmarks/src/main/java/org/apache/druid/benchmark/query/SearchBenchmark.java
@@ -282,7 +282,9 @@ public class SearchBenchmark
private static SearchQueryBuilder basicD(final BenchmarkSchemaInfo
basicSchema)
{
-final QuerySegmentSpec intervalSpec = new
MultipleIntervalSegmentSpec(Collections.singletonList(basicSchema.getDataInterval()));
+final QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(
+Collections.singletonList(basicSchema.getDataInterval())
+);
final List dimUniformFilterVals = new ArrayList<>();
final int resultNum = (int) (10 * 0.1);
@@ -296,9 +298,6 @@ public class SearchBenchmark
dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, null));
dimFilters.add(new SelectorDimFilter(dimName, "3", null));
dimFilters.add(new BoundDimFilter(dimName, "100", "1", true, true,
true, null, null));
-dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, null));
-dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, null));
-dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, null));
return Druids.newSearchQueryBuilder()
.dataSource("blah")
diff --git a/codestyle/checkstyle-suppressions.xml
b/codestyle/checkstyle-suppressions.xml
index bf5087c..0c8300a 100644
--- a/codestyle/checkstyle-suppressions.xml
+++ b/codestyle/checkstyle-suppressions.xml
@@ -50,13 +50,16 @@
-
+
-
+
+
+
diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml
index 16e8c14..d6ef3e8 100644
--- a/codestyle/checkstyle.xml
+++ b/codestyle/checkstyle.xml
@@ -221,6 +221,7 @@
+
+
+
+
+
+
+
+
diff --git
a/core/src/main/java/org/apache/druid/jackson/CommaListJoinDeserializer.java
b/core/src/main/java/org/apache/druid/jackson/CommaListJoinDeserializer.java
index bda1f36..b9910e6 100644
--- a/core/src/main/java/org/apache/druid/jackson/CommaListJoinDeserializer.java
+++ b/core/src/main/java/org/apache/druid/jackson/CommaListJoinDeserializer.java
@@ -20,7 +20,6 @@
package org.apache.druid.jackson;
import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer;
@@ -39,7 +38,7 @@ public class CommaListJoinDeserializer extends
StdScalarDeserializer deserialize(JsonParser jsonParser,
[incubator-druid] branch master updated: Fix a race in FileSessionCredentialsProvider (#6664)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b4a4669 Fix a race in FileSessionCredentialsProvider (#6664)
b4a4669 is described below
commit b4a46691282e4fd36bc158cdb040147e80380a24
Author: Roman Leventov
AuthorDate: Tue Nov 27 21:02:30 2018 +0100
Fix a race in FileSessionCredentialsProvider (#6664)
`sessionToken`, `accessKey` and `secretKey` must be updated atomically.
Another race is possible between the file updater and the Druid process
reading the file. It could be enforced only with mandatory file locking, but
file locking is advisory by default in Linux.
---
.../common/aws/FileSessionCredentialsProvider.java | 87 +-
1 file changed, 52 insertions(+), 35 deletions(-)
diff --git
a/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java
b/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java
index 3dbe64b..ad4000e 100644
---
a/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java
+++
b/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java
@@ -24,27 +24,29 @@ import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSSessionCredentials;
import org.apache.druid.java.util.common.concurrent.Execs;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Properties;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public class FileSessionCredentialsProvider implements AWSCredentialsProvider
{
- private final String sessionCredentials;
- private volatile String sessionToken;
- private volatile String accessKey;
- private volatile String secretKey;
-
private final ScheduledExecutorService scheduler =
Execs.scheduledSingleThreaded("FileSessionCredentialsProviderRefresh-%d");
+ private final String sessionCredentialsFile;
+
+ /**
+ * This field doesn't need to be volatile. From the Java Memory Model point
of view, volatile on this field changes
+ * nothing and doesn't provide any extra guarantees.
+ */
+ private AWSSessionCredentials awsSessionCredentials;
- public FileSessionCredentialsProvider(String sessionCredentials)
+ public FileSessionCredentialsProvider(String sessionCredentialsFile)
{
-this.sessionCredentials = sessionCredentials;
+this.sessionCredentialsFile = sessionCredentialsFile;
refresh();
scheduler.scheduleAtFixedRate(this::refresh, 1, 1, TimeUnit.HOURS); //
refresh every hour
@@ -53,26 +55,7 @@ public class FileSessionCredentialsProvider implements
AWSCredentialsProvider
@Override
public AWSCredentials getCredentials()
{
-return new AWSSessionCredentials()
-{
- @Override
- public String getSessionToken()
- {
-return sessionToken;
- }
-
- @Override
- public String getAWSAccessKeyId()
- {
-return accessKey;
- }
-
- @Override
- public String getAWSSecretKey()
- {
-return secretKey;
- }
-};
+return awsSessionCredentials;
}
@Override
@@ -80,16 +63,50 @@ public class FileSessionCredentialsProvider implements
AWSCredentialsProvider
{
try {
Properties props = new Properties();
- InputStream is = new FileInputStream(new File(sessionCredentials));
- props.load(is);
- is.close();
+ try (InputStream is =
Files.newInputStream(Paths.get(sessionCredentialsFile))) {
+props.load(is);
+ }
+
+ String sessionToken = props.getProperty("sessionToken");
+ String accessKey = props.getProperty("accessKey");
+ String secretKey = props.getProperty("secretKey");
- sessionToken = props.getProperty("sessionToken");
- accessKey = props.getProperty("accessKey");
- secretKey = props.getProperty("secretKey");
+ awsSessionCredentials = new Credentials(sessionToken, accessKey,
secretKey);
}
catch (IOException e) {
throw new RuntimeException("cannot refresh AWS credentials", e);
}
}
+
+ private static class Credentials implements AWSSessionCredentials
+ {
+private final String sessionToken;
+private final String accessKey;
+private final String secretKey;
+
+private Credentials(String sessionToken, String accessKey, String
secretKey)
+{
+ this.sessionToken = sessionToken;
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+}
+
+@Override
+public String getSessionToken()
+{
+ return session
[incubator-druid] branch master updated: Simplify DruidNodeDiscoveryProvider; add DruidNodeDiscovery.Listener.nodeViewInitialized() (#6606)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new ec38df7 Simplify DruidNodeDiscoveryProvider; add
DruidNodeDiscovery.Listener.nodeViewInitialized() (#6606)
ec38df7 is described below
commit ec38df75756b975461bf1234e26cd195c462107d
Author: Roman Leventov
AuthorDate: Sat Dec 1 01:12:56 2018 +0100
Simplify DruidNodeDiscoveryProvider; add
DruidNodeDiscovery.Listener.nodeViewInitialized() (#6606)
* Simplify DruidNodeDiscoveryProvider; add
DruidNodeDiscovery.Listener.nodeViewInitialized() method; prohibit and
eliminate some suboptimal Java 8 patterns
* Fix style
* Fix HttpEmitterTest.timeoutEmptyQueue()
* Add DruidNodeDiscovery.Listener.nodeViewInitialized() calls in tests
* Clarify code
---
.idea/inspectionProfiles/Druid.xml | 2 +
.../java/util/emitter/core/HttpPostEmitter.java| 31 +++---
.../data/input/impl/FileIteratingFirehoseTest.java | 3 +-
.../java/util/emitter/core/HttpEmitterTest.java| 12 ++-
.../indexing/kafka/LegacyKafkaIndexTaskRunner.java | 7 +-
.../overlord/hrtr/HttpRemoteTaskRunner.java| 16 +--
.../overlord/hrtr/HttpRemoteTaskRunnerTest.java| 1 +
.../apache/druid/query/filter/AndDimFilter.java| 3 +-
.../org/apache/druid/query/filter/OrDimFilter.java | 3 +-
.../apache/druid/query/groupby/GroupByQuery.java | 2 +-
.../druid/segment/IndexMergerNullHandlingTest.java | 2 +-
.../druid/client/HttpServerInventoryView.java | 22 ++---
.../CuratorDruidNodeDiscoveryProvider.java | 82
.../apache/druid/discovery/DruidNodeDiscovery.java | 17 ++--
.../discovery/DruidNodeDiscoveryProvider.java | 109 ++---
.../appenderator/AppenderatorDriverMetadata.java | 44 -
.../server/lookup/cache/LookupNodeDiscovery.java | 2 +-
.../server/router/TieredBrokerHostSelector.java| 5 +-
.../druid/client/HttpServerInventoryViewTest.java | 1 +
.../CuratorDruidNodeAnnouncerAndDiscoveryTest.java | 9 +-
.../discovery/DruidNodeDiscoveryProviderTest.java | 9 +-
.../coordinator/CuratorDruidCoordinatorTest.java | 3 +-
.../server/coordinator/HttpLoadQueuePeonTest.java | 1 +
.../router/TieredBrokerHostSelectorTest.java | 1 +
.../expression/builtin/CeilOperatorConversion.java | 21 ++--
.../druid/sql/calcite/rule/GroupByRules.java | 6 +-
26 files changed, 236 insertions(+), 178 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index 6bb556d..dfedeb4 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -76,6 +76,7 @@
+
@@ -229,6 +230,7 @@
+
diff --git
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
index 8b486cb..d2da5a7 100644
---
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
+++
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
@@ -140,7 +140,7 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
private final AtomicInteger allocatedBuffers = new AtomicInteger();
private final AtomicInteger droppedBuffers = new AtomicInteger();
- private volatile long lastFillTimeMillis;
+ private volatile long lastBatchFillTimeMillis;
private final ConcurrentTimeCounter batchFillingTimeCounter = new
ConcurrentTimeCounter();
private final Object startLock = new Object();
@@ -180,8 +180,8 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
emittingThread = new EmittingThread(config);
long firstBatchNumber = 1;
concurrentBatch.set(new Batch(this, acquireBuffer(), firstBatchNumber));
-// lastFillTimeMillis must not be 0, minHttpTimeoutMillis could be.
-lastFillTimeMillis = Math.max(config.minHttpTimeoutMillis, 1);
+// lastBatchFillTimeMillis must not be 0, minHttpTimeoutMillis could be.
+lastBatchFillTimeMillis = Math.max(config.minHttpTimeoutMillis, 1);
}
@Override
@@ -328,7 +328,7 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
if (elapsedTimeMillis > 0) {
// If elapsedTimeMillis is 0 or negative, it's likely because
System.currentTimeMillis() is not monotonic, so not
// accounting this time for determining batch sending timeout.
- lastFillTimeMillis = elapsedTimeMillis;
+ lastBatchFillTimeMillis = elapsedTimeMillis;
}
addBatchToEmitQueue(batch);
wakeUpEmittingThread();
@@ -663,7 +663,7 @@ public class HttpPostEmitter implements Flushable,
Closeable,
[incubator-druid] branch master updated: Double-checked locking bugs (#6662)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new bbb283f Double-checked locking bugs (#6662)
bbb283f is described below
commit bbb283fa34258942227e5690635753993fdcfd17
Author: Furkan KAMACI
AuthorDate: Fri Dec 7 19:10:29 2018 +0300
Double-checked locking bugs (#6662)
* Double-checked locking bug is fixed.
* @Nullable is removed since there is no need to use along with
@MonotonicNonNull.
* Static import is removed.
* Lazy initialization is implemented.
* Local variables used instead of volatile ones.
* Local variables used instead of volatile ones.
---
aws-common/pom.xml | 5 +++
.../aws/LazyFileSessionCredentialsProvider.java| 28 +++
pom.xml| 1 +
processing/pom.xml | 5 +++
.../aggregation/JavaScriptAggregatorFactory.java | 40 +++---
.../aggregation/post/JavaScriptPostAggregator.java | 40 ++
.../query/extraction/JavaScriptExtractionFn.java | 35 +--
.../druid/query/filter/JavaScriptDimFilter.java| 35 +--
8 files changed, 136 insertions(+), 53 deletions(-)
diff --git a/aws-common/pom.xml b/aws-common/pom.xml
index 4b941c4..941dc28 100644
--- a/aws-common/pom.xml
+++ b/aws-common/pom.xml
@@ -45,6 +45,11 @@
com.amazonaws
aws-java-sdk-s3
+
+org.checkerframework
+checker
+${checkerframework.version}
+
diff --git
a/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java
b/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java
index 7fc046b..483b32a 100644
---
a/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java
+++
b/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java
@@ -21,27 +21,43 @@ package org.apache.druid.common.aws;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
public class LazyFileSessionCredentialsProvider implements
AWSCredentialsProvider
{
- private AWSCredentialsConfig config;
- private FileSessionCredentialsProvider provider;
+ private final AWSCredentialsConfig config;
+
+ /**
+ * The field is declared volatile in order to ensure safe publication of the
object
+ * in {@link #getUnderlyingProvider()} without worrying about final modifiers
+ * on the fields of the created object
+ *
+ * @see https://github.com/apache/incubator-druid/pull/6662#discussion_r237013157";>
+ *
https://github.com/apache/incubator-druid/pull/6662#discussion_r237013157
+ */
+ @MonotonicNonNull
+ private volatile FileSessionCredentialsProvider provider;
public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config)
{
this.config = config;
}
+ @EnsuresNonNull("provider")
private FileSessionCredentialsProvider getUnderlyingProvider()
{
-if (provider == null) {
+FileSessionCredentialsProvider syncedProvider = provider;
+if (syncedProvider == null) {
synchronized (config) {
-if (provider == null) {
- provider = new
FileSessionCredentialsProvider(config.getFileSessionCredentials());
+syncedProvider = provider;
+if (syncedProvider == null) {
+ syncedProvider = new
FileSessionCredentialsProvider(config.getFileSessionCredentials());
+ provider = syncedProvider;
}
}
}
-return provider;
+return syncedProvider;
}
@Override
diff --git a/pom.xml b/pom.xml
index 4c2938e..46a1355 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,6 +100,7 @@
2.5.5
3.4.11
+2.5.7
apache.snapshots
Apache Snapshot Repository
https://repository.apache.org/snapshots
diff --git a/processing/pom.xml b/processing/pom.xml
index 35f6dd0..542c3f2 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -111,6 +111,11 @@
org.ow2.asm
asm-commons
+
+org.checkerframework
+checker
+${checkerframework.version}
+
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java
b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java
index e747ad6..b5e9ed1 100644
---
a/processing/src/main/java/org/apache/druid
[incubator-druid] branch master updated: FileUtils: Sync directory entry too on writeAtomically. (#6677)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b7709e1 FileUtils: Sync directory entry too on writeAtomically.
(#6677)
b7709e1 is described below
commit b7709e12451787f2b2ceff27239bbe20c19648e5
Author: Gian Merlino
AuthorDate: Sat Dec 8 08:12:59 2018 -0800
FileUtils: Sync directory entry too on writeAtomically. (#6677)
* FileUtils: Sync directory entry too on writeAtomically.
See the fsync(2) man page for why this is important:
https://linux.die.net/man/2/fsync
This also plumbs CompressionUtils's "zip" function through
writeAtomically, so the code for handling atomic local filesystem
writes is all done in the same place.
* Remove unused import.
* Avoid FileOutputStream.
* Allow non-atomic writes to overwrite.
* Add some comments. And no need to flush an unbuffered stream.
---
.../druid/java/util/common/CompressionUtils.java | 24 +++
.../apache/druid/java/util/common/FileUtils.java | 48 --
.../druid/java/util/common/FileUtilsTest.java | 10 -
...natorPollingBasicAuthenticatorCacheManager.java | 8 +++-
...rdinatorPollingBasicAuthorizerCacheManager.java | 8 +++-
.../druid/query/lookup/LookupSnapshotTaker.java| 8 +++-
6 files changed, 79 insertions(+), 27 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/java/util/common/CompressionUtils.java
b/core/src/main/java/org/apache/druid/java/util/common/CompressionUtils.java
index 79e010a..ce94a4a 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/CompressionUtils.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/CompressionUtils.java
@@ -41,7 +41,10 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
+import java.nio.file.StandardOpenOption;
import java.util.Enumeration;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -78,16 +81,19 @@ public class CompressionUtils
log.warn("No .zip suffix[%s], putting files from [%s] into it anyway.",
outputZipFile, directory);
}
-try (final FileOutputStream out = new FileOutputStream(outputZipFile)) {
- long bytes = zip(directory, out);
-
- // For explanation of why fsyncing here is a good practice:
- //
https://github.com/apache/incubator-druid/pull/5187#pullrequestreview-85188984
- if (fsync) {
-out.getChannel().force(true);
+if (fsync) {
+ return FileUtils.writeAtomically(outputZipFile, out -> zip(directory,
out));
+} else {
+ try (
+ final FileChannel fileChannel = FileChannel.open(
+ outputZipFile.toPath(),
+ StandardOpenOption.WRITE,
+ StandardOpenOption.CREATE
+ );
+ final OutputStream out = Channels.newOutputStream(fileChannel)
+ ) {
+return zip(directory, out);
}
-
- return bytes;
}
}
diff --git
a/core/src/main/java/org/apache/druid/java/util/common/FileUtils.java
b/core/src/main/java/org/apache/druid/java/util/common/FileUtils.java
index 97ac6cb..1ba63c1 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/FileUtils.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/FileUtils.java
@@ -24,16 +24,19 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
+import org.apache.druid.java.util.common.logger.Logger;
+import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.MappedByteBuffer;
+import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -41,6 +44,8 @@ import java.util.UUID;
public class FileUtils
{
+ private static final Logger log = new Logger(FileUtils.class);
+
/**
* Useful for retry functionality that doesn't want to stop Throwables, but
does want to retry on Exceptions
*/
@@ -182,22 +187,35 @@ public class FileUtils
*
* This method is not just thread-safe, but is also safe to use from
multiple processes on the same machine.
*/
- public static void writeAtomically(final File file, OutputStreamConsumer f)
throws IOException
+ public static T writeAtomically(final File file
[incubator-druid] branch master updated: add charset UTF-8 to log api (#6709)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 91e3cf7 add charset UTF-8 to log api (#6709)
91e3cf7 is described below
commit 91e3cf7196ced5a5f9e0128ffc040104dff897a3
Author: dongyifeng
AuthorDate: Wed Dec 12 23:31:04 2018 +0800
add charset UTF-8 to log api (#6709)
When I retrieve the task log in browser, the Chinese characters all end up
as garbage.
After adding charset UTF-8, it was correct.
---
codestyle/checkstyle.xml | 6 +
.../indexing/overlord/http/OverlordResource.java | 3 ++-
.../druid/indexing/worker/http/WorkerResource.java | 3 ++-
.../apache/druid/server/http/HttpMediaType.java| 27 ++
4 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml
index d6ef3e8..d42b705 100644
--- a/codestyle/checkstyle.xml
+++ b/codestyle/checkstyle.xml
@@ -198,6 +198,12 @@
+
+
+
+
+
+
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
index fd7fae4..0e230db 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordResource.java
@@ -58,6 +58,7 @@ import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.metadata.EntryExistsException;
+import org.apache.druid.server.http.HttpMediaType;
import org.apache.druid.server.http.security.ConfigResourceFilter;
import org.apache.druid.server.http.security.DatasourceResourceFilter;
import org.apache.druid.server.http.security.StateResourceFilter;
@@ -953,7 +954,7 @@ public class OverlordResource
@GET
@Path("/task/{taskid}/log")
- @Produces("text/plain")
+ @Produces(HttpMediaType.TEXT_PLAIN_UTF8)
@ResourceFilters(TaskResourceFilter.class)
public Response doGetLog(
@PathParam("taskid") final String taskid,
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java
index bef73ce..5251321 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java
@@ -34,6 +34,7 @@ import
org.apache.druid.indexing.worker.WorkerCuratorCoordinator;
import org.apache.druid.indexing.worker.WorkerTaskMonitor;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.server.http.HttpMediaType;
import org.apache.druid.server.http.security.ConfigResourceFilter;
import org.apache.druid.server.http.security.StateResourceFilter;
import org.apache.druid.tasklogs.TaskLogStreamer;
@@ -179,7 +180,7 @@ public class WorkerResource
@GET
@Path("/task/{taskid}/log")
- @Produces("text/plain")
+ @Produces(HttpMediaType.TEXT_PLAIN_UTF8)
@ResourceFilters(StateResourceFilter.class)
public Response doGetLog(
@PathParam("taskid") String taskid,
diff --git
a/server/src/main/java/org/apache/druid/server/http/HttpMediaType.java
b/server/src/main/java/org/apache/druid/server/http/HttpMediaType.java
new file mode 100644
index 000..6f69c3a
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/http/HttpMediaType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific langu
[incubator-druid] branch master updated: Adds bloom filter aggregator to 'druid-bloom-filters' extension (#6397)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new a6d81c0 Adds bloom filter aggregator to 'druid-bloom-filters' extension (#6397) a6d81c0 is described below commit a6d81c0d1654b28d15d922dbe0d128489978fc89 Author: Clint Wylie AuthorDate: Tue Jan 29 05:05:17 2019 -0800 Adds bloom filter aggregator to 'druid-bloom-filters' extension (#6397) * blooming aggs * partially address review * fix docs * minor test refactor after rebase * use copied bloomkfilter * add ByteBuffer methods to BloomKFilter to allow agg to use in place, simplify some things, more tests * add methods to BloomKFilter to get number of set bits, use in comparator, fixes * more docs * fix * fix style * simplify bloomfilter bytebuffer merge, change methods to allow passing buffer offsets * oof, more fixes * more sane docs example * fix it * do the right thing in the right place * formatting * fix * avoid conflict * typo fixes, faster comparator, docs for comparator behavior * unused imports * use buffer comparator instead of deserializing * striped readwrite lock for buffer agg, null handling comparator, other review changes * style fixes * style * remove sync for now * oops * consistency * inspect runtime shape of selector instead of selector plus, static comparator, add inner exception on serde exception * CardinalityBufferAggregator inspect selectors instead of selectorPluses * fix style * refactor away from using ColumnSelectorPlus and ColumnSelectorStrategyFactory to instead use specialized aggregators for each supported column type, other review comments * adjustment * fix teamcity error? * rename nil aggs to empty, change empty agg constructor signature, add comments * use stringutils base64 stuff to be chill with master * add aggregate combiner, comment --- .../development/extensions-core/bloom-filter.md| 104 +++- .../DerivativeDataSourceManager.java | 2 +- .../materializedview/MaterializedViewUtils.java| 2 +- .../druid/guice/BloomFilterSerializersModule.java | 12 +- .../bloom/BaseBloomFilterAggregator.java | 69 +++ .../bloom/BaseBloomFilterBufferAggregator.java | 66 +-- .../bloom/BloomFilterAggregateCombiner.java| 72 +++ .../bloom/BloomFilterAggregatorFactory.java| 296 ++ .../bloom/BloomFilterMergeAggregator.java | 57 ++ .../bloom/BloomFilterMergeAggregatorFactory.java | 84 +++ .../bloom/BloomFilterMergeBufferAggregator.java| 40 ++ .../query/aggregation/bloom/BloomFilterSerde.java | 69 +++ .../bloom/DoubleBloomFilterAggregator.java | 42 ++ .../bloom/DoubleBloomFilterBufferAggregator.java | 44 ++ .../bloom/EmptyBloomFilterAggregator.java | 37 ++ .../bloom/EmptyBloomFilterBufferAggregator.java| 44 ++ .../bloom/FloatBloomFilterAggregator.java | 42 ++ .../bloom/FloatBloomFilterBufferAggregator.java| 44 ++ .../bloom/LongBloomFilterAggregator.java | 42 ++ .../bloom/LongBloomFilterBufferAggregator.java | 44 ++ .../bloom/StringBloomFilterAggregator.java | 54 ++ .../bloom/StringBloomFilterBufferAggregator.java | 56 ++ .../apache/druid/query/filter/BloomKFilter.java| 278 - .../bloom/BloomFilterAggregatorTest.java | 656 + .../bloom/BloomFilterGroupByQueryTest.java | 177 ++ .../druid/query/filter/BloomKFilterTest.java | 541 + .../src/test/resources/sample.data.tsv | 13 + .../druid/query/aggregation/AggregatorUtil.java| 4 + .../cardinality/CardinalityBufferAggregator.java | 5 +- 29 files changed, 2935 insertions(+), 61 deletions(-) diff --git a/docs/content/development/extensions-core/bloom-filter.md b/docs/content/development/extensions-core/bloom-filter.md index 3b83ff0..f878e75 100644 --- a/docs/content/development/extensions-core/bloom-filter.md +++ b/docs/content/development/extensions-core/bloom-filter.md @@ -24,22 +24,44 @@ title: "Bloom Filter" # Bloom Filter -Make sure to [include](../../operations/including-extensions.html) `druid-bloom-filter` as an extension. +This extension adds the ability to both construct bloom filters from query results, and filter query results by testing +against a bloom filter. Make sure to [include](../../operations/including-extensions.html) `druid-bloom-filter` as an +extension. -BloomFilter is a probabilistic data structure for set members
[incubator-druid] branch master updated: Added an allocation rate metric #6604 (#6710)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2803fda Added an allocation rate metric #6604 (#6710)
2803fda is described below
commit 2803fda8b781ffd13739ce49779e0e0578332d91
Author: Egor Riashin
AuthorDate: Tue Jan 29 16:16:35 2019 +0300
Added an allocation rate metric #6604 (#6710)
Addressing #6604
---
.../util/metrics/AllocationMetricCollector.java| 90 +
.../util/metrics/AllocationMetricCollectors.java | 59 +++
.../apache/druid/java/util/metrics/JvmMonitor.java | 14 +++
.../metrics/AllocationMetricCollectorTest.java | 108 +
4 files changed, 271 insertions(+)
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/AllocationMetricCollector.java
b/core/src/main/java/org/apache/druid/java/util/metrics/AllocationMetricCollector.java
new file mode 100644
index 000..a0a9d7f
--- /dev/null
+++
b/core/src/main/java/org/apache/druid/java/util/metrics/AllocationMetricCollector.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.java.util.metrics;
+
+import it.unimi.dsi.fastutil.longs.Long2LongMap;
+import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import java.lang.management.ThreadMXBean;
+import java.lang.reflect.Method;
+
+class AllocationMetricCollector
+{
+ private static final Logger log = new
Logger(AllocationMetricCollector.class);
+
+ private static final int NO_DATA = -1;
+
+ private final Method getThreadAllocatedBytes;
+ private final ThreadMXBean threadMXBean;
+
+ private Long2LongMap previousResults;
+
+ AllocationMetricCollector(Method method, ThreadMXBean threadMXBean)
+ {
+this.getThreadAllocatedBytes = method;
+this.threadMXBean = threadMXBean;
+
+previousResults = new Long2LongOpenHashMap();
+previousResults.defaultReturnValue(NO_DATA);
+ }
+
+ /**
+ * Uses getThreadAllocatedBytes internally {@link
com.sun.management.ThreadMXBean#getThreadAllocatedBytes}.
+ *
+ * Tests show the call to getThreadAllocatedBytes for a single thread ID out
of 500 threads running takes around
+ * 9000 ns (in the worst case), which for 500 IDs should take
500*9000/1000/1000 = 4.5 ms to the max.
+ * AllocationMetricCollector takes linear time to calculate delta, for 500
threads it's negligible.
+ * See the default emitting period {@link
MonitorSchedulerConfig#getEmitterPeriod}.
+ *
+ * @return all threads summed allocated bytes delta
+ */
+ long calculateDelta()
+ {
+try {
+ long[] allThreadIds = threadMXBean.getAllThreadIds();
+ // the call time depends on number of threads, for 500 threads the
estimated time is 4 ms
+ long[] bytes = (long[]) getThreadAllocatedBytes.invoke(threadMXBean,
(Object) allThreadIds);
+ long sum = 0;
+ Long2LongMap newResults = new Long2LongOpenHashMap();
+ newResults.defaultReturnValue(NO_DATA);
+ for (int i = 0; i < allThreadIds.length; i++) {
+long threadId = allThreadIds[i];
+long previous = previousResults.get(threadId);
+long current = bytes[i];
+newResults.put(threadId, current);
+// a) some threads can be terminated and their ids won't be present
+// b) if new threads ids can collide with terminated threads ids then
the current allocation can be lesser than
+// before
+if (previous == NO_DATA || previous > current) {
+ sum += current;
+} else {
+ sum += current - previous;
+}
+ }
+ previousResults = newResults;
+ return sum;
+}
+catch (ReflectiveOperationException e) {
+ log.error(e, "Cannot calculate delta"); // it doesn't make sense after
initialization is complete
+}
+return 0;
+ }
+}
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/AllocationMetricCollectors.java
b/core/src/main/java/org/apache/druid/ja
[incubator-druid] branch master updated: Fix and document concurrency of EventReceiverFirehose and TimedShutoffFirehose; Refine concurrency specification of Firehose (#7038)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 10c9f6d Fix and document concurrency of EventReceiverFirehose and
TimedShutoffFirehose; Refine concurrency specification of Firehose (#7038)
10c9f6d is described below
commit 10c9f6d7086fc2bbccb8673bce55b1e711d131e7
Author: Roman Leventov
AuthorDate: Mon Mar 4 18:50:03 2019 -0300
Fix and document concurrency of EventReceiverFirehose and
TimedShutoffFirehose; Refine concurrency specification of Firehose (#7038)
`EventReceiverFirehoseFactory`
Fixed several concurrency bugs in `EventReceiverFirehoseFactory`:
- Race condition over putting an entry into `producerSequences` in
`checkProducerSequence()`.
- `Stopwatch` used to measure time across threads, but it's a
non-thread-safe class.
- Use `System.nanoTime()` instead of `System.currentTimeMillis()` because
the latter are [not suitable](https://stackoverflow.com/a/351571/648955) for
measuring time intervals.
- `close()` was not synchronized by could be called from multiple threads
concurrently.
Removed unnecessary `readLock` (protecting `hasMore()` and `nextRow()`
which are always called from a single thread). Removed unnecessary `volatile`
modifiers.
Documented threading model and concurrent control flow of
`EventReceiverFirehose` instances.
**Important:** please read the updated Javadoc for
`EventReceiverFirehose.addAll()`. It allows events from different requests
(batches) to be interleaved in the buffer. Is this OK?
`TimedShutoffFirehoseFactory`
- Fixed a race condition that was possible because `close()` that was not
properly synchronized.
Documented threading model and concurrent control flow of
`TimedShutoffFirehose` instances.
`Firehose`
Refined concurrency contract of `Firehose` based on `EventReceiverFirehose`
implementation. Importantly, now it states that `close()` doesn't affect
`hasMore()` and `nextRow()` and could be called concurrently with them. In
other words, specified that `close()` is for "row supply" side rather than "row
consume" side. However, I didn't check that other `Firehose` implementatations
adhere to this contract.
This issue is the result of reviewing `EventReceiverFirehose` and
`TimedShutoffFirehose` using [this
checklist](https://medium.com/@leventov/code-review-checklist-java-concurrency-49398c326154).
---
.idea/inspectionProfiles/Druid.xml | 4 +-
.../org/apache/druid/concurrent/LifecycleLock.java | 6 +-
.../java/org/apache/druid/concurrent/Threads.java | 59 +++
.../java/org/apache/druid/data/input/Firehose.java | 41 +-
.../org/apache/druid/data/input/FirehoseV2.java| 7 +-
.../org/apache/druid/utils/CloseableUtils.java | 49 +++
docs/content/ingestion/firehose.md | 5 +-
.../firehose/EventReceiverFirehoseFactory.java | 460 ++---
.../firehose/TimedShutoffFirehoseFactory.java | 54 +--
.../firehose/EventReceiverFirehoseIdleTest.java| 22 +-
.../firehose/EventReceiverFirehoseTest.java| 34 +-
11 files changed, 525 insertions(+), 216 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index 70a6a49..b2dcda3 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -46,6 +46,7 @@
+
@@ -119,7 +120,7 @@
-
+
@@ -356,6 +357,5 @@
-
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/druid/concurrent/LifecycleLock.java
b/core/src/main/java/org/apache/druid/concurrent/LifecycleLock.java
index ed6fea2..021c413 100644
--- a/core/src/main/java/org/apache/druid/concurrent/LifecycleLock.java
+++ b/core/src/main/java/org/apache/druid/concurrent/LifecycleLock.java
@@ -23,9 +23,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.AbstractQueuedSynchronizer;
/**
- * A synchronization tool for lifecycled objects (see {@link
org.apache.druid.java.util.common.lifecycle.Lifecycle}, that need
- * happens-before between start() and other methods and/or to check that the
object was successfully started in other
- * methods.
+ * A synchronization tool for lifecycled objects (see {@link
org.apache.druid.java.util.common.lifecycle.Lifecycle},
+ * that need happens-before between start() and other methods and/or to check
that the object was successfully started
+ * in other methods.
*
* Guarantees in terms of JMM: happens-before between {@link #exitStart()} and
{@link #awaitStarted()},
* exitStart() and {@link #canStop()}, if it returns
[incubator-druid] branch master updated: Remove unnecessary check for contains() in LoadRule (#7073)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 8804bd0 Remove unnecessary check for contains() in LoadRule (#7073)
8804bd0 is described below
commit 8804bd0dc10c4b0d3f63179501c96259b91d798e
Author: Samarth Jain
AuthorDate: Mon Mar 11 09:52:46 2019 -0700
Remove unnecessary check for contains() in LoadRule (#7073)
See https://github.com/apache/incubator-druid/issues/7072
---
.../java/org/apache/druid/server/coordinator/rules/LoadRule.java | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/rules/LoadRule.java
b/server/src/main/java/org/apache/druid/server/coordinator/rules/LoadRule.java
index 1de3479..3b2b642 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/rules/LoadRule.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/rules/LoadRule.java
@@ -74,10 +74,7 @@ public abstract class LoadRule implements Rule
currentReplicants.putAll(params.getSegmentReplicantLookup().getClusterTiers(segment.getId()));
final CoordinatorStats stats = new CoordinatorStats();
-
- if (params.getAvailableSegments().contains(segment)) {
-assign(params, segment, stats);
- }
+ assign(params, segment, stats);
drop(params, segment, stats);
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Locale problem is fixed which fails tests. (#7120)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 48bc523 Locale problem is fixed which fails tests. (#7120)
48bc523 is described below
commit 48bc523bdf36201263517cc431340fd4960e6ea3
Author: Furkan KAMACI
AuthorDate: Thu Mar 14 00:47:14 2019 +0300
Locale problem is fixed which fails tests. (#7120)
* Locale problem is fixed which fails tests.
* Forbidden apis definition is improved to prevent using
com.ibm.icu.text.SimpleDateFormat and com.ibm.icu.text.DateFormatSymbols
without using any Locale defined.
* Error message is improved.
---
codestyle/druid-forbidden-apis.txt | 5 +
.../java/org/apache/druid/query/extraction/TimeDimExtractionFn.java | 5 +++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/codestyle/druid-forbidden-apis.txt
b/codestyle/druid-forbidden-apis.txt
index 5eb5256..3d7da96 100644
--- a/codestyle/druid-forbidden-apis.txt
+++ b/codestyle/druid-forbidden-apis.txt
@@ -35,6 +35,11 @@ java.lang.Math#random() @ Use ThreadLocalRandom.current()
java.util.regex.Pattern#matches(java.lang.String,java.lang.CharSequence) @ Use
String.startsWith(), endsWith(), contains(), or compile and cache a Pattern
explicitly
org.apache.commons.io.FileUtils#getTempDirectory() @ Use
org.junit.rules.TemporaryFolder for tests instead
+@defaultMessage Use Locale.ENGLISH
+com.ibm.icu.text.DateFormatSymbols#()
+com.ibm.icu.text.SimpleDateFormat#()
+com.ibm.icu.text.SimpleDateFormat#(java.lang.String)
+
@defaultMessage For performance reasons, use the utf8Base64 / encodeBase64 /
encodeBase64String / decodeBase64 / decodeBase64String methods in StringUtils
org.apache.commons.codec.binary.Base64
com.google.common.io.BaseEncoding.base64
\ No newline at end of file
diff --git
a/processing/src/main/java/org/apache/druid/query/extraction/TimeDimExtractionFn.java
b/processing/src/main/java/org/apache/druid/query/extraction/TimeDimExtractionFn.java
index 8b58a81..1e82b3e 100644
---
a/processing/src/main/java/org/apache/druid/query/extraction/TimeDimExtractionFn.java
+++
b/processing/src/main/java/org/apache/druid/query/extraction/TimeDimExtractionFn.java
@@ -33,6 +33,7 @@ import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.text.ParseException;
import java.util.Date;
+import java.util.Locale;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.Supplier;
@@ -84,8 +85,8 @@ public class TimeDimExtractionFn extends DimExtractionFn
} else {
final ThreadLocal> threadLocal =
ThreadLocal.withInitial(
() -> {
-final SimpleDateFormat parser = new SimpleDateFormat(timeFormat);
-final SimpleDateFormat formatter = new
SimpleDateFormat(resultFormat);
+final SimpleDateFormat parser = new SimpleDateFormat(timeFormat,
Locale.ENGLISH);
+final SimpleDateFormat formatter = new
SimpleDateFormat(resultFormat, Locale.ENGLISH);
parser.setLenient(true);
return value -> {
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Avoid many unnecessary materializations of collections of 'all segments in cluster' cardinality (#7185)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new dfd27e0 Avoid many unnecessary materializations of collections of 'all segments in cluster' cardinality (#7185) dfd27e0 is described below commit dfd27e00c0559e89a77ad8abbae03eeb382c2d09 Author: Roman Leventov AuthorDate: Tue Mar 19 18:22:56 2019 -0300 Avoid many unnecessary materializations of collections of 'all segments in cluster' cardinality (#7185) * Avoid many unnecessary materializations of collections of 'all segments in cluster' cardinality * Fix DruidCoordinatorTest; Renamed DruidCoordinator.getReplicationStatus() to computeUnderReplicationCountsPerDataSourcePerTier() * More Javadocs, typos, refactor DruidCoordinatorRuntimeParams.createAvailableSegmentsSet() * Style * typo * Disable StaticPseudoFunctionalStyleMethod inspection because of too much false positives * Fixes --- .idea/inspectionProfiles/Druid.xml | 28 .../util/common/concurrent/ScheduledExecutors.java | 6 +- .../org/apache/druid/utils/CollectionUtils.java| 72 + .../ambari/metrics/AmbariMetricsEmitterModule.java | 19 +-- .../druid/indexing/worker/WorkerTaskMonitor.java | 7 +- .../org/apache/druid/client/BrokerServerView.java | 2 +- .../apache/druid/client/CoordinatorServerView.java | 2 +- .../java/org/apache/druid/client/DruidServer.java | 20 ++- .../druid/client/HttpServerInventoryView.java | 17 ++- .../apache/druid/client/ImmutableDruidServer.java | 37 ++--- .../org/apache/druid/curator/CuratorUtils.java | 6 + .../druid/metadata/MetadataSegmentManager.java | 9 ++ .../druid/metadata/SQLMetadataSegmentManager.java | 6 + .../druid/server/coordinator/BalancerStrategy.java | 4 +- .../server/coordinator/CostBalancerStrategy.java | 6 +- .../server/coordinator/CuratorLoadQueuePeon.java | 6 +- .../DiskNormalizedCostBalancerStrategy.java| 4 +- .../druid/server/coordinator/DruidCoordinator.java | 79 +- .../coordinator/DruidCoordinatorRuntimeParams.java | 58 ++-- .../server/coordinator/HttpLoadQueuePeon.java | 6 +- .../coordinator/ReservoirSegmentSampler.java | 2 +- .../server/coordinator/SegmentReplicantLookup.java | 2 +- .../helper/DruidCoordinatorBalancer.java | 15 +- .../helper/DruidCoordinatorCleanupUnneeded.java| 63 .../coordinator/helper/DruidCoordinatorLogger.java | 10 +- .../helper/DruidCoordinatorRuleRunner.java | 68 - .../helper/DruidCoordinatorSegmentInfoLoader.java | 38 - .../druid/server/coordinator/rules/Rule.java | 12 +- .../druid/server/http/CoordinatorResource.java | 2 +- .../apache/druid/server/http/ServersResource.java | 10 +- .../apache/druid/server/http/TiersResource.java| 2 +- .../druid/client/HttpServerInventoryViewTest.java | 2 +- .../client/BatchServerInventoryViewTest.java | 18 +-- .../coordinator/CostBalancerStrategyTest.java | 2 +- .../coordinator/CuratorDruidCoordinatorTest.java | 14 +- .../DiskNormalizedCostBalancerStrategyTest.java| 2 +- .../DruidCoordinatorBalancerProfiler.java | 12 +- .../coordinator/DruidCoordinatorBalancerTest.java | 4 +- .../DruidCoordinatorRuleRunnerTest.java| 30 ++-- .../server/coordinator/DruidCoordinatorTest.java | 162 - .../coordinator/ReservoirSegmentSamplerTest.java | 8 +- .../cost/CachingCostBalancerStrategyTest.java | 2 +- .../DruidCoordinatorCleanupOvershadowedTest.java | 11 +- .../rules/BroadcastDistributionRuleTest.java | 8 +- .../server/coordinator/rules/LoadRuleTest.java | 24 +-- .../druid/sql/calcite/schema/SystemSchema.java | 2 +- .../druid/sql/calcite/schema/DruidSchemaTest.java | 22 ++- 47 files changed, 549 insertions(+), 392 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index b2dcda3..77078ae 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -269,6 +269,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -278,6 +302,10 @@ + + + diff --git a/core/src/main/java/org/apache/druid/java/util/common/concurrent/ScheduledExecutors.java b/core/src/main/java/org/apache/druid/java/util/common/concurrent/ScheduledExecutors.java index a25d73e..2850c50 100644 --- a/core/src/main/java/org/apache/dru
[incubator-druid] branch master updated: More TeamCity and Structural Search inspection instructions (#7275)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new a4270da More TeamCity and Structural Search inspection instructions (#7275) a4270da is described below commit a4270da5f3de25f1182cfdf2ac0d9de7781312cf Author: Roman Leventov AuthorDate: Thu Mar 21 17:24:21 2019 -0300 More TeamCity and Structural Search inspection instructions (#7275) * More TeamCity and Structural Search inspection instructions * the -> a * an e-mail --- ci/README_TeamCity.md | 63 ++-- ci/inspections_change_apply.png | Bin 0 -> 706159 bytes ci/structural_search_dialog.png | Bin 0 -> 213511 bytes ci/structural_search_find.png | Bin 0 -> 545045 bytes ci/structural_search_inspection.png | Bin 0 -> 619809 bytes ci/structural_search_inspection_add.png | Bin 0 -> 1072641 bytes 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ci/README_TeamCity.md b/ci/README_TeamCity.md index 20728ce..a567234 100644 --- a/ci/README_TeamCity.md +++ b/ci/README_TeamCity.md @@ -17,16 +17,16 @@ ~ under the License. --> -## Overview +### Overview TeamCity is a continuous integration and deployment server responsible for static analysis of Druid source code. Each Github PR request for [Druid](https://teamcity.jetbrains.com/project.html?projectId=OpenSourceProjects_Druid) is checked by TeamCity automatically. -## Login +### Login One can log in to TeamCity either via credentials or as a guest to check static analysis result of any PR. -## Becoming a Project Administrator +### Becoming a Project Administrator Druid committers shall obtain a status of a [Druid project]( https://teamcity.jetbrains.com/project.html?projectId=OpenSourceProjects_Druid) administrator. First, the Druid committer needs to log in teamcity.jetbrains.com using his Github account. @@ -39,5 +39,60 @@ Then, somebody who is already a project administrator needs to do the following: 5. Press the "Assign roles" button in the bottom of the page 6. Select "Role: Project administrator" and "Scope: Open-source project -> Druid" in the inputs, press "Assign" -## Restarting a Build +### Restarting a Build A project administrator could restart a build by pressing the "Run" button on the build page. + +### Contacting TeamCity support regarding problems with builds + +Regarding any build problems, feel free to contact TeamCity support by writing an e-mail at +teamcity-supp...@jetbrains.com. There is an automatic system behind this e-mail address that creates a ticket and sends +you an e-mail with a link to the ticket. Note that: + + - Contacting TeamCity support regarding problems with open-source Druid project builds is *free*. Don't hestitate doing + that if you don't see how to resolve some problem with a build. + - You don't need to be a Druid committer to do this. Any Druid contributor can contact TeamCity support. + +### Creating a custom inspection from a Structural Search pattern + +1. Open a structural search dialog: `Edit` -> `Find` -> `Search Structurally...` + +2. Enter a pattern that you want to make an inspection, for example: + + +3. Press the `Find` button to test your pattern: + + +Note that even if currently the pattern finds nothing, it might still be a good idea to add it as an inspection to +prevent bugs creeping in the codebase in the future. However, test that your pattern doesn't contain mistakes by +deliberately adding code with a mistake that should be spotted by the pattern in any existing Druid class and testing +that your Structural Search pattern finds that newly added dummy mistake. + +4. Open `Preferences...` -> `Inspections`, navigate to `General` -> `Structural Search inspection`: + + +5. Click a button with a plus sign (`+`) in the bottom of the Options window, `Add Search Template...`: + + +6. Click `OK`. Then you will see a dialong window with title `Save Template` and a field to enter a "template name". +Enter in this field something reasonably short that yet would serve as a good message for people who add code that +is caught by this pattern, e. g. "Use Map.putIfAbsent() instead of containsKey() + put()", in this example case. Press +`OK` again. + +7. Move focus anywhere, e. g. by choosing any other inspection. Upon doing this, the `Apply` botton should become +active: +![Inspections change apply
[incubator-druid] branch master updated: Fix some IntelliJ inspections (#7273)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new bca40dc Fix some IntelliJ inspections (#7273)
bca40dc is described below
commit bca40dcdaf3177b95d01187f413e1c0c6865964b
Author: Roman Leventov
AuthorDate: Mon Mar 25 21:11:01 2019 -0300
Fix some IntelliJ inspections (#7273)
Prepare TeamCity for IntelliJ 2018.3.1 upgrade. Mostly removed redundant
exceptions declarations in `throws` clauses.
---
.idea/inspectionProfiles/Druid.xml | 3 +++
.../input/impl/prefetch/PrefetchSqlFirehoseFactory.java | 1 -
.../java/org/apache/druid/tasklogs/NoopTaskLogs.java| 3 +--
.../druid/segment/loading/NoopDataSegmentArchiver.java | 4 ++--
.../druid/segment/loading/NoopDataSegmentKiller.java| 6 ++
.../org/apache/druid/storage/azure/AzureTaskLogs.java | 2 +-
.../DerivativeDataSourceMetadataTest.java | 4 ++--
.../MaterializedViewSupervisorSpecTest.java | 4 ++--
.../druid/data/input/orc/OrcHadoopInputRowParser.java | 17 -
.../java/org/apache/druid/client/cache/RedisCache.java | 3 +--
.../FixedBucketsHistogramAggregatorFactory.java | 3 +--
.../druid/indexer/HadoopDruidIndexerMapperTest.java | 5 ++---
.../task/AppenderatorDriverRealtimeIndexTask.java | 2 +-
.../druid/indexing/common/task/HadoopIndexTask.java | 2 +-
.../overlord/supervisor/SupervisorResourceTest.java | 2 +-
.../org/apache/druid/segment/data/GenericIndexed.java | 2 +-
.../first/StringFirstBufferAggregatorTest.java | 4 ++--
.../last/StringLastBufferAggregatorTest.java| 4 ++--
.../org/apache/druid/client/cache/CaffeineCache.java| 3 +--
.../java/org/apache/druid/client/cache/MapCache.java| 3 +--
.../org/apache/druid/client/cache/MemcachedCache.java | 2 +-
.../curator/PotentiallyGzippedCompressionProvider.java | 4 ++--
.../server/security/AllowOptionsResourceFilter.java | 2 +-
.../apache/druid/segment/indexing/DataSchemaTest.java | 2 +-
.../appenderator/StreamAppenderatorDriverTest.java | 2 +-
.../firehose/EventReceiverFirehoseIdleTest.java | 2 +-
.../java/org/apache/druid/sql/http/ResultFormat.java| 2 +-
27 files changed, 43 insertions(+), 50 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index 77078ae..0b1817e 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -79,6 +79,9 @@
+
+
+
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchSqlFirehoseFactory.java
b/core/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchSqlFirehoseFactory.java
index 4ae948f..d81b3bd 100644
---
a/core/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchSqlFirehoseFactory.java
+++
b/core/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchSqlFirehoseFactory.java
@@ -147,7 +147,6 @@ public abstract class PrefetchSqlFirehoseFactory
@Override
public Firehose connect(InputRowParser> firehoseParser,
@Nullable File temporaryDirectory)
- throws IOException
{
if (objects == null) {
objects = ImmutableList.copyOf(Preconditions.checkNotNull(initObjects(),
"objects"));
diff --git a/core/src/main/java/org/apache/druid/tasklogs/NoopTaskLogs.java
b/core/src/main/java/org/apache/druid/tasklogs/NoopTaskLogs.java
index b26e712..a0dc877 100644
--- a/core/src/main/java/org/apache/druid/tasklogs/NoopTaskLogs.java
+++ b/core/src/main/java/org/apache/druid/tasklogs/NoopTaskLogs.java
@@ -24,7 +24,6 @@ import com.google.common.io.ByteSource;
import org.apache.druid.java.util.common.logger.Logger;
import java.io.File;
-import java.io.IOException;
public class NoopTaskLogs implements TaskLogs
{
@@ -43,7 +42,7 @@ public class NoopTaskLogs implements TaskLogs
}
@Override
- public void pushTaskReports(String taskid, File reportFile) throws
IOException
+ public void pushTaskReports(String taskid, File reportFile)
{
log.info("Not pushing reports for task: %s", taskid);
}
diff --git
a/core/src/test/java/org/apache/druid/segment/loading/NoopDataSegmentArchiver.java
b/core/src/test/java/org/apache/druid/segment/loading/NoopDataSegmentArchiver.java
index 8524a87..4d89ec5 100644
---
a/core/src/test/java/org/apache/druid/segment/loading/NoopDataSegmentArchiver.java
+++
b/core/src/test/java/org/apache/druid/segment/loading/NoopDataSegmentArchiver.java
@@ -30,14 +30,14 @@ public class NoopDataSegmentArchiver implements
DataSegmentArchiver
{
@Nullable
@Override
- public DataSegment archive(DataSegment segment) throws
SegmentLoadingException
+ public DataSegment archive(DataSegment se
[incubator-druid] branch master updated: Make 'field can be final' inspection a warning in IntelliJ config (#7301)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new b9354fb Make 'field can be final' inspection a warning in IntelliJ config (#7301) b9354fb is described below commit b9354fbddb7cf0e7d225c78a850e59ee27cccf8c Author: Roman Leventov AuthorDate: Tue Mar 26 20:45:03 2019 -0300 Make 'field can be final' inspection a warning in IntelliJ config (#7301) A small step towards #7227. --- .idea/inspectionProfiles/Druid.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index 0b1817e..9bc1a75 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -57,6 +57,7 @@ + @@ -305,10 +306,9 @@ - - - + - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Remove unnecessary collection (#7350)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4654e1e Remove unnecessary collection (#7350)
4654e1e is described below
commit 4654e1e851633820b460fc928502a71a8ac340ad
Author: Surekha
AuthorDate: Mon Apr 15 10:49:21 2019 -0700
Remove unnecessary collection (#7350)
From the discussion
[here](https://github.com/apache/incubator-druid/pull/6901#discussion_r265741002)
Remove the collection and filter datasources from the stream.
Also remove StreamingOutput and JsonFactory constructs.
---
.../apache/druid/server/http/MetadataResource.java | 34 --
1 file changed, 5 insertions(+), 29 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
index af106fb..da0abe7 100644
--- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
@@ -19,8 +19,6 @@
package org.apache.druid.server.http;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
@@ -51,14 +49,12 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
@@ -68,9 +64,7 @@ public class MetadataResource
{
private final MetadataSegmentManager metadataSegmentManager;
private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private final AuthConfig authConfig;
private final AuthorizerMapper authorizerMapper;
- private final ObjectMapper jsonMapper;
@Inject
public MetadataResource(
@@ -83,9 +77,7 @@ public class MetadataResource
{
this.metadataSegmentManager = metadataSegmentManager;
this.metadataStorageCoordinator = metadataStorageCoordinator;
-this.authConfig = authConfig;
this.authorizerMapper = authorizerMapper;
-this.jsonMapper = jsonMapper;
}
@GET
@@ -161,15 +153,11 @@ public class MetadataResource
// If we haven't polled the metadata store yet, use an empty list of
datasources.
Collection druidDataSources =
Optional.ofNullable(metadataSegmentManager.getDataSources())
.orElse(Collections.emptyList());
-
+Stream dataSourceStream =
druidDataSources.stream();
if (datasources != null && !datasources.isEmpty()) {
- druidDataSources = druidDataSources.stream()
- .filter(src ->
datasources.contains(src.getName()))
- .collect(Collectors.toSet());
+ dataSourceStream = dataSourceStream.filter(src ->
datasources.contains(src.getName()));
}
-final Stream metadataSegments = druidDataSources
-.stream()
-.flatMap(t -> t.getSegments().stream());
+final Stream metadataSegments = dataSourceStream.flatMap(t ->
t.getSegments().stream());
final Function> raGenerator =
segment -> Collections.singletonList(
AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSource()));
@@ -177,20 +165,8 @@ public class MetadataResource
final Iterable authorizedSegments =
AuthorizationUtils.filterAuthorizedResources(req,
metadataSegments::iterator, raGenerator, authorizerMapper);
-final StreamingOutput stream = outputStream -> {
- final JsonFactory jsonFactory = jsonMapper.getFactory();
- try (final JsonGenerator jsonGenerator =
jsonFactory.createGenerator(outputStream)) {
-jsonGenerator.writeStartArray();
-for (DataSegment ds : authorizedSegments) {
- jsonGenerator.writeObject(ds);
- jsonGenerator.flush();
-}
-jsonGenerator.writeEndArray();
- }
-};
-
-Response.ResponseBuilder builder = Response.status(Response.Status.OK);
-return builder.entity(stream).build();
+final Response.ResponseBuilder builder =
Response.status(Response.Status.OK);
+return builder.entity(authorizedSegments).build();
}
@GET
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Move dev-related files and instructions to dev/ directory; add committer's instructions (#7279)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new 9d38527 Move dev-related files and instructions to dev/ directory; add committer's instructions (#7279) 9d38527 is described below commit 9d385274e31abfbd19a24279902f4961fcc8e4e7 Author: Roman Leventov AuthorDate: Wed Apr 17 15:27:14 2019 +0200 Move dev-related files and instructions to dev/ directory; add committer's instructions (#7279) * Add committer_readme.md * Clean up * Update, add PR merge action checklist * Move dev-related docs and files except CONTRIBUTING.md to dev/ directory; More committer's intructions * Add some accents * Move TeamCity instruction images to teamcity-images/, edit CONTRIBUTING.md * Add links to tags --- CONTRIBUTING.md| 16 +- dev/committer-instructions.md | 176 + .../druid_intellij_formatting.xml | 0 eclipse.importorder => dev/eclipse.importorder | 0 .../eclipse_formatting.xml | 0 .../intellij-sdk-config.jpg| Bin INTELLIJ_SETUP.md => dev/intellij-setup.md | 3 +- .../teamcity-images}/inspections_change_apply.png | Bin .../teamcity-images}/structural_search_dialog.png | Bin .../teamcity-images}/structural_search_find.png| Bin .../structural_search_inspection.png | Bin .../structural_search_inspection_add.png | Bin ci/README_TeamCity.md => dev/teamcity.md | 10 +- distribution/src/assembly/source-assembly.xml | 4 +- 14 files changed, 195 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2fa1e22..38d2052 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,15 +23,17 @@ When submitting a pull request (PR), please use the following guidelines: - Make sure your code respects existing formatting conventions. In general, follow the same coding style as the code that you are modifying. -- For Intellij you can import our code style settings xml: [druid_intellij_formatting.xml](https://github.com/apache/incubator-druid/raw/master/druid_intellij_formatting.xml). -- For Eclipse you can import our code style settings xml: [eclipse_formatting.xml](https://github.com/apache/incubator-druid/raw/master/eclipse_formatting.xml). +- For Intellij you can import our code style settings xml: [`druid_intellij_formatting.xml`]( + https://github.com/apache/incubator-druid/raw/master/dev/druid_intellij_formatting.xml). +- For Eclipse you can import our code style settings xml: [`eclipse_formatting.xml`]( + https://github.com/apache/incubator-druid/raw/master/dev/eclipse_formatting.xml). - Do add/update documentation appropriately for the change you are making. - If you are introducing a new feature you may want to first write about your idea - for feedback to [d...@druid.apache.org](https://lists.apache.org/list.html?d...@druid.apache.org). - Non-trivial features should include unit tests covering the new functionality. + for feedback to [d...@druid.apache.org](https://lists.apache.org/list.html?d...@druid.apache.org). Or create an issue + using "Feature/Change" template. Non-trivial features should include unit tests covering the new functionality. Open + a "Proposal" issue for large changes. - Bugfixes should include a unit test or integration test reproducing the issue. - Do not use author tags/information in the code. -- Always include license header on each java file your create. See [this example](https://github.com/apache/incubator-druid/blob/master/core/src/main/java/org/apache/druid/metadata/PasswordProvider.java) - Try to keep pull requests short and submit separate ones for unrelated features, but feel free to combine simple bugfixes/tests into one pull request. - Keep the number of commits small and combine commits for related changes. @@ -39,6 +41,8 @@ When submitting a pull request (PR), please use the following guidelines: - Keep formatting changes in separate commits to make code reviews easier and distinguish them from actual code changes. +You can find more developers' resources in [`dev/`](dev) directory. + ## GitHub Workflow 1. Fork the apache/incubator-druid repository into your GitHub account @@ -136,7 +140,7 @@ When submitting a pull request (PR), please use the following guidelines: committer that merges your change will rebase and squash it into a single commit before committing it to master. -# FAQ +## FAQ ### Help! I merged changes from upstream and cannot figure out how to resolve conflicts when rebasing! diff --git a/dev/committer-instructions.md b/dev/committer-i
[incubator-druid] branch master updated: Introduce Non-TeamCity Warning inspection severity in IntelliJ's profile (#7499)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new 0a0fd63 Introduce Non-TeamCity Warning inspection severity in IntelliJ's profile (#7499) 0a0fd63 is described below commit 0a0fd63f4ed0783901bff80fbc103e99a6326aa7 Author: Roman Leventov AuthorDate: Fri Apr 19 15:20:08 2019 +0200 Introduce Non-TeamCity Warning inspection severity in IntelliJ's profile (#7499) In the IDE interface, "Non-TeamCity Warning" looks exactly like an ordinary warning, but TeamCity should be unaware of it. This may help to workaround these issues: https://youtrack.jetbrains.com/issue/IDEA-209789 and https://youtrack.jetbrains.com/issue/IDEA-209791, that block the upgrade of IntelliJ engine used in the TeamCity build. It seems like there may be a bug that leads to false positive error and the build fail in this PR: https://github.com/apache/incubator-druid/pull/6702. Removed the comment regarding "StaticPseudoFunctionalStyleMethod" inspection because the IntelliJ keeps removing it, see this issue: https://youtrack.jetbrains.com/issue/IDEA-211087 --- .idea/inspectionProfiles/Druid.xml | 6 ++ .idea/inspectionProfiles/profiles_settings.xml | 20 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index 9bc1a75..0a9adb1 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -306,8 +306,6 @@ - @@ -332,7 +330,7 @@ - + @@ -363,7 +361,7 @@ - + diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml index 67c3084..75b0a83 100644 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -2,5 +2,25 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Fix race between canHandle() and addSegment() in StorageLocation (#8114)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new adf7baf Fix race between canHandle() and addSegment() in
StorageLocation (#8114)
adf7baf is described below
commit adf7bafb9fda4ce8be47f47b5fbc554e92b4608b
Author: Jihoon Son
AuthorDate: Sat Jul 27 01:11:06 2019 -0700
Fix race between canHandle() and addSegment() in StorageLocation (#8114)
* Fix race between canHandle() and addSegment() in StorageLocation
* add comment
* add comments
* fix test
* address comments
* remove tag from javadoc
* address comments
* comparingLong
---
.../indexing/worker/IntermediaryDataManager.java | 76 +
.../loading/SegmentLoaderLocalCacheManager.java| 41 +++--
.../druid/segment/loading/StorageLocation.java | 109 -
...mentLoaderLocalCacheManagerConcurrencyTest.java | 181 +
.../druid/segment/loading/StorageLocationTest.java | 34 ++--
5 files changed, 338 insertions(+), 103 deletions(-)
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/IntermediaryDataManager.java
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/IntermediaryDataManager.java
index bfd202e..c47425a 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/IntermediaryDataManager.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/IntermediaryDataManager.java
@@ -45,6 +45,7 @@ import org.joda.time.Period;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -231,7 +232,6 @@ public class IntermediaryDataManager
* addSegment method.
*/
public void addSegment(String supervisorTaskId, String subTaskId,
DataSegment segment, File segmentFile)
- throws IOException
{
final Iterator iterator =
locationIterators.computeIfAbsent(
supervisorTaskId,
@@ -243,7 +243,7 @@ public class IntermediaryDataManager
public List findPartitionFiles(String supervisorTaskId, Interval
interval, int partitionId)
{
for (StorageLocation location : shuffleDataLocations) {
- final File partitionDir = getPartitionDir(location, supervisorTaskId,
interval, partitionId);
+ final File partitionDir = new File(location.getPath(),
getPartitionDir(supervisorTaskId, interval, partitionId));
if (partitionDir.exists()) {
supervisorTaskCheckTimes.put(supervisorTaskId, DateTimes.nowUtc());
final File[] segmentFiles = partitionDir.listFiles();
@@ -279,53 +279,65 @@ public class IntermediaryDataManager
String subTaskId,
DataSegment segment,
File segmentFile
- ) throws IOException
- {
-final StorageLocation location = findLocationForSegment(cyclicIterator,
numLocations, segment);
-final File destFile = new File(
-getPartitionDir(location, supervisorTaskId, segment.getInterval(),
segment.getShardSpec().getPartitionNum()),
-subTaskId
-);
-FileUtils.forceMkdirParent(destFile);
-final long copiedBytes =
Files.asByteSource(segmentFile).copyTo(Files.asByteSink(destFile));
-if (copiedBytes == 0) {
- throw new IOE(
- "0 bytes copied after copying a segment file from [%s] to [%s]",
- segmentFile.getAbsolutePath(),
- destFile.getAbsolutePath()
- );
-}
-location.addFile(destFile);
- }
-
- private static StorageLocation findLocationForSegment(
- Iterator cyclicIterator,
- int numLocations,
- DataSegment segment
)
{
for (int i = 0; i < numLocations; i++) {
final StorageLocation location = cyclicIterator.next();
- if (location.canHandle(segment)) {
-return location;
+ final File destFile = location.reserve(
+ getPartitionFilePath(
+ supervisorTaskId,
+ subTaskId,
+ segment.getInterval(),
+ segment.getShardSpec().getPartitionNum()
+ ),
+ segment.getId(),
+ segmentFile.length()
+ );
+ if (destFile != null) {
+try {
+ FileUtils.forceMkdirParent(destFile);
+ final long copiedBytes =
Files.asByteSource(segmentFile).copyTo(Files.asByteSink(destFile));
+ if (copiedBytes == 0) {
+throw new IOE(
+"0 bytes copied after copying a segment file from [%s] to
[%s]",
+segmentFile.getAbsolutePath(),
+destFile.getAbsolutePath()
+);
+ } else {
+return;
+ }
+}
+catch (IOException e) {
+ // Only log here to try other locations as well.
+
[incubator-druid] branch master updated: Enable Spotbugs: WMI_WRONG_MAP_ITERATOR (#8005)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new e016995 Enable Spotbugs: WMI_WRONG_MAP_ITERATOR (#8005)
e016995 is described below
commit e016995d1fb6085a48e2e0d0a28504ed955f0293
Author: Fokko Driesprong
AuthorDate: Tue Jul 30 18:51:53 2019 +0200
Enable Spotbugs: WMI_WRONG_MAP_ITERATOR (#8005)
* WMI_WRONG_MAP_ITERATOR
* Fixed missing loop
---
codestyle/spotbugs-exclude.xml | 1 -
.../apache/druid/java/util/metrics/KeyedDiff.java | 10 +-
.../MaterializedViewSupervisor.java| 14 +-
.../DataSourceOptimizerMonitor.java| 8 +-
.../druid/indexing/overlord/RemoteTaskRunner.java | 503 ++---
.../overlord/hrtr/HttpRemoteTaskRunner.java| 78 ++--
.../overlord/supervisor/SupervisorManager.java | 4 +-
.../query/lookup/LookupListeningResource.java | 6 +-
8 files changed, 287 insertions(+), 337 deletions(-)
diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml
index eb112b2..ca9fb08 100644
--- a/codestyle/spotbugs-exclude.xml
+++ b/codestyle/spotbugs-exclude.xml
@@ -88,5 +88,4 @@
-
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/KeyedDiff.java
b/core/src/main/java/org/apache/druid/java/util/metrics/KeyedDiff.java
index e4bfd04..c1a3bae 100644
--- a/core/src/main/java/org/apache/druid/java/util/metrics/KeyedDiff.java
+++ b/core/src/main/java/org/apache/druid/java/util/metrics/KeyedDiff.java
@@ -41,12 +41,12 @@ public class KeyedDiff
}
}
- public static Map subtract(Map xs, Map ys)
+ public static Map subtract(Map lhs, Map rhs)
{
-assert xs.keySet().equals(ys.keySet());
-final Map zs = new HashMap();
-for (String k : xs.keySet()) {
- zs.put(k, xs.get(k) - ys.get(k));
+assert lhs.keySet().equals(rhs.keySet());
+final Map zs = new HashMap<>();
+for (Map.Entry k : lhs.entrySet()) {
+ zs.put(k.getKey(), k.getValue() - rhs.get(k.getKey()));
}
return zs;
}
diff --git
a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
index 1ed80ca..378ed50 100644
---
a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
+++
b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
@@ -371,16 +371,12 @@ public class MaterializedViewSupervisor implements
Supervisor
Map toDropInterval = new
HashMap<>(difference.entriesOnlyOnRight());
// if some intervals are in running tasks and the versions are the same,
remove it from toBuildInterval
// if some intervals are in running tasks, but the versions are different,
stop the task.
-for (Interval interval : runningVersion.keySet()) {
- if (toBuildInterval.containsKey(interval)
- && toBuildInterval.get(interval).equals(runningVersion.get(interval))
- ) {
+for (Map.Entry version : runningVersion.entrySet()) {
+ final Interval interval = version.getKey();
+ final String host = version.getValue();
+ if (toBuildInterval.containsKey(interval) &&
toBuildInterval.get(interval).equals(host)) {
toBuildInterval.remove(interval);
-
- } else if (
- toBuildInterval.containsKey(interval)
- &&
!toBuildInterval.get(interval).equals(runningVersion.get(interval))
- ) {
+ } else if (toBuildInterval.containsKey(interval) &&
!toBuildInterval.get(interval).equals(host)) {
if (taskMaster.getTaskQueue().isPresent()) {
taskMaster.getTaskQueue().get().shutdown(runningTasks.get(interval).getId(),
"version mismatch");
runningTasks.remove(interval);
diff --git
a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
index 0720957..08b0b25 100644
---
a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
+++
b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
@@ -41,7 +41,7 @@ public class DataSourceOptimizerMonitor extends
AbstractMonitor
@Override
public boolean doMonitor(ServiceEmitter emitt
[incubator-druid] branch master updated: Spotbugs fix DM_NUMBER_CTOR (#8072)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0f8c902 Spotbugs fix DM_NUMBER_CTOR (#8072)
0f8c902 is described below
commit 0f8c902e94aca8c55fd8b10d74398f6528b357e0
Author: Fokko Driesprong
AuthorDate: Tue Jul 30 19:08:27 2019 +0200
Spotbugs fix DM_NUMBER_CTOR (#8072)
---
codestyle/spotbugs-exclude.xml| 1 -
.../main/java/org/apache/druid/data/input/influx/InfluxParser.java| 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml
index ca9fb08..fe8a6de 100644
--- a/codestyle/spotbugs-exclude.xml
+++ b/codestyle/spotbugs-exclude.xml
@@ -37,7 +37,6 @@
-
diff --git
a/extensions-contrib/influx-extensions/src/main/java/org/apache/druid/data/input/influx/InfluxParser.java
b/extensions-contrib/influx-extensions/src/main/java/org/apache/druid/data/input/influx/InfluxParser.java
index a14e5e1..42fca8b 100644
---
a/extensions-contrib/influx-extensions/src/main/java/org/apache/druid/data/input/influx/InfluxParser.java
+++
b/extensions-contrib/influx-extensions/src/main/java/org/apache/druid/data/input/influx/InfluxParser.java
@@ -124,7 +124,7 @@ public class InfluxParser implements Parser
private Object parseNumber(String raw)
{
if (raw.endsWith("i")) {
- return new Long(raw.substring(0, raw.length() - 1));
+ return Long.valueOf(raw.substring(0, raw.length() - 1));
}
return new Double(raw);
@@ -161,7 +161,7 @@ public class InfluxParser implements Parser
dest.put(TIMESTAMP_KEY, 0L);
} else {
timestamp = timestamp.substring(0, timestamp.length() - 6);
- long timestampMillis = new Long(timestamp);
+ final long timestampMillis = Long.valueOf(timestamp);
dest.put(TIMESTAMP_KEY, timestampMillis);
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Add SuppressWarnings SS_SHOULD_BE_STATIC (#8138)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new faf5110 Add SuppressWarnings SS_SHOULD_BE_STATIC (#8138)
faf5110 is described below
commit faf51107d590eb6b947fc6f0f3164f5866b671b2
Author: Fokko Driesprong
AuthorDate: Wed Jul 31 18:44:42 2019 +0200
Add SuppressWarnings SS_SHOULD_BE_STATIC (#8138)
* Spotbugs: SS_SHOULD_BE_STATIC (#8073)
* Add SuppressWarnings SS_SHOULD_BE_STATIC
Fixes #8073
* Fix the voilation
* Make them non-final
* Remove @Nonnull
---
codestyle/spotbugs-exclude.xml| 1 -
.../apache/druid/indexing/worker/config/WorkerConfig.java | 15 ++-
.../druid/server/security/AnonymousAuthenticator.java | 2 +-
3 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml
index fe8a6de..3ee2330 100644
--- a/codestyle/spotbugs-exclude.xml
+++ b/codestyle/spotbugs-exclude.xml
@@ -83,7 +83,6 @@
-
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java
index 65d09e7..4bfa6ee 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java
@@ -25,32 +25,29 @@ import org.apache.druid.utils.JvmUtils;
import org.joda.time.Period;
import javax.validation.constraints.Min;
-import javax.validation.constraints.NotNull;
/**
*/
public class WorkerConfig
{
@JsonProperty
- @NotNull
- private final String ip = DruidNode.getDefaultHost();
+ private String ip = DruidNode.getDefaultHost();
@JsonProperty
- @NotNull
- private final String version = "0";
+ private String version = "0";
@JsonProperty
@Min(1)
- private final int capacity = Math.max(1,
JvmUtils.getRuntimeInfo().getAvailableProcessors() - 1);
+ private int capacity = Math.max(1,
JvmUtils.getRuntimeInfo().getAvailableProcessors() - 1);
@JsonProperty
- private final long intermediaryPartitionDiscoveryPeriodSec = 60L;
+ private long intermediaryPartitionDiscoveryPeriodSec = 60L;
@JsonProperty
- private final long intermediaryPartitionCleanupPeriodSec = 300L;
+ private long intermediaryPartitionCleanupPeriodSec = 300L;
@JsonProperty
- private final Period intermediaryPartitionTimeout = new Period("P1D");
+ private Period intermediaryPartitionTimeout = new Period("P1D");
public String getIp()
{
diff --git
a/server/src/main/java/org/apache/druid/server/security/AnonymousAuthenticator.java
b/server/src/main/java/org/apache/druid/server/security/AnonymousAuthenticator.java
index 7cfe76c..a110189 100644
---
a/server/src/main/java/org/apache/druid/server/security/AnonymousAuthenticator.java
+++
b/server/src/main/java/org/apache/druid/server/security/AnonymousAuthenticator.java
@@ -40,8 +40,8 @@ import java.util.Map;
@JsonTypeName("anonymous")
public class AnonymousAuthenticator implements Authenticator
{
+ private static final String DEFAULT_IDENTITY = "defaultUser";
private final AuthenticationResult anonymousResult;
- private final String DEFAULT_IDENTITY = "defaultUser";
@JsonCreator
public AnonymousAuthenticator(
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Fixed a bug in HttpPostEmitter leading to ClassCastException (#8205)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new bcaffe2 Fixed a bug in HttpPostEmitter leading to ClassCastException
(#8205)
bcaffe2 is described below
commit bcaffe2bc0c6e915c50df9fcb1c191d326a7dfd5
Author: Artiom Darie
AuthorDate: Thu Aug 1 20:36:23 2019 +0300
Fixed a bug in HttpPostEmitter leading to ClassCastException (#8205)
* Issue 8206: Fixed class cast exception in case of batch recovery
* Issue 8206: Added HttpPostEmitterTest license header
* Issue 8206: Updated comments accordingly to code review.
* Issue 8206: Updated HttpPostEmitterTest accordingly to new modifications.
---
.../apache/druid/java/util/emitter/core/Batch.java | 3 +
.../java/util/emitter/core/HttpPostEmitter.java| 18 ++--
.../util/emitter/core/HttpPostEmitterTest.java | 98 ++
3 files changed, 110 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/java/util/emitter/core/Batch.java
b/core/src/main/java/org/apache/druid/java/util/emitter/core/Batch.java
index 5bc598a..fbcfb23 100644
--- a/core/src/main/java/org/apache/druid/java/util/emitter/core/Batch.java
+++ b/core/src/main/java/org/apache/druid/java/util/emitter/core/Batch.java
@@ -92,6 +92,9 @@ class Batch extends AbstractQueuedLongSynchronizer
* Ordering number of this batch, as they filled & emitted in {@link
HttpPostEmitter} serially, starting from 0.
* It's a boxed Long rather than primitive long, because we want to minimize
the number of allocations done in
* {@link HttpPostEmitter#onSealExclusive} and so the probability of {@link
OutOfMemoryError}.
+ *
+ * See {@link HttpPostEmitter#concurrentBatch} which may store this object.
+ *
* @see HttpPostEmitter#onSealExclusive
* @see HttpPostEmitter#concurrentBatch
*/
diff --git
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
index 85ad787..1036fb9 100644
---
a/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
+++
b/core/src/main/java/org/apache/druid/java/util/emitter/core/HttpPostEmitter.java
@@ -109,10 +109,10 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
private final AtomicInteger approximateBuffersToReuseCount = new
AtomicInteger();
/**
- * concurrentBatch.get() == null means the service is closed.
concurrentBatch.get() is the instance of Integer,
- * it means that some thread has failed with a serious error during {@link
#onSealExclusive} (with the batch number
- * corresponding to the Integer object) and {@link #tryRecoverCurrentBatch}
needs to be called. Otherwise (i. e.
- * normally), an instance of {@link Batch} is stored in this atomic
reference.
+ * concurrentBatch.get() == null means the service is closed.
concurrentBatch.get() is the instance of Long (i. e. the
+ * type of {@link Batch#batchNumber}), it means that some thread has failed
with a serious error during {@link
+ * #onSealExclusive} (with the batch number corresponding to the Long
object) and {@link #tryRecoverCurrentBatch}
+ * needs to be called. Otherwise (i. e. normally), an instance of {@link
Batch} is stored in this atomic reference.
*/
private final AtomicReference concurrentBatch = new
AtomicReference<>();
@@ -251,8 +251,8 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
while (true) {
Object batchObj = concurrentBatch.get();
- if (batchObj instanceof Integer) {
-tryRecoverCurrentBatch((Integer) batchObj);
+ if (batchObj instanceof Long) {
+tryRecoverCurrentBatch((Long) batchObj);
continue;
}
if (batchObj == null) {
@@ -342,7 +342,7 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
}
}
- private void tryRecoverCurrentBatch(Integer failedBatchNumber)
+ private void tryRecoverCurrentBatch(Long failedBatchNumber)
{
log.info("Trying to recover currentBatch");
long nextBatchNumber =
ConcurrentAwaitableCounter.nextCount(failedBatchNumber);
@@ -535,8 +535,8 @@ public class HttpPostEmitter implements Flushable,
Closeable, Emitter
if (batch instanceof Batch) {
((Batch) batch).sealIfFlushNeeded();
} else {
- // batch == null means that HttpPostEmitter is terminated. Batch
object could also be Integer, if some
- // thread just failed with a serious error in onSealExclusive(), in
this case we don't want to shutdown
+ // batch == null means that HttpPostEmitter is terminated. Batch
object might also be a Long object if some
+ // thread just failed with a se
[incubator-druid] branch master updated: Spotbugs: NP_NONNULL_PARAM_VIOLATION (#8129)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 91743ee Spotbugs: NP_NONNULL_PARAM_VIOLATION (#8129)
91743ee is described below
commit 91743eeebe56cdb93e4b38abb71c4be43566ad8e
Author: Fokko Driesprong
AuthorDate: Fri Aug 2 18:20:22 2019 +0200
Spotbugs: NP_NONNULL_PARAM_VIOLATION (#8129)
---
codestyle/spotbugs-exclude.xml | 2 --
.../org/apache/druid/collections/SerializablePair.java | 5 -
.../theta/SketchEstimatePostAggregator.java| 10 ++
.../aggregation/histogram/ApproximateHistogram.java| 15 ++-
.../histogram/ApproximateHistogramErrorBenchmark.java | 2 +-
.../histogram/ApproximateHistogramTest.java| 18 +-
.../variance/VarianceAggregatorCollector.java | 2 +-
.../variance/VarianceAggregatorFactory.java| 6 --
.../variance/VarianceFoldingAggregatorFactory.java | 4 +++-
.../query/aggregation/DoubleMaxAggregatorFactory.java | 2 +-
.../query/aggregation/DoubleMinAggregatorFactory.java | 2 +-
.../query/aggregation/DoubleSumAggregatorFactory.java | 2 +-
.../query/aggregation/FloatMaxAggregatorFactory.java | 2 +-
.../query/aggregation/FloatMinAggregatorFactory.java | 2 +-
.../query/aggregation/FloatSumAggregatorFactory.java | 2 +-
.../query/aggregation/LongMaxAggregatorFactory.java| 2 +-
.../query/aggregation/LongMinAggregatorFactory.java| 2 +-
.../query/aggregation/LongSumAggregatorFactory.java| 2 +-
.../apache/druid/query/aggregation/PostAggregator.java | 1 +
.../query/aggregation/SerializablePairLongString.java | 4 +++-
.../aggregation/last/StringLastBufferAggregator.java | 2 +-
.../aggregation/post/ArithmeticPostAggregator.java | 4 +++-
.../aggregation/post/ExpressionPostAggregator.java | 2 +-
.../aggregation/post/FieldAccessPostAggregator.java| 6 +-
.../org/apache/druid/query/groupby/GroupByQuery.java | 10 +-
.../groupby/epinephelinae/ByteBufferHashTable.java | 6 --
.../groupby/epinephelinae/GroupByRowProcessor.java | 1 +
.../query/groupby/epinephelinae/ParallelCombiner.java | 5 -
.../apache/druid/segment/DimensionHandlerUtils.java| 2 +-
.../org/apache/druid/segment/DimensionIndexer.java | 2 +-
.../apache/druid/segment/DoubleDimensionIndexer.java | 2 +-
.../apache/druid/segment/FloatDimensionIndexer.java| 2 +-
.../main/java/org/apache/druid/segment/IndexSpec.java | 10 ++
.../org/apache/druid/segment/LongDimensionIndexer.java | 2 +-
.../apache/druid/segment/StringDimensionIndexer.java | 4 ++--
.../druid/segment/data/GenericIndexedWriter.java | 2 +-
.../druid/segment/incremental/IncrementalIndex.java| 2 +-
.../server/coordination/ChangeRequestsSnapshot.java| 14 +++---
.../server/coordination/DataSegmentChangeHandler.java | 6 --
.../server/coordination/DataSegmentChangeRequest.java | 4 +++-
.../server/coordination/SegmentChangeRequestDrop.java | 4 +++-
.../server/coordination/SegmentChangeRequestLoad.java | 4 +++-
.../server/coordination/SegmentChangeRequestNoop.java | 8 ++--
43 files changed, 116 insertions(+), 75 deletions(-)
diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml
index 3ee2330..effec8d 100644
--- a/codestyle/spotbugs-exclude.xml
+++ b/codestyle/spotbugs-exclude.xml
@@ -60,8 +60,6 @@
-
-
diff --git
a/core/src/main/java/org/apache/druid/collections/SerializablePair.java
b/core/src/main/java/org/apache/druid/collections/SerializablePair.java
index ce98933..46b3995 100644
--- a/core/src/main/java/org/apache/druid/collections/SerializablePair.java
+++ b/core/src/main/java/org/apache/druid/collections/SerializablePair.java
@@ -23,10 +23,12 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.java.util.common.Pair;
+import javax.annotation.Nullable;
+
public class SerializablePair extends Pair
{
@JsonCreator
- public SerializablePair(@JsonProperty("lhs") T1 lhs, @JsonProperty("rhs") T2
rhs)
+ public SerializablePair(@JsonProperty("lhs") T1 lhs, @JsonProperty("rhs")
@Nullable T2 rhs)
{
super(lhs, rhs);
}
@@ -38,6 +40,7 @@ public class SerializablePair extends Pair
}
@JsonProperty
+ @Nullable
public T2 getRhs()
{
return rhs;
diff --git
a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java
b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java
index b52c74d..a725c64 100644
[incubator-druid] branch master updated: Enum of ResponseContext keys (#8157)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 3f3162b Enum of ResponseContext keys (#8157)
3f3162b is described below
commit 3f3162b85e269749bf0aa848f435409b3a1db8fe
Author: Eugene Sevastianov
AuthorDate: Sat Aug 3 12:05:21 2019 +0300
Enum of ResponseContext keys (#8157)
* Refactored ResponseContext and aggregated its keys into Enum
* Added unit tests for ResponseContext and refactored the serialization
* Removed unused methods
* Fixed code style
* Fixed code style
* Fixed code style
* Made SerializationResult static
* Updated according to the PR discussion:
Renamed an argument
Updated comparator
Replaced Pair usage with Map.Entry
Added a comment about quadratic complexity
Removed boolean field with an expression
Renamed SerializationResult field
Renamed the method merge to add and renamed several context keys
Renamed field and method related to scanRowsLimit
Updated a comment
Simplified a block of code
Renamed a variable
* Added JsonProperty annotation to renamed ScanQuery field
* Extension-friendly context key implementation
* Refactored ResponseContext: updated delegate type, comments and exceptions
Reducing serialized context length by removing some of its'
collection elements
* Fixed tests
* Simplified response context truncation during serialization
* Extracted a method of removing elements from a response context and
added some comments
* Fixed typos and updated comments
---
.../movingaverage/MovingAverageQueryRunner.java| 18 +-
.../druid/query/CPUTimeMetricQueryRunner.java | 1 +
.../main/java/org/apache/druid/query/Druids.java | 2 +-
.../ReportTimelineMissingSegmentQueryRunner.java | 11 +-
.../org/apache/druid/query/RetryQueryRunner.java | 4 +-
.../query/context/ConcurrentResponseContext.java | 4 +-
.../query/context/DefaultResponseContext.java | 4 +-
.../druid/query/context/ResponseContext.java | 413 ++---
.../org/apache/druid/query/scan/ScanQuery.java | 20 +-
.../apache/druid/query/scan/ScanQueryEngine.java | 28 +-
.../query/scan/ScanQueryLimitRowIterator.java | 2 +-
.../druid/query/scan/ScanQueryQueryToolChest.java | 2 +-
.../druid/query/scan/ScanQueryRunnerFactory.java | 24 +-
.../query/spec/SpecificSegmentQueryRunner.java | 15 +-
.../druid/segment/StringDimensionHandler.java | 2 +-
.../apache/druid/query/RetryQueryRunnerTest.java | 80 ++--
.../apache/druid/query/UnionQueryRunnerTest.java | 14 +-
.../druid/query/context/ResponseContextTest.java | 332 +
.../DataSourceMetadataQueryTest.java | 2 +-
.../query/scan/ScanQueryRunnerFactoryTest.java | 8 +-
.../query/spec/SpecificSegmentQueryRunnerTest.java | 2 +-
.../timeboundary/TimeBoundaryQueryRunnerTest.java | 4 +-
.../druid/query/topn/TopNQueryRunnerTest.java | 4 +-
.../druid/client/CachingClusteredClient.java | 8 +-
.../org/apache/druid/client/DirectDruidClient.java | 8 +-
.../druid/query/ResultLevelCachingQueryRunner.java | 2 +-
.../org/apache/druid/server/QueryResource.java | 28 +-
.../CachingClusteredClientFunctionalityTest.java | 6 +-
.../druid/client/CachingClusteredClientTest.java | 6 +-
29 files changed, 846 insertions(+), 208 deletions(-)
diff --git
a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQueryRunner.java
b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQueryRunner.java
index 645a3b1..3d704de 100644
---
a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQueryRunner.java
+++
b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/MovingAverageQueryRunner.java
@@ -68,10 +68,6 @@ import java.util.stream.Collectors;
*/
public class MovingAverageQueryRunner implements QueryRunner
{
-
- public static final String QUERY_FAIL_TIME = "queryFailTime";
- public static final String QUERY_TOTAL_BYTES_GATHERED =
"queryTotalBytesGathered";
-
private final QuerySegmentWalker walker;
private final RequestLogger requestLogger;
@@ -127,8 +123,11 @@ public class MovingAverageQueryRunner implements
QueryRunner
GroupByQuery gbq = builder.build();
ResponseContext gbqResponseContext = ResponseContext.createEmpty();
- gbqResponseContext.put(QUERY_FAIL_TIME, System.currentTimeMillis() +
Que
[incubator-druid] branch master updated: Enable toggling request logging on/off for different query types (#7562)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new e252abe Enable toggling request logging on/off for different query
types (#7562)
e252abe is described below
commit e252abedc5d7bd20c3c1dad7daf4cbdeb99ec761
Author: Lucas Capistrant
AuthorDate: Tue Aug 6 07:47:30 2019 -0500
Enable toggling request logging on/off for different query types (#7562)
* Enable ability to toggle SegmentMetadata request logging on/off
* Move SegmentMetadata query log filter to FilteredRequestLogger
* Update documentation to reflect the segment metadata flag moving to the
filtered request logger
* Modify patch to allow blacklist of query types to not log to request
logger
* Address styling and naming requests following latest code review
* Fix indentation on multiple locations per Druid style rules
---
docs/content/configuration/index.md| 1 +
.../server/log/FileRequestLoggerProvider.java | 7 ++-
.../server/log/FilteredRequestLoggerProvider.java | 30 --
.../server/log/FilteredRequestLoggerTest.java | 65 +-
4 files changed, 96 insertions(+), 7 deletions(-)
diff --git a/docs/content/configuration/index.md
b/docs/content/configuration/index.md
index 0e9e663..fcccac7 100644
--- a/docs/content/configuration/index.md
+++ b/docs/content/configuration/index.md
@@ -378,6 +378,7 @@ For native query, only request logs where query/time is
above the threshold are
||---|---|
|`druid.request.logging.queryTimeThresholdMs`|Threshold value for query/time
in milliseconds.|0 i.e no filtering|
|`druid.request.logging.sqlQueryTimeThresholdMs`|Threshold value for
sqlQuery/time in milliseconds.|0 i.e no filtering|
+|`druid.request.logging.mutedQueryTypes` | Query requests of these types are
not logged. Query types are defined as string objects corresponding to the
"queryType" value for the specified query in the Druid's [native JSON query
API](http://druid.apache.org/docs/latest/querying/querying.html). Misspelled
query types will be ignored. Example to ignore scan and timeBoundary queries:
["scan", "timeBoundary"]| []|
|`druid.request.logging.delegate.type`|Type of delegate request logger to log
requests.|none|
Composite Request Logging
diff --git
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
index be88caf..0d993d6 100644
---
a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java
@@ -58,7 +58,12 @@ public class FileRequestLoggerProvider implements
RequestLoggerProvider
@Override
public RequestLogger get()
{
-FileRequestLogger logger = new FileRequestLogger(jsonMapper,
factory.create(1, "RequestLogger-%s"), dir, filePattern);
+FileRequestLogger logger = new FileRequestLogger(
+jsonMapper,
+factory.create(1, "RequestLogger-%s"),
+dir,
+filePattern
+);
log.debug(new Exception("Stack trace"), "Creating %s at", logger);
return logger;
}
diff --git
a/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
b/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
index 3bee5f4..beee88e 100644
---
a/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
+++
b/server/src/main/java/org/apache/druid/server/log/FilteredRequestLoggerProvider.java
@@ -21,13 +21,16 @@ package org.apache.druid.server.log;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.ImmutableList;
import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.query.Query;
import org.apache.druid.server.RequestLogLine;
import javax.validation.constraints.NotNull;
import java.io.IOException;
+import java.util.List;
/**
*/
@@ -46,13 +49,17 @@ public class FilteredRequestLoggerProvider implements
RequestLoggerProvider
@JsonProperty
private long sqlQueryTimeThresholdMs = 0;
+ @JsonProperty
+ private List mutedQueryTypes = ImmutableList.of();
+
@Override
public RequestLogger get()
{
FilteredRequestLogger logger = new FilteredRequestLogger(
delegate.get(),
queryTimeThresholdMs,
-sqlQueryTimeThresholdMs
+sqlQueryTimeThresholdMs,
+mutedQueryTypes
[incubator-druid] branch master updated: Bump httpcore from 4.4.4 to 4.4.11 (#7870)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new f7cf2f7 Bump httpcore from 4.4.4 to 4.4.11 (#7870) f7cf2f7 is described below commit f7cf2f7cad430d7d94bf3f8993af81552290430d Author: Benedict Jin AuthorDate: Sat Aug 10 00:53:20 2019 +0800 Bump httpcore from 4.4.4 to 4.4.11 (#7870) * Bump httpcore from 4.4.4 to 4.4.11 * Update the version of httpcore in licenses.yaml --- licenses.yaml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index f6f8337..1ebe9b5 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -676,7 +676,7 @@ name: Apache HttpCore license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.4.4 +version: 4.4.11 libraries: - org.apache.httpcomponents: httpcore diff --git a/pom.xml b/pom.xml index dcebf1a..f6a2a25 100644 --- a/pom.xml +++ b/pom.xml @@ -640,7 +640,7 @@ org.apache.httpcomponents httpcore -4.4.4 +4.4.11 org.apache.hadoop - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (e64070b -> 781873b)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from e64070b Web-console: remove specific column filters (#8343) add 781873b Fix resource leak (#8337) No new revisions were added by this update. Summary of changes: .../SequenceInputStreamResponseHandler.java| 15 +++- ...hiteListBasedDruidToTimelineEventConverter.java | 11 +++--- .../cassandra/CassandraDataSegmentPusher.java | 7 ++-- .../druid/storage/google/GoogleTaskLogs.java | 45 +++--- .../org/apache/druid/guice/PropertiesModule.java | 4 +- .../groupby/epinephelinae/SpillingGrouper.java | 6 ++- .../druid/segment/data/GenericIndexedWriter.java | 10 ++--- .../org/apache/druid/server/security/TLSUtils.java | 44 - 8 files changed, 83 insertions(+), 59 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (781873b -> 818bf49)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from 781873b Fix resource leak (#8337) add 818bf49 Enable Spotbugs NP_NONNULL_RETURN_VIOLATION (#8234) No new revisions were added by this update. Summary of changes: codestyle/spotbugs-exclude.xml| 1 - .../druid/query/aggregation/TimestampAggregatorFactory.java | 1 + .../org/apache/druid/query/groupby/GroupByQueryEngine.java| 1 + .../main/java/org/apache/druid/segment/AbstractSegment.java | 3 +++ .../druid/segment/ColumnSelectorBitmapIndexSelector.java | 2 ++ .../main/java/org/apache/druid/segment/DimensionHandler.java | 3 +++ .../src/main/java/org/apache/druid/segment/IndexMerger.java | 1 + .../src/main/java/org/apache/druid/segment/MMappedIndex.java | 11 --- .../org/apache/druid/segment/ReferenceCountingSegment.java| 7 +++ .../src/main/java/org/apache/druid/segment/Segment.java | 1 + .../java/org/apache/druid/segment/StringDimensionIndexer.java | 1 + .../main/java/org/apache/druid/segment/VirtualColumns.java| 3 ++- .../apache/druid/segment/incremental/IncrementalIndex.java| 1 + .../segment/incremental/SpatialDimensionRowTransformer.java | 4 .../org/apache/druid/segment/serde/ComplexMetricSerde.java| 1 + 15 files changed, 32 insertions(+), 9 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (818bf49 -> ffcbd1e)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from 818bf49 Enable Spotbugs NP_NONNULL_RETURN_VIOLATION (#8234) add ffcbd1e Ensure ReferenceCountingSegment.decrement() is invoked correctly (#8323) No new revisions were added by this update. Summary of changes: .../appenderator/SinkQuerySegmentWalker.java | 171 +++-- 1 file changed, 86 insertions(+), 85 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (d117bfb -> 33f0753)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from d117bfb Handle exception thrown in log while trying to call sun.misc.VM.maxDirectMemory() which is not available in Java 11 (#8352) add 33f0753 Add Checkstyle for constant name static final (#8060) No new revisions were added by this update. Summary of changes: .../druid/benchmark/FlattenJSONBenchmark.java | 18 +- .../FloatCompressionBenchmarkFileGenerator.java|4 +- .../druid/benchmark/GenericIndexedBenchmark.java |6 +- .../IncrementalIndexRowTypeBenchmark.java | 34 +- .../LongCompressionBenchmarkFileGenerator.java |8 +- .../benchmark/StupidPoolConcurrencyBenchmark.java |4 +- .../query/timecompare/TimeCompareBenchmark.java|8 +- .../coordinator/CostBalancerStrategyBenchmark.java |6 +- codestyle/checkstyle-suppressions.xml |7 + codestyle/checkstyle.xml |5 +- .../ReferenceCountingResourceHolder.java |6 +- .../druid/data/input/impl/TimestampSpec.java |6 +- .../druid/jackson/CommaListJoinSerializer.java |4 +- .../java/util/common/granularity/Granularity.java |8 +- .../java/util/common/io/smoosh/FileSmoosher.java |4 +- .../util/common/parsers/JSONToLowerParser.java |6 +- .../java/util/emitter/core/HttpPostEmitter.java|4 +- .../druid/java/util/http/client/Request.java |4 +- .../apache/druid/java/util/metrics/SigarUtil.java |4 +- .../partition/HashBasedNumberedShardSpec.java |4 +- .../druid/collections/SerializablePairTest.java| 10 +- .../PrefetchableTextFilesFirehoseFactoryTest.java | 26 +- .../java/util/common/CompressionUtilsTest.java | 106 +- .../druid/java/util/common/RetryUtilsTest.java | 10 +- .../java/util/common/lifecycle/LifecycleTest.java |6 +- .../common/parsers/FlatTextFormatParserTest.java | 20 +- .../util/common/parsers/JSONPathParserTest.java| 28 +- .../druid/java/util/emitter/core/EmitterTest.java | 34 +- .../java/util/emitter/core/HttpEmitterTest.java|4 +- .../emitter/core/HttpPostEmitterStressTest.java|4 +- .../util/emitter/core/HttpPostEmitterTest.java |4 +- .../emitter/core/ParametrizedUriEmitterTest.java | 14 +- .../metadata/DefaultPasswordProviderTest.java | 28 +- .../EnvironmentVariablePasswordProviderTest.java |6 +- .../MetadataStorageConnectorConfigTest.java|8 +- .../org/apache/druid/timeline/DataSegmentTest.java | 18 +- .../SegmentWithOvershadowedStatusTest.java | 12 +- .../storage/azure/AzureDataSegmentPullerTest.java | 12 +- .../storage/azure/AzureDataSegmentPusherTest.java | 20 +- .../druid/storage/azure/AzureTaskLogsTest.java | 44 +- .../distinctcount/ConciseBitMapFactory.java|4 +- .../distinctcount/JavaBitMapFactory.java |4 +- .../distinctcount/RoaringBitMapFactory.java|4 +- .../DistinctCountGroupByQueryTest.java |8 +- .../DistinctCountTimeseriesQueryTest.java |8 +- .../distinctcount/DistinctCountTopNQueryTest.java |8 +- .../DerivativeDataSourceManager.java | 12 +- .../materializedview/DatasourceOptimizerTest.java |6 +- .../MaterializedViewQueryQueryToolChestTest.java | 10 +- .../MaterializedViewQueryTest.java | 30 +- .../movingaverage/MovingAverageIterableTest.java | 34 +- .../druid/segment/MapVirtualColumnGroupByTest.java |4 +- .../druid/segment/MapVirtualColumnSelectTest.java |8 +- .../druid/segment/MapVirtualColumnTopNTest.java|4 +- .../quantiles/DoublesSketchComplexMetricSerde.java |6 +- .../bloom/BloomFilterAggregatorTest.java | 116 +- .../bloom/BloomFilterGroupByQueryTest.java |6 +- .../bloom/sql/BloomFilterSqlAggregatorTest.java|4 +- .../query/filter/sql/BloomDimFilterSqlTest.java| 12 +- .../ApproximateHistogramGroupByQueryTest.java | 26 +- .../ApproximateHistogramTopNQueryTest.java | 28 +- .../FixedBucketsHistogramGroupByQueryTest.java | 26 +- .../FixedBucketsHistogramTopNQueryTest.java| 28 +- .../query/lookup/TestKafkaExtractionCluster.java | 20 +- .../indexing/kafka/KafkaIndexTaskClientTest.java | 10 +- .../indexing/kafka/KafkaRecordSupplierTest.java| 74 +- .../druid/indexing/kafka/KafkaSamplerSpecTest.java | 10 +- .../kafka/supervisor/KafkaSupervisorTest.java | 22 +- .../kinesis/KinesisIndexTaskClientTest.java| 10 +- .../kinesis/KinesisRecordSupplierTest.java | 40 +- .../indexing/kinesis/KinesisSamplerSpecTest.java |8 +- .../kinesis/supervisor/KinesisSupervisorTest.java | 808 +-- .../namespace/NamespacedExtractorModuleTest.java | 10
[incubator-druid] branch master updated (c73a489 -> f4e2902)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from c73a489 bump master version to 0.17.0-incubating-SNAPSHOT (#8421) add f4e2902 Add Checkstyle check for String literal equality (#8386) No new revisions were added by this update. Summary of changes: codestyle/checkstyle.xml | 3 +++ 1 file changed, 3 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: #8156 : StructuralSearchInspection, Prohibit check on Thread.ge… (#8394)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new ade8d19 #8156 : StructuralSearchInspection, Prohibit check on
Thread.ge… (#8394)
ade8d19 is described below
commit ade8d1922d338d05ed48a2c8ef49c277378ca456
Author: SandishKumarHN
AuthorDate: Sun Sep 22 04:12:05 2019 -0700
#8156 : StructuralSearchInspection, Prohibit check on Thread.ge… (#8394)
* StructuralSearchInspection, Prohibit check on Thread.getState()
* review changes - 1
* review changes 2
* review changes 3
* test fix
* review changes-2
* review changes-3
---
.idea/inspectionProfiles/Druid.xml| 8
.../org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java | 1 +
.../segment/realtime/firehose/EventReceiverFirehoseIdleTest.java | 4 +---
.../segment/realtime/firehose/EventReceiverFirehoseTest.java | 4 +---
4 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index bac71e7..5791c1c 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -311,6 +311,14 @@
+
+
+
+
+
+
+
+
diff --git
a/core/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java
b/core/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java
index ecb2711..2fbb933 100644
---
a/core/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java
+++
b/core/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java
@@ -179,6 +179,7 @@ final class DeadlockDetectingFailOnTimeout extends Statement
* problem or if the thread cannot be determined. The return value is never
equal
* to {@code mainThread}.
*/
+ @SuppressWarnings("SSBasedInspection") // Prohibit check on Thread.getState()
private Thread getStuckThread(Thread mainThread)
{
List threadsInGroup =
getThreadsInGroup(mainThread.getThreadGroup());
diff --git
a/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseIdleTest.java
b/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseIdleTest.java
index 97edb6a..878a7d4 100644
---
a/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseIdleTest.java
+++
b/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseIdleTest.java
@@ -100,9 +100,7 @@ public class EventReceiverFirehoseIdleTest
private void awaitDelayedExecutorThreadTerminated() throws
InterruptedException
{
-while (firehose.getDelayedCloseExecutor().getState() !=
Thread.State.TERMINATED) {
- Thread.sleep(50);
-}
+firehose.getDelayedCloseExecutor().join();
}
@Test(timeout = 40_000L)
diff --git
a/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseTest.java
b/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseTest.java
index e9c7ee3..683e870 100644
---
a/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseTest.java
+++
b/server/src/test/java/org/apache/druid/segment/realtime/firehose/EventReceiverFirehoseTest.java
@@ -275,9 +275,7 @@ public class EventReceiverFirehoseTest
private void awaitDelayedExecutorThreadTerminated() throws
InterruptedException
{
-while (firehose.getDelayedCloseExecutor().getState() !=
Thread.State.TERMINATED) {
- Thread.sleep(50);
-}
+firehose.getDelayedCloseExecutor().join();
}
@Test(timeout = 60_000L)
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Add new items to concurrency code review checklist (#8493)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new e7cc968 Add new items to concurrency code review checklist (#8493) e7cc968 is described below commit e7cc96874932bd63b99324f011e7eebd7f6054ed Author: Roman Leventov AuthorDate: Sun Sep 22 14:19:37 2019 +0300 Add new items to concurrency code review checklist (#8493) --- dev/code-review/concurrency.md | 99 +++--- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/dev/code-review/concurrency.md b/dev/code-review/concurrency.md index 279d463..9bbb82a 100644 --- a/dev/code-review/concurrency.md +++ b/dev/code-review/concurrency.md @@ -28,9 +28,10 @@ Design - Producer-consumer - Instance confinement - Thread/Task/Serial thread confinement + - Active object Documentation - - [Thread-safety is justified in comments?]( + - [Thread safety is justified in comments?]( https://github.com/code-review-checklists/java-concurrency#justify-document) - [Class (method, field) has concurrent access documentation?]( https://github.com/code-review-checklists/java-concurrency#justify-document) @@ -40,25 +41,38 @@ Documentation https://github.com/code-review-checklists/java-concurrency#threading-flow-model) - [Class is documented as immutable, thread-safe, or not thread-safe?]( https://github.com/code-review-checklists/java-concurrency#immutable-thread-safe) - - [Applied concurrency patterns are pronounced?]( + - [Used concurrency patterns are pronounced?]( https://github.com/code-review-checklists/java-concurrency#name-patterns) - [`@GuardedBy` annotation is used?](https://github.com/code-review-checklists/java-concurrency#guarded-by) - - [Safety of benign races is explained?]( + - [Safety of a benign race (e. g. unbalanced synchronization) is explained?]( https://github.com/code-review-checklists/java-concurrency#document-benign-race) - - [Each use of `volatile` is justified?]( - https://github.com/code-review-checklists/java-concurrency#justify-volatile) - - [Field that is neither `volatile` nor annotated with `@GuardedBy` has a comment?]( + - [Each use of `volatile` is justified?](https://github.com/code-review-checklists/java-concurrency#justify-volatile) + - [Each field that is neither `volatile` nor annotated with `@GuardedBy` has a comment?]( https://github.com/code-review-checklists/java-concurrency#plain-field) +Insufficient synchronization + - [Static methods and fields are thread-safe?]( + https://github.com/code-review-checklists/java-concurrency#static-thread-safe) + - [Thread *doesn't* wait in a loop for a non-volatile field to be updated by another thread?]( + https://github.com/code-review-checklists/java-concurrency#non-volatile-visibility) + - [Read access to a non-volatile, concurrently updatable primitive field is protected?]( + https://github.com/code-review-checklists/java-concurrency#non-volatile-protection) + - [`@GET`/`@POST` methods, Jetty Filters and Handlers are thread-safe?]( + https://github.com/code-review-checklists/java-concurrency#server-framework-sync) + - [Calls to `DateFormat.parse()` and `format()` are synchronized?]( + https://github.com/code-review-checklists/java-concurrency#dateformat) + Excessive thread safety - [No "extra" (pseudo) thread safety?](https://github.com/code-review-checklists/java-concurrency#pseudo-safety) - [No atomics on which only `get()` and `set()` are called?]( https://github.com/code-review-checklists/java-concurrency#redundant-atomics) - [Class (method) needs to be thread-safe?]( https://github.com/code-review-checklists/java-concurrency#unneeded-thread-safety) + - [`ReentrantLock` (`ReentrantReadWriteLock`, `Semaphore`) needs to be fair?]( + https://github.com/code-review-checklists/java-concurrency#unneeded-fairness) Race conditions - - [No `put()` or `remove()` calls on a `ConcurrentHashMap` after `get()` or `containsKey()`?]( + - [No `put()` or `remove()` calls on a `ConcurrentMap` (or Cache) after `get()` or `containsKey()`?]( https://github.com/code-review-checklists/java-concurrency#chm-race) - [No point accesses to a non-thread-safe collection outside of critical sections?]( https://github.com/code-review-checklists/java-concurrency#unsafe-concurrent-point-read) @@ -68,19 +82,43 @@ Race conditions https://github.com/code-review-checklists/java-concurrency#concurrent-mutation-race) - [No separate getters to an atomically updated state?]( https://github.com/code-review-checklists/java-concurrency#moving-state-race) - - [No state used for making decisions or preparing data inside a critical section is read outside?]( + - [No *check-then-act* race conditions (state used inside a critical section is read outside of it)?]( + https://github.
[incubator-druid] branch master updated (4c215b4 -> bba262a)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from 4c215b4 Web console: misc bug fixes and tidy up (#8654) add bba262a Fix resource leaks and suppress an incorrect LGTM alert (#8589) No new revisions were added by this update. Summary of changes: .../apache/druid/java/util/common/io/smoosh/FileSmoosher.java | 2 +- .../src/main/java/org/apache/druid/indexer/JobHelper.java | 11 ++- 2 files changed, 7 insertions(+), 6 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] 01/01: Add an item to concurrency checklist
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch leventov-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git commit 1129200fbdd4250a3cf411150afa1a2f46321821 Author: Roman Leventov AuthorDate: Sat Oct 19 16:31:08 2019 +0300 Add an item to concurrency checklist Add an item to concurrency checklist about assertions in parallel threads and async code in tests --- dev/code-review/concurrency.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/code-review/concurrency.md b/dev/code-review/concurrency.md index 9bbb82a..2637bdf 100644 --- a/dev/code-review/concurrency.md +++ b/dev/code-review/concurrency.md @@ -108,6 +108,8 @@ Testing https://github.com/code-review-checklists/java-concurrency#coordinate-test-workers) - [There are more test threads than CPUs (if possible for the test)?]( https://github.com/code-review-checklists/java-concurrency#test-workers-interleavings) + - [Assertions in parallel threads and asynchronous code are handled properly?]( + https://github.com/code-review-checklists/java-concurrency#concurrent-assert) Locks - [Can use `LifecycleLock` instead of a standard lock in a lifecycled object?](#use-lifecycle-lock) @@ -245,4 +247,4 @@ daemon threads already. [#](#use-execs) TE.D2. Is it possible to use one of the static factory methods in Druid's `Execs` utility class to create an `ExecutorService` instead of Java's standard `ExecutorServices`? This is recommended because `Execs` configure -ThreadFactories to create daemon threads by default, as required by the previous item. \ No newline at end of file +ThreadFactories to create daemon threads by default, as required by the previous item. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch leventov-patch-1 created (now 1129200)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch leventov-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. at 1129200 Add an item to concurrency checklist This branch includes the following new commits: new 1129200 Add an item to concurrency checklist The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[druid] branch master updated: Added CronScheduler support as a proof to clock drift while emitting metrics (#10448)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new d0c2ede Added CronScheduler support as a proof to clock drift while
emitting metrics (#10448)
d0c2ede is described below
commit d0c2ede50c94bef89b72261fe3e553c60e7f0013
Author: Ayush Kulshrestha
AuthorDate: Wed Nov 25 17:01:38 2020 +0530
Added CronScheduler support as a proof to clock drift while emitting
metrics (#10448)
Co-authored-by: Ayush Kulshrestha
---
core/pom.xml | 4 +
.../druid/java/util/metrics/AbstractMonitor.java | 17 ++
.../druid/java/util/metrics/CompoundMonitor.java | 15 +
.../apache/druid/java/util/metrics/Monitor.java| 7 +
.../druid/java/util/metrics/MonitorScheduler.java | 75 +++--
.../java/util/metrics/MonitorSchedulerTest.java| 304 -
licenses.yaml | 10 +
pom.xml| 5 +
server/pom.xml | 4 +
.../apache/druid/server/metrics/MetricsModule.java | 7 +-
10 files changed, 423 insertions(+), 25 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index e0ccfa1..72fea54 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -224,6 +224,10 @@
org.antlr
antlr4-runtime
+
+ io.timeandspace
+ cron-scheduler
+
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/AbstractMonitor.java
b/core/src/main/java/org/apache/druid/java/util/metrics/AbstractMonitor.java
index 029dd47..4fbefb8 100644
--- a/core/src/main/java/org/apache/druid/java/util/metrics/AbstractMonitor.java
+++ b/core/src/main/java/org/apache/druid/java/util/metrics/AbstractMonitor.java
@@ -22,11 +22,16 @@ package org.apache.druid.java.util.metrics;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import java.util.concurrent.Future;
+
+
/**
*/
public abstract class AbstractMonitor implements Monitor
{
private volatile boolean started = false;
+
+ private volatile Future scheduledFuture;
@Override
public void start()
@@ -51,4 +56,16 @@ public abstract class AbstractMonitor implements Monitor
}
public abstract boolean doMonitor(ServiceEmitter emitter);
+
+ @Override
+ public Future getScheduledFuture()
+ {
+return scheduledFuture;
+ }
+
+ @Override
+ public void setScheduledFuture(Future scheduledFuture)
+ {
+this.scheduledFuture = scheduledFuture;
+ }
}
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/CompoundMonitor.java
b/core/src/main/java/org/apache/druid/java/util/metrics/CompoundMonitor.java
index 9811f58..6649312 100644
--- a/core/src/main/java/org/apache/druid/java/util/metrics/CompoundMonitor.java
+++ b/core/src/main/java/org/apache/druid/java/util/metrics/CompoundMonitor.java
@@ -24,10 +24,13 @@ import
org.apache.druid.java.util.emitter.service.ServiceEmitter;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.Future;
public abstract class CompoundMonitor implements Monitor
{
private final List monitors;
+
+ private volatile Future scheduledFuture;
public CompoundMonitor(List monitors)
{
@@ -61,5 +64,17 @@ public abstract class CompoundMonitor implements Monitor
return shouldReschedule(Lists.transform(monitors, monitor ->
monitor.monitor(emitter)));
}
+ @Override
+ public Future getScheduledFuture()
+ {
+return scheduledFuture;
+ }
+
+ @Override
+ public void setScheduledFuture(Future scheduledFuture)
+ {
+this.scheduledFuture = scheduledFuture;
+ }
+
public abstract boolean shouldReschedule(List reschedules);
}
diff --git a/core/src/main/java/org/apache/druid/java/util/metrics/Monitor.java
b/core/src/main/java/org/apache/druid/java/util/metrics/Monitor.java
index 2ccd5db..8a3975e 100644
--- a/core/src/main/java/org/apache/druid/java/util/metrics/Monitor.java
+++ b/core/src/main/java/org/apache/druid/java/util/metrics/Monitor.java
@@ -21,6 +21,9 @@ package org.apache.druid.java.util.metrics;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import java.util.concurrent.Future;
+
+
/**
*/
public interface Monitor
@@ -35,4 +38,8 @@ public interface Monitor
* @return true if this monitor needs to continue monitoring. False
otherwise.
*/
boolean monitor(ServiceEmitter emitter);
+
+ Future getScheduledFuture();
+
+ void setScheduledFuture(Future scheduledFuture);
}
diff --git
a/core/src/main/java/org/apache/druid/java/util/metrics/MonitorScheduler.java
b/core/src/main/java/org/apache/druid/java/util/metrics/MonitorScheduler.java
index 2adbe95..961f823 100644
---
a/core/src/main/java/org/apache/druid/java/util/metrics/MonitorScheduler.java
+++
b/core/src/main/java/org/apache/druid/java/util/metr
[incubator-druid] branch master updated (b8ceee4 -> 3e9723e)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from b8ceee4 Removed 'if' condition. (#8768) add 3e9723e Add an item to concurrency checklist about assertions in parall… (#8701) No new revisions were added by this update. Summary of changes: dev/code-review/concurrency.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (517c146 -> fca23d0)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from 517c146 Upgrade joda-time to 2.10.5 (#8821) add fca23d0 use copy-on-write list in InMemoryAppender (#8808) No new revisions were added by this update. Summary of changes: .../java/org/apache/druid/testing/junit/LoggerCaptureRule.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (fca23d0 -> a9aa416)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from fca23d0 use copy-on-write list in InMemoryAppender (#8808) add a9aa416 In DirectDruidClient, don't run Future cancellation listener in… (#8700) No new revisions were added by this update. Summary of changes: .../org/apache/druid/client/DirectDruidClient.java | 77 +- .../apache/druid/client/DirectDruidClientTest.java | 4 +- 2 files changed, 48 insertions(+), 33 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated (d0a6fe7 -> 3916461)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. from d0a6fe7 fix bug with sqlOuterLimit, use sqlOuterLimit in web console (#8919) add 3916461 Fix double-checked locking in predicate suppliers in BoundDimFi… (#8974) No new revisions were added by this update. Summary of changes: .../apache/druid/query/filter/BoundDimFilter.java | 522 + .../org/apache/druid/query/filter/InDimFilter.java | 197 +++- 2 files changed, 298 insertions(+), 421 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[druid] branch master updated (b287711 -> 4716e0b)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/druid.git. from b287711 Suppress CVE-2019-20330 for htrace-core-4.0.1 (#9189) add 4716e0b Fix concurrency of ComplexMetrics.java (#9134) No new revisions were added by this update. Summary of changes: .../apache/druid/segment/serde/ComplexMetrics.java | 43 +++--- 1 file changed, 29 insertions(+), 14 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[druid] branch master updated (57765a4 -> 53bb45f)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/druid.git. from 57765a4 Allow overriding default JoinableFactory in SpecificSegmentsQuerySegmentWalker (#9330) add 53bb45f Forbid easily misused HashSet and HashMap constructors (#9165) No new revisions were added by this update. Summary of changes: .../query/CachingClusteredClientBenchmark.java | 3 ++- .../VersionedIntervalTimelineBenchmark.java| 8 +++ codestyle/druid-forbidden-apis.txt | 8 +++ .../main/java/org/apache/druid/math/expr/Expr.java | 2 +- .../org/apache/druid/math/expr/ExprMacroTable.java | 4 ++-- .../org/apache/druid/utils/CollectionUtils.java| 26 ++ .../input/avro/InlineSchemasAvroBytesDecoder.java | 4 ++-- .../lookup/KafkaLookupExtractorFactoryTest.java| 6 ++--- .../batch/parallel/ParallelIndexPhaseRunner.java | 4 ++-- .../batch/parallel/PartialSegmentMergeTask.java| 3 ++- .../overlord/hrtr/HttpRemoteTaskRunner.java| 3 +-- .../indexing/overlord/http/OverlordResource.java | 3 ++- .../indexing/common/task/CompactionTaskTest.java | 7 +++--- .../RecordSupplierInputSourceTest.java | 4 ++-- pom.xml| 3 +++ processing/pom.xml | 4 .../druid/query/groupby/having/HavingSpecUtil.java | 4 ++-- .../org/apache/druid/query/topn/TopNBinaryFn.java | 3 ++- .../join/table/IndexedTableJoinMatcher.java| 3 +++ .../druid/segment/filter/BaseFilterTest.java | 3 ++- .../druid/client/HttpServerInventoryView.java | 3 +-- .../server/coordinator/duty/CompactSegments.java | 4 +++- .../duty/NewestSegmentFirstIterator.java | 4 ++-- .../apache/druid/server/http/RouterResource.java | 4 ++-- 24 files changed, 85 insertions(+), 35 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[druid] branch master updated (73a0181 -> e7eb45e)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/druid.git. from 73a0181 Fix handling for columns that appear multiple times in join conditions (#9362) add e7eb45e Run IntelliJ inspections on Travis (#9179) No new revisions were added by this update. Summary of changes: .idea/scopes/JavaInspectionsScope.xml | 3 +++ .travis.yml | 11 +++ .../org/apache/druid/common/guava/ThreadRenamingCallable.java | 2 +- .../apache/druid/data/input/IntermediateRowParsingReader.java | 2 +- .../org/apache/druid/emitter/influxdb/InfluxdbEmitter.java| 4 ++-- .../druid/emitter/influxdb/InfluxdbEmitterConfigTest.java | 11 +-- .../query/movingaverage/averagers/DoubleSumAveragerTest.java | 2 +- .../druid/data/input/parquet/DecimalParquetInputTest.java | 2 +- .../epinephelinae/vector/GroupByVectorColumnSelector.java | 2 ++ .../druid/segment/ColumnSelectorBitmapIndexSelector.java | 2 +- .../java/org/apache/druid/segment/data/ColumnarDoubles.java | 1 + .../java/org/apache/druid/segment/data/ColumnarFloats.java| 1 + .../java/org/apache/druid/segment/data/ColumnarLongs.java | 1 + .../druid/query/aggregation/StringColumnAggregationTest.java | 2 +- .../query/aggregation/mean/DoubleMeanAggregationTest.java | 2 +- .../java/org/apache/druid/query/scan/ScanQueryRunnerTest.java | 3 +-- .../java/org/apache/druid/client/cache/CaffeineCacheTest.java | 2 +- .../druid/metadata/SqlSegmentsMetadataManagerEmptyTest.java | 2 +- .../test/java/org/apache/druid/server/RequestLogLineTest.java | 6 +++--- 19 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 .idea/scopes/JavaInspectionsScope.xml - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] 01/01: Add instruction about skipping up-to-date checks when running integration tests
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch leventov-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git commit e8ef50bc4cd5947d3ed3002f206e268c2e095e23 Author: Roman Leventov AuthorDate: Thu Jun 6 15:47:09 2019 +0200 Add instruction about skipping up-to-date checks when running integration tests --- integration-tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/README.md b/integration-tests/README.md index 55ec185..37435f0 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -79,6 +79,9 @@ To run only a single test using mvn run the following command: mvn verify -P integration-tests -Dit.test= ``` +Add `-rf :druid-integration-tests` when running integration tests for the second time or later without changing +the code of core modules in between to skip up-to-date checks for the whole module dependency tree. + Running Tests Using A Configuration File for Any Cluster --- - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch leventov-patch-1 created (now e8ef50b)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch leventov-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. at e8ef50b Add instruction about skipping up-to-date checks when running integration tests This branch includes the following new commits: new e8ef50b Add instruction about skipping up-to-date checks when running integration tests The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Druid basic authentication class composition config (#7789)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0802702 Druid basic authentication class composition config (#7789)
0802702 is described below
commit 080270283a40e5d83eefb909decb9c167e7aa420
Author: Eugene Sevastyanov
AuthorDate: Thu Jun 6 16:51:37 2019 +0300
Druid basic authentication class composition config (#7789)
* Druid basic authentication class composition config.
* Added comments
* Reduced nulls
* Used noop implementations to get rid of null
* Added docs for no-op metadata storage updaters
* Fixed BasicAuthClassCompositionConfig javadoc
* Removed incorrect comments
---
.../basic/BasicAuthClassCompositionConfig.java | 128 +++
.../security/basic/BasicSecurityDruidModule.java | 181 ++---
.../cache/NoopBasicAuthenticatorCacheNotifier.java | 34
...opBasicAuthenticatorMetadataStorageUpdater.java | 73 +
.../db/cache/NoopBasicAuthorizerCacheNotifier.java | 34
.../NoopBasicAuthorizerMetadataStorageUpdater.java | 99 +++
6 files changed, 493 insertions(+), 56 deletions(-)
diff --git
a/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/BasicAuthClassCompositionConfig.java
b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/BasicAuthClassCompositionConfig.java
new file mode 100644
index 000..dc531a1
--- /dev/null
+++
b/extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/BasicAuthClassCompositionConfig.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.security.basic;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Basic authentication storage/cache/resource handler config.
+ * BasicAuthClassCompositionConfig provides options to specify
authenticator/authorizer classes of user/role managers,
+ * caches and notifiers. If a field in this class is non-null then the
corresponding class is instantiated
+ * regardless of what type of Druid component runs it (see {@link
BasicSecurityDruidModule}).
+ * Hence every Druid component might be a user/role manager and notify others
by sending notifications.
+ * Every field must be a valid class name (appropriate for the corresponding
goal) or null.
+ */
+public class BasicAuthClassCompositionConfig
+{
+ @JsonProperty
+ private final String authenticatorMetadataStorageUpdater;
+
+ @JsonProperty
+ private final String authenticatorCacheManager;
+
+ @JsonProperty
+ private final String authenticatorResourceHandler;
+
+ @JsonProperty
+ private final String authenticatorCacheNotifier;
+
+ @JsonProperty
+ private final String authorizerMetadataStorageUpdater;
+
+ @JsonProperty
+ private final String authorizerCacheManager;
+
+ @JsonProperty
+ private final String authorizerResourceHandler;
+
+ @JsonProperty
+ private final String authorizerCacheNotifier;
+
+ @JsonCreator
+ public BasicAuthClassCompositionConfig(
+ @JsonProperty("authenticatorMetadataStorageUpdater") String
authenticatorMetadataStorageUpdater,
+ @JsonProperty("authenticatorCacheManager") String
authenticatorCacheManager,
+ @JsonProperty("authenticatorResourceHandler") String
authenticatorResourceHandler,
+ @JsonProperty("authenticatorCacheNotifier") String
authenticatorCacheNotifier,
+ @JsonProperty("authorizerMetadataStorageUpdater") String
authorizerMetadataStorageUpdater,
+ @JsonProperty("authorizerCacheManager") String authorizerCacheManager,
+ @JsonProperty("authorizerResourceHandler") String
authorizerResourceHandler,
+ @JsonProperty("authorizerCacheNotifier") String authorizerCacheNotifier
+ )
+ {
+this.authenticatorMetadataStorageUpdater =
authenticatorMetadataStorageUpdater;
+
[incubator-druid] branch master updated: Optimize overshadowed segments computation (#7595)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new ea752ef Optimize overshadowed segments computation (#7595)
ea752ef is described below
commit ea752ef562b5987ff1cb0a03469855097a4bec9e
Author: Surekha
AuthorDate: Fri Jun 7 10:15:54 2019 -0700
Optimize overshadowed segments computation (#7595)
* Move the overshadowed segment computation to SQLMetadataSegmentManager's
poll
* rename method in MetadataSegmentManager
* Fix tests
* PR comments
* PR comments
* PR comments
* fix indentation
* fix tests
* fix test
* add test for SegmentWithOvershadowedStatus serde format
* PR comments
* PR comments
* fix test
* remove snapshot updates outside poll
* PR comments
* PR comments
* PR comments
* removed unused import
---
.../timeline/SegmentWithOvershadowedStatus.java| 7 +
.../org/apache/druid/utils/CollectionUtils.java| 15 ++
.../SegmentWithOvershadowedStatusTest.java | 180 +
.../MaterializedViewSupervisor.java| 2 +-
.../common/actions/SegmentListActionsTest.java | 2 +-
.../results/auth_test_sys_schema_segments.json | 2 +-
.../apache/druid/client/DataSourcesSnapshot.java | 114 +
.../druid/client/ImmutableDruidDataSource.java | 40 -
.../druid/metadata/MetadataSegmentManager.java | 27 +++-
.../druid/metadata/SQLMetadataSegmentManager.java | 132 +++
.../druid/server/coordinator/DruidCoordinator.java | 35 +++-
.../coordinator/DruidCoordinatorRuntimeParams.java | 22 ++-
.../helper/DruidCoordinatorRuleRunner.java | 16 +-
.../druid/server/http/DataSourcesResource.java | 2 +-
.../apache/druid/server/http/MetadataResource.java | 25 ++-
.../metadata/SQLMetadataSegmentManagerTest.java| 2 +-
.../coordinator/CuratorDruidCoordinatorTest.java | 6 +
.../DruidCoordinatorRuleRunnerTest.java| 19 ++-
.../server/coordinator/DruidCoordinatorTest.java | 22 +--
.../druid/sql/calcite/schema/SystemSchema.java | 13 +-
20 files changed, 511 insertions(+), 172 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
b/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
index e86daea..3f2972f 100644
---
a/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
+++
b/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
@@ -21,6 +21,7 @@ package org.apache.druid.timeline;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonUnwrapped;
/**
* DataSegment object plus the overshadowed status for the segment. An
immutable object.
@@ -31,6 +32,12 @@ import com.fasterxml.jackson.annotation.JsonProperty;
public class SegmentWithOvershadowedStatus implements
Comparable
{
private final boolean overshadowed;
+ /**
+ * dataSegment is serialized "unwrapped", i.e. it's properties are included
as properties of
+ * enclosing class. If in future, if {@Code SegmentWithOvershadowedStatus}
were to extend {@link DataSegment},
+ * there will be no change in the serialized format.
+ */
+ @JsonUnwrapped
private final DataSegment dataSegment;
@JsonCreator
diff --git a/core/src/main/java/org/apache/druid/utils/CollectionUtils.java
b/core/src/main/java/org/apache/druid/utils/CollectionUtils.java
index 0ba595c..af3cf07 100644
--- a/core/src/main/java/org/apache/druid/utils/CollectionUtils.java
+++ b/core/src/main/java/org/apache/druid/utils/CollectionUtils.java
@@ -19,10 +19,14 @@
package org.apache.druid.utils;
+import com.google.common.collect.Maps;
+
import java.util.AbstractCollection;
import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
import java.util.Spliterator;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -68,6 +72,17 @@ public final class CollectionUtils
};
}
+ /**
+ * Returns a transformed map from the given input map where the value is
modified based on the given valueMapper
+ * function.
+ */
+ public static Map mapValues(Map map, Function
valueMapper)
+ {
+final Map result = Maps.newHashMapWithExpectedSize(map.size());
+map.forEach((k, v) -> result.put(k, valueMapper.apply(v)));
+return result;
+ }
+
private CollectionUtils()
{
}
diff --git
a/core/src/test/java/org/apache/druid/timeline/SegmentWithOvershadowedStatusTest.java
b/core/src/test/java/or
[incubator-druid] branch master updated: Use map.putIfAbsent() or map.computeIfAbsent() as appropriate instead of containsKey() + put() (#7764)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 3bee6ad Use map.putIfAbsent() or map.computeIfAbsent() as appropriate
instead of containsKey() + put() (#7764)
3bee6ad is described below
commit 3bee6adcf7338b501687f11ea866608e873c5b0a
Author: Sashidhar Thallam
AuthorDate: Fri Jun 14 21:29:36 2019 +0530
Use map.putIfAbsent() or map.computeIfAbsent() as appropriate instead of
containsKey() + put() (#7764)
* https://github.com/apache/incubator-druid/issues/7316 Use
Map.putIfAbsent() instead of containsKey() + put()
* fixing indentation
* Using map.computeIfAbsent() instead of map.putIfAbsent() where appropriate
* fixing checkstyle
* Changing the recommendation text
* Reverting auto changes made by IDE
* Implementing recommendation: A ConcurrentHashMap on which
computeIfAbsent() is called should be assigned into variables of
ConcurrentHashMap type, not ConcurrentMap
* Removing unused import
---
.idea/inspectionProfiles/Druid.xml | 7 +-
.../druid/storage/s3/S3DataSegmentMoverTest.java | 4 +-
.../indexer/DetermineHashedPartitionsJob.java | 4 +-
.../ActionBasedUsedSegmentChecker.java | 5 +-
.../druid/indexing/common/task/IndexTask.java | 8 +-
.../firehose/IngestSegmentFirehoseFactory.java | 149 +++---
.../druid/indexing/overlord/ForkingTaskRunner.java | 548 ++---
.../metadata/SQLMetadataSupervisorManager.java | 4 +-
.../druid/server/http/DataSourcesResource.java | 4 +-
.../druid/client/CachingClusteredClientTest.java | 4 +-
.../appenderator/StreamAppenderatorDriverTest.java | 4 +-
.../java/org/apache/druid/sql/SqlLifecycle.java| 4 +-
.../util/SpecificSegmentsQuerySegmentWalker.java | 4 +-
13 files changed, 365 insertions(+), 384 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml
b/.idea/inspectionProfiles/Druid.xml
index 9770c11..ed890c8 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -306,6 +306,11 @@
+
+
+
+
+
@@ -400,4 +405,4 @@
-
\ No newline at end of file
+
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
index ec30aa9..34496d9 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java
@@ -261,9 +261,7 @@ public class S3DataSegmentMoverTest
@Override
public PutObjectResult putObject(String bucketName, String key, File file)
{
- if (!storage.containsKey(bucketName)) {
-storage.put(bucketName, new HashSet<>());
- }
+ storage.putIfAbsent(bucketName, new HashSet<>());
storage.get(bucketName).add(key);
return new PutObjectResult();
}
diff --git
a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DetermineHashedPartitionsJob.java
b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DetermineHashedPartitionsJob.java
index c83bc08..17f5172 100644
---
a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DetermineHashedPartitionsJob.java
+++
b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DetermineHashedPartitionsJob.java
@@ -307,9 +307,7 @@ public class DetermineHashedPartitionsJob implements Jobby
.getSegmentGranularity()
.bucket(DateTimes.utc(inputRow.getTimestampFromEpoch()));
-if (!hyperLogLogs.containsKey(interval)) {
- hyperLogLogs.put(interval,
HyperLogLogCollector.makeLatestCollector());
-}
+hyperLogLogs.computeIfAbsent(interval, intv ->
HyperLogLogCollector.makeLatestCollector());
} else {
final Optional maybeInterval = config.getGranularitySpec()
.bucketInterval(DateTimes.utc(inputRow.getTimestampFromEpoch()));
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/appenderator/ActionBasedUsedSegmentChecker.java
b/indexing-service/src/main/java/org/apache/druid/indexing/appenderator/ActionBasedUsedSegmentChecker.java
index a1eb90f..96ce6ae 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/appenderator/ActionBasedUsedSegmentChecker.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/appenderator/ActionBasedUsedSegmentChecker.java
@@ -50,9 +50,8 @@ public class ActionBasedUsedSegmentChecker implements
Use
[incubator-druid] branch master updated: Use only com.google.errorprone.annotations.concurrent.GuardedBy, not javax.annotations.concurrent.GuardedBy (#7889)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new 01881e3 Use only com.google.errorprone.annotations.concurrent.GuardedBy, not javax.annotations.concurrent.GuardedBy (#7889) 01881e3 is described below commit 01881e3a987ad34cce6456cf6e5766b2629ebde1 Author: SandishKumarHN AuthorDate: Mon Jun 17 06:58:51 2019 -0700 Use only com.google.errorprone.annotations.concurrent.GuardedBy, not javax.annotations.concurrent.GuardedBy (#7889) --- codestyle/druid-forbidden-apis.txt | 5 - core/pom.xml | 4 core/src/main/java/org/apache/druid/common/config/Log4jShutdown.java | 2 +- .../druid/java/util/common/concurrent/DirectExecutorService.java | 3 +-- .../apache/druid/java/util/emitter/core/ParametrizedUriEmitter.java | 2 +- .../aggregation/tdigestsketch/TDigestBuildSketchAggregator.java | 3 ++- .../aggregation/tdigestsketch/TDigestMergeSketchAggregator.java | 2 +- .../java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java | 2 +- .../java/org/apache/druid/segment/incremental/IncrementalIndex.java | 2 +- .../druid/curator/discovery/CuratorDruidNodeDiscoveryProvider.java | 2 +- sql/src/main/java/org/apache/druid/sql/avatica/DruidConnection.java | 2 +- sql/src/main/java/org/apache/druid/sql/avatica/DruidStatement.java | 2 +- 12 files changed, 19 insertions(+), 12 deletions(-) diff --git a/codestyle/druid-forbidden-apis.txt b/codestyle/druid-forbidden-apis.txt index 3d7da96..b06e502 100644 --- a/codestyle/druid-forbidden-apis.txt +++ b/codestyle/druid-forbidden-apis.txt @@ -42,4 +42,7 @@ com.ibm.icu.text.SimpleDateFormat#(java.lang.String) @defaultMessage For performance reasons, use the utf8Base64 / encodeBase64 / encodeBase64String / decodeBase64 / decodeBase64String methods in StringUtils org.apache.commons.codec.binary.Base64 -com.google.common.io.BaseEncoding.base64 \ No newline at end of file +com.google.common.io.BaseEncoding.base64 + +@defaultMessage Use com.google.errorprone.annotations.concurrent.GuardedBy +javax.annotations.concurrent.GuardedBy \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index f847d67..1f7cd6b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -284,6 +284,10 @@ jetty-servlet test + + com.google.errorprone + error_prone_annotations + diff --git a/core/src/main/java/org/apache/druid/common/config/Log4jShutdown.java b/core/src/main/java/org/apache/druid/common/config/Log4jShutdown.java index 918970c..1aa6022 100644 --- a/core/src/main/java/org/apache/druid/common/config/Log4jShutdown.java +++ b/core/src/main/java/org/apache/druid/common/config/Log4jShutdown.java @@ -19,12 +19,12 @@ package org.apache.druid.common.config; +import com.google.errorprone.annotations.concurrent.GuardedBy; import org.apache.druid.java.util.common.ISE; import org.apache.logging.log4j.core.LifeCycle; import org.apache.logging.log4j.core.util.Cancellable; import org.apache.logging.log4j.core.util.ShutdownCallbackRegistry; -import javax.annotation.concurrent.GuardedBy; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; diff --git a/core/src/main/java/org/apache/druid/java/util/common/concurrent/DirectExecutorService.java b/core/src/main/java/org/apache/druid/java/util/common/concurrent/DirectExecutorService.java index c692445..89d950d 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/concurrent/DirectExecutorService.java +++ b/core/src/main/java/org/apache/druid/java/util/common/concurrent/DirectExecutorService.java @@ -19,10 +19,9 @@ package org.apache.druid.java.util.common.concurrent; - import com.google.common.util.concurrent.AbstractListeningExecutorService; +import com.google.errorprone.annotations.concurrent.GuardedBy; -import javax.annotation.concurrent.GuardedBy; import java.util.Collections; import java.util.List; import java.util.concurrent.RejectedExecutionException; diff --git a/core/src/main/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitter.java b/core/src/main/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitter.java index 8a41362..b72572a 100644 --- a/core/src/main/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitter.java +++ b/core/src/main/java/org/apache/druid/java/util/emitter/core/ParametrizedUriEmitter.java @@ -21,6 +21,7 @@ package org.apache.druid.java.util.emitter.core; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.concurrent.GuardedBy; import org.apache.druid.java.util.common.StringUtils; import
[incubator-druid] branch master updated: Add Spotbugs (#7894)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new 48f20fe Add Spotbugs (#7894) 48f20fe is described below commit 48f20fe7542ffa62d88d2756aa3e97ebdbb8eb36 Author: Fokko Driesprong AuthorDate: Thu Jun 20 20:06:52 2019 +0200 Add Spotbugs (#7894) * Add Spotbugs Exclude all the issues for now, so we can add them one by one. (cherry picked from commit ceda4754dc8c703d1e0de85b48cd5f5409cfd5b7) * Add additional rules to the list * More rules * More rules * Add comments to the xml * Move the spotbugs-exclude.xml to codestyle/ --- .travis.yml| 2 +- codestyle/spotbugs-exclude.xml | 100 + pom.xml| 16 +++ 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8ccad05..8215525 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ matrix: - NAME="strict compilation" install: true # Strict compilation requires more than 2 GB - script: MAVEN_OPTS='-Xmx3000m' mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B --fail-at-end + script: MAVEN_OPTS='-Xmx3000m' mvn clean -Pstrict -pl '!benchmarks' compile test-compile spotbugs:check -B --fail-at-end # packaging check - env: diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml new file mode 100644 index 000..749053a --- /dev/null +++ b/codestyle/spotbugs-exclude.xml @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 53b420e..ff52815 100644 --- a/pom.xml +++ b/pom.xml @@ -1010,6 +1010,22 @@ +com.github.spotbugs +spotbugs-maven-plugin +3.1.12 + + + +com.github.spotbugs +spotbugs +3.1.12 + + + + codestyle/spotbugs-exclude.xml + + + org.apache.maven.plugins maven-pmd-plugin 3.8 - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Use ComplexMetrics.registerSerde() across the codebase (#7925)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new b9c6a26 Use ComplexMetrics.registerSerde() across the codebase (#7925)
b9c6a26 is described below
commit b9c6a26c0eeb1b159ac04f9f105f53d32762ac68
Author: Xue Yu
AuthorDate: Tue Jun 25 16:39:04 2019 +0800
Use ComplexMetrics.registerSerde() across the codebase (#7925)
* refactor complexmetric registerserde
* fix error
* feedback address
---
.../druid/benchmark/FilterPartitionBenchmark.java | 4 +--
.../benchmark/FilteredAggregatorBenchmark.java | 4 +--
.../benchmark/GroupByTypeInterfaceBenchmark.java | 4 +--
.../benchmark/TopNTypeInterfaceBenchmark.java | 4 +--
.../druid/benchmark/datagen/SegmentGenerator.java | 4 +--
.../indexing/IncrementalIndexReadBenchmark.java| 4 +--
.../indexing/IndexIngestionBenchmark.java | 2 +-
.../benchmark/indexing/IndexMergeBenchmark.java| 4 +--
.../benchmark/indexing/IndexPersistBenchmark.java | 4 +--
.../druid/benchmark/query/GroupByBenchmark.java| 5 ++--
.../druid/benchmark/query/ScanBenchmark.java | 5 ++--
.../druid/benchmark/query/SearchBenchmark.java | 5 ++--
.../druid/benchmark/query/SelectBenchmark.java | 4 +--
.../druid/benchmark/query/TimeseriesBenchmark.java | 4 +--
.../druid/benchmark/query/TopNBenchmark.java | 4 +--
.../query/timecompare/TimeCompareBenchmark.java| 4 +--
.../momentsketch/MomentSketchModule.java | 4 +--
.../tdigestsketch/TDigestSketchModule.java | 4 +--
.../datasketches/hll/HllSketchModule.java | 12 +++--
.../quantiles/DoublesSketchModule.java | 4 +--
.../datasketches/theta/SketchModule.java | 14 +++
.../theta/oldapi/OldApiSketchModule.java | 29 +-
.../tuple/ArrayOfDoublesSketchModule.java | 20 +++
.../druid/guice/BloomFilterSerializersModule.java | 4 +--
.../histogram/ApproximateHistogramDruidModule.java | 9 ++-
.../query/aggregation/stats/DruidStatsModule.java | 5 +---
.../apache/druid/jackson/AggregatorsModule.java| 20 +--
.../apache/druid/segment/serde/ComplexMetrics.java | 11 +---
.../apache/druid/segment/SchemalessIndexTest.java | 4 +--
.../java/org/apache/druid/segment/TestIndex.java | 4 +--
30 files changed, 56 insertions(+), 153 deletions(-)
diff --git
a/benchmarks/src/main/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
b/benchmarks/src/main/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
index 810d90f..ecf5027 100644
---
a/benchmarks/src/main/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
+++
b/benchmarks/src/main/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
@@ -149,9 +149,7 @@ public class FilterPartitionBenchmark
{
log.info("SETUP CALLED AT " + System.currentTimeMillis());
-if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
- ComplexMetrics.registerSerde("hyperUnique", new
HyperUniquesSerde(HyperLogLogHash.getDefault()));
-}
+ComplexMetrics.registerSerde("hyperUnique", () -> new
HyperUniquesSerde(HyperLogLogHash.getDefault()));
schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schema);
diff --git
a/benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java
b/benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java
index e156d24..4aafcc3 100644
---
a/benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java
+++
b/benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java
@@ -145,9 +145,7 @@ public class FilteredAggregatorBenchmark
{
log.info("SETUP CALLED AT " + System.currentTimeMillis());
-if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
- ComplexMetrics.registerSerde("hyperUnique", new
HyperUniquesSerde(HyperLogLogHash.getDefault()));
-}
+ComplexMetrics.registerSerde("hyperUnique", () -> new
HyperUniquesSerde(HyperLogLogHash.getDefault()));
schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schema);
diff --git
a/benchmarks/src/main/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java
b/benchmarks/src/main/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java
index f1b2001..dd66cc4 100644
---
a/benchmarks/src/main/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java
+++
b/benchmarks/src/main/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java
@@ -265,9 +265,7 @@ public class GroupByTypeInterfaceBenchmark
{
log.info("SETUP CALLED AT %d", System.currentTimeMillis());
-
[incubator-druid] branch master updated: Add the pull-request template (#7206)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git The following commit(s) were added to refs/heads/master by this push: new 46ea5b8 Add the pull-request template (#7206) 46ea5b8 is described below commit 46ea5b88b7346d20e1f379ddc08bb266e6dc2854 Author: Roman Leventov AuthorDate: Thu Jun 27 15:51:25 2019 +0300 Add the pull-request template (#7206) * Add the pull-request template * Rewording * Replaced checklist link, added Rat exclusion * Update the PR template. Add Concurrency Checklist to the repository * Merge Description and Design sections. Softer language. Removed requirement to test in production environment. Added a committer's instruction to justify addition of meta tags. * Rephrase item about comments * Add license header * Add item to concurrency checklist --- .github/pull_request_template.md | 65 + dev/code-review/concurrency.md | 199 +++ dev/committer-instructions.md| 11 ++- pom.xml | 4 +- 4 files changed, 274 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000..44d1c61 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,65 @@ +Fixes #. + +(Replace with the id of the issue fixed in this PR. Remove the above line if there is no corresponding +issue. Don't reference the issue in the title of this pull-request.) + +(If you are a committer, follow the PR action item checklist for committers: +https://github.com/apache/incubator-druid/blob/master/dev/committer-instructions.md#pr-and-issue-action-item-checklist-for-committers.) + +### Description + +Describe the goal of this PR, what problem are you fixing. If there is a corresponding issue (referenced above), it's +not necessary to repeat the description here, however, you may choose to keep one summary sentence. + +Describe your patch: what did you change in code? How did you fix the problem? + +If there are several relatively logically separate changes in this PR, create a mini-section for each of them. For +example: + Fixed the bug ... + Renamed the class ... + Added a forbidden-apis entry ... + +In each section, please describe design decisions made, including: + - Choice of algorithms + - Behavioral aspects. What configuration values are acceptable? How are corner cases and error conditions handled, such + as when there are insufficient resources? + - Class organization and design (how the logic is split between classes, inheritance, composition, design patterns) + - Method organization and design (how the logic is split between methods, parameters and return types) + - Naming (class, method, API, configuration, HTTP endpoint, names of emitted metrics) + +It's good to describe an alternative design (or mention an alternative name) for every design (or naming) decision point +and compare the alternatives with the designs that you've implemented (or the names you've chosen) to highlight the +advantages of the chosen designs and names. + +If there was a discussion of the design of the feature implemented in this PR elsewhere (e. g. a "Proposal" issue, any +other issue, or a thread in the development mailing list), link to that discussion from this PR description and explain +what have changed in your final design compared to your original proposal or the consensus version in the end of the +discussion. If something hasn't changed since the original discussion, you can omit a detailed discussion of those +aspects of the design here, perhaps apart from brief mentioning for the sake of readability of this PR description. + +Some of the aspects mentioned above may be omitted for simple and small changes. + + + +This PR has: +- [ ] been self-reviewed. + - [ ] using the [concurrency checklist]( + https://github.com/apache/incubator-druid/blob/master/dev/code-review/concurrency.md) (Remove this item if the PR + doesn't have any relation to concurrency.) +- [ ] added documentation for new or modified features or behaviors. +- [ ] added Javadocs for most classes and all non-trivial methods. Linked related entities via Javadoc links. +- [ ] added comments explaining the "why" and the intent of the code wherever would not be obvious for an unfamiliar +reader. +- [ ] added unit tests or modified existing tests to cover new code paths. +- [ ] added integration tests. +- [ ] been tested in a test Druid cluster. + +Check the items by putting "x" in the brackets for the done things. Not all of these items apply to every PR. Remove the +items which are not done or not relevant to the PR. None of the items from the checklist above are
[incubator-druid] 01/01: Using gender-neutral preposition
This is an automated email from the ASF dual-hosted git repository. leventov pushed a commit to branch leventov-patch-2 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git commit 49f1309bc074b2b5e3f855af82931428c2ef4ac0 Author: Roman Leventov AuthorDate: Mon Jul 15 17:15:47 2019 +0300 Using gender-neutral preposition --- dev/teamcity.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/teamcity.md b/dev/teamcity.md index 1fb15b3..da732c4 100644 --- a/dev/teamcity.md +++ b/dev/teamcity.md @@ -29,7 +29,7 @@ One can log in to TeamCity either via credentials or as a guest to check static ### Becoming a Project Administrator Druid committers shall obtain a status of a [Druid project]( https://teamcity.jetbrains.com/project.html?projectId=OpenSourceProjects_Druid) -administrator. First, the Druid committer needs to log in teamcity.jetbrains.com using his Github account. +administrator. First, the Druid committer needs to log in teamcity.jetbrains.com using their Github account. Then, somebody who is already a project administrator needs to do the following: 1. Follow the "Administration" link in the top-right corner of the page @@ -95,4 +95,4 @@ active: ``` -Don't forget to include this change in your commit when making a pull-request. \ No newline at end of file +Don't forget to include this change in your commit when making a pull-request. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch leventov-patch-2 created (now 49f1309)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch leventov-patch-2 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. at 49f1309 Using gender-neutral preposition This branch includes the following new commits: new 49f1309 Using gender-neutral preposition The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch leventov-patch-1 deleted (was e8ef50b)
This is an automated email from the ASF dual-hosted git repository. leventov pushed a change to branch leventov-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-druid.git. was e8ef50b Add instruction about skipping up-to-date checks when running integration tests The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: #7858 Throwing UnsupportedOperationException from ImmutableDrui… (#7933)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 72496d3 #7858 Throwing UnsupportedOperationException from
ImmutableDrui… (#7933)
72496d3 is described below
commit 72496d371290850cbc2acee8be82cce17fa3480e
Author: Sashidhar Thallam
AuthorDate: Fri Jul 19 01:05:19 2019 +0530
#7858 Throwing UnsupportedOperationException from ImmutableDrui… (#7933)
* #7858 Throwing UnsupportedOperationException from
ImmutableDruidDataSource's equals() and hashCode() methods.
* 1. Turning ImmutableDruidDataSource into a data container. 2. Adding a
Util method to be used in tests for checking equality of
ImmutableDruidDataSource objects.
* Removing unused method
* Fixing assert equals
* Fixing assert equals in TestUtils.java
* Adding java doc comments, Using ExpectedException in tests
* Fixing test cases
* Fixed expected exception message in tests
* fixed line width
* line width fix
* code style fixes
* code indentation fixes
* fixing method name
---
.../druid/client/ImmutableDruidDataSource.java | 48 ---
.../druid/client/ImmutableDruidDataSourceTest.java | 84 ++-
.../druid/server/http/DataSourcesResourceTest.java | 15 ++--
.../utils/ImmutableDruidDataSourceTestUtils.java | 94 ++
4 files changed, 204 insertions(+), 37 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/client/ImmutableDruidDataSource.java
b/server/src/main/java/org/apache/druid/client/ImmutableDruidDataSource.java
index 7ce3fd6..d92f90f 100644
--- a/server/src/main/java/org/apache/druid/client/ImmutableDruidDataSource.java
+++ b/server/src/main/java/org/apache/druid/client/ImmutableDruidDataSource.java
@@ -22,6 +22,7 @@ package org.apache.druid.client;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
@@ -30,10 +31,10 @@ import org.apache.druid.timeline.SegmentId;
import java.util.Collection;
import java.util.Map;
-import java.util.Objects;
/**
- * An immutable collection of metadata of segments ({@link DataSegment}
objects), belonging to a particular data source.
+ * An immutable collection of metadata of segments ({@link DataSegment}
objects), belonging to a particular data
+ * source.
*
* @see DruidDataSource - a mutable counterpart of this class
*/
@@ -120,11 +121,42 @@ public class ImmutableDruidDataSource
+ "'}";
}
+ /**
+ * ImmutableDruidDataSource should be considered a container, not a data
class. The idea is the same as behind
+ * prohibiting/limiting equals() (and therefore usage as HashSet/HashMap
keys) of DataSegment: see
+ * https://github.com/apache/incubator-druid/issues/6358. When somebody
wants to deduplicate ImmutableDruidDataSource
+ * objects, they would need to put them into a Map and resolve conflicts by name
+ * manually.
+ *
+ * See https://github.com/apache/incubator-druid/issues/7858
+ */
@Override
public boolean equals(Object o)
{
-// Note: this method is not well-defined. It should instead just throw
UnsupportedOperationsException.
-// See https://github.com/apache/incubator-druid/issues/7858.
+throw new UnsupportedOperationException("ImmutableDruidDataSource
shouldn't be used as the key in containers");
+ }
+
+ /**
+ * ImmutableDruidDataSource should be considered a container, not a data
class. The idea is the same as behind
+ * prohibiting/limiting hashCode() (and therefore usage as HashSet/HashMap
keys) of DataSegment: see
+ * https://github.com/apache/incubator-druid/issues/6358. When somebody
wants to deduplicate ImmutableDruidDataSource
+ * objects, they would need to put them into a Map and resolve conflicts by name
+ * manually.
+ *
+ * See https://github.com/apache/incubator-druid/issues/7858
+ */
+ @Override
+ public int hashCode()
+ {
+throw new UnsupportedOperationException("ImmutableDruidDataSource
shouldn't be used as the key in containers");
+ }
+
+ /**
+ * This method should only be used in tests.
+ */
+ @VisibleForTesting
+ public boolean equalsForTesting(Object o)
+ {
if (this == o) {
return true;
}
@@ -144,12 +176,4 @@ public class ImmutableDruidDataSource
return this.idToSegments.equals(that.idToSegments);
}
-
- @Override
- public int hashCode()
- {
-// Note: this method is not well-defined. It should
[incubator-druid] branch master updated: Java 9 compatible specialized class compilation (#7477)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 30fed78 Java 9 compatible specialized class compilation (#7477)
30fed78 is described below
commit 30fed78daf7b4f7e9f7946046facee8aa61e8d5b
Author: Xavier Léauté
AuthorDate: Mon Apr 29 09:44:28 2019 -0700
Java 9 compatible specialized class compilation (#7477)
* Java 9 compatible specialized class compilation
We currently use Unsafe.defineClass to compile specialized classes,
which has been removed in Java 9 and above. This change switches to
MethodHandles.Lookup.defineClass at runtime, which provides similar
functionality in newer JDK versions.
* add comments
* fix incorrect comment
* add unsafe utility class
* make comments java-doc style
* fix checkstyle errors
* rename unsafe -> unsafeutil
* move defineClass method to utility class
* rename unsafeutil -> unsafeutils to match other utility class names
* remove extra lookup method
* add utiliy class docs
* more comments
* minor comments and formatting
---
.../druid/java/util/common/ByteBufferUtils.java| 16 +-
.../druid/java/util/common/DefineClassUtils.java | 200 +
.../apache/druid/java/util/common/UnsafeUtils.java | 79
.../SpecializationService.java | 36 +---
4 files changed, 294 insertions(+), 37 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java
b/core/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java
index 02ac521..61a9211 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/ByteBufferUtils.java
@@ -24,7 +24,6 @@ import org.apache.druid.utils.JvmUtils;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
-import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
@@ -94,6 +93,9 @@ public class ByteBufferUtils
}
}
+ /**
+ * NB: while Druid no longer support Java 7, this method would still work
with that version as well.
+ */
private static MethodHandle unmapJava7Or8(MethodHandles.Lookup lookup)
throws ReflectiveOperationException
{
// "Compile" a MethodHandle that is roughly equivalent to the following
lambda:
@@ -128,14 +130,12 @@ public class ByteBufferUtils
private static MethodHandle unmapJava9(MethodHandles.Lookup lookup) throws
ReflectiveOperationException
{
-Class unsafeClass = Class.forName("sun.misc.Unsafe");
-MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner",
-
MethodType.methodType(void.class, ByteBuffer.class)
+MethodHandle unmapper = lookup.findVirtual(
+UnsafeUtils.theUnsafeClass(),
+"invokeCleaner",
+MethodType.methodType(void.class, ByteBuffer.class)
);
-Field f = unsafeClass.getDeclaredField("theUnsafe");
-f.setAccessible(true);
-Object theUnsafe = f.get(null);
-return unmapper.bindTo(theUnsafe);
+return unmapper.bindTo(UnsafeUtils.theUnsafe());
}
/**
diff --git
a/core/src/main/java/org/apache/druid/java/util/common/DefineClassUtils.java
b/core/src/main/java/org/apache/druid/java/util/common/DefineClassUtils.java
new file mode 100644
index 000..dba354d
--- /dev/null
+++ b/core/src/main/java/org/apache/druid/java/util/common/DefineClassUtils.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.java.util.common;
+
+import org.apache.druid.utils.JvmUtils;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.security.ProtectionDomain;
+
+/*
[incubator-druid] branch master updated: EmitterModule: Throw an error on invalid emitter types. (#7328)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 7b8bc9a EmitterModule: Throw an error on invalid emitter types.
(#7328)
7b8bc9a is described below
commit 7b8bc9a5efac2055cd04760efdb0a7beced7726b
Author: Gian Merlino
AuthorDate: Mon Apr 29 10:23:53 2019 -0700
EmitterModule: Throw an error on invalid emitter types. (#7328)
* EmitterModule: Throw an error on invalid emitter types.
The current behavior of silently using the "noop" emitter is unhelpful
and makes it difficult to debug config typos.
* Add comments.
---
.../apache/druid/server/emitter/EmitterModule.java | 10 +--
.../druid/server/emitter/EmitterModuleTest.java| 32 --
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/server/emitter/EmitterModule.java
b/server/src/main/java/org/apache/druid/server/emitter/EmitterModule.java
index 95e4eb3..5b3cf3b 100644
--- a/server/src/main/java/org/apache/druid/server/emitter/EmitterModule.java
+++ b/server/src/main/java/org/apache/druid/server/emitter/EmitterModule.java
@@ -19,6 +19,7 @@
package org.apache.druid.server.emitter;
+import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Binder;
@@ -130,13 +131,16 @@ public class EmitterModule implements Module
{
final List> emitterBindings =
injector.findBindingsByType(new TypeLiteral(){});
- emitter = findEmitter(emitterType, emitterBindings);
-
- if (emitter == null) {
+ if (Strings.isNullOrEmpty(emitterType)) {
+// If the emitter is unspecified, we want to default to the no-op
emitter. Include empty string here too, just
+// in case nulls are translated to empty strings at some point
somewhere in the system.
emitter = findEmitter(NoopEmitterModule.EMITTER_TYPE, emitterBindings);
+ } else {
+emitter = findEmitter(emitterType, emitterBindings);
}
if (emitter == null) {
+// If the requested emitter couldn't be found, throw an error. It
might mean a typo, or a missing extension.
List knownTypes = new ArrayList<>();
for (Binding binding : emitterBindings) {
final Annotation annotation = binding.getKey().getAnnotation();
diff --git
a/server/src/test/java/org/apache/druid/server/emitter/EmitterModuleTest.java
b/server/src/test/java/org/apache/druid/server/emitter/EmitterModuleTest.java
index 0acd085..dc48f47 100644
---
a/server/src/test/java/org/apache/druid/server/emitter/EmitterModuleTest.java
+++
b/server/src/test/java/org/apache/druid/server/emitter/EmitterModuleTest.java
@@ -31,9 +31,13 @@ import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.ServerModule;
import org.apache.druid.jackson.JacksonModule;
import org.apache.druid.java.util.emitter.core.Emitter;
+import org.apache.druid.java.util.emitter.core.NoopEmitter;
import org.apache.druid.java.util.emitter.core.ParametrizedUriEmitter;
+import org.hamcrest.CoreMatchers;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import javax.validation.Validation;
import javax.validation.Validator;
@@ -41,6 +45,8 @@ import java.util.Properties;
public class EmitterModuleTest
{
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
@Test
public void testParametrizedUriEmitterConfig()
@@ -55,10 +61,30 @@ public class EmitterModuleTest
props.setProperty("druid.emitter.parametrized.httpEmitting.maxBatchSize",
"4");
props.setProperty("druid.emitter.parametrized.httpEmitting.flushTimeOut",
"1000");
-final Emitter emitter =
-makeInjectorWithProperties(props).getInstance(Emitter.class);
+final Emitter emitter =
makeInjectorWithProperties(props).getInstance(Emitter.class);
+
// Testing that ParametrizedUriEmitter is successfully deserialized from
the above config
-Assert.assertTrue(emitter instanceof ParametrizedUriEmitter);
+Assert.assertThat(emitter,
CoreMatchers.instanceOf(ParametrizedUriEmitter.class));
+ }
+
+ @Test
+ public void testMissingEmitterType()
+ {
+final Properties props = new Properties();
+props.setProperty("druid.emitter", "");
+
+final Emitter emitter =
makeInjectorWithProperties(props).getInstance(Emitter.class);
+Assert.assertThat(emitter, CoreMatchers.instanceOf(NoopEmitter.class));
+ }
+
+ @Test
+ public void testInvalidEmitterType()
+ {
+final Properties props = new Properties();
+props.setProperty("druid.emitter", &qu
[incubator-druid] branch master updated: AggregatorFactory: Clarify methods that return other AggregatorFactories. (#7293)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new f776b94 AggregatorFactory: Clarify methods that return other
AggregatorFactories. (#7293)
f776b94 is described below
commit f776b9408962b9006cfcfe4d6c1794751972cc8e
Author: Gian Merlino
AuthorDate: Mon Apr 29 10:27:30 2019 -0700
AggregatorFactory: Clarify methods that return other AggregatorFactories.
(#7293)
---
.../druid/query/aggregation/AggregatorFactory.java | 50 ++
1 file changed, 42 insertions(+), 8 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java
b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java
index 6ec9fde..6b0f4a1 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java
@@ -94,20 +94,45 @@ public abstract class AggregatorFactory implements Cacheable
}
/**
- * Returns an AggregatorFactory that can be used to combine the output of
aggregators from this factory. This
- * generally amounts to simply creating a new factory that is the same as
the current except with its input
- * column renamed to the same as the output column.
+ * Returns an AggregatorFactory that can be used to combine the output of
aggregators from this factory. It is used
+ * when we know we have some values that were produced with this aggregator
factory, and want to do some additional
+ * combining of them. This happens, for example, when merging query results
from two different segments, or two
+ * different servers.
+ *
+ * For simple aggregators, the combining factory may be computed by simply
creating a new factory that is the same as
+ * the current, except with its input column renamed to the same as the
output column. For example, this aggregator:
+ *
+ * {"type": "longSum", "fieldName": "foo", "name": "bar"}
+ *
+ * Would become:
+ *
+ * {"type": "longSum", "fieldName": "bar", "name": "bar"}
+ *
+ * Sometimes, the type or other parameters of the combining aggregator will
be different from the original aggregator.
+ * For example, the {@link CountAggregatorFactory} getCombiningFactory
method will return a
+ * {@link LongSumAggregatorFactory}, because counts are combined by summing.
+ *
+ * No matter what, `foo.getCombiningFactory()` and
`foo.getCombiningFactory().getCombiningFactory()` should return
+ * the same result.
*
* @return a new Factory that can be used for operations on top of data
output from the current factory.
*/
public abstract AggregatorFactory getCombiningFactory();
/**
- * Returns an AggregatorFactory that can be used to merge the output of
aggregators from this factory and
- * other factory.
- * This method is relevant only for AggregatorFactory which can be used at
ingestion time.
+ * Returns an AggregatorFactory that can be used to combine the output of
aggregators from this factory and
+ * another factory. It is used when we have some values produced by this
aggregator factory, and some values produced
+ * by the "other" aggregator factory, and we want to do some additional
combining of them. This happens, for example,
+ * when compacting two segments together that both have a metric column with
the same name. (Even though the name of
+ * the column is the same, the aggregator factory used to create it may be
different from segment to segment.)
+ *
+ * This method may throw {@link AggregatorFactoryNotMergeableException},
meaning that "this" and "other" are not
+ * compatible and values from one cannot sensibly be combined with values
from the other.
*
* @return a new Factory that can be used for merging the output of
aggregators from this factory and other.
+ *
+ * @see #getCombiningFactory() which is equivalent to {@code
foo.getMergingFactory(foo)} (when "this" and "other"
+ * are the same instance).
*/
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws
AggregatorFactoryNotMergeableException
{
@@ -120,9 +145,15 @@ public abstract class AggregatorFactory implements
Cacheable
}
/**
- * Gets a list of all columns that this AggregatorFactory will scan
+ * Used by {@link org.apache.druid.query.groupby.strategy.GroupByStrategyV1}
when running nested groupBys, to
+ * "transfer" values from this aggreagtor to an incremental index that the
outer query will run on. This method
+ * only exists due to the design of GroupByStrat
[incubator-druid] branch master updated: Add is_overshadowed column to sys.segments table (#7425)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 15d19f3 Add is_overshadowed column to sys.segments table (#7425)
15d19f3 is described below
commit 15d19f3059a990f022b1cb98bdd4e6a82cdf9a06
Author: Surekha
AuthorDate: Wed May 1 09:00:57 2019 -0700
Add is_overshadowed column to sys.segments table (#7425)
* Add is_overshadowed column to sys.segments table
* update docs
* Rename class and variables
* PR comments
* PR comments
* remove unused variables in MetadataResource
* move constants together
* add getFullyOvershadowedSegments method to ImmutableDruidDataSource
* Fix compareTo of SegmentWithOvershadowedStatus
* PR comment
* PR comments
* PR comments
* PR comments
* PR comments
* fix issue with already consumed stream
* minor refactoring
* PR comments
---
.../timeline/SegmentWithOvershadowedStatus.java| 90 ++
docs/content/querying/sql.md | 1 +
.../druid/client/ImmutableDruidDataSource.java | 40 ++
.../helper/DruidCoordinatorRuleRunner.java | 30 +---
.../apache/druid/server/http/MetadataResource.java | 64 +--
.../sql/calcite/schema/MetadataSegmentView.java| 69 -
.../druid/sql/calcite/schema/SystemSchema.java | 64 ---
.../druid/sql/calcite/schema/SystemSchemaTest.java | 44 +++
8 files changed, 289 insertions(+), 113 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
b/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
new file mode 100644
index 000..e86daea
--- /dev/null
+++
b/core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.timeline;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * DataSegment object plus the overshadowed status for the segment. An
immutable object.
+ *
+ * SegmentWithOvershadowedStatus's {@link #compareTo} method considers only
the {@link SegmentId}
+ * of the DataSegment object.
+ */
+public class SegmentWithOvershadowedStatus implements
Comparable
+{
+ private final boolean overshadowed;
+ private final DataSegment dataSegment;
+
+ @JsonCreator
+ public SegmentWithOvershadowedStatus(
+ @JsonProperty("dataSegment") DataSegment dataSegment,
+ @JsonProperty("overshadowed") boolean overshadowed
+ )
+ {
+this.dataSegment = dataSegment;
+this.overshadowed = overshadowed;
+ }
+
+ @JsonProperty
+ public boolean isOvershadowed()
+ {
+return overshadowed;
+ }
+
+ @JsonProperty
+ public DataSegment getDataSegment()
+ {
+return dataSegment;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+if (this == o) {
+ return true;
+}
+if (!(o instanceof SegmentWithOvershadowedStatus)) {
+ return false;
+}
+final SegmentWithOvershadowedStatus that = (SegmentWithOvershadowedStatus)
o;
+if (!dataSegment.equals(that.dataSegment)) {
+ return false;
+}
+if (overshadowed != (that.overshadowed)) {
+ return false;
+}
+return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+int result = dataSegment.hashCode();
+result = 31 * result + Boolean.hashCode(overshadowed);
+return result;
+ }
+
+ @Override
+ public int compareTo(SegmentWithOvershadowedStatus o)
+ {
+return dataSegment.getId().compareTo(o.dataSegment.getId());
+ }
+}
diff --git a/docs/content/querying/sql.md b/docs/content/querying/sql.md
index 9e97138..1fc1243 100644
--- a/docs/content/querying/sql.md
+++ b/docs/content/querying/sql.md
@@ -612,6 +612,7 @@ Note that a segment can be served by more than one stream
ingestio
[incubator-druid] branch master updated: Improve parallelism of zookeeper based segment change processing (#7088)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new afbcb9c Improve parallelism of zookeeper based segment change
processing (#7088)
afbcb9c is described below
commit afbcb9c07f21c12f241f1dc6575589ef14cda836
Author: Samarth Jain
AuthorDate: Fri May 3 06:58:42 2019 -0700
Improve parallelism of zookeeper based segment change processing (#7088)
* V1 - improve parallelism of zookeeper based segment change processing
* Create zk nodes in batches. Address code review comments.
Introduce various configs.
* Add documentation for the newly added configs
* Fix test failures
* Fix more test failures
* Remove prinstacktrace statements
* Address code review comments
* Use a single queue
* Address code review comments
Since we have a separate load peon for every historical, just having a
single SegmentChangeProcessor
task per historical is enough. This commit also gets rid of the associated
config druid.coordinator.loadqueuepeon.curator.numCreateThreads
* Resolve merge conflict
* Fix compilation failure
* Remove batching since we already have a dynamic config
maxSegmentsInNodeLoadingQueue that provides that control
* Fix NPE in test
* Remove documentation for configs that are no longer needed
* Address code review comments
* Address more code review comments
* Fix checkstyle issue
* Address code review comments
* Code review comments
* Add back monitor node remove executor
* Cleanup code to isolate null checks and minor refactoring
* Change param name since it conflicts with member variable name
---
docs/content/configuration/index.md| 4 +-
.../druid/segment/loading/SegmentLoaderConfig.java | 3 +-
.../druid/server/coordination/ZkCoordinator.java | 119 +++---
.../server/coordinator/CuratorLoadQueuePeon.java | 420 ++---
.../server/coordinator/DruidCoordinatorConfig.java | 6 +
.../server/coordination/ZkCoordinatorTest.java | 3 +-
.../coordinator/CuratorDruidCoordinatorTest.java | 3 +-
.../server/coordinator/DruidCoordinatorTest.java | 3 +-
.../server/coordinator/HttpLoadQueuePeonTest.java | 3 +-
.../server/coordinator/LoadQueuePeonTest.java | 203 +-
.../server/coordinator/LoadQueuePeonTester.java| 24 +-
.../coordinator/TestDruidCoordinatorConfig.java| 7 +-
.../helper/DruidCoordinatorSegmentKillerTest.java | 3 +-
.../java/org/apache/druid/cli/CliCoordinator.java | 13 +-
14 files changed, 429 insertions(+), 385 deletions(-)
diff --git a/docs/content/configuration/index.md
b/docs/content/configuration/index.md
index 91edb82..29c11fa 100644
--- a/docs/content/configuration/index.md
+++ b/docs/content/configuration/index.md
@@ -1254,8 +1254,8 @@ These Historical configurations can be defined in the
`historical/runtime.proper
|`druid.segmentCache.dropSegmentDelayMillis`|How long a process delays before
completely dropping segment.|3 (30 seconds)|
|`druid.segmentCache.infoDir`|Historical processes keep track of the segments
they are serving so that when the process is restarted they can reload the same
segments without waiting for the Coordinator to reassign. This path defines
where this metadata is kept. Directory will be created if
needed.|${first_location}/info_dir|
|`druid.segmentCache.announceIntervalMillis`|How frequently to announce
segments while segments are loading from cache. Set this value to zero to wait
for all segments to be loaded before announcing.|5000 (5 seconds)|
-|`druid.segmentCache.numLoadingThreads`|How many segments to drop or load
concurrently from from deep storage.|10|
-|`druid.segmentCache.numBootstrapThreads`|How many segments to load
concurrently from local storage at startup.|Same as numLoadingThreads|
+|`druid.segmentCache.numLoadingThreads`|How many segments to drop or load
concurrently from deep storage. Note that the work of loading segments involves
downloading segments from deep storage, decompressing them and loading them to
a memory mapped location. So the work is not all I/O Bound. Depending on CPU
and network load, one could possibly increase this config to a higher
value.|Number of cores|
+|`druid.coordinator.loadqueuepeon.curator.numCallbackThreads`|Number of
threads for executing callback actions associated with loading or dropping of
segments. One might want to increase this number when noticing clusters are
lagging behind w.r.t. balancing segments across historical nodes.|2|
In `druid.segmentCache.locations`, *freeSpacePercent* was added because
*maxSize* setting is only a theoretical limit and assumes that much space
[incubator-druid] branch master updated: Fix old-console/cluster.html; Remove redundant throws in ITBasicAuthConfigurationTest (#7589)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new a00d3e1 Fix old-console/cluster.html; Remove redundant throws in
ITBasicAuthConfigurationTest (#7589)
a00d3e1 is described below
commit a00d3e19313ef80233670a67fddb2dd7abcdfb5d
Author: Roman Leventov
AuthorDate: Mon May 6 18:08:49 2019 +0200
Fix old-console/cluster.html; Remove redundant throws in
ITBasicAuthConfigurationTest (#7589)
* Fix old-console/cluster.html; Remove redundant throws in
ITBasicAuthConfigurationTest
* Remove comment in cluster.html
---
.../tests/security/ITBasicAuthConfigurationTest.java | 2 +-
web-console/old-console/cluster.html | 18 --
2 files changed, 1 insertion(+), 19 deletions(-)
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java
index dfa3791..67e1f37 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/security/ITBasicAuthConfigurationTest.java
@@ -120,7 +120,7 @@ public class ITBasicAuthConfigurationTest
private CoordinatorResourceTestClient coordinatorClient;
@BeforeMethod
- public void before() throws Exception
+ public void before()
{
// ensure that auth_test segments are loaded completely, we use them for
testing system schema tables
RetryUtil.retryUntilTrue(
diff --git a/web-console/old-console/cluster.html
b/web-console/old-console/cluster.html
index 89b8296..2a5cec7 100644
--- a/web-console/old-console/cluster.html
+++ b/web-console/old-console/cluster.html
@@ -47,24 +47,6 @@
Loading segment data... this may take a few
minutes
-
-
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[incubator-druid] branch master updated: Add checkstyle for "Local variable names shouldn't start with capital" (#7681)
This is an automated email from the ASF dual-hosted git repository.
leventov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 26fad7e Add checkstyle for "Local variable names shouldn't start with
capital" (#7681)
26fad7e is described below
commit 26fad7e06a61ae76a670a0ced0690767ed76c6c7
Author: Merlin Lee <39320777+lml2...@users.noreply.github.com>
AuthorDate: Fri May 24 00:40:28 2019 +0800
Add checkstyle for "Local variable names shouldn't start with capital"
(#7681)
* Add checkstyle for "Local variable names shouldn't start with capital"
* Adjust some local variables to constants
* Replace StringUtils.LINE_SEPARATOR with System.lineSeparator()
---
.../druid/benchmark/VSizeSerdeBenchmark.java | 4 +-
codestyle/checkstyle.xml | 8 ++
.../druid/data/input/impl/TimestampSpecTest.java | 4 +-
.../twitter/TwitterSpritzerFirehoseFactory.java| 4 +-
.../ambari/metrics/AmbariMetricsEmitter.java | 4 +-
.../druid/emitter/graphite/GraphiteEmitter.java| 4 +-
.../kinesis/supervisor/KinesisSupervisorTest.java | 24 ++--
.../druid/hll/HyperLogLogCollectorBenchmark.java | 18 +--
.../EqualDistributionWorkerSelectStrategyTest.java | 8 +-
.../query/PrioritizedExecutorServiceTest.java | 10 +-
.../druid/query/topn/TopNQueryRunnerTest.java | 4 +-
.../org/apache/druid/server/StatusResource.java| 8 +-
.../discovery/ServerDiscoverySelectorTest.java | 20 ++--
.../druid/server/http/DataSourcesResourceTest.java | 128 ++---
14 files changed, 128 insertions(+), 120 deletions(-)
diff --git
a/benchmarks/src/main/java/org/apache/druid/benchmark/VSizeSerdeBenchmark.java
b/benchmarks/src/main/java/org/apache/druid/benchmark/VSizeSerdeBenchmark.java
index 8c049a3..e96c36c 100644
---
a/benchmarks/src/main/java/org/apache/druid/benchmark/VSizeSerdeBenchmark.java
+++
b/benchmarks/src/main/java/org/apache/druid/benchmark/VSizeSerdeBenchmark.java
@@ -81,9 +81,9 @@ public class VSizeSerdeBenchmark
File base = new
File(this.getClass().getClassLoader().getResource("").toURI());
dummy = new File(base, "dummy");
try (Writer writer = Files.newBufferedWriter(dummy.toPath(),
StandardCharsets.UTF_8)) {
- String EMPTY_STRING = "";
+ String emptyStr = "";
for (int i = 0; i < values + 10; i++) {
-writer.write(EMPTY_STRING);
+writer.write(emptyStr);
}
}
diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml
index 12e9532..424f2a3 100644
--- a/codestyle/checkstyle.xml
+++ b/codestyle/checkstyle.xml
@@ -311,5 +311,13 @@ codestyle/checkstyle.xml.
"/>
+
+
+
+
+
+
+
+
diff --git
a/core/src/test/java/org/apache/druid/data/input/impl/TimestampSpecTest.java
b/core/src/test/java/org/apache/druid/data/input/impl/TimestampSpecTest.java
index ffbca5c..b736d77 100644
--- a/core/src/test/java/org/apache/druid/data/input/impl/TimestampSpecTest.java
+++ b/core/src/test/java/org/apache/druid/data/input/impl/TimestampSpecTest.java
@@ -51,7 +51,7 @@ public class TimestampSpecTest
@Test
public void testContextualTimestampList()
{
-String DATE_FORMAT = "-MM-dd'T'HH:mm:ss";
+String dateFormat = "-MM-dd'T'HH:mm:ss";
String[] dates = new String[]{
"2000-01-01T05:00:00",
"2000-01-01T05:00:01",
@@ -59,7 +59,7 @@ public class TimestampSpecTest
"2000-01-01T05:00:02",
"2000-01-01T05:00:03",
};
-TimestampSpec spec = new TimestampSpec("TIMEstamp", DATE_FORMAT, null);
+TimestampSpec spec = new TimestampSpec("TIMEstamp", dateFormat, null);
DateTimes.UtcFormatter formatter =
DateTimes.wrapFormatter(ISODateTimeFormat.dateHourMinuteSecond());
diff --git
a/examples/src/main/java/org/apache/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
b/examples/src/main/java/org/apache/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
index 65bdacd..300db0d 100644
---
a/examples/src/main/java/org/apache/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
+++
b/examples/src/main/java/org/apache/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
@@ -84,6 +84,7 @@ public class TwitterSpritzerFirehoseFactory implements
FirehoseFactory]*>(.*?)", Pattern.CASE_INSENSITIVE);
+ private static final int DEFAULT_QUEUE_SIZE = 2000;
/**
* max events to receive, -1 is infinite, 0 means nothing is delivered; use
this to prevent
@@ -140,9 +141,8 @@ public class TwitterSpritzerFirehoseFactory implements
FirehoseFactory que
