paul-rogers commented on a change in pull request #1987: DRILL-7589: Set
temporary tests folder for UDF_DIRECTORY_LOCAL, fix allocators closing in
BloomFilterTest, fix permissions issue for TestGracefulShutdown tests
URL: https://github.com/apache/drill/pull/1987#discussion_r380485298
##########
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/work/filter/BloomFilterTest.java
##########
@@ -133,214 +135,227 @@ public boolean hasFailed() {
}
}
-
@Test
public void testNotExist() throws Exception {
- Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet(),
ClassPathScanner.fromPrescan(c));
- bit.run();
- DrillbitContext bitContext = bit.getContext();
- FunctionImplementationRegistry registry =
bitContext.getFunctionImplementationRegistry();
- FragmentContextImpl context = new FragmentContextImpl(bitContext,
BitControl.PlanFragment.getDefaultInstance(), null, registry);
- BufferAllocator bufferAllocator = bitContext.getAllocator();
- //create RecordBatch
- VarCharVector vector = new VarCharVector(SchemaBuilder.columnSchema("a",
TypeProtos.MinorType.VARCHAR, TypeProtos.DataMode.REQUIRED), bufferAllocator);
- vector.allocateNew();
- int valueCount = 3;
- VarCharVector.Mutator mutator = vector.getMutator();
- mutator.setSafe(0, "a".getBytes());
- mutator.setSafe(1, "b".getBytes());
- mutator.setSafe(2, "c".getBytes());
- mutator.setValueCount(valueCount);
- VectorContainer vectorContainer = new VectorContainer();
- TypedFieldId fieldId = vectorContainer.add(vector);
- RecordBatch recordBatch = new TestRecordBatch(vectorContainer);
- //construct hash64
- ValueVectorReadExpression exp = new ValueVectorReadExpression(fieldId);
- LogicalExpression[] expressions = new LogicalExpression[1];
- expressions[0] = exp;
- TypedFieldId[] fieldIds = new TypedFieldId[1];
- fieldIds[0] = fieldId;
- ValueVectorHashHelper valueVectorHashHelper = new
ValueVectorHashHelper(recordBatch, context);
- ValueVectorHashHelper.Hash64 hash64 =
valueVectorHashHelper.getHash64(expressions, fieldIds);
-
- //construct BloomFilter
- int numBytes = BloomFilter.optimalNumOfBytes(3, 0.03);
-
- BloomFilter bloomFilter = new BloomFilter(numBytes, bufferAllocator);
- for (int i = 0; i < valueCount; i++) {
- long hashCode = hash64.hash64Code(i, 0, 0);
- bloomFilter.insert(hashCode);
+ int userPort = QueryTestUtil.getFreePortNumber(31170, 300);
+ int bitPort = QueryTestUtil.getFreePortNumber(31180, 300);
+ ClusterFixtureBuilder clusterFixtureBuilder =
ClusterFixture.bareBuilder(dirTestWatcher)
+ .configProperty(ExecConstants.INITIAL_USER_PORT, userPort)
+ .configProperty(ExecConstants.INITIAL_BIT_PORT, bitPort)
+ .configProperty(ExecConstants.ALLOW_LOOPBACK_ADDRESS_BINDING, true);
+ try (ClusterFixture cluster = clusterFixtureBuilder.build()) {
+ Drillbit bit = cluster.drillbit();
+ DrillbitContext bitContext = bit.getContext();
+ FunctionImplementationRegistry registry =
bitContext.getFunctionImplementationRegistry();
+ FragmentContextImpl context = new FragmentContextImpl(bitContext,
BitControl.PlanFragment.getDefaultInstance(), null, registry);
+ BufferAllocator bufferAllocator = bitContext.getAllocator();
+ //create RecordBatch
+ VarCharVector vector = new VarCharVector(SchemaBuilder.columnSchema("a",
TypeProtos.MinorType.VARCHAR, TypeProtos.DataMode.REQUIRED), bufferAllocator);
+ vector.allocateNew();
+ int valueCount = 3;
+ VarCharVector.Mutator mutator = vector.getMutator();
+ mutator.setSafe(0, "a".getBytes());
+ mutator.setSafe(1, "b".getBytes());
+ mutator.setSafe(2, "c".getBytes());
+ mutator.setValueCount(valueCount);
+ VectorContainer vectorContainer = new VectorContainer();
+ TypedFieldId fieldId = vectorContainer.add(vector);
+ RecordBatch recordBatch = new TestRecordBatch(vectorContainer);
+ //construct hash64
+ ValueVectorReadExpression exp = new ValueVectorReadExpression(fieldId);
+ LogicalExpression[] expressions = new LogicalExpression[1];
+ expressions[0] = exp;
+ TypedFieldId[] fieldIds = new TypedFieldId[1];
+ fieldIds[0] = fieldId;
+ ValueVectorHashHelper valueVectorHashHelper = new
ValueVectorHashHelper(recordBatch, context);
+ ValueVectorHashHelper.Hash64 hash64 =
valueVectorHashHelper.getHash64(expressions, fieldIds);
+
+ //construct BloomFilter
+ int numBytes = BloomFilter.optimalNumOfBytes(3, 0.03);
+
+ BloomFilter bloomFilter = new BloomFilter(numBytes, bufferAllocator);
+ for (int i = 0; i < valueCount; i++) {
+ long hashCode = hash64.hash64Code(i, 0, 0);
+ bloomFilter.insert(hashCode);
+ }
+
+ //-----------------create probe side RecordBatch---------------------
+ VarCharVector probeVector = new
VarCharVector(SchemaBuilder.columnSchema("a", TypeProtos.MinorType.VARCHAR,
TypeProtos.DataMode.REQUIRED), bufferAllocator);
+ probeVector.allocateNew();
+ int probeValueCount = 1;
+ VarCharVector.Mutator mutator1 = probeVector.getMutator();
+ mutator1.setSafe(0, "f".getBytes());
+ mutator1.setValueCount(probeValueCount);
+ VectorContainer probeVectorContainer = new VectorContainer();
+ TypedFieldId probeFieldId = probeVectorContainer.add(probeVector);
+ RecordBatch probeRecordBatch = new TestRecordBatch(probeVectorContainer);
+ ValueVectorReadExpression probExp = new
ValueVectorReadExpression(probeFieldId);
+ LogicalExpression[] probExpressions = new LogicalExpression[1];
+ probExpressions[0] = probExp;
+ TypedFieldId[] probeFieldIds = new TypedFieldId[1];
+ probeFieldIds[0] = probeFieldId;
+ ValueVectorHashHelper probeValueVectorHashHelper = new
ValueVectorHashHelper(probeRecordBatch, context);
+ ValueVectorHashHelper.Hash64 probeHash64 =
probeValueVectorHashHelper.getHash64(probExpressions, probeFieldIds);
+ long hashCode = probeHash64.hash64Code(0, 0, 0);
+ boolean contain = bloomFilter.find(hashCode);
+ Assert.assertFalse(contain);
+ bloomFilter.getContent().close();
+ vectorContainer.clear();
+ probeVectorContainer.clear();
+ context.close();
}
-
- //-----------------create probe side RecordBatch---------------------
- VarCharVector probeVector = new
VarCharVector(SchemaBuilder.columnSchema("a", TypeProtos.MinorType.VARCHAR,
TypeProtos.DataMode.REQUIRED), bufferAllocator);
- probeVector.allocateNew();
- int probeValueCount = 1;
- VarCharVector.Mutator mutator1 = probeVector.getMutator();
- mutator1.setSafe(0, "f".getBytes());
- mutator1.setValueCount(probeValueCount);
- VectorContainer probeVectorContainer = new VectorContainer();
- TypedFieldId probeFieldId = probeVectorContainer.add(probeVector);
- RecordBatch probeRecordBatch = new TestRecordBatch(probeVectorContainer);
- ValueVectorReadExpression probExp = new
ValueVectorReadExpression(probeFieldId);
- LogicalExpression[] probExpressions = new LogicalExpression[1];
- probExpressions[0] = probExp;
- TypedFieldId[] probeFieldIds = new TypedFieldId[1];
- probeFieldIds[0] = probeFieldId;
- ValueVectorHashHelper probeValueVectorHashHelper = new
ValueVectorHashHelper(probeRecordBatch, context);
- ValueVectorHashHelper.Hash64 probeHash64 =
probeValueVectorHashHelper.getHash64(probExpressions, probeFieldIds);
- long hashCode = probeHash64.hash64Code(0, 0, 0);
- boolean contain = bloomFilter.find(hashCode);
- Assert.assertFalse(contain);
- bloomFilter.getContent().close();
- vectorContainer.clear();
- probeVectorContainer.clear();
- context.close();
- bitContext.close();
- bit.close();
}
@Test
public void testExist() throws Exception {
-
- Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet(),
ClassPathScanner.fromPrescan(c));
- bit.run();
- DrillbitContext bitContext = bit.getContext();
- FunctionImplementationRegistry registry =
bitContext.getFunctionImplementationRegistry();
- FragmentContextImpl context = new FragmentContextImpl(bitContext,
BitControl.PlanFragment.getDefaultInstance(), null, registry);
- BufferAllocator bufferAllocator = bitContext.getAllocator();
- //create RecordBatch
- VarCharVector vector = new VarCharVector(SchemaBuilder.columnSchema("a",
TypeProtos.MinorType.VARCHAR, TypeProtos.DataMode.REQUIRED), bufferAllocator);
- vector.allocateNew();
- int valueCount = 3;
- VarCharVector.Mutator mutator = vector.getMutator();
- mutator.setSafe(0, "a".getBytes());
- mutator.setSafe(1, "b".getBytes());
- mutator.setSafe(2, "c".getBytes());
- mutator.setValueCount(valueCount);
- VectorContainer vectorContainer = new VectorContainer();
- TypedFieldId fieldId = vectorContainer.add(vector);
- RecordBatch recordBatch = new TestRecordBatch(vectorContainer);
- //construct hash64
- ValueVectorReadExpression exp = new ValueVectorReadExpression(fieldId);
- LogicalExpression[] expressions = new LogicalExpression[1];
- expressions[0] = exp;
- TypedFieldId[] fieldIds = new TypedFieldId[1];
- fieldIds[0] = fieldId;
- ValueVectorHashHelper valueVectorHashHelper = new
ValueVectorHashHelper(recordBatch, context);
- ValueVectorHashHelper.Hash64 hash64 =
valueVectorHashHelper.getHash64(expressions, fieldIds);
-
- //construct BloomFilter
- int numBytes = BloomFilter.optimalNumOfBytes(3, 0.03);
-
- BloomFilter bloomFilter = new BloomFilter(numBytes, bufferAllocator);
- for (int i = 0; i < valueCount; i++) {
- long hashCode = hash64.hash64Code(i, 0, 0);
- bloomFilter.insert(hashCode);
+ int userPort = QueryTestUtil.getFreePortNumber(31170, 300);
+ int bitPort = QueryTestUtil.getFreePortNumber(31180, 300);
+ ClusterFixtureBuilder clusterFixtureBuilder =
ClusterFixture.bareBuilder(dirTestWatcher)
+ .configProperty(ExecConstants.INITIAL_USER_PORT, userPort)
+ .configProperty(ExecConstants.INITIAL_BIT_PORT, bitPort)
+ .configProperty(ExecConstants.ALLOW_LOOPBACK_ADDRESS_BINDING, true);
+ try (ClusterFixture cluster = clusterFixtureBuilder.build()) {
+ Drillbit bit = cluster.drillbit();
+ DrillbitContext bitContext = bit.getContext();
+ FunctionImplementationRegistry registry =
bitContext.getFunctionImplementationRegistry();
+ FragmentContextImpl context = new FragmentContextImpl(bitContext,
BitControl.PlanFragment.getDefaultInstance(), null, registry);
+ BufferAllocator bufferAllocator = bitContext.getAllocator();
+ //create RecordBatch
+ VarCharVector vector = new VarCharVector(SchemaBuilder.columnSchema("a",
TypeProtos.MinorType.VARCHAR, TypeProtos.DataMode.REQUIRED), bufferAllocator);
+ vector.allocateNew();
+ int valueCount = 3;
+ VarCharVector.Mutator mutator = vector.getMutator();
+ mutator.setSafe(0, "a".getBytes());
+ mutator.setSafe(1, "b".getBytes());
+ mutator.setSafe(2, "c".getBytes());
+ mutator.setValueCount(valueCount);
+ VectorContainer vectorContainer = new VectorContainer();
+ TypedFieldId fieldId = vectorContainer.add(vector);
+ RecordBatch recordBatch = new TestRecordBatch(vectorContainer);
+ //construct hash64
+ ValueVectorReadExpression exp = new ValueVectorReadExpression(fieldId);
+ LogicalExpression[] expressions = new LogicalExpression[1];
+ expressions[0] = exp;
+ TypedFieldId[] fieldIds = new TypedFieldId[1];
+ fieldIds[0] = fieldId;
+ ValueVectorHashHelper valueVectorHashHelper = new
ValueVectorHashHelper(recordBatch, context);
+ ValueVectorHashHelper.Hash64 hash64 =
valueVectorHashHelper.getHash64(expressions, fieldIds);
+
+ //construct BloomFilter
+ int numBytes = BloomFilter.optimalNumOfBytes(3, 0.03);
+
+ BloomFilter bloomFilter = new BloomFilter(numBytes, bufferAllocator);
+ for (int i = 0; i < valueCount; i++) {
+ long hashCode = hash64.hash64Code(i, 0, 0);
+ bloomFilter.insert(hashCode);
+ }
+
+ //-----------------create probe side RecordBatch---------------------
+ VarCharVector probeVector = new
VarCharVector(SchemaBuilder.columnSchema("a", TypeProtos.MinorType.VARCHAR,
TypeProtos.DataMode.REQUIRED), bufferAllocator);
Review comment:
This code looks highly redundant. In general copy/paste is a poor form of
reuse. Can any of this be pulled out into a method? If not a method, can you
create a "fixture" class to hold the common bits?
Also if you use the `RowSet` mechanism to create your vectors, you can use
some handy utilities such as `RowSetUtilities.setFromInt()` to set (just about)
any data type from an int value. See `TestCopier` for an example where this is
used.
The result should be that each test case reduces to a few setup lines.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
