This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new b827d7f DRILL-7368: Fix Iceberg Metastore failure when filter column
contains nulls
b827d7f is described below
commit b827d7f29590dd120fda1a6b653fa34c5701f664
Author: Arina Ielchiieva <arina.yelchiy...@gmail.com>
AuthorDate: Tue Sep 10 15:30:10 2019 +0300
DRILL-7368: Fix Iceberg Metastore failure when filter column contains nulls
---
exec/java-exec/pom.xml | 6 ++
.../drill/exec/ops/ExecutorFragmentContext.java | 4 +-
.../org/apache/drill/exec/ops/FragmentContext.java | 37 +++++---
.../apache/drill/exec/ops/FragmentContextImpl.java | 43 +++++----
.../org/apache/drill/exec/ops/QueryContext.java | 5 -
.../apache/drill/exec/server/DrillbitContext.java | 7 +-
.../org/apache/drill/test/OperatorFixture.java | 6 ++
.../apache/drill/test/PhysicalOpUnitTestBase.java | 2 +-
metastore/iceberg-metastore/pom.xml | 2 +-
.../drill/metastore/iceberg/operate/Overwrite.java | 2 +-
.../components/tables/TestBasicRequests.java | 105 +++++++++++++--------
11 files changed, 136 insertions(+), 83 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a80bc48..a9ac685 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -547,6 +547,12 @@
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.drill.metastore</groupId>
+ <artifactId>drill-iceberg-metastore</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
index dbf440c..df782b1 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
@@ -36,8 +36,10 @@ import java.util.Set;
* {@link org.apache.drill.exec.work.fragment.FragmentExecutor}.
*/
public interface ExecutorFragmentContext extends RootFragmentContext {
+
/**
* Returns the root allocator for the Drillbit.
+ *
* @return The root allocator for the Drillbit.
*/
BufferAllocator getRootAllocator();
@@ -58,7 +60,7 @@ public interface ExecutorFragmentContext extends
RootFragmentContext {
QueryProfileStoreContext getProfileStoreContext();
- WorkEventBus getWorkEventbus();
+ WorkEventBus getWorkEventBus();
Set<Map.Entry<UserServer.BitToUserConnection,
UserServer.BitToUserConnectionConfig>> getUserConnections();
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 46f94f0..414d583 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.metastore.MetastoreRegistry;
import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.config.DrillConfig;
@@ -96,9 +98,10 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
/**
* Returns the statement type (e.g. SELECT, CTAS, ANALYZE) from the query
context.
+ *
* @return query statement type {@link SqlStatementType}, if known.
*/
- public SqlStatementType getSQLStatementType();
+ SqlStatementType getSQLStatementType();
/**
* Get this node's identity.
@@ -145,12 +148,12 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
/**
* @return ID {@link java.util.UUID} of the current query
*/
- public QueryId getQueryId();
+ QueryId getQueryId();
/**
* @return The string representation of the ID {@link java.util.UUID} of the
current query
*/
- public String getQueryIdString();
+ String getQueryIdString();
OperatorContext newOperatorContext(PhysicalOperator popConfig);
@@ -171,22 +174,32 @@ public interface FragmentContext extends UdfUtilities,
AutoCloseable {
@Override
void close();
+
/**
- * add a RuntimeFilter when the RuntimeFilter receiver belongs to the same
MinorFragment
- * @param runtimeFilter
+ * Add a RuntimeFilter when the RuntimeFilter receiver belongs to the same
MinorFragment.
+ *
+ * @param runtimeFilter runtime filter
*/
- public void addRuntimeFilter(RuntimeFilterWritable runtimeFilter);
+ void addRuntimeFilter(RuntimeFilterWritable runtimeFilter);
- public RuntimeFilterWritable getRuntimeFilter(long rfIdentifier);
+ RuntimeFilterWritable getRuntimeFilter(long rfIdentifier);
/**
- * get the RuntimeFilter with a blocking wait, if the waiting option is
enabled
- * @param rfIdentifier
- * @param maxWaitTime
- * @param timeUnit
+ * Get the RuntimeFilter with a blocking wait, if the waiting option is
enabled.
+ *
+ * @param rfIdentifier runtime filter identifier
+ * @param maxWaitTime max wait time
+ * @param timeUnit time unit
* @return the RFW or null
*/
- public RuntimeFilterWritable getRuntimeFilter(long rfIdentifier, long
maxWaitTime, TimeUnit timeUnit);
+ RuntimeFilterWritable getRuntimeFilter(long rfIdentifier, long maxWaitTime,
TimeUnit timeUnit);
+
+ /**
+ * Get instance of Metastore registry to obtain Metastore instance if needed.
+ *
+ * @return Metastore registry
+ */
+ MetastoreRegistry getMetastoreRegistry();
interface ExecutorState {
/**
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
index afbcd1c..cb35318 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.ops;
import io.netty.buffer.DrillBuf;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -69,10 +71,9 @@ import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.work.batch.IncomingBuffers;
import org.apache.drill.exec.work.filter.RuntimeFilterWritable;
+import org.apache.drill.metastore.MetastoreRegistry;
import org.apache.drill.shaded.guava.com.google.common.base.Function;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
/**
* <p>
@@ -104,8 +105,8 @@ import
org.apache.drill.shaded.guava.com.google.common.collect.Maps;
public class FragmentContextImpl extends BaseFragmentContext implements
ExecutorFragmentContext {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FragmentContextImpl.class);
- private final Map<DrillbitEndpoint, AccountingDataTunnel> tunnels =
Maps.newHashMap();
- private final List<OperatorContextImpl> contexts = Lists.newLinkedList();
+ private final Map<DrillbitEndpoint, AccountingDataTunnel> tunnels = new
HashMap<>();
+ private final List<OperatorContextImpl> contexts = new LinkedList<>();
private final DrillbitContext context;
private final UserClientConnection connection; // is null if this context is
for non-root fragment
@@ -150,10 +151,10 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
/**
* Create a FragmentContext instance for non-root fragment.
*
- * @param dbContext DrillbitContext.
- * @param fragment Fragment implementation.
- * @param funcRegistry FunctionImplementationRegistry.
- * @throws ExecutionSetupException
+ * @param dbContext DrillbitContext
+ * @param fragment Fragment implementation
+ * @param funcRegistry FunctionImplementationRegistry
+ * @throws ExecutionSetupException when unable to init fragment context
*/
public FragmentContextImpl(final DrillbitContext dbContext, final
PlanFragment fragment,
final FunctionImplementationRegistry
funcRegistry) throws ExecutionSetupException {
@@ -163,12 +164,12 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
/**
* Create a FragmentContext instance for root fragment.
*
- * @param dbContext DrillbitContext.
- * @param fragment Fragment implementation.
- * @param queryContext QueryContext.
- * @param connection UserClientConnection.
- * @param funcRegistry FunctionImplementationRegistry.
- * @throws ExecutionSetupException
+ * @param dbContext DrillbitContext
+ * @param fragment Fragment implementation
+ * @param queryContext QueryContext
+ * @param connection UserClientConnection
+ * @param funcRegistry FunctionImplementationRegistry
+ * @throws ExecutionSetupException when unable to init fragment context
*/
public FragmentContextImpl(final DrillbitContext dbContext, final
PlanFragment fragment, final QueryContext queryContext,
final UserClientConnection connection, final
FunctionImplementationRegistry funcRegistry)
@@ -216,7 +217,7 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
stats = new FragmentStats(allocator, fragment.getAssignment());
bufferManager = new BufferManagerImpl(this.allocator);
- constantValueHolderCache = Maps.newHashMap();
+ constantValueHolderCache = new HashMap<>();
enableRuntimeFilter =
this.getOptions().getOption(ExecConstants.HASHJOIN_ENABLE_RUNTIME_FILTER_KEY).bool_val;
enableRFWaiting =
this.getOptions().getOption(ExecConstants.HASHJOIN_RUNTIME_FILTER_WAITING_ENABLE_KEY).bool_val
&& enableRuntimeFilter;
if (enableRFWaiting) {
@@ -354,8 +355,7 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
@Override
public String getFragIdString() {
final FragmentHandle handle = getHandle();
- final String frag = handle != null ? handle.getMajorFragmentId() + ":" +
handle.getMinorFragmentId() : "0:0";
- return frag;
+ return handle != null ? handle.getMajorFragmentId() + ":" +
handle.getMinorFragmentId() : "0:0";
}
@Override
@@ -566,7 +566,7 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
@Override
public ValueHolder getConstantValueHolder(String value, MinorType type,
Function<DrillBuf, ValueHolder> holderInitializer) {
if (!constantValueHolderCache.containsKey(value)) {
- constantValueHolderCache.put(value, Maps.<MinorType,
ValueHolder>newHashMap());
+ constantValueHolderCache.put(value, new HashMap<>());
}
Map<MinorType, ValueHolder> holdersByType =
constantValueHolderCache.get(value);
@@ -589,7 +589,7 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
}
@Override
- public WorkEventBus getWorkEventbus() {
+ public WorkEventBus getWorkEventBus() {
return context.getWorkBus();
}
@@ -620,4 +620,9 @@ public class FragmentContextImpl extends
BaseFragmentContext implements Executor
+ " Calling from non-root fragment");
return queryContext.getSQLStatementType();
}
+
+ @Override
+ public MetastoreRegistry getMetastoreRegistry() {
+ return context.getMetastoreRegistry();
+ }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index fb1b4a9..97db9b2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -52,7 +52,6 @@ import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.exec.util.Utilities;
-import org.apache.drill.metastore.Metastore;
import org.apache.drill.shaded.guava.com.google.common.base.Function;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
@@ -390,8 +389,4 @@ public class QueryContext implements AutoCloseable,
OptimizerRulesContext, Schem
public boolean isSkipProfileWrite() {
return skipProfileWrite;
}
-
- public Metastore getMetastore() {
- return drillbitContext.getMetastore();
- }
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index b3cd6f0..79d3bed 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -44,7 +44,6 @@ import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.foreman.rm.ResourceManager;
import org.apache.drill.exec.work.foreman.rm.ResourceManagerBuilder;
-import org.apache.drill.metastore.Metastore;
import org.apache.drill.metastore.MetastoreRegistry;
import java.util.Collection;
@@ -300,14 +299,14 @@ public class DrillbitContext implements AutoCloseable {
getFunctionImplementationRegistry().close();
getRemoteFunctionRegistry().close();
getCompiler().close();
- metastoreRegistry.close();
+ getMetastoreRegistry().close();
}
public ResourceManager getResourceManager() {
return resourceManager;
}
- public Metastore getMetastore() {
- return metastoreRegistry.get();
+ public MetastoreRegistry getMetastoreRegistry() {
+ return metastoreRegistry;
}
}
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
index cb71071..ef6a9c2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.test;
+import org.apache.drill.metastore.MetastoreRegistry;
import org.apache.drill.shaded.guava.com.google.common.base.Function;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
@@ -334,6 +335,11 @@ public class OperatorFixture extends BaseFixture
implements AutoCloseable {
public ValueHolder getConstantValueHolder(String value,
TypeProtos.MinorType type, Function<DrillBuf, ValueHolder> holderInitializer) {
return null;
}
+
+ @Override
+ public MetastoreRegistry getMetastoreRegistry() {
+ return null;
+ }
}
private final SystemOptionManager options;
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/test/PhysicalOpUnitTestBase.java
b/exec/java-exec/src/test/java/org/apache/drill/test/PhysicalOpUnitTestBase.java
index 27a4ad6..172bd46 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/test/PhysicalOpUnitTestBase.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/test/PhysicalOpUnitTestBase.java
@@ -250,7 +250,7 @@ public class PhysicalOpUnitTestBase extends ExecTest {
}
@Override
- public WorkEventBus getWorkEventbus() {
+ public WorkEventBus getWorkEventBus() {
return null;
}
diff --git a/metastore/iceberg-metastore/pom.xml
b/metastore/iceberg-metastore/pom.xml
index 41d4690..5a2746b 100644
--- a/metastore/iceberg-metastore/pom.xml
+++ b/metastore/iceberg-metastore/pom.xml
@@ -33,7 +33,7 @@
<name>metastore/Drill Iceberg Metastore</name>
<properties>
- <iceberg.version>1b0b9c2</iceberg.version>
+ <iceberg.version>4d8ae52</iceberg.version>
<caffeine.version>2.7.0</caffeine.version>
</properties>
diff --git
a/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/operate/Overwrite.java
b/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/operate/Overwrite.java
index 67f88fd..9fc4b50 100644
---
a/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/operate/Overwrite.java
+++
b/metastore/iceberg-metastore/src/main/java/org/apache/drill/metastore/iceberg/operate/Overwrite.java
@@ -47,7 +47,7 @@ public class Overwrite implements IcebergOperation {
transaction.newOverwrite()
.overwriteByRowFilter(filter)
.addFile(dataFile)
- .validateAddedFiles()
+ .validateAddedFilesMatchOverwriteFilter()
.commit();
}
}
diff --git
a/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/components/tables/TestBasicRequests.java
b/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/components/tables/TestBasicRequests.java
index 9862c9b..c5608d7 100644
---
a/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/components/tables/TestBasicRequests.java
+++
b/metastore/iceberg-metastore/src/test/java/org/apache/drill/metastore/iceberg/components/tables/TestBasicRequests.java
@@ -18,10 +18,10 @@
package org.apache.drill.metastore.iceberg.components.tables;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.metastore.components.tables.BasicTablesTransformer;
import org.apache.drill.metastore.components.tables.BasicTablesRequests;
-import org.apache.drill.metastore.components.tables.TableMetadataUnit;
+import org.apache.drill.metastore.components.tables.BasicTablesTransformer;
import org.apache.drill.metastore.components.tables.MetastoreTableInfo;
+import org.apache.drill.metastore.components.tables.TableMetadataUnit;
import org.apache.drill.metastore.components.tables.Tables;
import org.apache.drill.metastore.expressions.FilterExpression;
import org.apache.drill.metastore.iceberg.IcebergBaseTest;
@@ -317,69 +317,90 @@ public class TestBasicRequests extends IcebergBaseTest {
* @param tables Drill Metastore Tables instance
*/
private static void prepareData(Tables tables) {
- nationTable = basicUnit().toBuilder()
- .tableName("nation")
- .metadataType(MetadataType.TABLE.name())
- .metadataKey(MetadataInfo.GENERAL_INFO_KEY)
- .build();
+ TableMetadataUnit basicUnit = basicUnit();
+
+ nationTable = BaseTableMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName("nation")
+ .metadataType(MetadataType.TABLE.name())
+ .metadataKey(MetadataInfo.GENERAL_INFO_KEY)
+ .build())
+ .build()
+ .toMetadataUnit();
nationTableInfo = TableInfo.builder().metadataUnit(nationTable).build();
- TableMetadataUnit nationSegment1 = nationTable.toBuilder()
- .metadataType(MetadataType.SEGMENT.name())
+ TableMetadataUnit basicSegment = SegmentMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName(nationTableInfo.name())
+ .metadataType(MetadataType.SEGMENT.name())
+ .build())
+ .build()
+ .toMetadataUnit();
+
+ TableMetadataUnit nationSegment1 = basicSegment.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/d3")
.column("n_nation")
.lastModifiedTime(1L)
.build();
- TableMetadataUnit nationSegment2 = nationTable.toBuilder()
- .metadataType(MetadataType.SEGMENT.name())
+ TableMetadataUnit nationSegment2 = basicSegment.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/d4")
.column("n_nation")
.lastModifiedTime(2L)
.build();
- TableMetadataUnit nationSegment3 = nationTable.toBuilder()
- .metadataType(MetadataType.SEGMENT.name())
+ TableMetadataUnit nationSegment3 = basicSegment.toBuilder()
.metadataKey("part_int=4")
.location("/tmp/nation/part_int=4/d5")
.column("n_nation")
.lastModifiedTime(3L)
.build();
- TableMetadataUnit nationPartition1 = nationTable.toBuilder()
- .metadataType(MetadataType.PARTITION.name())
+ TableMetadataUnit basicPartition = PartitionMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName(nationTableInfo.name())
+ .metadataType(MetadataType.PARTITION.name())
+ .build())
+ .build()
+ .toMetadataUnit();
+
+ TableMetadataUnit nationPartition1 = basicPartition.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/d5")
.column("n_nation")
.build();
- TableMetadataUnit nationPartition2 = nationTable.toBuilder()
- .metadataType(MetadataType.PARTITION.name())
+ TableMetadataUnit nationPartition2 = basicPartition.toBuilder()
.metadataKey("part_int=4")
.location("/tmp/nation/part_int=4/d5")
.column("n_nation")
.build();
- TableMetadataUnit nationPartition3 = nationTable.toBuilder()
- .metadataType(MetadataType.PARTITION.name())
+ TableMetadataUnit nationPartition3 = basicPartition.toBuilder()
.metadataKey("part_int=4")
.column("n_region")
.location("/tmp/nation/part_int=4/d6")
.build();
- TableMetadataUnit nationFile1 = nationTable.toBuilder()
- .metadataType(MetadataType.FILE.name())
+ TableMetadataUnit basicFile = FileMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName(nationTableInfo.name())
+ .metadataType(MetadataType.FILE.name())
+ .build())
+ .build()
+ .toMetadataUnit();
+
+ TableMetadataUnit nationFile1 = basicFile.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/part_varchar=g")
.path("/tmp/nation/part_int=3/part_varchar=g/0_0_0.parquet")
.lastModifiedTime(1L)
.build();
- TableMetadataUnit nationFile2 = nationTable.toBuilder()
- .metadataType(MetadataType.FILE.name())
+ TableMetadataUnit nationFile2 = basicFile.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/part_varchar=g")
.path("/tmp/nation/part_int=3/part_varchar=g/0_0_1.parquet")
@@ -387,51 +408,57 @@ public class TestBasicRequests extends IcebergBaseTest {
.lastModifiedTime(2L)
.build();
- TableMetadataUnit nationFile3 = nationTable.toBuilder()
- .metadataType(MetadataType.FILE.name())
+ TableMetadataUnit nationFile3 = basicFile.toBuilder()
.metadataKey("part_int=4")
.location("/tmp/nation/part_int=4/part_varchar=g")
.path("/tmp/nation/part_int=4/part_varchar=g/0_0_0.parquet")
.lastModifiedTime(3L)
.build();
- TableMetadataUnit nationRowGroup1 = nationTable.toBuilder()
- .metadataType(MetadataType.ROW_GROUP.name())
+ TableMetadataUnit basicRowGroup = RowGroupMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName(nationTableInfo.name())
+ .metadataType(MetadataType.ROW_GROUP.name())
+ .build())
+ .build()
+ .toMetadataUnit();
+
+ TableMetadataUnit nationRowGroup1 = basicRowGroup.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/part_varchar=g")
.path("/tmp/nation/part_int=3/part_varchar=g/0_0_0.parquet")
.rowGroupIndex(1)
.build();
- TableMetadataUnit nationRowGroup2 = nationTable.toBuilder()
- .metadataType(MetadataType.ROW_GROUP.name())
+ TableMetadataUnit nationRowGroup2 = basicRowGroup.toBuilder()
.metadataKey("part_int=3")
.location("/tmp/nation/part_int=3/part_varchar=g")
.path("/tmp/nation/part_int=3/part_varchar=g/0_0_0.parquet")
.rowGroupIndex(2)
.build();
- TableMetadataUnit nationRowGroup3 = nationTable.toBuilder()
- .metadataType(MetadataType.ROW_GROUP.name())
+ TableMetadataUnit nationRowGroup3 = basicRowGroup.toBuilder()
.metadataKey("part_int=4")
.location("/tmp/nation/part_int=4/part_varchar=g")
.path("/tmp/nation/part_int=4/part_varchar=g/0_0_0.parquet")
.rowGroupIndex(1)
.build();
- TableMetadataUnit nationRowGroup4 = nationTable.toBuilder()
- .metadataType(MetadataType.ROW_GROUP.name())
+ TableMetadataUnit nationRowGroup4 = basicRowGroup.toBuilder()
.metadataKey("part_int=4")
.location("/tmp/nation/part_int=4/part_varchar=g")
.path("/tmp/nation/part_int=4/part_varchar=g/0_0_0.parquet")
.rowGroupIndex(2)
.build();
- TableMetadataUnit regionTable = basicUnit().toBuilder()
- .tableName("region")
- .metadataType(MetadataType.TABLE.name())
- .metadataKey(MetadataInfo.GENERAL_INFO_KEY)
- .build();
+ TableMetadataUnit regionTable = BaseTableMetadata.builder()
+ .metadataUnit(basicUnit.toBuilder()
+ .tableName("region")
+ .metadataType(MetadataType.TABLE.name())
+ .metadataKey(MetadataInfo.GENERAL_INFO_KEY)
+ .build())
+ .build()
+ .toMetadataUnit();
tables.modify()
.overwrite(nationTable,
@@ -471,7 +498,7 @@ public class TestBasicRequests extends IcebergBaseTest {
"\"statisticsKind\":{\"name\":\"approx_count_distinct\"}}"))
.lastModifiedTime(System.currentTimeMillis())
.partitionKeys(Collections.singletonMap("dir0", "2018"))
- .additionalMetadata("test table metadata")
+ .additionalMetadata("additional test metadata")
.metadataIdentifier("part_int=3/part_varchar=g/0_0_0.parquet")
.column("`id`")
.locations(Arrays.asList("/tmp/nation/1", "/tmp/nation/2"))
