This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit b44b7127306f68c250beafb5a4e7980303fd5cf1
Author: Volodymyr Vysotskyi <vvo...@gmail.com>
AuthorDate: Thu Aug 22 19:01:15 2019 +0300
DRILL-7356: Introduce session options for the Drill Metastore
closes #1846
---
.../java/org/apache/drill/exec/ExecConstants.java | 63 ++++++++++++++++++++++
.../exec/server/options/SystemOptionManager.java | 9 +++-
.../java-exec/src/main/resources/drill-module.conf | 10 +++-
3 files changed, 80 insertions(+), 2 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 6f3f17d..549d374 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -1044,4 +1044,67 @@ public final class ExecConstants {
public static final RangeLongValidator QUERY_MAX_ROWS_VALIDATOR = new
RangeLongValidator(QUERY_MAX_ROWS, 0, Integer.MAX_VALUE,
new OptionDescription("The maximum number of rows that the query will
return. This can be only set at a SYSTEM level by an admin. (Drill 1.16+)"));
+ /**
+ * Option that enables Drill Metastore usage.
+ */
+ public static final String METASTORE_ENABLED = "metastore.enabled";
+ public static final BooleanValidator METASTORE_ENABLED_VALIDATOR = new
BooleanValidator(METASTORE_ENABLED,
+ new OptionDescription("Enables Drill Metastore usage to be able to store
table metadata " +
+ "during ANALYZE TABLE commands execution and to be able to read
table metadata during regular " +
+ "queries execution or when querying some INFORMATION_SCHEMA tables.
" +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for specifying maximum level depth for collecting metadata
+ * which will be stored in the Drill Metastore. For example, when {@code
FILE} level value
+ * is set, {@code ROW_GROUP} level metadata won't be collected and stored
into the Metastore.
+ */
+ public static final String METASTORE_METADATA_STORE_DEPTH_LEVEL =
"metastore.metadata.store.depth_level";
+ public static final EnumeratedStringValidator
METASTORE_METADATA_STORE_DEPTH_LEVEL_VALIDATOR = new
EnumeratedStringValidator(METASTORE_METADATA_STORE_DEPTH_LEVEL,
+ new OptionDescription("Specifies maximum level depth for collecting
metadata. " +
+ "This option is not active for now. Default is 'ROW_GROUP'. (Drill
1.17+)"),
+ "TABLE", "SEGMENT", "PARTITION", "FILE", "ROW_GROUP");
+
+ /**
+ * Option for enabling schema usage, stored to the Metastore.
+ */
+ public static final String METASTORE_USE_SCHEMA_METADATA =
"metastore.metadata.use_schema";
+ public static final BooleanValidator METASTORE_USE_SCHEMA_METADATA_VALIDATOR
= new BooleanValidator(METASTORE_USE_SCHEMA_METADATA,
+ new OptionDescription("Enables schema usage, stored to the Metastore. " +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for enabling statistics usage, stored in the Metastore, at the
planning stage.
+ */
+ public static final String METASTORE_USE_STATISTICS_METADATA =
"metastore.metadata.use_statistics";
+ public static final BooleanValidator
METASTORE_USE_STATISTICS_METADATA_VALIDATOR = new
BooleanValidator(METASTORE_USE_STATISTICS_METADATA,
+ new OptionDescription("Enables statistics usage, stored in the
Metastore, at the planning stage. " +
+ "This option is not active for now. Default is false. (Drill
1.17+)"));
+
+ /**
+ * Option for collecting schema and / or column statistics for every table
after CTAS and CTTAS execution.
+ */
+ public static final String METASTORE_CTAS_AUTO_COLLECT_METADATA =
"metastore.metadata.ctas.auto-collect";
+ public static final EnumeratedStringValidator
METASTORE_CTAS_AUTO_COLLECT_METADATA_VALIDATOR = new
EnumeratedStringValidator(METASTORE_CTAS_AUTO_COLLECT_METADATA,
+ new OptionDescription("Specifies whether schema and / or column
statistics will be " +
+ "automatically collected for every table after CTAS and CTTAS. " +
+ "This option is not active for now. Default is 'NONE'. (Drill
1.17+)"),
+ "NONE", "ALL", "SCHEMA");
+
+ /**
+ * Option for allowing using file metadata cache if required metadata is
absent in the Metastore.
+ */
+ public static final String METASTORE_FALLBACK_TO_FILE_METADATA =
"metastore.metadata.fallback_to_file_metadata";
+ public static final BooleanValidator
METASTORE_FALLBACK_TO_FILE_METADATA_VALIDATOR = new
BooleanValidator(METASTORE_FALLBACK_TO_FILE_METADATA,
+ new OptionDescription("Allows using file metadata cache for the case
when required metadata is absent in the Metastore. " +
+ "This option is not active for now. Default is true. (Drill
1.17+)"));
+
+ /**
+ * Option for specifying the number of attempts for retrying query planning
after detecting that query metadata is changed.
+ */
+ public static final String METASTORE_RETRIVAL_RETRY_ATTEMPTS =
"metastore.retrival.retry_attempts";
+ public static final IntegerValidator
METASTORE_RETRIVAL_RETRY_ATTEMPTS_VALIDATOR = new
IntegerValidator(METASTORE_RETRIVAL_RETRY_ATTEMPTS,
+ new OptionDescription("Specifies the number of attempts for retrying
query planning after detecting that query metadata is changed. " +
+ "If the number of retries was exceeded, query will be planned
without metadata information from the Metastore. " +
+ "This option is not active for now. Default is 5. (Drill 1.17+)"));
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 88bba32..d4289fb 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -287,7 +287,14 @@ public class SystemOptionManager extends BaseOptionManager
implements AutoClosea
new OptionDefinition(ExecConstants.RM_QUERY_TAGS_VALIDATOR, new
OptionMetaData(OptionValue.AccessibleScopes.SESSION_AND_QUERY, false, false)),
new
OptionDefinition(ExecConstants.RM_QUEUES_WAIT_FOR_PREFERRED_NODES_VALIDATOR),
new OptionDefinition(ExecConstants.TDIGEST_COMPRESSION_VALIDATOR),
- new OptionDefinition(ExecConstants.QUERY_MAX_ROWS_VALIDATOR, new
OptionMetaData(OptionValue.AccessibleScopes.ALL, true, false))
+ new OptionDefinition(ExecConstants.QUERY_MAX_ROWS_VALIDATOR, new
OptionMetaData(OptionValue.AccessibleScopes.ALL, true, false)),
+ new OptionDefinition(ExecConstants.METASTORE_ENABLED_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_METADATA_STORE_DEPTH_LEVEL_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_USE_SCHEMA_METADATA_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_USE_STATISTICS_METADATA_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_CTAS_AUTO_COLLECT_METADATA_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_FALLBACK_TO_FILE_METADATA_VALIDATOR),
+ new
OptionDefinition(ExecConstants.METASTORE_RETRIVAL_RETRY_ATTEMPTS_VALIDATOR)
};
CaseInsensitiveMap<OptionDefinition> map = Arrays.stream(definitions)
diff --git a/exec/java-exec/src/main/resources/drill-module.conf
b/exec/java-exec/src/main/resources/drill-module.conf
index bb8f292..422d850 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -691,5 +691,13 @@ drill.exec.options: {
# ========= rm related options ===========
exec.rm.queryTags: "",
exec.rm.queues.wait_for_preferred_nodes: true,
- exec.statistics.tdigest_compression: 100
+ exec.statistics.tdigest_compression: 100,
+ # ========= Metastore related options ===========
+ metastore.enabled: false,
+ metastore.retrival.retry_attempts: 5
+ metastore.metadata.store.depth_level: "ROW_GROUP",
+ metastore.metadata.use_schema: false,
+ metastore.metadata.use_statistics: false,
+ metastore.metadata.ctas.auto-collect: "NONE",
+ metastore.metadata.fallback_to_file_metadata: true
}
