DRILL-5547: Linking config options with system option manager closes #868
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a51c98b8 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a51c98b8 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a51c98b8 Branch: refs/heads/master Commit: a51c98b8bf210bbe9d3f4018361d937252d1226d Parents: a344202 Author: Jyothsna Donapati <jyothsnadonapati@Skatkam-598.local> Authored: Wed Jun 7 11:15:08 2017 -0700 Committer: Paul Rogers <prog...@maprtech.com> Committed: Fri Aug 25 20:50:10 2017 -0700 ---------------------------------------------------------------------- .../org/apache/drill/exec/ExecConstants.java | 173 +++++++-------- .../exec/compile/ClassCompilerSelector.java | 6 +- .../planner/fragment/SimpleParallelizer.java | 5 +- .../exec/planner/physical/PlannerSettings.java | 74 +++---- .../planner/sql/handlers/DefaultSqlHandler.java | 4 +- .../planner/sql/handlers/SetOptionHandler.java | 21 +- .../rpc/user/InboundImpersonationManager.java | 4 +- .../apache/drill/exec/rpc/user/UserSession.java | 6 +- .../org/apache/drill/exec/server/Drillbit.java | 3 +- .../drill/exec/server/DrillbitContext.java | 2 +- .../exec/server/options/BaseOptionManager.java | 1 + .../server/options/DrillConfigIterator.java | 7 +- .../server/options/FallbackOptionManager.java | 27 ++- .../exec/server/options/OptionValidator.java | 10 +- .../drill/exec/server/options/OptionValue.java | 46 ++-- .../server/options/SessionOptionManager.java | 10 +- .../server/options/SystemOptionManager.java | 97 ++++++--- .../exec/server/options/TypeValidators.java | 131 ++++++++---- .../drill/exec/server/rest/StatusResources.java | 2 +- .../exec/store/sys/ExtendedOptionIterator.java | 155 ++++++++++++++ .../drill/exec/store/sys/OptionIterator.java | 7 +- .../drill/exec/store/sys/SystemTable.java | 8 + .../drill/exec/testing/ExecutionControls.java | 14 +- .../exec/util/MemoryAllocationUtilities.java | 4 +- .../src/main/resources/drill-module.conf | 147 +++++++++++++ .../java/org/apache/drill/PlanningBase.java | 2 +- .../java/org/apache/drill/QueryTestUtil.java | 3 +- .../java/org/apache/drill/exec/ExecTest.java | 2 +- .../exec/compile/CodeCompilerTestFactory.java | 2 +- .../exec/compile/TestClassTransformation.java | 13 +- .../compile/bytecode/ReplaceMethodInvoke.java | 2 +- .../TestInboundImpersonationPrivileges.java | 2 +- .../partitionsender/TestPartitionSender.java | 11 +- .../exec/testing/ControlsInjectionUtil.java | 3 +- .../work/metadata/TestMetadataProvider.java | 8 +- .../org/apache/drill/test/FixtureBuilder.java | 9 + .../org/apache/drill/test/OperatorFixture.java | 9 +- .../apache/drill/test/TestConfigLinkage.java | 209 +++++++++++++++++++ .../drill/jdbc/test/TestJdbcMetadata.java | 4 +- 39 files changed, 979 insertions(+), 264 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index a88875f..95ee00e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -29,8 +29,8 @@ import org.apache.drill.exec.server.options.TypeValidators.PowerOfTwoLongValidat import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator; import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator; import org.apache.drill.exec.server.options.TypeValidators.StringValidator; +import org.apache.drill.exec.server.options.TypeValidators.MaxWidthValidator; import org.apache.drill.exec.testing.ExecutionControls; -import org.apache.drill.exec.util.ImpersonationUtil; public interface ExecConstants { String ZK_RETRY_TIMES = "drill.exec.zk.retry.count"; @@ -88,22 +88,22 @@ public interface ExecConstants { // External Sort Runtime options - BooleanValidator EXTERNAL_SORT_DISABLE_MANAGED_OPTION = new BooleanValidator("exec.sort.disable_managed", false); + BooleanValidator EXTERNAL_SORT_DISABLE_MANAGED_OPTION = new BooleanValidator("exec.sort.disable_managed"); // Hash Aggregate Options String HASHAGG_NUM_PARTITIONS = "drill.exec.hashagg.num_partitions"; String HASHAGG_NUM_PARTITIONS_KEY = "exec.hashagg.num_partitions"; - LongValidator HASHAGG_NUM_PARTITIONS_VALIDATOR = new RangeLongValidator(HASHAGG_NUM_PARTITIONS_KEY, 1, 128, 32); // 1 means - no spilling + LongValidator HASHAGG_NUM_PARTITIONS_VALIDATOR = new RangeLongValidator(HASHAGG_NUM_PARTITIONS_KEY, 1, 128); // 1 means - no spilling String HASHAGG_MAX_MEMORY = "drill.exec.hashagg.mem_limit"; String HASHAGG_MAX_MEMORY_KEY = "exec.hashagg.mem_limit"; - LongValidator HASHAGG_MAX_MEMORY_VALIDATOR = new RangeLongValidator(HASHAGG_MAX_MEMORY_KEY, 0, Integer.MAX_VALUE, 0); + LongValidator HASHAGG_MAX_MEMORY_VALIDATOR = new RangeLongValidator(HASHAGG_MAX_MEMORY_KEY, 0, Integer.MAX_VALUE); // min batches is used for tuning (each partition needs so many batches when planning the number of partitions, // or reserve this number when calculating whether the remaining available memory is too small and requires a spill.) // Low value may OOM (e.g., when incoming rows become wider), higher values use fewer partitions but are safer String HASHAGG_MIN_BATCHES_PER_PARTITION = "drill.exec.hashagg.min_batches_per_partition"; String HASHAGG_MIN_BATCHES_PER_PARTITION_KEY = "drill.exec.hashagg.min_batches_per_partition"; - LongValidator HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR = new RangeLongValidator(HASHAGG_MIN_BATCHES_PER_PARTITION_KEY, 2, 5, 3); + LongValidator HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR = new RangeLongValidator(HASHAGG_MIN_BATCHES_PER_PARTITION_KEY, 2, 5); String HASHAGG_SPILL_DIRS = "drill.exec.hashagg.spill.directories"; String HASHAGG_SPILL_FILESYSTEM = "drill.exec.hashagg.spill.fs"; @@ -186,70 +186,70 @@ public interface ExecConstants { String DEFAULT_TEMPORARY_WORKSPACE = "drill.exec.default_temporary_workspace"; String OUTPUT_FORMAT_OPTION = "store.format"; - OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet"); + OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION); String PARQUET_BLOCK_SIZE = "store.parquet.block-size"; String PARQUET_WRITER_USE_SINGLE_FS_BLOCK = "store.parquet.writer.use_single_fs_block"; OptionValidator PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR = new BooleanValidator( - PARQUET_WRITER_USE_SINGLE_FS_BLOCK, false); - OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_BLOCK_SIZE, Integer.MAX_VALUE, 512 * 1024 * 1024); + PARQUET_WRITER_USE_SINGLE_FS_BLOCK); + OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_BLOCK_SIZE, Integer.MAX_VALUE); String PARQUET_PAGE_SIZE = "store.parquet.page-size"; - OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024); + OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGE_SIZE, Integer.MAX_VALUE); String PARQUET_DICT_PAGE_SIZE = "store.parquet.dictionary.page-size"; - OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_DICT_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024); + OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_DICT_PAGE_SIZE, Integer.MAX_VALUE); String PARQUET_WRITER_COMPRESSION_TYPE = "store.parquet.compression"; OptionValidator PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR = new EnumeratedStringValidator( - PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "snappy", "gzip", "none"); + PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "gzip", "none"); String PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING = "store.parquet.enable_dictionary_encoding"; OptionValidator PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR = new BooleanValidator( - PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false); + PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); String PARQUET_VECTOR_FILL_THRESHOLD = "store.parquet.vector_fill_threshold"; - OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l, 85l); + OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l); String PARQUET_VECTOR_FILL_CHECK_THRESHOLD = "store.parquet.vector_fill_check_threshold"; - OptionValidator PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_CHECK_THRESHOLD, 100l, 10l); + OptionValidator PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_CHECK_THRESHOLD, 100l); String PARQUET_NEW_RECORD_READER = "store.parquet.use_new_reader"; - OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER, false); + OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER); String PARQUET_READER_INT96_AS_TIMESTAMP = "store.parquet.reader.int96_as_timestamp"; - OptionValidator PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR = new BooleanValidator(PARQUET_READER_INT96_AS_TIMESTAMP, false); + OptionValidator PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR = new BooleanValidator(PARQUET_READER_INT96_AS_TIMESTAMP); String PARQUET_PAGEREADER_ASYNC = "store.parquet.reader.pagereader.async"; - OptionValidator PARQUET_PAGEREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ASYNC, true); + OptionValidator PARQUET_PAGEREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ASYNC); // Number of pages the Async Parquet page reader will read before blocking String PARQUET_PAGEREADER_QUEUE_SIZE = "store.parquet.reader.pagereader.queuesize"; - OptionValidator PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGEREADER_QUEUE_SIZE, Integer.MAX_VALUE, 2); + OptionValidator PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGEREADER_QUEUE_SIZE, Integer.MAX_VALUE); String PARQUET_PAGEREADER_ENFORCETOTALSIZE = "store.parquet.reader.pagereader.enforceTotalSize"; - OptionValidator PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ENFORCETOTALSIZE, false); + OptionValidator PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ENFORCETOTALSIZE); String PARQUET_COLUMNREADER_ASYNC = "store.parquet.reader.columnreader.async"; - OptionValidator PARQUET_COLUMNREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_COLUMNREADER_ASYNC, false); + OptionValidator PARQUET_COLUMNREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_COLUMNREADER_ASYNC); // Use a buffering reader for parquet page reader String PARQUET_PAGEREADER_USE_BUFFERED_READ = "store.parquet.reader.pagereader.bufferedread"; - OptionValidator PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_BUFFERED_READ, true); + OptionValidator PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_BUFFERED_READ); // Size in MiB of the buffer the Parquet page reader will use to read from disk. Default is 1 MiB String PARQUET_PAGEREADER_BUFFER_SIZE = "store.parquet.reader.pagereader.buffersize"; - OptionValidator PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR = new LongValidator(PARQUET_PAGEREADER_BUFFER_SIZE, 1*1024*1024); + OptionValidator PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR = new LongValidator(PARQUET_PAGEREADER_BUFFER_SIZE); // try to use fadvise if available String PARQUET_PAGEREADER_USE_FADVISE = "store.parquet.reader.pagereader.usefadvise"; - OptionValidator PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_FADVISE, false); + OptionValidator PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_FADVISE); - OptionValidator COMPILE_SCALAR_REPLACEMENT = new BooleanValidator("exec.compile.scalar_replacement", false); + OptionValidator COMPILE_SCALAR_REPLACEMENT = new BooleanValidator("exec.compile.scalar_replacement"); String JSON_ALL_TEXT_MODE = "store.json.all_text_mode"; - BooleanValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE, false); - BooleanValidator JSON_EXTENDED_TYPES = new BooleanValidator("store.json.extended_types", false); - BooleanValidator JSON_WRITER_UGLIFY = new BooleanValidator("store.json.writer.uglify", false); - BooleanValidator JSON_WRITER_SKIPNULLFIELDS = new BooleanValidator("store.json.writer.skip_null_fields", true); + BooleanValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE); + BooleanValidator JSON_EXTENDED_TYPES = new BooleanValidator("store.json.extended_types"); + BooleanValidator JSON_WRITER_UGLIFY = new BooleanValidator("store.json.writer.uglify"); + BooleanValidator JSON_WRITER_SKIPNULLFIELDS = new BooleanValidator("store.json.writer.skip_null_fields"); String JSON_READER_SKIP_INVALID_RECORDS_FLAG = "store.json.reader.skip_invalid_records"; - BooleanValidator JSON_SKIP_MALFORMED_RECORDS_VALIDATOR = new BooleanValidator(JSON_READER_SKIP_INVALID_RECORDS_FLAG, false); + BooleanValidator JSON_SKIP_MALFORMED_RECORDS_VALIDATOR = new BooleanValidator(JSON_READER_SKIP_INVALID_RECORDS_FLAG); String JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG = "store.json.reader.print_skipped_invalid_record_number"; - BooleanValidator JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR = new BooleanValidator(JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG, false); + BooleanValidator JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR = new BooleanValidator(JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG); DoubleValidator TEXT_ESTIMATED_ROW_SIZE = new RangeDoubleValidator( - "store.text.estimated_row_size_bytes", 1, Long.MAX_VALUE, 100.0); + "store.text.estimated_row_size_bytes", 1, Long.MAX_VALUE); /** * The column label (for directory levels) in results when querying files in a directory @@ -259,59 +259,61 @@ public interface ExecConstants { * |- baz - b.parquet */ String FILESYSTEM_PARTITION_COLUMN_LABEL = "drill.exec.storage.file.partition.column.label"; - OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL, "dir"); + OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL); /** * Implicit file columns */ String IMPLICIT_FILENAME_COLUMN_LABEL = "drill.exec.storage.implicit.filename.column.label"; - OptionValidator IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILENAME_COLUMN_LABEL, "filename"); + OptionValidator IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILENAME_COLUMN_LABEL); String IMPLICIT_SUFFIX_COLUMN_LABEL = "drill.exec.storage.implicit.suffix.column.label"; - OptionValidator IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_SUFFIX_COLUMN_LABEL, "suffix"); + OptionValidator IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_SUFFIX_COLUMN_LABEL); String IMPLICIT_FQN_COLUMN_LABEL = "drill.exec.storage.implicit.fqn.column.label"; - OptionValidator IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FQN_COLUMN_LABEL, "fqn"); + OptionValidator IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FQN_COLUMN_LABEL); String IMPLICIT_FILEPATH_COLUMN_LABEL = "drill.exec.storage.implicit.filepath.column.label"; - OptionValidator IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILEPATH_COLUMN_LABEL, "filepath"); + OptionValidator IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILEPATH_COLUMN_LABEL); String JSON_READ_NUMBERS_AS_DOUBLE = "store.json.read_numbers_as_double"; - BooleanValidator JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(JSON_READ_NUMBERS_AS_DOUBLE, false); + BooleanValidator JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(JSON_READ_NUMBERS_AS_DOUBLE); String MONGO_ALL_TEXT_MODE = "store.mongo.all_text_mode"; - OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE, false); + OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE); String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double"; - OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false); + OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE); String MONGO_BSON_RECORD_READER = "store.mongo.bson.record.reader"; - OptionValidator MONGO_BSON_RECORD_READER_VALIDATOR = new BooleanValidator(MONGO_BSON_RECORD_READER, true); + OptionValidator MONGO_BSON_RECORD_READER_VALIDATOR = new BooleanValidator(MONGO_BSON_RECORD_READER); - BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false); + BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type"); // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare // in core which is not right. Move this option and above two mongo plugin related options once we have the feature. String HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS = "store.hive.optimize_scan_with_native_readers"; OptionValidator HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR = - new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, false); + new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS); String SLICE_TARGET = "planner.slice_target"; long SLICE_TARGET_DEFAULT = 100000l; - PositiveLongValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE, - SLICE_TARGET_DEFAULT); + PositiveLongValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE + ); String CAST_TO_NULLABLE_NUMERIC = "drill.exec.functions.cast_empty_string_to_null"; - BooleanValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC, false); + BooleanValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC); /** * HashTable runtime settings */ String MIN_HASH_TABLE_SIZE_KEY = "exec.min_hash_table_size"; - PositiveLongValidator MIN_HASH_TABLE_SIZE = new PositiveLongValidator(MIN_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY, HashTable.DEFAULT_INITIAL_CAPACITY); + PositiveLongValidator MIN_HASH_TABLE_SIZE = new PositiveLongValidator(MIN_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY); String MAX_HASH_TABLE_SIZE_KEY = "exec.max_hash_table_size"; - PositiveLongValidator MAX_HASH_TABLE_SIZE = new PositiveLongValidator(MAX_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY, HashTable.MAXIMUM_CAPACITY); + PositiveLongValidator MAX_HASH_TABLE_SIZE = new PositiveLongValidator(MAX_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY); /** * Limits the maximum level of parallelization to this factor time the number of Drillbits */ + String CPU_LOAD_AVERAGE_KEY = "planner.cpu_load_average"; + DoubleValidator CPU_LOAD_AVERAGE = new DoubleValidator(CPU_LOAD_AVERAGE_KEY); String MAX_WIDTH_PER_NODE_KEY = "planner.width.max_per_node"; - OptionValidator MAX_WIDTH_PER_NODE = new PositiveLongValidator(MAX_WIDTH_PER_NODE_KEY, Integer.MAX_VALUE, (long) Math.ceil(Runtime.getRuntime().availableProcessors() * 0.70)); + MaxWidthValidator MAX_WIDTH_PER_NODE = new MaxWidthValidator(MAX_WIDTH_PER_NODE_KEY); /** * The maximum level or parallelization any stage of the query can do. Note that while this @@ -319,19 +321,19 @@ public interface ExecConstants { * number of we want to do things like speed results return. */ String MAX_WIDTH_GLOBAL_KEY = "planner.width.max_per_query"; - OptionValidator MAX_WIDTH_GLOBAL = new PositiveLongValidator(MAX_WIDTH_GLOBAL_KEY, Integer.MAX_VALUE, 1000); + OptionValidator MAX_WIDTH_GLOBAL = new PositiveLongValidator(MAX_WIDTH_GLOBAL_KEY, Integer.MAX_VALUE); /** * Factor by which a node with endpoint affinity will be favored while creating assignment */ String AFFINITY_FACTOR_KEY = "planner.affinity_factor"; - OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d); + OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY); String EARLY_LIMIT0_OPT_KEY = "planner.enable_limit0_optimization"; - BooleanValidator EARLY_LIMIT0_OPT = new BooleanValidator(EARLY_LIMIT0_OPT_KEY, false); + BooleanValidator EARLY_LIMIT0_OPT = new BooleanValidator(EARLY_LIMIT0_OPT_KEY); String ENABLE_MEMORY_ESTIMATION_KEY = "planner.memory.enable_memory_estimation"; - OptionValidator ENABLE_MEMORY_ESTIMATION = new BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY, false); + OptionValidator ENABLE_MEMORY_ESTIMATION = new BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY); /** * Maximum query memory per node (in MB). Re-plan with cheaper operators if memory estimation exceeds this limit. @@ -340,7 +342,7 @@ public interface ExecConstants { */ String MAX_QUERY_MEMORY_PER_NODE_KEY = "planner.memory.max_query_memory_per_node"; LongValidator MAX_QUERY_MEMORY_PER_NODE = new RangeLongValidator( - MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE, 2 * 1024 * 1024 * 1024L); + MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE); /** * Minimum memory alocated to each buffered operator instance. @@ -349,7 +351,7 @@ public interface ExecConstants { */ String MIN_MEMORY_PER_BUFFERED_OP_KEY = "planner.memory.min_memory_per_buffered_op"; LongValidator MIN_MEMORY_PER_BUFFERED_OP = new RangeLongValidator( - MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE, 40 * 1024 * 1024L); + MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE); /** * Extra query memory per node for non-blocking operators. @@ -360,51 +362,51 @@ public interface ExecConstants { */ String NON_BLOCKING_OPERATORS_MEMORY_KEY = "planner.memory.non_blocking_operators_memory"; OptionValidator NON_BLOCKING_OPERATORS_MEMORY = new PowerOfTwoLongValidator( - NON_BLOCKING_OPERATORS_MEMORY_KEY, 1 << 11, 1 << 6); + NON_BLOCKING_OPERATORS_MEMORY_KEY, 1 << 11); String HASH_JOIN_TABLE_FACTOR_KEY = "planner.memory.hash_join_table_factor"; - OptionValidator HASH_JOIN_TABLE_FACTOR = new DoubleValidator(HASH_JOIN_TABLE_FACTOR_KEY, 1.1d); + OptionValidator HASH_JOIN_TABLE_FACTOR = new DoubleValidator(HASH_JOIN_TABLE_FACTOR_KEY); String HASH_AGG_TABLE_FACTOR_KEY = "planner.memory.hash_agg_table_factor"; - OptionValidator HASH_AGG_TABLE_FACTOR = new DoubleValidator(HASH_AGG_TABLE_FACTOR_KEY, 1.1d); + OptionValidator HASH_AGG_TABLE_FACTOR = new DoubleValidator(HASH_AGG_TABLE_FACTOR_KEY); String AVERAGE_FIELD_WIDTH_KEY = "planner.memory.average_field_width"; - OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE, 8); + OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE); - BooleanValidator ENABLE_QUEUE = new BooleanValidator("exec.queue.enable", false); - LongValidator LARGE_QUEUE_SIZE = new PositiveLongValidator("exec.queue.large", 1000, 10); - LongValidator SMALL_QUEUE_SIZE = new PositiveLongValidator("exec.queue.small", 100000, 100); + BooleanValidator ENABLE_QUEUE = new BooleanValidator("exec.queue.enable"); + LongValidator LARGE_QUEUE_SIZE = new PositiveLongValidator("exec.queue.large", 1000); + LongValidator SMALL_QUEUE_SIZE = new PositiveLongValidator("exec.queue.small", 100000); LongValidator QUEUE_THRESHOLD_SIZE = new PositiveLongValidator("exec.queue.threshold", - Long.MAX_VALUE, 30000000); + Long.MAX_VALUE); LongValidator QUEUE_TIMEOUT = new PositiveLongValidator("exec.queue.timeout_millis", - Long.MAX_VALUE, 60 * 1000 * 5); + Long.MAX_VALUE); String ENABLE_VERBOSE_ERRORS_KEY = "exec.errors.verbose"; - OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY, false); + OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY); String ENABLE_NEW_TEXT_READER_KEY = "exec.storage.enable_new_text_reader"; - OptionValidator ENABLE_NEW_TEXT_READER = new BooleanValidator(ENABLE_NEW_TEXT_READER_KEY, true); + OptionValidator ENABLE_NEW_TEXT_READER = new BooleanValidator(ENABLE_NEW_TEXT_READER_KEY); String BOOTSTRAP_STORAGE_PLUGINS_FILE = "bootstrap-storage-plugins.json"; String DRILL_SYS_FILE_SUFFIX = ".sys.drill"; String ENABLE_WINDOW_FUNCTIONS = "window.enable"; - OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS, true); + OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS); String DRILLBIT_CONTROL_INJECTIONS = "drill.exec.testing.controls"; OptionValidator DRILLBIT_CONTROLS_VALIDATOR = - new ExecutionControls.ControlsOptionValidator(DRILLBIT_CONTROL_INJECTIONS, ExecutionControls.DEFAULT_CONTROLS, 1); + new ExecutionControls.ControlsOptionValidator(DRILLBIT_CONTROL_INJECTIONS, 1); String NEW_VIEW_DEFAULT_PERMS_KEY = "new_view_default_permissions"; OptionValidator NEW_VIEW_DEFAULT_PERMS_VALIDATOR = - new StringValidator(NEW_VIEW_DEFAULT_PERMS_KEY, "700"); + new StringValidator(NEW_VIEW_DEFAULT_PERMS_KEY); String CTAS_PARTITIONING_HASH_DISTRIBUTE = "store.partition.hash_distribute"; - BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE, false); + BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE); String ENABLE_BULK_LOAD_TABLE_LIST_KEY = "exec.enable_bulk_load_table_list"; - BooleanValidator ENABLE_BULK_LOAD_TABLE_LIST = new BooleanValidator(ENABLE_BULK_LOAD_TABLE_LIST_KEY, false); + BooleanValidator ENABLE_BULK_LOAD_TABLE_LIST = new BooleanValidator(ENABLE_BULK_LOAD_TABLE_LIST_KEY); /** * When getting Hive Table information with exec.enable_bulk_load_table_list set to true, @@ -412,20 +414,20 @@ public interface ExecConstants { * at a time. (The number of tables can get to be quite large.) */ String BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY = "exec.bulk_load_table_list.bulk_size"; - PositiveLongValidator BULK_LOAD_TABLE_LIST_BULK_SIZE = new PositiveLongValidator(BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY, Integer.MAX_VALUE, 1000); + PositiveLongValidator BULK_LOAD_TABLE_LIST_BULK_SIZE = new PositiveLongValidator(BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY, Integer.MAX_VALUE); /** * Option whose value is a comma separated list of admin usernames. Admin users are users who have special privileges * such as changing system options. */ String ADMIN_USERS_KEY = "security.admin.users"; - StringValidator ADMIN_USERS_VALIDATOR = new StringValidator(ADMIN_USERS_KEY, ImpersonationUtil.getProcessUserName(), true); + StringValidator ADMIN_USERS_VALIDATOR = new StringValidator(ADMIN_USERS_KEY, true); /** * Option whose value is a comma separated list of admin usergroups. */ String ADMIN_USER_GROUPS_KEY = "security.admin.user_groups"; - StringValidator ADMIN_USER_GROUPS_VALIDATOR = new StringValidator(ADMIN_USER_GROUPS_KEY, "", true); + StringValidator ADMIN_USER_GROUPS_VALIDATOR = new StringValidator(ADMIN_USER_GROUPS_KEY, true); /** * Option whose value is a string representing list of inbound impersonation policies. * @@ -440,16 +442,17 @@ public interface ExecConstants { */ String IMPERSONATION_POLICIES_KEY = "exec.impersonation.inbound_policies"; StringValidator IMPERSONATION_POLICY_VALIDATOR = - new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY, "[]"); + new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY); + /** * Web settings */ String WEB_LOGS_MAX_LINES = "web.logs.max_lines"; - OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, 10000); + OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE); String CODE_GEN_EXP_IN_METHOD_SIZE = "exec.java.compiler.exp_in_method_size"; - LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE, 50); + LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE); /** * Timeout for create prepare statement request. If the request exceeds this timeout, then request is timed out. @@ -457,10 +460,10 @@ public interface ExecConstants { */ String CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS = "prepare.statement.create_timeout_ms"; OptionValidator CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR = - new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS, Integer.MAX_VALUE, 10000); + new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS, Integer.MAX_VALUE); String DYNAMIC_UDF_SUPPORT_ENABLED = "exec.udf.enable_dynamic_support"; - BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR = new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED, true, true); + BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR = new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED); /** * Option to save query profiles. If false, no query profile will be saved @@ -468,7 +471,7 @@ public interface ExecConstants { */ String ENABLE_QUERY_PROFILE_OPTION = "exec.query_profile.save"; BooleanValidator ENABLE_QUERY_PROFILE_VALIDATOR = new BooleanValidator( - ENABLE_QUERY_PROFILE_OPTION, true, false); + ENABLE_QUERY_PROFILE_OPTION); /** * Profiles are normally written after the last client message to reduce latency. @@ -478,16 +481,16 @@ public interface ExecConstants { */ String QUERY_PROFILE_DEBUG_OPTION = "exec.query_profile.debug_mode"; BooleanValidator QUERY_PROFILE_DEBUG_VALIDATOR = new BooleanValidator( - QUERY_PROFILE_DEBUG_OPTION, false, false); + QUERY_PROFILE_DEBUG_OPTION); String USE_DYNAMIC_UDFS_KEY = "exec.udf.use_dynamic"; - BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY, true); + BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY); String QUERY_TRANSIENT_STATE_UPDATE_KEY = "exec.query.progress.update"; - BooleanValidator QUERY_TRANSIENT_STATE_UPDATE = new BooleanValidator(QUERY_TRANSIENT_STATE_UPDATE_KEY, true); + BooleanValidator QUERY_TRANSIENT_STATE_UPDATE = new BooleanValidator(QUERY_TRANSIENT_STATE_UPDATE_KEY); String PERSISTENT_TABLE_UMASK = "exec.persistent_table.umask"; - StringValidator PERSISTENT_TABLE_UMASK_VALIDATOR = new StringValidator(PERSISTENT_TABLE_UMASK, "002"); + StringValidator PERSISTENT_TABLE_UMASK_VALIDATOR = new StringValidator(PERSISTENT_TABLE_UMASK); /** * Enables batch iterator (operator) validation. Validation is normally enabled @@ -496,7 +499,7 @@ public interface ExecConstants { * a "production" Drill instance. */ String ENABLE_ITERATOR_VALIDATION_OPTION = "debug.validate_iterators"; - BooleanValidator ENABLE_ITERATOR_VALIDATOR = new BooleanValidator(ENABLE_ITERATOR_VALIDATION_OPTION, false); + BooleanValidator ENABLE_ITERATOR_VALIDATOR = new BooleanValidator(ENABLE_ITERATOR_VALIDATION_OPTION); /** * Boot-time config option to enable validation. Primarily used for tests. @@ -511,7 +514,7 @@ public interface ExecConstants { * each batch passed to each iterator. */ String ENABLE_VECTOR_VALIDATION_OPTION = "debug.validate_vectors"; - BooleanValidator ENABLE_VECTOR_VALIDATOR = new BooleanValidator(ENABLE_VECTOR_VALIDATION_OPTION, false); + BooleanValidator ENABLE_VECTOR_VALIDATOR = new BooleanValidator(ENABLE_VECTOR_VALIDATION_OPTION); /** * Boot-time config option to enable vector validation. Primarily used for http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java index 92b8430..9b6e86e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java @@ -75,12 +75,12 @@ public class ClassCompilerSelector { public static final String JAVA_COMPILER_OPTION = "exec.java_compiler"; public static final String JAVA_COMPILER_JANINO_MAXSIZE_OPTION = "exec.java_compiler_janino_maxsize"; - public static final OptionValidator JAVA_COMPILER_JANINO_MAXSIZE = new LongValidator(JAVA_COMPILER_JANINO_MAXSIZE_OPTION, 256*1024); + public static final OptionValidator JAVA_COMPILER_JANINO_MAXSIZE = new LongValidator(JAVA_COMPILER_JANINO_MAXSIZE_OPTION); public static final String JAVA_COMPILER_DEBUG_OPTION = "exec.java_compiler_debug"; - public static final OptionValidator JAVA_COMPILER_DEBUG = new BooleanValidator(JAVA_COMPILER_DEBUG_OPTION, true); + public static final OptionValidator JAVA_COMPILER_DEBUG = new BooleanValidator(JAVA_COMPILER_DEBUG_OPTION); - public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION, CompilerPolicy.DEFAULT.toString()) { + public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION) { @Override public void validate(final OptionValue v, final OptionSet manager) { super.validate(v, manager); http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java index c389e89..4584bd5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java @@ -72,7 +72,10 @@ public class SimpleParallelizer implements ParallelizationParameters { OptionManager optionManager = context.getOptions(); long sliceTarget = optionManager.getOption(ExecConstants.SLICE_TARGET).num_val; this.parallelizationThreshold = sliceTarget > 0 ? sliceTarget : 1; - this.maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val.intValue(); + double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE); + final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE); + // compute the maxwidth + this.maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth); this.maxGlobalWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_GLOBAL_KEY).num_val.intValue(); this.affinityFactor = optionManager.getOption(ExecConstants.AFFINITY_FACTOR_KEY).float_val.intValue(); } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 15314ba..36285a5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -51,42 +51,42 @@ public class PlannerSettings implements Context{ // max off heap memory for planning (16G) private static final long MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 16l * 1024 * 1024 * 1024; - public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", true); - public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false); - public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true); - public static final OptionValidator STREAMAGG = new BooleanValidator("planner.enable_streamagg", true); - public static final OptionValidator HASHJOIN = new BooleanValidator("planner.enable_hashjoin", true); - public static final OptionValidator MERGEJOIN = new BooleanValidator("planner.enable_mergejoin", true); - public static final OptionValidator NESTEDLOOPJOIN = new BooleanValidator("planner.enable_nestedloopjoin", true); - public static final OptionValidator MULTIPHASE = new BooleanValidator("planner.enable_multiphase_agg", true); - public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join", true); - public static final OptionValidator BROADCAST_THRESHOLD = new PositiveLongValidator("planner.broadcast_threshold", MAX_BROADCAST_THRESHOLD, 10000000); - public static final OptionValidator BROADCAST_FACTOR = new RangeDoubleValidator("planner.broadcast_factor", 0, Double.MAX_VALUE, 1.0d); - public static final OptionValidator NESTEDLOOPJOIN_FACTOR = new RangeDoubleValidator("planner.nestedloopjoin_factor", 0, Double.MAX_VALUE, 100.0d); - public static final OptionValidator NLJOIN_FOR_SCALAR = new BooleanValidator("planner.enable_nljoin_for_scalar_only", true); - public static final OptionValidator JOIN_ROW_COUNT_ESTIMATE_FACTOR = new RangeDoubleValidator("planner.join.row_count_estimate_factor", 0, Double.MAX_VALUE, 1.0d); - public static final OptionValidator MUX_EXCHANGE = new BooleanValidator("planner.enable_mux_exchange", true); - public static final OptionValidator DEMUX_EXCHANGE = new BooleanValidator("planner.enable_demux_exchange", false); - public static final OptionValidator PARTITION_SENDER_THREADS_FACTOR = new LongValidator("planner.partitioner_sender_threads_factor", 2); - public static final OptionValidator PARTITION_SENDER_MAX_THREADS = new LongValidator("planner.partitioner_sender_max_threads", 8); - public static final OptionValidator PARTITION_SENDER_SET_THREADS = new LongValidator("planner.partitioner_sender_set_threads", -1); - public static final OptionValidator PRODUCER_CONSUMER = new BooleanValidator("planner.add_producer_consumer", false); - public static final OptionValidator PRODUCER_CONSUMER_QUEUE_SIZE = new LongValidator("planner.producer_consumer_queue_size", 10); - public static final OptionValidator HASH_SINGLE_KEY = new BooleanValidator("planner.enable_hash_single_key", true); - public static final OptionValidator HASH_JOIN_SWAP = new BooleanValidator("planner.enable_hashjoin_swap", true); - public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100, 10d); + public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding"); + public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges"); + public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg"); + public static final OptionValidator STREAMAGG = new BooleanValidator("planner.enable_streamagg"); + public static final OptionValidator HASHJOIN = new BooleanValidator("planner.enable_hashjoin"); + public static final OptionValidator MERGEJOIN = new BooleanValidator("planner.enable_mergejoin"); + public static final OptionValidator NESTEDLOOPJOIN = new BooleanValidator("planner.enable_nestedloopjoin"); + public static final OptionValidator MULTIPHASE = new BooleanValidator("planner.enable_multiphase_agg"); + public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join"); + public static final OptionValidator BROADCAST_THRESHOLD = new PositiveLongValidator("planner.broadcast_threshold", MAX_BROADCAST_THRESHOLD); + public static final OptionValidator BROADCAST_FACTOR = new RangeDoubleValidator("planner.broadcast_factor", 0, Double.MAX_VALUE); + public static final OptionValidator NESTEDLOOPJOIN_FACTOR = new RangeDoubleValidator("planner.nestedloopjoin_factor", 0, Double.MAX_VALUE); + public static final OptionValidator NLJOIN_FOR_SCALAR = new BooleanValidator("planner.enable_nljoin_for_scalar_only"); + public static final OptionValidator JOIN_ROW_COUNT_ESTIMATE_FACTOR = new RangeDoubleValidator("planner.join.row_count_estimate_factor", 0, Double.MAX_VALUE); + public static final OptionValidator MUX_EXCHANGE = new BooleanValidator("planner.enable_mux_exchange"); + public static final OptionValidator DEMUX_EXCHANGE = new BooleanValidator("planner.enable_demux_exchange"); + public static final OptionValidator PARTITION_SENDER_THREADS_FACTOR = new LongValidator("planner.partitioner_sender_threads_factor"); + public static final OptionValidator PARTITION_SENDER_MAX_THREADS = new LongValidator("planner.partitioner_sender_max_threads"); + public static final OptionValidator PARTITION_SENDER_SET_THREADS = new LongValidator("planner.partitioner_sender_set_threads"); + public static final OptionValidator PRODUCER_CONSUMER = new BooleanValidator("planner.add_producer_consumer"); + public static final OptionValidator PRODUCER_CONSUMER_QUEUE_SIZE = new LongValidator("planner.producer_consumer_queue_size"); + public static final OptionValidator HASH_SINGLE_KEY = new BooleanValidator("planner.enable_hash_single_key"); + public static final OptionValidator HASH_JOIN_SWAP = new BooleanValidator("planner.enable_hashjoin_swap"); + public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100); public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type"; - public static final BooleanValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false); - public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt", true); - public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning", true); + public static final BooleanValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY); + public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt"); + public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning"); public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit", - INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES); + INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES); public static final String UNIONALL_DISTRIBUTE_KEY = "planner.enable_unionall_distribute"; - public static final BooleanValidator UNIONALL_DISTRIBUTE = new BooleanValidator(UNIONALL_DISTRIBUTE_KEY, false); + public static final BooleanValidator UNIONALL_DISTRIBUTE = new BooleanValidator(UNIONALL_DISTRIBUTE_KEY); public static final OptionValidator IDENTIFIER_MAX_LENGTH = new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */, - Integer.MAX_VALUE, DEFAULT_IDENTIFIER_MAX_LENGTH); + Integer.MAX_VALUE); public static final DoubleValidator FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR = new MinRangeDoubleValidator("planner.filter.min_selectivity_estimate_factor", @@ -96,15 +96,15 @@ public class PlannerSettings implements Context{ 0.0, 1.0, 1.0d, "planner.filter.min_selectivity_estimate_factor"); public static final String TYPE_INFERENCE_KEY = "planner.enable_type_inference"; - public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY, true); + public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY); public static final LongValidator IN_SUBQUERY_THRESHOLD = - new PositiveLongValidator("planner.in_subquery_threshold", Integer.MAX_VALUE, 20); /* Same as Calcite's default IN List subquery size */ + new PositiveLongValidator("planner.in_subquery_threshold", Integer.MAX_VALUE); /* Same as Calcite's default IN List subquery size */ - public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY = "planner.store.parquet.rowgroup.filter.pushdown"; - public static final BooleanValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING = new BooleanValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY, true); + public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY = "planner.store.parquet.rowgroup.filter.pushdown.enabled"; + public static final BooleanValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING = new BooleanValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY); public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY = "planner.store.parquet.rowgroup.filter.pushdown.threshold"; public static final PositiveLongValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD = new PositiveLongValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY, - Long.MAX_VALUE, 10000); + Long.MAX_VALUE); public static final String QUOTING_IDENTIFIERS_KEY = "planner.parser.quoting_identifiers"; public static final EnumeratedStringValidator QUOTING_IDENTIFIERS = new EnumeratedStringValidator( @@ -132,10 +132,10 @@ public class PlannerSettings implements Context{ Note: once hash and merge joins will allow non-equi join conditions, the need to turn off join optimization may go away. */ - public static final BooleanValidator JOIN_OPTIMIZATION = new BooleanValidator("planner.enable_join_optimization", true); + public static final BooleanValidator JOIN_OPTIMIZATION = new BooleanValidator("planner.enable_join_optimization"); // for testing purpose public static final String FORCE_2PHASE_AGGR_KEY = "planner.force_2phase_aggr"; - public static final BooleanValidator FORCE_2PHASE_AGGR = new BooleanValidator(FORCE_2PHASE_AGGR_KEY, false); + public static final BooleanValidator FORCE_2PHASE_AGGR = new BooleanValidator(FORCE_2PHASE_AGGR_KEY); public OptionManager options = null; public FunctionImplementationRegistry functionImplementationRegistry = null; http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index e03a40c..bdf12dd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -448,8 +448,8 @@ public class DefaultSqlHandler extends AbstractSqlHandler { log("Not enough memory for this plan", phyRelNode, logger, null); logger.debug("Re-planning without hash operations."); - queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false)); - queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false)); + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false, OptionValue.OptionScope.QUERY)); + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false, OptionValue.OptionScope.QUERY)); try { final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits); http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java index f278989..95115f6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java @@ -31,6 +31,7 @@ import org.apache.drill.exec.planner.sql.DirectPlan; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.calcite.sql.SqlLiteral; @@ -64,15 +65,19 @@ public class SetOptionHandler extends AbstractSqlHandler { final String scope = option.getScope(); final OptionValue.OptionType type; + final OptionValue.OptionScope optionScope; if (scope == null) { // No scope mentioned assumed SESSION type = OptionType.SESSION; + optionScope = OptionScope.SESSION; } else { switch (scope.toLowerCase()) { case "session": type = OptionType.SESSION; + optionScope = OptionScope.SESSION; break; case "system": type = OptionType.SYSTEM; + optionScope = OptionScope.SYSTEM; break; default: throw UserException.validationError() @@ -99,7 +104,7 @@ public class SetOptionHandler extends AbstractSqlHandler { // Currently, we convert multi-part identifier to a string. final String name = option.getName().toString(); if (value != null) { // SET option - final OptionValue optionValue = createOptionValue(name, type, (SqlLiteral) value); + final OptionValue optionValue = createOptionValue(name, type, (SqlLiteral) value, optionScope); options.setOption(optionValue); } else { // RESET option if ("ALL".equalsIgnoreCase(name)) { @@ -113,36 +118,36 @@ public class SetOptionHandler extends AbstractSqlHandler { } private static OptionValue createOptionValue(final String name, final OptionValue.OptionType type, - final SqlLiteral literal) { + final SqlLiteral literal, final OptionValue.OptionScope scope) { final Object object = literal.getValue(); final SqlTypeName typeName = literal.getTypeName(); switch (typeName) { case DECIMAL: { final BigDecimal bigDecimal = (BigDecimal) object; if (bigDecimal.scale() == 0) { - return OptionValue.createLong(type, name, bigDecimal.longValue()); + return OptionValue.createLong(type, name, bigDecimal.longValue(), scope); } else { - return OptionValue.createDouble(type, name, bigDecimal.doubleValue()); + return OptionValue.createDouble(type, name, bigDecimal.doubleValue(), scope); } } case DOUBLE: case FLOAT: - return OptionValue.createDouble(type, name, ((BigDecimal) object).doubleValue()); + return OptionValue.createDouble(type, name, ((BigDecimal) object).doubleValue(), scope); case SMALLINT: case TINYINT: case BIGINT: case INTEGER: - return OptionValue.createLong(type, name, ((BigDecimal) object).longValue()); + return OptionValue.createLong(type, name, ((BigDecimal) object).longValue(), scope); case VARBINARY: case VARCHAR: case CHAR: - return OptionValue.createString(type, name, ((NlsString) object).getValue()); + return OptionValue.createString(type, name, ((NlsString) object).getValue(), scope); case BOOLEAN: - return OptionValue.createBoolean(type, name, (Boolean) object); + return OptionValue.createBoolean(type, name, (Boolean) object, scope); default: throw UserException.validationError() http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java index b64ed14..9c94065 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java @@ -86,8 +86,8 @@ public class InboundImpersonationManager { */ public static class InboundImpersonationPolicyValidator extends StringValidator { - public InboundImpersonationPolicyValidator(String name, String def) { - super(name, def, true); + public InboundImpersonationPolicyValidator(String name) { + super(name, true); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java index fc647c8..c30d1b3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java @@ -41,6 +41,8 @@ import org.apache.drill.exec.proto.UserBitShared.UserCredentials; import org.apache.drill.exec.proto.UserProtos.UserProperties; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; +import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.apache.drill.exec.server.options.SessionOptionManager; import com.google.common.collect.Maps; @@ -247,8 +249,8 @@ public class UserSession implements AutoCloseable { * @param value option value */ public void setSessionOption(String name, String value) { - OptionValue.Kind optionKind = SystemOptionManager.getValidator(name).getKind(); - OptionValue optionValue = OptionValue.createOption(optionKind, OptionValue.OptionType.SESSION, name, value); + OptionValue.Kind optionKind = ((SessionOptionManager) sessionOptions).getFallbackOptionManager().getValidator(name).getKind(); + OptionValue optionValue = OptionValue.createOption(optionKind, OptionType.SESSION, name, value, OptionScope.SESSION); sessionOptions.setOption(optionValue); } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java index 7ed9220..6f22b76 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java @@ -34,6 +34,7 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.server.rest.WebServer; import org.apache.drill.exec.service.ServiceEngine; import org.apache.drill.exec.store.StoragePluginRegistry; @@ -226,7 +227,7 @@ public class Drillbit implements AutoCloseable { } final OptionValue optionValue = OptionValue.createOption( - defaultValue.kind, OptionType.SYSTEM, optionName, optionString); + defaultValue.kind, OptionType.SYSTEM, optionName, optionString, OptionScope.SYSTEM); optionManager.setOption(optionValue); } } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 973b97c..fb3ccf8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -104,7 +104,7 @@ public class DrillbitContext implements AutoCloseable { this.reader = new PhysicalPlanReader(context.getConfig(), classpathScan, lpPersistence, endpoint, storagePlugins); this.operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); - this.systemOptions = new SystemOptionManager(lpPersistence, provider); + this.systemOptions = new SystemOptionManager(lpPersistence, provider,context.getConfig()); this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), classpathScan, systemOptions); this.compiler = new CodeCompiler(context.getConfig(), systemOptions); http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java index 299b221..9eba2a3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java @@ -56,4 +56,5 @@ public abstract class BaseOptionManager implements OptionSet { public String getOption(StringValidator validator) { return getOptionSafe(validator).string_val; } + } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java index 34ebf80..2b3fdfc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java @@ -22,6 +22,7 @@ import java.util.Map.Entry; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import com.typesafe.config.ConfigValue; @@ -57,17 +58,17 @@ public class DrillConfigIterator implements Iterable<OptionValue> { OptionValue optionValue = null; switch(cv.valueType()) { case BOOLEAN: - optionValue = OptionValue.createBoolean(OptionType.BOOT, name, (Boolean) cv.unwrapped()); + optionValue = OptionValue.createBoolean(OptionType.BOOT, name, (Boolean) cv.unwrapped(), OptionScope.BOOT); break; case LIST: case OBJECT: case STRING: - optionValue = OptionValue.createString(OptionType.BOOT, name, cv.render()); + optionValue = OptionValue.createString(OptionType.BOOT, name, cv.render(),OptionScope.BOOT); break; case NUMBER: - optionValue = OptionValue.createLong(OptionType.BOOT, name, ((Number) cv.unwrapped()).longValue()); + optionValue = OptionValue.createLong(OptionType.BOOT, name, ((Number) cv.unwrapped()).longValue(),OptionScope.BOOT); break; case NULL: http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java index 373b0d2..301fa34 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java @@ -108,7 +108,7 @@ public abstract class FallbackOptionManager extends BaseOptionManager implements @Override public void setOption(OptionValue value) { - final OptionValidator validator = SystemOptionManager.getValidator(value.name); + final OptionValidator validator = getSystemOptionManager().getValidator(value.name); validator.validate(value, this); // validate the option @@ -120,7 +120,7 @@ public abstract class FallbackOptionManager extends BaseOptionManager implements @Override public void deleteOption(final String name, final OptionType type) { - SystemOptionManager.getValidator(name); // ensure the option exists + getSystemOptionManager().getValidator(name); // ensure the option exists // fallback if unable to delete locally if (!deleteLocalOption(name, type)) { @@ -144,4 +144,27 @@ public abstract class FallbackOptionManager extends BaseOptionManager implements } return list; } + + public OptionManager getFallback() { + return fallback; + } + + /** + * {@link FragmentOptionManager} and {@link SessionOptionManager} use {@link SystemOptionManager} as the fall back + * manager so for both FragmentOptionManager and SessionOptionManager fallback is the SystemOptionManager so it is + * returned. But in case of {@link QueryOptionManager}, it uses {@link SessionOptionManager} as the fallback manager + * and since SessionOptionManager uses SystemOptionManager as fallback, SystemOptionManager can be fetched from the + * SessionOptionManager. + */ + public SystemOptionManager getSystemOptionManager() { + final SystemOptionManager systemOptionManager; + if(fallback instanceof SessionOptionManager) { + final SessionOptionManager sessionOptionManager = (SessionOptionManager) fallback; + systemOptionManager = sessionOptionManager.getFallbackOptionManager(); + } + else { + systemOptionManager = (SystemOptionManager) fallback; + } + return systemOptionManager; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java index 951cbc4..7122a19 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.options; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.server.options.OptionValue.Kind; @@ -28,7 +29,7 @@ public abstract class OptionValidator { // the error messages produced by the validator private final String optionName; private final boolean isAdminOption; - + public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options."; /** By default, if admin option value is not specified, it would be set to false.*/ public OptionValidator(String optionName) { this(optionName, false); @@ -106,4 +107,11 @@ public abstract class OptionValidator { */ public abstract Kind getKind(); + /** + * Loads the default option value for this validator. + * + * @return default option value + */ + public abstract void loadDefault(DrillConfig bootConfig); + } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java index 3c07608..5883480 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java @@ -22,14 +22,22 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; import org.apache.drill.exec.store.sys.PersistentStore; +import com.google.common.base.Preconditions; /** + * <p> * An {@link OptionValue option value} is used by an {@link OptionManager} to store a run-time setting. This setting, * for example, could affect a query in execution stage. Instances of this class are JSON serializable and can be stored * in a {@link PersistentStore persistent store} (see {@link SystemOptionManager#options}), or * in memory (see {@link InMemoryOptionManager#options}). + * </p> + *<p> OptionType defines the permission level of the option.If it can be set at System level or Session level or so on. + * Whereas OptionScope defines the scope at which the option is being set. If the option is being set at the BOOT time + * the scope of the option is BOOT. If it is set at SYSTEM level the scope is SYSTEM. Although they look similar there + * is a fine level which differentiates both of them which is at which level of hierarchy they can be set and + * at what at level of hierarchy they were actually set. + * </p> */ @JsonInclude(Include.NON_NULL) public class OptionValue implements Comparable<OptionValue> { @@ -42,6 +50,10 @@ public class OptionValue implements Comparable<OptionValue> { BOOLEAN, LONG, STRING, DOUBLE } + public enum OptionScope { + BOOT, SYSTEM, SESSION, QUERY + } + public final String name; public final Kind kind; public final OptionType type; @@ -49,33 +61,34 @@ public class OptionValue implements Comparable<OptionValue> { public final String string_val; public final Boolean bool_val; public final Double float_val; + public final OptionScope scope; - public static OptionValue createLong(OptionType type, String name, long val) { - return new OptionValue(Kind.LONG, type, name, val, null, null, null); + public static OptionValue createLong(OptionType type, String name, long val, OptionScope scope) { + return new OptionValue(Kind.LONG, type, name, val, null, null, null, scope); } - public static OptionValue createBoolean(OptionType type, String name, boolean bool) { - return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null); + public static OptionValue createBoolean(OptionType type, String name, boolean bool, OptionScope scope) { + return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null, scope); } - public static OptionValue createString(OptionType type, String name, String val) { - return new OptionValue(Kind.STRING, type, name, null, val, null, null); + public static OptionValue createString(OptionType type, String name, String val, OptionScope scope) { + return new OptionValue(Kind.STRING, type, name, null, val, null, null, scope); } - public static OptionValue createDouble(OptionType type, String name, double val) { - return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val); + public static OptionValue createDouble(OptionType type, String name, double val, OptionScope scope) { + return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val, scope); } - public static OptionValue createOption(Kind kind, OptionType type, String name, String val) { + public static OptionValue createOption(Kind kind, OptionType type, String name, String val, OptionScope scope) { switch (kind) { case BOOLEAN: - return createBoolean(type, name, Boolean.valueOf(val)); + return createBoolean(type, name, Boolean.valueOf(val), scope); case LONG: - return createLong(type, name, Long.valueOf(val)); + return createLong(type, name, Long.valueOf(val), scope); case STRING: - return createString(type, name, val); + return createString(type, name, val, scope); case DOUBLE: - return createDouble(type, name, Double.valueOf(val)); + return createDouble(type, name, Double.valueOf(val), scope); default: return null; } @@ -88,16 +101,17 @@ public class OptionValue implements Comparable<OptionValue> { @JsonProperty("num_val") Long num_val, @JsonProperty("string_val") String string_val, @JsonProperty("bool_val") Boolean bool_val, - @JsonProperty("float_val") Double float_val) { + @JsonProperty("float_val") Double float_val, + @JsonProperty("scope") OptionScope scope) { Preconditions.checkArgument(num_val != null || string_val != null || bool_val != null || float_val != null); this.kind = kind; this.type = type; this.name = name; - this.float_val = float_val; this.num_val = num_val; this.string_val = string_val; this.bool_val = bool_val; + this.scope = scope; } public String getName() { http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java index 38f8556..512c8e9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java @@ -61,7 +61,7 @@ public class SessionOptionManager extends InMemoryOptionManager { return false; } final String name = value.name; - final OptionValidator validator = SystemOptionManager.getValidator(name); // if set, validator must exist. + final OptionValidator validator = getFallbackOptionManager().getValidator(name); // if set, validator must exist. final boolean shortLived = validator.isShortLived(); if (shortLived) { final int start = session.getQueryCount() + 1; // start from the next query @@ -83,7 +83,7 @@ public class SessionOptionManager extends InMemoryOptionManager { final int start = shortLivedOptions.get(name).getLeft(); // option is not in effect if queryNumber < start if (queryNumber < start) { - return SystemOptionManager.getValidator(name).getDefault(); + return getFallbackOptionManager().getValidator(name).getDefault(); // reset if queryNumber <= end } else { options.remove(name); @@ -120,4 +120,10 @@ public class SessionOptionManager extends InMemoryOptionManager { boolean supportsOptionType(OptionType type) { return type == OptionType.SESSION; } + + /* Gets fallback manager and returns it as SystemOptionManager */ + public SystemOptionManager getFallbackOptionManager() { + final SystemOptionManager systemOptionManager = (SystemOptionManager) getFallback(); + return systemOptionManager; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 3392a21..d2dfc2a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -26,7 +26,9 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import com.typesafe.config.ConfigException; import org.apache.commons.collections.IteratorUtils; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.map.CaseInsensitiveMap; @@ -48,12 +50,31 @@ import com.google.common.collect.Sets; * Only one instance of this class exists per drillbit. Options set at the system level affect the entire system and * persist between restarts. */ + +/** + * <p> All the system options are externalized into conf file. While adding a new system option + * a validator should be added and the default value for the option should be set in + * the conf files(example : drill-module.conf) under the namespace drill.exec.options. + * </p> + * + * <p> + * The SystemOptionManager loads all the validators and the default values for the options are + * fetched from the config. The validators are populated with the default values fetched from + * the config. If the option is not set in the conf files config option is missing exception + * will be thrown. + * </p> + * + * <p> + * If the option is set using ALTER, the value that is set will be returned. Else the default value + * that is loaded into validator from the config will be returned. + * </p> + */ + public class SystemOptionManager extends BaseOptionManager implements OptionManager, AutoCloseable { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class); - private static final CaseInsensitiveMap<OptionValidator> VALIDATORS; - - static { + private CaseInsensitiveMap<OptionValidator> VALIDATORS; + public void populateValidators() { final OptionValidator[] validators = new OptionValidator[]{ PlannerSettings.CONSTANT_FOLDING, PlannerSettings.EXCHANGE, @@ -177,8 +198,9 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana ExecConstants.USE_DYNAMIC_UDFS, ExecConstants.QUERY_TRANSIENT_STATE_UPDATE, ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR, - ExecConstants.ENABLE_ITERATOR_VALIDATOR, - ExecConstants.ENABLE_VECTOR_VALIDATOR + ExecConstants.CPU_LOAD_AVERAGE, + ExecConstants.ENABLE_VECTOR_VALIDATOR, + ExecConstants.ENABLE_ITERATOR_VALIDATOR }; final Map<String, OptionValidator> tmp = new HashMap<>(); for (final OptionValidator validator : validators) { @@ -194,6 +216,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana private final PersistentStoreProvider provider; + private final DrillConfig bootConfig; /** * Persistent store for options that have been changed from default. * NOTE: CRUD operations must use lowercase keys. @@ -201,10 +224,35 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana private PersistentStore<OptionValue> options; public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider) { + this(lpPersistence,provider,null); + } + + public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider, final DrillConfig bootConfig) { this.provider = provider; - this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class) - .name("sys.options") - .build(); + this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class) + .name("sys.options") + .build(); + this.bootConfig = bootConfig; + populateValidators(); + populateDefaultValues(); + + } + + /** + * Gets the {@link OptionValidator} associated with the name. + * + * @param name name of the option + * @return the associated validator + * @throws UserException - if the validator is not found + */ + public OptionValidator getValidator(final String name) { + final OptionValidator validator = VALIDATORS.get(name); + if (validator == null) { + throw UserException.validationError() + .message(String.format("The option '%s' does not exist.", name.toLowerCase())) + .build(logger); + } + return validator; } /** @@ -237,23 +285,6 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana return this; } - /** - * Gets the {@link OptionValidator} associated with the name. - * - * @param name name of the option - * @return the associated validator - * @throws UserException - if the validator is not found - */ - public static OptionValidator getValidator(final String name) { - final OptionValidator validator = VALIDATORS.get(name); - if (validator == null) { - throw UserException.validationError() - .message(String.format("The option '%s' does not exist.", name)) - .build(logger); - } - return validator; - } - @Override public Iterator<OptionValue> iterator() { final Map<String, OptionValue> buildList = CaseInsensitiveMap.newHashMap(); @@ -276,7 +307,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana return value; } - // otherwise, return default. + // If option is not set return the default set in the validator. final OptionValidator validator = getValidator(name); return validator.getDefault(); } @@ -288,7 +319,6 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana final OptionValidator validator = getValidator(name); validator.validate(value, this); // validate the option - if (options.get(name) == null && value.equals(validator.getDefault())) { return; // if the option is not overridden, ignore setting option to default } @@ -315,6 +345,19 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana } } + public void populateDefaultValues() { + + // populate the options from the config + final Map<String, OptionValidator> tmp = new HashMap<>(); + for (OptionValidator validator: VALIDATORS.values()) { + try { + validator.loadDefault(bootConfig); + } catch (ConfigException.Missing e) { + throw new IllegalStateException("Config Option is missing"+ validator.getOptionName()); + } + } + } + @Override public OptionList getOptionList() { return (OptionList) IteratorUtils.toList(iterator());
