Drill configuration changes + Moved bootstrap-storage-plugins.json from "conf/" to "java-exec" jar. + Cleared drill-override.conf and added drill-override-example.conf + Updated maven assembly descriptor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/41f66160 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/41f66160 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/41f66160 Branch: refs/heads/master Commit: 41f661609ca716553e9ad1e099f75cf17678397c Parents: fd1be92 Author: Aditya Kishore <[email protected]> Authored: Fri Jun 6 16:31:31 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sun Jun 8 19:13:05 2014 -0700 ---------------------------------------------------------------------- distribution/src/assemble/bin.xml | 2 +- .../resources/bootstrap-storage-plugins.json | 70 --------- .../src/resources/drill-override-example.conf | 146 +++++++++++++++++++ distribution/src/resources/drill-override.conf | 138 +++--------------- .../resources/bootstrap-storage-plugins.json | 70 +++++++++ .../src/main/resources/drill-module.conf | 8 +- 6 files changed, 241 insertions(+), 193 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index ed44c84..b0a07df 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -145,7 +145,7 @@ <outputDirectory>conf</outputDirectory> </file> <file> - <source>src/resources/bootstrap-storage-plugins.json</source> + <source>src/resources/drill-override-example.conf</source> <outputDirectory>conf</outputDirectory> </file> </files> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/bootstrap-storage-plugins.json ---------------------------------------------------------------------- diff --git a/distribution/src/resources/bootstrap-storage-plugins.json b/distribution/src/resources/bootstrap-storage-plugins.json deleted file mode 100644 index 3b1cbd0..0000000 --- a/distribution/src/resources/bootstrap-storage-plugins.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "storage":{ - dfs: { - type: "file", - connection: "file:///", - workspaces: { - "root" : { - location: "/", - writable: false - }, - "tmp" : { - location: "/tmp", - writable: true, - storageformat: "csv" - } - }, - formats: { - "psv" : { - type: "text", - extensions: [ "tbl" ], - delimiter: "|" - }, - "csv" : { - type: "text", - extensions: [ "csv" ], - delimiter: "," - }, - "tsv" : { - type: "text", - extensions: [ "tsv" ], - delimiter: "\t" - }, - "parquet" : { - type: "parquet" - }, - "json" : { - type: "json" - } - } - }, - cp: { - type: "file", - connection: "classpath:///" - } - - /*, - hive : { - type:"hive", - config : - { - "hive.metastore.uris" : "", - "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true", - "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh", - "fs.default.name" : "file:///", - "hive.metastore.sasl.enabled" : "false" - } - } - */ - - /*, - hbase : { - type:"hbase", - config : { - "hbase.zookeeper.quorum" : "localhost", - "hbase.zookeeper.property.clientPort" : 2181 - } - } - */ - } -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/drill-override-example.conf ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf new file mode 100644 index 0000000..9d87f76 --- /dev/null +++ b/distribution/src/resources/drill-override-example.conf @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. +# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" + +drill.exec: { + cluster-id: "drillbits1" + rpc: { + user: { + server: { + port: 31010 + threads: 1 + } + client: { + threads: 1 + } + }, + bit: { + server: { + port : 31011, + retry:{ + count: 7200, + delay: 500 + }, + threads: 1 + } + }, + use.ip : false + }, + operator: { + packages += "org.apache.drill.exec.physical.config" + }, + optimizer: { + implementation: "org.apache.drill.exec.opt.IdentityOptimizer" + }, + functions: ["org.apache.drill.expr.fn.impl"], + storage: { + packages += "org.apache.drill.exec.store", + file: { + text: { + buffer.size: 262144, + batch.size: 4000 + }, + partition.column.label: "dir" + } + }, + metrics : { + context: "drillbit", + jmx: { + enabled : true + }, + log: { + enabled : false, + interval : 60 + } + }, + zk: { + connect: "localhost:2181", + root: "drill", + refresh: 500, + timeout: 5000, + retry: { + count: 7200, + delay: 500 + } + }, + http: { + enabled: true, + port: 8047 + }, + functions: ["org.apache.drill.expr.fn.impl"], + network: { + start: 35000 + }, + work: { + max.width.per.endpoint: 5, + global.max.width: 100, + affinity.factor: 1.2, + executor.threads: 4 + }, + sys.store.provider: { + class: "org.apache.drill.exec.store.sys.local.LocalPStoreProvider", + local: { + path: "/tmp/drill", + write: true + } + } + trace: { + directory: "/tmp/drill-trace", + filesystem: "file:///" + }, + tmp: { + directories: ["/tmp/drill"], + filesystem: "drill-local:///" + }, + buffer:{ + impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer", + size: "100", + spooling: { + delete: false, + size: 100000000 + } + }, + cache.hazel.subnets: ["*.*.*.*"], + sort: { + purge.threshold : 100, + external: { + batch.size : 4000, + spill: { + batch.size : 4000, + group.size : 100, + threshold : 200, + directories : [ "/tmp/drill/spill" ], + fs : "file:///" + } + } + }, + memory: { + top.max: 1000000000000, + operator: { + max: 20000000000, + initial: 10000000 + }, + fragment: { + max: 20000000000, + initial: 20000000 + } + }, + debug.error_on_leak: true +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/distribution/src/resources/drill-override.conf ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-override.conf b/distribution/src/resources/drill-override.conf index 472296b..b484ea3 100644 --- a/distribution/src/resources/drill-override.conf +++ b/distribution/src/resources/drill-override.conf @@ -1,123 +1,25 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -// This file tells Drill to consider this module when class path scanning. -// This file can also include any supplementary configuration information. -// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. +# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" +# See 'drill-override-example.conf' for example configurations drill.exec: { - cluster-id: "drillbits1" - rpc: { - user: { - server: { - port: 31010 - threads: 1 - } - client: { - threads: 1 - } - }, - bit: { - server: { - port : 31011, - retry:{ - count: 7200, - delay: 500 - }, - threads: 1 - } - }, - use.ip : false - }, - operator: { - packages += "org.apache.drill.exec.physical.config" - }, - optimizer: { - implementation: "org.apache.drill.exec.opt.IdentityOptimizer" - }, - functions: ["org.apache.drill.expr.fn.impl"], - storage: { - packages += "org.apache.drill.exec.store", - file: { - text: { - buffer.size: 262144, - batch.size: 4000 - }, - partition.column.label: "dir" - } - }, - metrics : { - context: "drillbit", - jmx: { - enabled : true - }, - log: { - enabled : false, - interval : 60 - } - }, - zk: { - connect: "localhost:2181", - root: "/drill", - refresh: 500, - timeout: 5000, - retry: { - count: 7200, - delay: 500 - } - }, - functions: ["org.apache.drill.expr.fn.impl"], - network: { - start: 35000 - }, - work: { - max.width.per.endpoint: 5, - global.max.width: 100, - affinity.factor: 1.2, - executor.threads: 4 - }, - trace: { - directory: "/tmp/drill-trace", - filesystem: "file:///" - }, - tmp: { - directories: ["/tmp/drill"], - filesystem: "drill-local:///" - }, - buffer:{ - impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer", - size: "100", - spooling: { - delete: false, - size: 100000000 - } - }, - cache.hazel.subnets: ["*.*.*.*"], - sort: { - purge.threshold : 100, - external: { - batch.size : 4000, - spill: { - batch.size : 4000, - group.size : 100, - threshold : 200, - directories : [ "/tmp/drill/spill" ], - fs : "file:///" - } - } - } + cluster-id: "drillbits1", + zk.connect: "localhost:2181" } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json new file mode 100644 index 0000000..3b1cbd0 --- /dev/null +++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json @@ -0,0 +1,70 @@ +{ + "storage":{ + dfs: { + type: "file", + connection: "file:///", + workspaces: { + "root" : { + location: "/", + writable: false + }, + "tmp" : { + location: "/tmp", + writable: true, + storageformat: "csv" + } + }, + formats: { + "psv" : { + type: "text", + extensions: [ "tbl" ], + delimiter: "|" + }, + "csv" : { + type: "text", + extensions: [ "csv" ], + delimiter: "," + }, + "tsv" : { + type: "text", + extensions: [ "tsv" ], + delimiter: "\t" + }, + "parquet" : { + type: "parquet" + }, + "json" : { + type: "json" + } + } + }, + cp: { + type: "file", + connection: "classpath:///" + } + + /*, + hive : { + type:"hive", + config : + { + "hive.metastore.uris" : "", + "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true", + "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh", + "fs.default.name" : "file:///", + "hive.metastore.sasl.enabled" : "false" + } + } + */ + + /*, + hbase : { + type:"hbase", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } + } + */ + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/41f66160/exec/java-exec/src/main/resources/drill-module.conf ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 9ce22c7..7f399b2 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -60,7 +60,7 @@ drill.exec: { partition.column.label: "dir" } }, - metrics : { + metrics : { context: "drillbit", jmx: { enabled : true @@ -72,13 +72,13 @@ drill.exec: { }, zk: { connect: "localhost:2181", - root: "drill/happy", + root: "drill", refresh: 500, timeout: 5000, retry: { count: 7200, delay: 500 - } + } }, http: { enabled: true, @@ -111,7 +111,7 @@ drill.exec: { }, buffer:{ impl: "org.apache.drill.exec.work.batch.UnlimitedRawBatchBuffer", - size: "20", + size: "100", spooling: { delete: false, size: 100000000
