KYLIN-1672 support kylin on cdh 5.7 Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3909d8d6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3909d8d6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3909d8d6 Branch: refs/heads/yang21-cdh5.7 Commit: 3909d8d6f52d07b2f912dc13d5d89c02ce2ee717 Parents: 7b95bae Author: Lynne Jiang <lynne...@hotmail.com> Authored: Mon May 16 03:33:27 2016 -0700 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Nov 10 13:31:13 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 3 + dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0 .../kylin/engine/mr/steps/MockupMapContext.java | 15 +- examples/test_case_data/sandbox/core-site.xml | 146 +++--- examples/test_case_data/sandbox/hbase-site.xml | 162 ++---- examples/test_case_data/sandbox/hdfs-site.xml | 259 ++-------- examples/test_case_data/sandbox/mapred-site.xml | 398 ++++++--------- examples/test_case_data/sandbox/yarn-site.xml | 496 ++----------------- pom.xml | 16 +- server/pom.xml | 36 ++ .../storage/hbase/steps/MockupMapContext.java | 19 +- tool/pom.xml | 12 + 12 files changed, 431 insertions(+), 1131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index ed86bdb..ebd8cde 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=1000000 kylin.job.step.timeout=7200 +# for secure cdh, filtering hive dependency is risky, so filter nothing +kylin.job.dependency.filterlist=[^,]+ + ### CUBE ### # 'auto', 'inmem', 'layer' or 'random' for testing http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/dev-support/test_all_against_hdp_2_2_4_2_2.sh ---------------------------------------------------------------------- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh old mode 100644 new mode 100755 http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java index 847071d..9900465 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java @@ -77,6 +77,7 @@ public class MockupMapContext { outKV[0] = key; outKV[1] = value; } + } @Override @@ -99,6 +100,7 @@ public class MockupMapContext { throw new NotImplementedException(); } + @Override public float getProgress() { throw new NotImplementedException(); @@ -195,17 +197,17 @@ public class MockupMapContext { } @Override - public RawComparator<?> getSortComparator() { + public boolean userClassesTakesPrecedence() { throw new NotImplementedException(); } @Override - public String getJar() { + public RawComparator<?> getSortComparator() { throw new NotImplementedException(); } @Override - public RawComparator<?> getGroupingComparator() { + public String getJar() { throw new NotImplementedException(); } @@ -221,7 +223,7 @@ public class MockupMapContext { @Override public boolean getProfileEnabled() { - throw new NotImplementedException(); + return false; } @Override @@ -308,6 +310,11 @@ public class MockupMapContext { public RawComparator<?> getCombinerKeyGroupingComparator() { throw new NotImplementedException(); } + + @Override + public RawComparator<?> getGroupingComparator() { + return null; + } }); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/examples/test_case_data/sandbox/core-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml index 9aa588c..6162406 100644 --- a/examples/test_case_data/sandbox/core-site.xml +++ b/examples/test_case_data/sandbox/core-site.xml @@ -14,152 +14,146 @@ See the License for the specific language governing permissions and limitations under the License. --> +<!--Autogenerated by Cloudera Manager--> <configuration> - <property> <name>fs.defaultFS</name> - <value>hdfs://sandbox.hortonworks.com:8020</value> - <final>true</final> + <value>hdfs://quickstart.cloudera:8020</value> </property> - <property> <name>fs.trash.interval</name> - <value>360</value> + <value>1</value> </property> - <property> - <name>ha.failover-controller.active-standby-elector.zk.op.retries</name> - <value>120</value> + <name>io.compression.codecs</name> + <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value> </property> - <property> - <name>hadoop.http.authentication.simple.anonymous.allowed</name> - <value>true</value> + <name>hadoop.security.authentication</name> + <value>simple</value> </property> - <property> - <name>hadoop.proxyuser.falcon.groups</name> - <value>users</value> + <name>hadoop.security.authorization</name> + <value>false</value> + </property> + <property> + <name>hadoop.rpc.protection</name> + <value>authentication</value> + </property> + <property> + <name>hadoop.security.auth_to_local</name> + <value>DEFAULT</value> </property> - <property> - <name>hadoop.proxyuser.falcon.hosts</name> + <name>hadoop.proxyuser.oozie.hosts</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hbase.groups</name> - <value>users</value> + <name>hadoop.proxyuser.oozie.groups</name> + <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hbase.hosts</name> + <name>hadoop.proxyuser.mapred.hosts</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hcat.groups</name> + <name>hadoop.proxyuser.mapred.groups</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hcat.hosts</name> - <value>sandbox.hortonworks.com</value> + <name>hadoop.proxyuser.flume.hosts</name> + <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hive.groups</name> - <value>users</value> + <name>hadoop.proxyuser.flume.groups</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.HTTP.hosts</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.HTTP.groups</name> + <value>*</value> </property> - <property> <name>hadoop.proxyuser.hive.hosts</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.hue.groups</name> + <name>hadoop.proxyuser.hive.groups</name> <value>*</value> </property> - <property> <name>hadoop.proxyuser.hue.hosts</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.oozie.groups</name> + <name>hadoop.proxyuser.hue.groups</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.oozie.hosts</name> - <value>sandbox.hortonworks.com</value> + <name>hadoop.proxyuser.httpfs.hosts</name> + <value>*</value> </property> - <property> - <name>hadoop.proxyuser.root.groups</name> + <name>hadoop.proxyuser.httpfs.groups</name> <value>*</value> </property> - <property> - <name>hadoop.proxyuser.root.hosts</name> + <name>hadoop.proxyuser.hdfs.groups</name> <value>*</value> </property> - <property> - <name>hadoop.security.auth_to_local</name> - <value>DEFAULT</value> + <name>hadoop.proxyuser.hdfs.hosts</name> + <value>*</value> </property> - <property> - <name>hadoop.security.authentication</name> - <value>simple</value> + <name>hadoop.proxyuser.yarn.hosts</name> + <value>*</value> </property> - <property> - <name>hadoop.security.authorization</name> - <value>false</value> + <name>hadoop.proxyuser.yarn.groups</name> + <value>*</value> </property> - <property> - <name>io.compression.codecs</name> - <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value> + <name>hadoop.security.group.mapping</name> + <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value> </property> - <property> - <name>io.file.buffer.size</name> - <value>131072</value> + <name>hadoop.security.instrumentation.requires.admin</name> + <value>false</value> </property> - <property> - <name>io.serializations</name> - <value>org.apache.hadoop.io.serializer.WritableSerialization</value> + <name>net.topology.script.file.name</name> + <value>/etc/hadoop/conf.cloudera.yarn/topology.py</value> </property> - <property> - <name>ipc.client.connect.max.retries</name> - <value>50</value> + <name>io.file.buffer.size</name> + <value>65536</value> </property> - <property> - <name>ipc.client.connection.maxidletime</name> - <value>30000</value> + <name>hadoop.ssl.enabled</name> + <value>false</value> + </property> + <property> + <name>hadoop.ssl.require.client.cert</name> + <value>false</value> + <final>true</final> </property> - <property> - <name>ipc.client.idlethreshold</name> - <value>8000</value> + <name>hadoop.ssl.keystores.factory.class</name> + <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value> + <final>true</final> </property> - <property> - <name>ipc.server.tcpnodelay</name> - <value>true</value> + <name>hadoop.ssl.server.conf</name> + <value>ssl-server.xml</value> + <final>true</final> </property> - <property> - <name>mapreduce.jobtracker.webinterface.trusted</name> - <value>false</value> + <name>hadoop.ssl.client.conf</name> + <value>ssl-client.xml</value> + <final>true</final> </property> - </configuration> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/examples/test_case_data/sandbox/hbase-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 734908e..58c6223 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -15,180 +15,104 @@ limitations under the License. --> <configuration> - - <property> - <name>dfs.domain.socket.path</name> - <value>/var/lib/hadoop-hdfs/dn_socket</value> - </property> - <property> - <name>hbase.client.keyvalue.maxsize</name> - <value>10485760</value> - </property> - - <property> - <name>hbase.client.scanner.caching</name> - <value>100</value> + <name>hbase.rootdir</name> + <value>hdfs://quickstart.cloudera:8020/hbase</value> </property> - <property> - <name>hbase.cluster.distributed</name> + <name>hbase.replication</name> <value>true</value> </property> - - <property> - <name>hbase.coprocessor.master.classes</name> - <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value> - </property> - <property> - <name>hbase.coprocessor.region.classes</name> - <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value> + <name>hbase.client.write.buffer</name> + <value>2097152</value> </property> - - <property> - <name>hbase.defaults.for.version.skip</name> - <value>true</value> - </property> - <property> - <name>hbase.hregion.majorcompaction</name> - <value>604800000</value> + <name>hbase.client.pause</name> + <value>100</value> </property> - <property> - <name>hbase.hregion.majorcompaction.jitter</name> - <value>0.50</value> + <name>hbase.client.retries.number</name> + <value>35</value> </property> - <property> - <name>hbase.hregion.max.filesize</name> - <value>10737418240</value> + <name>hbase.client.scanner.caching</name> + <value>100</value> </property> - <property> - <name>hbase.hregion.memstore.block.multiplier</name> - <value>4</value> + <name>hbase.client.keyvalue.maxsize</name> + <value>10485760</value> </property> - <property> - <name>hbase.hregion.memstore.flush.size</name> - <value>134217728</value> - </property> - - <property> - <name>hbase.hregion.memstore.mslab.enabled</name> + <name>hbase.ipc.client.allowsInterrupt</name> <value>true</value> </property> - <property> - <name>hbase.hstore.blockingStoreFiles</name> + <name>hbase.client.primaryCallTimeout.get</name> <value>10</value> </property> - - <property> - <name>hbase.hstore.compactionThreshold</name> - <value>3</value> - </property> - <property> - <name>hbase.local.dir</name> - <value>${hbase.tmp.dir}/local</value> + <name>hbase.client.primaryCallTimeout.multiget</name> + <value>10</value> </property> - <property> - <name>hbase.master.info.bindAddress</name> - <value>0.0.0.0</value> + <name>hbase.regionserver.thrift.http</name> + <value>false</value> </property> - <property> - <name>hbase.master.info.port</name> - <value>60010</value> + <name>hbase.thrift.support.proxyuser</name> + <value>false</value> </property> - <property> - <name>hbase.master.port</name> + <name>hbase.rpc.timeout</name> <value>60000</value> </property> - - <property> - <name>hbase.regionserver.global.memstore.lowerLimit</name> - <value>0.38</value> - </property> - <property> - <name>hbase.regionserver.global.memstore.upperLimit</name> - <value>0.4</value> - </property> - - <property> - <name>hbase.regionserver.handler.count</name> - <value>60</value> + <name>hbase.snapshot.enabled</name> + <value>true</value> </property> - <property> - <name>hbase.regionserver.info.port</name> - <value>60030</value> + <name>hbase.snapshot.master.timeoutMillis</name> + <value>60000</value> </property> - <property> - <name>hbase.rootdir</name> - <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value> + <name>hbase.snapshot.region.timeout</name> + <value>60000</value> </property> - <property> - <name>hbase.rpc.protection</name> - <value>PRIVACY</value> + <name>hbase.snapshot.master.timeout.millis</name> + <value>60000</value> </property> - <property> <name>hbase.security.authentication</name> <value>simple</value> </property> - <property> - <name>hbase.security.authorization</name> - <value>true</value> + <name>hbase.rpc.protection</name> + <value>authentication</value> </property> - <property> - <name>hbase.superuser</name> - <value>hbase</value> + <name>zookeeper.session.timeout</name> + <value>60000</value> </property> - <property> - <name>hbase.tmp.dir</name> - <value>/hadoop/hbase</value> + <name>zookeeper.znode.parent</name> + <value>/hbase</value> </property> - <property> - <name>hbase.zookeeper.property.clientPort</name> - <value>2181</value> + <name>zookeeper.znode.rootserver</name> + <value>root-region-server</value> </property> - <property> <name>hbase.zookeeper.quorum</name> - <value>sandbox.hortonworks.com</value> + <value>quickstart.cloudera</value> </property> - <property> - <name>hbase.zookeeper.useMulti</name> - <value>true</value> - </property> - - <property> - <name>hfile.block.cache.size</name> - <value>0.40</value> - </property> - - <property> - <name>zookeeper.session.timeout</name> - <value>30000</value> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> </property> - <property> - <name>zookeeper.znode.parent</name> - <value>/hbase-unsecure</value> + <name>hbase.rest.ssl.enabled</name> + <value>false</value> </property> - </configuration> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/examples/test_case_data/sandbox/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml index 1175fff..05854bd 100644 --- a/examples/test_case_data/sandbox/hdfs-site.xml +++ b/examples/test_case_data/sandbox/hdfs-site.xml @@ -15,271 +15,68 @@ limitations under the License. --> <configuration> - - <property> - <name>dfs.block.access.token.enable</name> - <value>false</value> - </property> - - <property> - <name>dfs.block.size</name> - <value>34217472</value> - </property> - - <property> - <name>dfs.blockreport.initialDelay</name> - <value>120</value> - </property> - - <property> - <name>dfs.blocksize</name> - <value>134217728</value> - </property> - - <property> - <name>dfs.client.read.shortcircuit</name> - <value>true</value> - </property> - - <property> - <name>dfs.client.read.shortcircuit.streams.cache.size</name> - <value>4096</value> - </property> - - <property> - <name>dfs.cluster.administrators</name> - <value>hdfs</value> - </property> - - <property> - <name>dfs.datanode.address</name> - <value>0.0.0.0:50010</value> - </property> - - <property> - <name>dfs.datanode.balance.bandwidthPerSec</name> - <value>6250000</value> - </property> - - <property> - <name>dfs.datanode.data.dir</name> - <value>/hadoop/hdfs/data</value> - <final>true</final> - </property> - - <property> - <name>dfs.datanode.data.dir.perm</name> - <value>750</value> - </property> - <property> - <name>dfs.datanode.du.reserved</name> - <value>1073741824</value> - </property> - - <property> - <name>dfs.datanode.failed.volumes.tolerated</name> - <value>0</value> - <final>true</final> - </property> - - <property> - <name>dfs.datanode.http.address</name> - <value>0.0.0.0:50075</value> - </property> - - <property> - <name>dfs.datanode.https.address</name> - <value>0.0.0.0:50475</value> - </property> - - <property> - <name>dfs.datanode.ipc.address</name> - <value>0.0.0.0:8010</value> - </property> - - <property> - <name>dfs.datanode.max.transfer.threads</name> - <value>1024</value> - </property> - - <property> - <name>dfs.datanode.max.xcievers</name> - <value>1024</value> - </property> - - <property> - <name>dfs.domain.socket.path</name> - <value>/var/lib/hadoop-hdfs/dn_socket</value> - </property> - - <property> - <name>dfs.heartbeat.interval</name> - <value>3</value> + <name>dfs.namenode.name.dir</name> + <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value> </property> - <property> - <name>dfs.hosts.exclude</name> - <value>/etc/hadoop/conf/dfs.exclude</value> + <name>dfs.namenode.servicerpc-address</name> + <value>quickstart.cloudera:8022</value> </property> - <property> - <name>dfs.http.policy</name> - <value>HTTP_ONLY</value> + <name>dfs.https.address</name> + <value>quickstart.cloudera:50470</value> </property> - <property> <name>dfs.https.port</name> <value>50470</value> </property> - - <property> - <name>dfs.journalnode.edits.dir</name> - <value>/hadoop/hdfs/journalnode</value> - </property> - - <property> - <name>dfs.journalnode.http-address</name> - <value>0.0.0.0:8480</value> - </property> - - <property> - <name>dfs.journalnode.https-address</name> - <value>0.0.0.0:8481</value> - </property> - - <property> - <name>dfs.namenode.accesstime.precision</name> - <value>3600000</value> - </property> - - <property> - <name>dfs.namenode.avoid.read.stale.datanode</name> - <value>true</value> - </property> - - <property> - <name>dfs.namenode.avoid.write.stale.datanode</name> - <value>true</value> - </property> - - <property> - <name>dfs.namenode.checkpoint.dir</name> - <value>/hadoop/hdfs/namesecondary</value> - </property> - - <property> - <name>dfs.namenode.checkpoint.edits.dir</name> - <value>${dfs.namenode.checkpoint.dir}</value> - </property> - - <property> - <name>dfs.namenode.checkpoint.period</name> - <value>21600</value> - </property> - - <property> - <name>dfs.namenode.checkpoint.txns</name> - <value>1000000</value> - </property> - - <property> - <name>dfs.namenode.handler.count</name> - <value>100</value> - </property> - <property> <name>dfs.namenode.http-address</name> - <value>sandbox.hortonworks.com:50070</value> - <final>true</final> - </property> - - <property> - <name>dfs.namenode.https-address</name> - <value>sandbox.hortonworks.com:50470</value> - </property> - - <property> - <name>dfs.namenode.name.dir</name> - <value>/hadoop/hdfs/namenode</value> - <final>true</final> - </property> - - <property> - <name>dfs.namenode.name.dir.restore</name> - <value>true</value> - </property> - - <property> - <name>dfs.namenode.safemode.threshold-pct</name> - <value>1.0f</value> + <value>quickstart.cloudera:50070</value> </property> - <property> - <name>dfs.namenode.secondary.http-address</name> - <value>sandbox.hortonworks.com:50090</value> + <name>dfs.replication</name> + <value>1</value> </property> - <property> - <name>dfs.namenode.stale.datanode.interval</name> - <value>30000</value> + <name>dfs.blocksize</name> + <value>134217728</value> </property> - <property> - <name>dfs.namenode.startup.delay.block.deletion.sec</name> - <value>3600</value> + <name>dfs.client.use.datanode.hostname</name> + <value>false</value> </property> - <property> - <name>dfs.namenode.write.stale.datanode.ratio</name> - <value>1.0f</value> + <name>fs.permissions.umask-mode</name> + <value>022</value> </property> - <property> - <name>dfs.nfs.exports.allowed.hosts</name> - <value>* rw</value> + <name>dfs.namenode.acls.enabled</name> + <value>false</value> </property> - <property> - <name>dfs.nfs3.dump.dir</name> - <value>/tmp/.hdfs-nfs</value> + <name>dfs.client.use.legacy.blockreader</name> + <value>false</value> </property> - <property> - <name>dfs.permissions.enabled</name> - <value>true</value> - </property> - - <property> - <name>dfs.permissions.superusergroup</name> - <value>hdfs</value> + <name>dfs.client.read.shortcircuit</name> + <value>false</value> </property> - <property> - <name>dfs.replication</name> - <value>1</value> + <name>dfs.domain.socket.path</name> + <value>/var/run/hdfs-sockets/dn</value> </property> - <property> - <name>dfs.replication.max</name> - <value>50</value> + <name>dfs.client.read.shortcircuit.skip.checksum</name> + <value>false</value> </property> - <property> - <name>dfs.support.append</name> - <value>true</value> - <final>true</final> + <name>dfs.client.domain.socket.data.traffic</name> + <value>false</value> </property> - <property> - <name>dfs.webhdfs.enabled</name> + <name>dfs.datanode.hdfs-blocks-metadata.enabled</name> <value>true</value> - <final>true</final> </property> - - <property> - <name>fs.permissions.umask-mode</name> - <value>022</value> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/examples/test_case_data/sandbox/mapred-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml index 18f6feb..c9b1ca4 100644 --- a/examples/test_case_data/sandbox/mapred-site.xml +++ b/examples/test_case_data/sandbox/mapred-site.xml @@ -15,241 +15,165 @@ limitations under the License. --> <configuration> - - <property> - <name>io.sort.mb</name> - <value>128</value> - </property> - - <property> - <name>mapred.child.java.opts</name> - <value>-Xmx200m</value> - </property> - - <property> - <name>mapred.job.map.memory.mb</name> - <value>512</value> - </property> - - <property> - <name>mapred.job.reduce.memory.mb</name> - <value>512</value> - </property> - - <property> - <name>mapreduce.admin.map.child.java.opts</name> - <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> - </property> - - <property> - <name>mapreduce.admin.reduce.child.java.opts</name> - <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> - </property> - - <property> - <name>mapreduce.admin.user.env</name> - <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> - </property> - - <property> - <name>mapreduce.am.max-attempts</name> - <value>2</value> - </property> - - <property> - <name>mapreduce.application.classpath</name> - <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value> - </property> - - <property> - <name>mapreduce.application.framework.path</name> - <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value> - </property> - - <property> - <name>mapreduce.cluster.administrators</name> - <value>hadoop</value> - </property> - - <property> - <name>mapreduce.framework.name</name> - <value>yarn</value> - </property> - - <property> - <name>mapreduce.job.emit-timeline-data</name> - <value>false</value> - </property> - - <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource --> - <property> - <name>mapreduce.job.reduce.slowstart.completedmaps</name> - <value>1</value> - </property> - - <property> - <name>mapreduce.jobhistory.address</name> - <value>sandbox.hortonworks.com:10020</value> - </property> - - <property> - <name>mapreduce.jobhistory.bind-host</name> - <value>0.0.0.0</value> - </property> - - <property> - <name>mapreduce.jobhistory.done-dir</name> - <value>/mr-history/done</value> - </property> - - <property> - <name>mapreduce.jobhistory.intermediate-done-dir</name> - <value>/mr-history/tmp</value> - </property> - - <property> - <name>mapreduce.jobhistory.webapp.address</name> - <value>sandbox.hortonworks.com:19888</value> - </property> - - <property> - <name>mapreduce.map.java.opts</name> - <value>-Xmx512m</value> - </property> - - <property> - <name>mapreduce.map.log.level</name> - <value>INFO</value> - </property> - - <property> - <name>mapreduce.map.memory.mb</name> - <value>512</value> - </property> - - <property> - <name>mapreduce.map.output.compress</name> - <value>false</value> - </property> - - <property> - <name>mapreduce.map.sort.spill.percent</name> - <value>0.7</value> - </property> - - <property> - <name>mapreduce.map.speculative</name> - <value>false</value> - </property> - - <property> - <name>mapreduce.output.fileoutputformat.compress</name> - <value>false</value> - </property> - - <property> - <name>mapreduce.output.fileoutputformat.compress.type</name> - <value>BLOCK</value> - </property> - - <property> - <name>mapreduce.reduce.input.buffer.percent</name> - <value>0.0</value> - </property> - - <property> - <name>mapreduce.reduce.java.opts</name> - <value>-Xmx200m</value> - </property> - - <property> - <name>mapreduce.reduce.log.level</name> - <value>INFO</value> - </property> - - <property> - <name>mapreduce.reduce.memory.mb</name> - <value>512</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name> - <value>1</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name> - <value>1000</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name> - <value>30000</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.input.buffer.percent</name> - <value>0.7</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.merge.percent</name> - <value>0.66</value> - </property> - - <property> - <name>mapreduce.reduce.shuffle.parallelcopies</name> - <value>30</value> - </property> - - <property> - <name>mapreduce.reduce.speculative</name> - <value>false</value> - </property> - - <property> - <name>mapreduce.shuffle.port</name> - <value>13562</value> - </property> - - <property> - <name>mapreduce.task.io.sort.factor</name> - <value>100</value> - </property> - - <property> - <name>mapreduce.task.io.sort.mb</name> - <value>128</value> - </property> - - <property> - <name>mapreduce.task.timeout</name> - <value>300000</value> - </property> - - <property> - <name>yarn.app.mapreduce.am.admin-command-opts</name> - <value>-Dhdp.version=${hdp.version}</value> - </property> - - <property> - <name>yarn.app.mapreduce.am.command-opts</name> - <value>-Xmx512m</value> - </property> - - <property> - <name>yarn.app.mapreduce.am.log.level</name> - <value>INFO</value> - </property> - - <property> - <name>yarn.app.mapreduce.am.resource.mb</name> - <value>512</value> - </property> - - <property> - <name>yarn.app.mapreduce.am.staging-dir</name> - <value>/user</value> - </property> - +<property> + <name>mapreduce.job.split.metainfo.maxsize</name> + <value>10000000</value> +</property> +<property> + <name>mapreduce.job.counters.max</name> + <value>120</value> +</property> +<property> + <name>mapreduce.output.fileoutputformat.compress</name> + <value>false</value> +</property> +<property> + <name>mapreduce.output.fileoutputformat.compress.type</name> + <value>BLOCK</value> +</property> +<property> + <name>mapreduce.output.fileoutputformat.compress.codec</name> + <value>org.apache.hadoop.io.compress.DefaultCodec</value> +</property> +<property> + <name>mapreduce.map.output.compress.codec</name> + <value>org.apache.hadoop.io.compress.SnappyCodec</value> +</property> +<property> + <name>mapreduce.map.output.compress</name> + <value>true</value> +</property> +<property> + <name>zlib.compress.level</name> + <value>DEFAULT_COMPRESSION</value> +</property> +<property> + <name>mapreduce.task.io.sort.factor</name> + <value>64</value> +</property> +<property> + <name>mapreduce.map.sort.spill.percent</name> + <value>0.8</value> +</property> +<property> + <name>mapreduce.reduce.shuffle.parallelcopies</name> + <value>10</value> +</property> +<property> + <name>mapreduce.task.timeout</name> + <value>600000</value> +</property> +<property> + <name>mapreduce.client.submit.file.replication</name> + <value>1</value> +</property> +<property> + <name>mapreduce.job.reduces</name> + <value>1</value> +</property> +<property> + <name>mapreduce.task.io.sort.mb</name> + <value>16</value> +</property> +<property> + <name>mapreduce.map.speculative</name> + <value>false</value> +</property> +<property> + <name>mapreduce.reduce.speculative</name> + <value>false</value> +</property> +<property> + <name>mapreduce.job.reduce.slowstart.completedmaps</name> + <value>0.8</value> +</property> +<property> + <name>mapreduce.jobhistory.address</name> + <value>quickstart.cloudera:10020</value> +</property> +<property> + <name>mapreduce.jobhistory.webapp.address</name> + <value>quickstart.cloudera:19888</value> +</property> +<property> + <name>mapreduce.jobhistory.webapp.https.address</name> + <value>quickstart.cloudera:19890</value> +</property> +<property> + <name>mapreduce.jobhistory.admin.address</name> + <value>quickstart.cloudera:10033</value> +</property> +<property> + <name>mapreduce.framework.name</name> + <value>yarn</value> +</property> +<property> + <name>yarn.app.mapreduce.am.staging-dir</name> + <value>/user</value> +</property> +<property> + <name>mapreduce.am.max-attempts</name> + <value>2</value> +</property> +<property> + <name>yarn.app.mapreduce.am.resource.mb</name> + <value>128</value> +</property> +<property> + <name>yarn.app.mapreduce.am.resource.cpu-vcores</name> + <value>1</value> +</property> +<property> + <name>mapreduce.job.ubertask.enable</name> + <value>false</value> +</property> +<property> + <name>yarn.app.mapreduce.am.command-opts</name> + <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value> +</property> +<property> + <name>mapreduce.map.java.opts</name> + <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value> +</property> +<property> + <name>mapreduce.reduce.java.opts</name> + <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value> +</property> +<property> + <name>yarn.app.mapreduce.am.admin.user.env</name> + <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value> +</property> +<property> + <name>mapreduce.map.memory.mb</name> + <value>128</value> +</property> +<property> + <name>mapreduce.map.cpu.vcores</name> + <value>1</value> +</property> +<property> + <name>mapreduce.reduce.memory.mb</name> + <value>128</value> +</property> +<property> + <name>mapreduce.reduce.cpu.vcores</name> + <value>1</value> +</property> +<property> + <name>mapreduce.job.heap.memory-mb.ratio</name> + <value>0.8</value> +</property> +<property> + <name>mapreduce.application.classpath</name> + <value>/tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf</value> +</property> +<property> + <name>mapreduce.admin.user.env</name> + <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value> +</property> +<property> + <name>mapreduce.shuffle.max.connections</name> + <value>80</value> +</property> </configuration> + http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/examples/test_case_data/sandbox/yarn-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml index 8256158..8988d4a 100644 --- a/examples/test_case_data/sandbox/yarn-site.xml +++ b/examples/test_case_data/sandbox/yarn-site.xml @@ -15,520 +15,128 @@ limitations under the License. --> <configuration> - - <property> - <name>hadoop.registry.rm.enabled</name> - <value>false</value> - </property> - - <property> - <name>hadoop.registry.zk.quorum</name> - <value>sandbox.hortonworks.com:2181</value> - </property> - <property> <name>yarn.acl.enable</name> - <value>false</value> - </property> - - <property> - <name>yarn.admin.acl</name> - <value></value> - </property> - - <property> - <name>yarn.application.classpath</name> - <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value> - </property> - - <property> - <name>yarn.client.nodemanager-connect.max-wait-ms</name> - <value>60000</value> - </property> - - <property> - <name>yarn.client.nodemanager-connect.retry-interval-ms</name> - <value>10000</value> - </property> - - <property> - <name>yarn.http.policy</name> - <value>HTTP_ONLY</value> - </property> - - <property> - <name>yarn.log-aggregation-enable</name> - <value>true</value> - </property> - - <property> - <name>yarn.log-aggregation.retain-seconds</name> - <value>2592000</value> - </property> - - <property> - <name>yarn.log.server.url</name> - <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value> - </property> - - <property> - <name>yarn.node-labels.fs-store.retry-policy-spec</name> - <value>2000, 500</value> - </property> - - <property> - <name>yarn.node-labels.fs-store.root-dir</name> - <value>/system/yarn/node-labels</value> - </property> - - <property> - <name>yarn.node-labels.manager-class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value> - </property> - - <property> - <name>yarn.nodemanager.address</name> - <value>0.0.0.0:45454</value> - </property> - - <property> - <name>yarn.nodemanager.admin-env</name> - <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value> - </property> - - <property> - <name>yarn.nodemanager.aux-services</name> - <value>mapreduce_shuffle</value> - </property> - - <property> - <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name> - <value>org.apache.hadoop.mapred.ShuffleHandler</value> - </property> - - <property> - <name>yarn.nodemanager.bind-host</name> - <value>0.0.0.0</value> - </property> - - <property> - <name>yarn.nodemanager.container-executor.class</name> - <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value> - </property> - - <property> - <name>yarn.nodemanager.container-monitor.interval-ms</name> - <value>3000</value> - </property> - - <property> - <name>yarn.nodemanager.delete.debug-delay-sec</name> - <value>0</value> - </property> - - <property> - <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name> - <value>90</value> - </property> - - <property> - <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name> - <value>1000</value> - </property> - - <property> - <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name> - <value>0.25</value> - </property> - - <property> - <name>yarn.nodemanager.health-checker.interval-ms</name> - <value>135000</value> - </property> - - <property> - <name>yarn.nodemanager.health-checker.script.timeout-ms</name> - <value>60000</value> - </property> - - <property> - <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name> - <value>hadoop-yarn</value> - </property> - - <property> - <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name> - <value>false</value> - </property> - - <property> - <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name> - <value>false</value> - </property> - - <property> - <name>yarn.nodemanager.linux-container-executor.group</name> - <value>hadoop</value> - </property> - - <property> - <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name> - <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value> - </property> - - <property> - <name>yarn.nodemanager.local-dirs</name> - <value>/hadoop/yarn/local</value> - </property> - - <property> - <name>yarn.nodemanager.log-aggregation.compression-type</name> - <value>gz</value> - </property> - - <property> - <name>yarn.nodemanager.log-aggregation.debug-enabled</name> - <value>false</value> - </property> - - <property> - <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name> - <value>30</value> - </property> - - <property> - <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name> - <value>-1</value> - </property> - - <property> - <name>yarn.nodemanager.log-dirs</name> - <value>/hadoop/yarn/log</value> - </property> - - <property> - <name>yarn.nodemanager.log.retain-second</name> - <value>604800</value> - </property> - - <property> - <name>yarn.nodemanager.pmem-check-enabled</name> - <value>false</value> - </property> - - <property> - <name>yarn.nodemanager.recovery.dir</name> - <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value> - </property> - - <property> - <name>yarn.nodemanager.recovery.enabled</name> <value>true</value> </property> - - <property> - <name>yarn.nodemanager.remote-app-log-dir</name> - <value>/app-logs</value> - </property> - - <property> - <name>yarn.nodemanager.remote-app-log-dir-suffix</name> - <value>logs</value> - </property> - - <property> - <name>yarn.nodemanager.resource.cpu-vcores</name> - <value>8</value> - </property> - - <property> - <name>yarn.nodemanager.resource.memory-mb</name> - <value>9216</value> - </property> - - <property> - <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name> - <value>100</value> - </property> - - <property> - <name>yarn.nodemanager.vmem-check-enabled</name> - <value>false</value> - </property> - <property> - <name>yarn.nodemanager.vmem-pmem-ratio</name> - <value>10</value> + <name>yarn.admin.acl</name> + <value>*</value> </property> - <property> <name>yarn.resourcemanager.address</name> - <value>sandbox.hortonworks.com:8050</value> + <value>quickstart.cloudera:8032</value> </property> - <property> <name>yarn.resourcemanager.admin.address</name> - <value>sandbox.hortonworks.com:8141</value> - </property> - - <property> - <name>yarn.resourcemanager.am.max-attempts</name> - <value>2</value> - </property> - - <property> - <name>yarn.resourcemanager.bind-host</name> - <value>0.0.0.0</value> - </property> - - <property> - <name>yarn.resourcemanager.connect.max-wait.ms</name> - <value>900000</value> - </property> - - <property> - <name>yarn.resourcemanager.connect.retry-interval.ms</name> - <value>30000</value> - </property> - - <property> - <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name> - <value>2000, 500</value> - </property> - - <property> - <name>yarn.resourcemanager.fs.state-store.uri</name> - <value></value> - </property> - - <property> - <name>yarn.resourcemanager.ha.enabled</name> - <value>false</value> - </property> - - <property> - <name>yarn.resourcemanager.hostname</name> - <value>sandbox.hortonworks.com</value> - </property> - - <property> - <name>yarn.resourcemanager.nodes.exclude-path</name> - <value>/etc/hadoop/conf/yarn.exclude</value> - </property> - - <property> - <name>yarn.resourcemanager.recovery.enabled</name> - <value>true</value> + <value>quickstart.cloudera:8033</value> </property> - - <property> - <name>yarn.resourcemanager.resource-tracker.address</name> - <value>sandbox.hortonworks.com:8025</value> - </property> - <property> <name>yarn.resourcemanager.scheduler.address</name> - <value>sandbox.hortonworks.com:8030</value> + <value>quickstart.cloudera:8030</value> </property> - <property> - <name>yarn.resourcemanager.scheduler.class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> - </property> - - <property> - <name>yarn.resourcemanager.state-store.max-completed-applications</name> - <value>${yarn.resourcemanager.max-completed-applications}</value> - </property> - - <property> - <name>yarn.resourcemanager.store.class</name> - <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> - </property> - - <property> - <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name> - <value>10</value> - </property> - - <property> - <name>yarn.resourcemanager.system-metrics-publisher.enabled</name> - <value>true</value> + <name>yarn.resourcemanager.resource-tracker.address</name> + <value>quickstart.cloudera:8031</value> </property> - <property> <name>yarn.resourcemanager.webapp.address</name> - <value>sandbox.hortonworks.com:8088</value> + <value>quickstart.cloudera:8088</value> </property> - - <property> - <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name> - <value>false</value> - </property> - <property> <name>yarn.resourcemanager.webapp.https.address</name> - <value>localhost:8090</value> + <value>quickstart.cloudera:8090</value> </property> - <property> - <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name> - <value>*</value> + <name>yarn.resourcemanager.client.thread-count</name> + <value>50</value> </property> - <property> - <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name> - <value>*</value> + <name>yarn.resourcemanager.scheduler.client.thread-count</name> + <value>50</value> </property> - <property> - <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name> - <value>*</value> + <name>yarn.resourcemanager.admin.client.thread-count</name> + <value>1</value> </property> - <property> - <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name> - <value>*</value> + <name>yarn.scheduler.minimum-allocation-mb</name> + <value>1</value> </property> - <property> - <name>yarn.resourcemanager.work-preserving-recovery.enabled</name> - <value>true</value> + <name>yarn.scheduler.increment-allocation-mb</name> + <value>512</value> </property> - <property> - <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name> - <value>10000</value> + <name>yarn.scheduler.maximum-allocation-mb</name> + <value>2816</value> </property> - <property> - <name>yarn.resourcemanager.zk-acl</name> - <value>world:anyone:rwcda</value> + <name>yarn.scheduler.minimum-allocation-vcores</name> + <value>1</value> </property> - <property> - <name>yarn.resourcemanager.zk-address</name> - <value>localhost:2181</value> + <name>yarn.scheduler.increment-allocation-vcores</name> + <value>1</value> </property> - <property> - <name>yarn.resourcemanager.zk-num-retries</name> - <value>1000</value> + <name>yarn.scheduler.maximum-allocation-vcores</name> + <value>2</value> </property> - <property> - <name>yarn.resourcemanager.zk-retry-interval-ms</name> + <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name> <value>1000</value> </property> - <property> - <name>yarn.resourcemanager.zk-state-store.parent-path</name> - <value>/rmstore</value> + <name>yarn.am.liveness-monitor.expiry-interval-ms</name> + <value>600000</value> </property> - - <property> - <name>yarn.resourcemanager.zk-timeout-ms</name> - <value>10000</value> - </property> - - <property> - <name>yarn.scheduler.maximum-allocation-mb</name> - <value>9216</value> - </property> - <property> - <name>yarn.scheduler.minimum-allocation-mb</name> - <value>1536</value> + <name>yarn.resourcemanager.am.max-attempts</name> + <value>2</value> </property> - <property> - <name>yarn.timeline-service.address</name> - <value>sandbox.hortonworks.com:10200</value> + <name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name> + <value>600000</value> </property> - <property> - <name>yarn.timeline-service.bind-host</name> - <value>0.0.0.0</value> + <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name> + <value>1000</value> </property> - <property> - <name>yarn.timeline-service.client.max-retries</name> - <value>30</value> + <name>yarn.nm.liveness-monitor.expiry-interval-ms</name> + <value>600000</value> </property> - <property> - <name>yarn.timeline-service.client.retry-interval-ms</name> - <value>1000</value> + <name>yarn.resourcemanager.resource-tracker.client.thread-count</name> + <value>50</value> </property> - <property> - <name>yarn.timeline-service.enabled</name> - <value>true</value> + <name>yarn.application.classpath</name> + <value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value> </property> - <property> - <name>yarn.timeline-service.generic-application-history.store-class</name> - <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value> + <name>yarn.resourcemanager.scheduler.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value> </property> - <property> - <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name> + <name>yarn.scheduler.fair.user-as-default-queue</name> <value>true</value> </property> - - <property> - <name>yarn.timeline-service.http-authentication.type</name> - <value>simple</value> - </property> - <property> - <name>yarn.timeline-service.leveldb-timeline-store.path</name> - <value>/hadoop/yarn/timeline</value> + <name>yarn.scheduler.fair.preemption</name> + <value>false</value> </property> - <property> - <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name> - <value>104857600</value> + <name>yarn.scheduler.fair.sizebasedweight</name> + <value>false</value> </property> - <property> - <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name> - <value>10000</value> + <name>yarn.scheduler.fair.assignmultiple</name> + <value>false</value> </property> - <property> - <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name> + <name>yarn.resourcemanager.max-completed-applications</name> <value>10000</value> </property> - - <property> - <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name> - <value>300000</value> - </property> - - <property> - <name>yarn.timeline-service.store-class</name> - <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value> - </property> - - <property> - <name>yarn.timeline-service.ttl-enable</name> - <value>true</value> - </property> - - <property> - <name>yarn.timeline-service.ttl-ms</name> - <value>2678400000</value> - </property> - - <property> - <name>yarn.timeline-service.webapp.address</name> - <value>sandbox.hortonworks.com:8188</value> - </property> - - <property> - <name>yarn.timeline-service.webapp.https.address</name> - <value>sandbox.hortonworks.com:8190</value> - </property> - </configuration> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1df14f4..9b84f23 100644 --- a/pom.xml +++ b/pom.xml @@ -46,19 +46,19 @@ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <!-- Hadoop versions --> - <hadoop2.version>2.7.1</hadoop2.version> - <yarn.version>2.7.1</yarn.version> + <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version> + <yarn.version>2.6.0-cdh5.7.0</yarn.version> <!-- Hive versions --> - <hive.version>1.2.1</hive.version> - <hive-hcatalog.version>1.2.1</hive-hcatalog.version> + <hive.version>1.1.0-cdh5.7.0</hive.version> + <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version> <!-- HBase versions --> - <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version> + <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version> <kafka.version>0.8.1</kafka.version> <!-- Hadoop deps, keep compatible with hadoop2.version --> - <zookeeper.version>3.4.6</zookeeper.version> + <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version> <curator.version>2.7.1</curator.version> <jackson.version>2.2.4</jackson.version> <jsr305.version>3.0.1</jsr305.version> @@ -811,6 +811,10 @@ <id>conjars</id> <url>http://conjars.org/repo/</url> </repository> + <repository> + <id>cloudera</id> + <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> + </repository> </repositories> <build> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index 43453a8..d5f7ced 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -112,6 +112,10 @@ <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> <exclusion> <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> @@ -131,6 +135,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -146,6 +154,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -161,6 +173,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -176,6 +192,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -199,6 +219,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -214,6 +238,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -229,6 +257,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> @@ -262,6 +294,10 @@ <groupId>javax.servlet.jsp</groupId> <artifactId>jsp-api</artifactId> </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java index d5c3f60..5adf327 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java @@ -100,11 +100,6 @@ public class MockupMapContext { } @Override - public float getProgress() { - throw new NotImplementedException(); - } - - @Override public Counter getCounter(Enum<?> counterName) { throw new NotImplementedException(); } @@ -165,6 +160,11 @@ public class MockupMapContext { } @Override + public boolean userClassesTakesPrecedence() { + return false; + } + + @Override public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException { throw new NotImplementedException(); } @@ -214,10 +214,6 @@ public class MockupMapContext { throw new NotImplementedException(); } - @Override - public boolean getTaskCleanupNeeded() { - throw new NotImplementedException(); - } @Override public boolean getProfileEnabled() { @@ -230,11 +226,6 @@ public class MockupMapContext { } @Override - public IntegerRanges getProfileTaskRange(boolean isMap) { - throw new NotImplementedException(); - } - - @Override public String getUser() { throw new NotImplementedException(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3909d8d6/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index e3d7bfa..dcc052c 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -49,6 +49,18 @@ <!--Env--> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <version>${yarn.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + <version>${yarn.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <scope>provided</scope>
