kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Repository: kylin
Updated Branches:
refs/heads/master 08276a3b3 -> 6c32fd6a2
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c32fd6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c32fd6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c32fd6a
Branch: refs/heads/master
Commit: 6c32fd6a2e72183e8d4f132da8ee771ef6cc33c9
Parents: 08276a3
Author: shaofengshi
Authored: Wed May 18 09:52:32 2016 +0800
Committer: shaofengshi
Committed: Wed May 18 10:28:15 2016 +0800
--
.../kylin/job/engine/JobEngineConfig.java | 9 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../sandbox/kylin_job_conf_inmem.xml| 99
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
4 files changed, 105 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c32fd6a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index baa3bfe..64e801e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -77,13 +77,12 @@ public class JobEngineConfig {
* @return the job config file path
* @throws IOException
*/
-public String getHadoopJobConfFilePath(String jobType, String capacity)
throws IOException {
-String suffix;
+public String getHadoopJobConfFilePath(String jobType) throws IOException {
+String suffix = null;
if(!StringUtils.isEmpty(jobType)) {
-suffix = jobType + "_" + capacity;
-} else {
-suffix = capacity;
+suffix = jobType;
}
+
String path = getHadoopJobConfFilePath(suffix, true);
if (StringUtils.isEmpty(path)) {
path = getHadoopJobConfFilePath(jobType, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c32fd6a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 841c402..151d867 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
public void appendMapReduceParameters(StringBuilder buf, String jobType,
String capacity) {
try {
-String jobConf = config.getHadoopJobConfFilePath(jobType,
capacity);
+String jobConf = config.getHadoopJobConfFilePath(jobType);
if (jobConf != null && jobConf.length() > 0) {
buf.append(" -conf ").append(jobConf);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c32fd6a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
--
diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
new file mode 100644
index 000..5f2cfba
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
@@ -0,0 +1,99 @@
+
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+mapreduce.map.maxattempts
+2
+
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+
+mapreduce.map.memory.mb
+512
+
+
+
+
+mapreduce.map.java.opts
+-Xmx384m
+
+
+
+
+mapreduce.task.io.sort.mb
+100
+
+
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c32fd6a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardin
kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Repository: kylin
Updated Branches:
refs/heads/v1.5.2-release e80b13ff1 -> a63f959eb
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a63f959e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a63f959e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a63f959e
Branch: refs/heads/v1.5.2-release
Commit: a63f959eb33a2638733817ac796a1f7c8cf89608
Parents: e80b13f
Author: shaofengshi
Authored: Wed May 18 09:52:32 2016 +0800
Committer: shaofengshi
Committed: Wed May 18 10:39:35 2016 +0800
--
.../kylin/job/engine/JobEngineConfig.java | 9 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../sandbox/kylin_job_conf_inmem.xml| 99
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
4 files changed, 105 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index baa3bfe..64e801e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -77,13 +77,12 @@ public class JobEngineConfig {
* @return the job config file path
* @throws IOException
*/
-public String getHadoopJobConfFilePath(String jobType, String capacity)
throws IOException {
-String suffix;
+public String getHadoopJobConfFilePath(String jobType) throws IOException {
+String suffix = null;
if(!StringUtils.isEmpty(jobType)) {
-suffix = jobType + "_" + capacity;
-} else {
-suffix = capacity;
+suffix = jobType;
}
+
String path = getHadoopJobConfFilePath(suffix, true);
if (StringUtils.isEmpty(path)) {
path = getHadoopJobConfFilePath(jobType, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 841c402..151d867 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
public void appendMapReduceParameters(StringBuilder buf, String jobType,
String capacity) {
try {
-String jobConf = config.getHadoopJobConfFilePath(jobType,
capacity);
+String jobConf = config.getHadoopJobConfFilePath(jobType);
if (jobConf != null && jobConf.length() > 0) {
buf.append(" -conf ").append(jobConf);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
--
diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
new file mode 100644
index 000..5f2cfba
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
@@ -0,0 +1,99 @@
+
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+mapreduce.map.maxattempts
+2
+
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+
+mapreduce.map.memory.mb
+512
+
+
+
+
+mapreduce.map.java.opts
+-Xmx384m
+
+
+
+
+mapreduce.task.io.sort.mb
+100
+
+
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
b/source-hive/src/main/java/org/apache/kylin/so
[1/2] kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1077 [created] 66ad13860
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/209068b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/209068b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/209068b9
Branch: refs/heads/KYLIN-1077
Commit: 209068b943bf4a90efe4df618e1aaf5cbfe49cde
Parents: 1b54a40
Author: shaofengshi
Authored: Fri Apr 15 16:11:44 2016 +0800
Committer: shaofengshi
Committed: Sat Apr 16 09:05:40 2016 +0800
--
build/conf/kylin_job_conf_inmem.xml | 98
.../apache/kylin/common/KylinConfigBase.java| 19
.../kylin/job/engine/JobEngineConfig.java | 44 ++---
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 3 +-
.../kylin/engine/mr/JobBuilderSupport.java | 13 ++-
.../kylin/engine/mr/steps/InMemCuboidJob.java | 11 ---
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
7 files changed, 140 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/209068b9/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml
b/build/conf/kylin_job_conf_inmem.xml
new file mode 100644
index 000..55bf9ed
--- /dev/null
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -0,0 +1,98 @@
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+ mapred.compress.map.output
+ true
+ Compress map outputs
+
+
+
+ mapred.map.output.compression.codec
+ org.apache.hadoop.io.compress.SnappyCodec
+ The compression codec to use for map outputs
+
+
+
+
+ mapred.output.compress
+ true
+ Compress the output of a MapReduce
job
+
+
+
+ mapred.output.compression.codec
+ org.apache.hadoop.io.compress.SnappyCodec
+ The compression codec to use for job outputs
+
+
+
+
+ mapred.output.compression.type
+ BLOCK
+ The compression type to use for job
outputs
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+mapred.task.timeout
+360
+Set task timeout to 1 hour
+
+
+
+
+ mapreduce.map.memory.mb
+ 3072
+
+
+
+
+ mapreduce.map.java.opts
+ -Xmx2700m
+
+
+
+
+ mapreduce.task.io.sort.mb
+ 200
+
+
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/209068b9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 51aa8aa..4d65c1d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -531,25 +531,6 @@ abstract public class KylinConfigBase implements
Serializable {
return percent;
}
-public Map getCubingInMemMRJobConfOverride() {
-// in-mem cubing requires big memory, however dev env (sandbox) may
not have that much
-String defaultOverride = isDevEnv() ? "" :
"mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072;
mapreduce.task.io.sort.mb=200";
-String override =
getOptional("kylin.job.cubing.inmem.mrjob_conf_override", defaultOverride);
-
-Map result = Maps.newHashMap();
-for (String pair : override.split(";")) {
-int cut = pair.indexOf('=');
-if (cut < 0)
-continue;
-String k = pair.substring(0, cut).trim();
-String v = pair.substring(cut + 1).trim();
-if (k.isEmpty() || v.isEmpty())
-continue;
-result.put(k, v);
-}
-return result;
-}
-
public String getHbaseDefaultCompressionCodec() {
return getOptional("kylin.hbase.default.compression.codec", "");
}
http://git-wip-us.ap
[1/3] kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Repository: kylin
Updated Branches:
refs/heads/master fbf2b3814 -> dc1671ca5
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/209068b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/209068b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/209068b9
Branch: refs/heads/master
Commit: 209068b943bf4a90efe4df618e1aaf5cbfe49cde
Parents: 1b54a40
Author: shaofengshi
Authored: Fri Apr 15 16:11:44 2016 +0800
Committer: shaofengshi
Committed: Sat Apr 16 09:05:40 2016 +0800
--
build/conf/kylin_job_conf_inmem.xml | 98
.../apache/kylin/common/KylinConfigBase.java| 19
.../kylin/job/engine/JobEngineConfig.java | 44 ++---
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 3 +-
.../kylin/engine/mr/JobBuilderSupport.java | 13 ++-
.../kylin/engine/mr/steps/InMemCuboidJob.java | 11 ---
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
7 files changed, 140 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/209068b9/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml
b/build/conf/kylin_job_conf_inmem.xml
new file mode 100644
index 000..55bf9ed
--- /dev/null
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -0,0 +1,98 @@
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+ mapred.compress.map.output
+ true
+ Compress map outputs
+
+
+
+ mapred.map.output.compression.codec
+ org.apache.hadoop.io.compress.SnappyCodec
+ The compression codec to use for map outputs
+
+
+
+
+ mapred.output.compress
+ true
+ Compress the output of a MapReduce
job
+
+
+
+ mapred.output.compression.codec
+ org.apache.hadoop.io.compress.SnappyCodec
+ The compression codec to use for job outputs
+
+
+
+
+ mapred.output.compression.type
+ BLOCK
+ The compression type to use for job
outputs
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+mapred.task.timeout
+360
+Set task timeout to 1 hour
+
+
+
+
+ mapreduce.map.memory.mb
+ 3072
+
+
+
+
+ mapreduce.map.java.opts
+ -Xmx2700m
+
+
+
+
+ mapreduce.task.io.sort.mb
+ 200
+
+
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/209068b9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 51aa8aa..4d65c1d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -531,25 +531,6 @@ abstract public class KylinConfigBase implements
Serializable {
return percent;
}
-public Map getCubingInMemMRJobConfOverride() {
-// in-mem cubing requires big memory, however dev env (sandbox) may
not have that much
-String defaultOverride = isDevEnv() ? "" :
"mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072;
mapreduce.task.io.sort.mb=200";
-String override =
getOptional("kylin.job.cubing.inmem.mrjob_conf_override", defaultOverride);
-
-Map result = Maps.newHashMap();
-for (String pair : override.split(";")) {
-int cut = pair.indexOf('=');
-if (cut < 0)
-continue;
-String k = pair.substring(0, cut).trim();
-String v = pair.substring(cut + 1).trim();
-if (k.isEmpty() || v.isEmpty())
-continue;
-result.put(k, v);
-}
-return result;
-}
-
public String getHbaseDefaultCompressionCodec() {
return getOptional("kylin.hbase.default.compression.codec", "");
}
http://git-wip-us.apache.
[1/4] kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/1.5.x-HBase1.x bfafc52a4 -> d97b525f2 (forced update)
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a63f959e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a63f959e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a63f959e
Branch: refs/heads/1.5.x-HBase1.x
Commit: a63f959eb33a2638733817ac796a1f7c8cf89608
Parents: e80b13f
Author: shaofengshi
Authored: Wed May 18 09:52:32 2016 +0800
Committer: shaofengshi
Committed: Wed May 18 10:39:35 2016 +0800
--
.../kylin/job/engine/JobEngineConfig.java | 9 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../sandbox/kylin_job_conf_inmem.xml| 99
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
4 files changed, 105 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index baa3bfe..64e801e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -77,13 +77,12 @@ public class JobEngineConfig {
* @return the job config file path
* @throws IOException
*/
-public String getHadoopJobConfFilePath(String jobType, String capacity)
throws IOException {
-String suffix;
+public String getHadoopJobConfFilePath(String jobType) throws IOException {
+String suffix = null;
if(!StringUtils.isEmpty(jobType)) {
-suffix = jobType + "_" + capacity;
-} else {
-suffix = capacity;
+suffix = jobType;
}
+
String path = getHadoopJobConfFilePath(suffix, true);
if (StringUtils.isEmpty(path)) {
path = getHadoopJobConfFilePath(jobType, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 841c402..151d867 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
public void appendMapReduceParameters(StringBuilder buf, String jobType,
String capacity) {
try {
-String jobConf = config.getHadoopJobConfFilePath(jobType,
capacity);
+String jobConf = config.getHadoopJobConfFilePath(jobType);
if (jobConf != null && jobConf.length() > 0) {
buf.append(" -conf ").append(jobConf);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
--
diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
new file mode 100644
index 000..5f2cfba
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
@@ -0,0 +1,99 @@
+
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+mapreduce.map.maxattempts
+2
+
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+
+mapreduce.map.memory.mb
+512
+
+
+
+
+mapreduce.map.java.opts
+-Xmx384m
+
+
+
+
+mapreduce.task.io.sort.mb
+100
+
+
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
b/source-hive/src/main/java/org
[01/11] kylin git commit: KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/1.5.x-CDH5.7 319d4ae98 -> 0c517678a (forced update)
KYLIN-1566 use a separate kylin_job_conf.xml for in-mem cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a63f959e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a63f959e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a63f959e
Branch: refs/heads/1.5.x-CDH5.7
Commit: a63f959eb33a2638733817ac796a1f7c8cf89608
Parents: e80b13f
Author: shaofengshi
Authored: Wed May 18 09:52:32 2016 +0800
Committer: shaofengshi
Committed: Wed May 18 10:39:35 2016 +0800
--
.../kylin/job/engine/JobEngineConfig.java | 9 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 +-
.../sandbox/kylin_job_conf_inmem.xml| 99
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
4 files changed, 105 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index baa3bfe..64e801e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -77,13 +77,12 @@ public class JobEngineConfig {
* @return the job config file path
* @throws IOException
*/
-public String getHadoopJobConfFilePath(String jobType, String capacity)
throws IOException {
-String suffix;
+public String getHadoopJobConfFilePath(String jobType) throws IOException {
+String suffix = null;
if(!StringUtils.isEmpty(jobType)) {
-suffix = jobType + "_" + capacity;
-} else {
-suffix = capacity;
+suffix = jobType;
}
+
String path = getHadoopJobConfFilePath(suffix, true);
if (StringUtils.isEmpty(path)) {
path = getHadoopJobConfFilePath(jobType, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 841c402..151d867 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
public void appendMapReduceParameters(StringBuilder buf, String jobType,
String capacity) {
try {
-String jobConf = config.getHadoopJobConfFilePath(jobType,
capacity);
+String jobConf = config.getHadoopJobConfFilePath(jobType);
if (jobConf != null && jobConf.length() > 0) {
buf.append(" -conf ").append(jobConf);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
--
diff --git a/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
new file mode 100644
index 000..5f2cfba
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin_job_conf_inmem.xml
@@ -0,0 +1,99 @@
+
+
+
+
+
+mapreduce.job.split.metainfo.maxsize
+-1
+The maximum permissible size of the split metainfo file.
+The JobTracker won't attempt to read split metainfo files bigger
than
+the configured value. No limits if set to -1.
+
+
+
+
+mapreduce.map.maxattempts
+2
+
+
+
+
+
+mapreduce.job.max.split.locations
+2000
+No description
+
+
+
+dfs.replication
+2
+Block replication
+
+
+
+
+mapreduce.map.memory.mb
+512
+
+
+
+
+mapreduce.map.java.opts
+-Xmx384m
+
+
+
+
+mapreduce.task.io.sort.mb
+100
+
+
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a63f959e/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
b/source-hive/src/main/java/org/apa
