http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java index db02f34..db9fc80 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java @@ -1,246 +1,246 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.io.Serializable; -import java.nio.ByteBuffer; - -/** - * @author yangli9 - */ -public class ByteArray implements Comparable<ByteArray>, Serializable { - - private static final long serialVersionUID = 1L; - - public static final ByteArray EMPTY = new ImmutableByteArray(); - - public static ByteArray allocate(int length) { - return new ByteArray(new byte[length]); - } - - public static ByteArray copyOf(byte[] array, int offset, int length) { - byte[] space = new byte[length]; - System.arraycopy(array, offset, space, 0, length); - return new ByteArray(space, 0, length); - } - - // ============================================================================ - - private byte[] data; - private int offset; - private int length; - - public ByteArray() { - this(null, 0, 0); - } - - public ByteArray(int capacity) { - this(new byte[capacity], 0, capacity); - } - - public ByteArray(byte[] data) { - this(data, 0, data == null ? 0 : data.length); - } - - public ByteArray(byte[] data, int offset, int length) { - this.data = data; - this.offset = offset; - this.length = length; - } - - public byte[] array() { - return data; - } - - public int offset() { - return offset; - } - - public int length() { - return length; - } - - public void set(byte[] array) { - set(array, 0, array.length); - } - - public void set(byte[] array, int offset, int length) { - this.data = array; - this.offset = offset; - this.length = length; - } - - public void set(ByteArray o) { - set(o.data, o.offset, o.length); - } - - public void set(int offset, int length) { - this.offset = offset; - this.length = length; - } - - public void setLength(int length) { - this.length = length; - } - - public ByteArray copy() { - ByteArray copy; - if (data != null) { - copy = new ByteArray(length); - } else { - copy = new ByteArray(null); - } - copy.copyFrom(this); - return copy; - } - - //notice this will have a length header - public void exportData(ByteBuffer out) { - BytesUtil.writeByteArray(this.data, this.offset, this.length, out); - } - - public static ByteArray importData(ByteBuffer in) { - byte[] bytes = BytesUtil.readByteArray(in); - return new ByteArray(bytes); - } - - public void copyFrom(ByteArray other) { - if (other.data != null) { - System.arraycopy(other.array(), other.offset, data, offset, other.length); - } - this.length = other.length; - } - - public ByteBuffer asBuffer() { - if (data == null) - return null; - else if (offset == 0 && length == data.length) - return ByteBuffer.wrap(data); - else - return ByteBuffer.wrap(data, offset, length).slice(); - } - - public byte[] toBytes() { - return Bytes.copy(this.array(), this.offset(), this.length()); - } - - @Override - public int hashCode() { - if (data == null) { - return 0; - } else { - if (length <= Bytes.SIZEOF_LONG && length > 0) { - // to avoid hash collision of byte arrays those are converted from nearby integers/longs, which is the case for kylin dictionary - long value = BytesUtil.readLong(data, offset, length); - return (int) (value ^ (value >>> 32)); - } - return Bytes.hashCode(data, offset, length); - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ByteArray o = (ByteArray) obj; - if (this.data == null && o.data == null) - return true; - else if (this.data == null || o.data == null) - return false; - else - return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length); - } - - @Override - public int compareTo(ByteArray o) { - if (this.data == null && o.data == null) - return 0; - else if (this.data == null) - return -1; - else if (o.data == null) - return 1; - else - return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length); - } - - public String toReadableText() { - if (data == null) { - return null; - } else { - return BytesUtil.toHex(data, offset, length); - } - } - - @Override - public String toString() { - if (data == null) - return null; - else - return Bytes.toStringBinary(data, offset, length); - } - - // ============================================================================ - - public static class ImmutableByteArray extends ByteArray { - - private static final long serialVersionUID = 1L; - - public ImmutableByteArray() { - super(); - } - - public ImmutableByteArray(byte[] data, int offset, int length) { - super(data, offset, length); - } - - public ImmutableByteArray(byte[] data) { - super(data); - } - - @Override - public void set(byte[] array) { - throw new UnsupportedOperationException(); - } - - @Override - public void set(byte[] array, int offset, int length) { - throw new UnsupportedOperationException(); - } - - @Override - public void set(ByteArray o) { - throw new UnsupportedOperationException(); - } - - @Override - public void setLength(int length) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFrom(ByteArray other) { - throw new UnsupportedOperationException(); - } - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.io.Serializable; +import java.nio.ByteBuffer; + +/** + * @author yangli9 + */ +public class ByteArray implements Comparable<ByteArray>, Serializable { + + private static final long serialVersionUID = 1L; + + public static final ByteArray EMPTY = new ImmutableByteArray(); + + public static ByteArray allocate(int length) { + return new ByteArray(new byte[length]); + } + + public static ByteArray copyOf(byte[] array, int offset, int length) { + byte[] space = new byte[length]; + System.arraycopy(array, offset, space, 0, length); + return new ByteArray(space, 0, length); + } + + // ============================================================================ + + private byte[] data; + private int offset; + private int length; + + public ByteArray() { + this(null, 0, 0); + } + + public ByteArray(int capacity) { + this(new byte[capacity], 0, capacity); + } + + public ByteArray(byte[] data) { + this(data, 0, data == null ? 0 : data.length); + } + + public ByteArray(byte[] data, int offset, int length) { + this.data = data; + this.offset = offset; + this.length = length; + } + + public byte[] array() { + return data; + } + + public int offset() { + return offset; + } + + public int length() { + return length; + } + + public void set(byte[] array) { + set(array, 0, array.length); + } + + public void set(byte[] array, int offset, int length) { + this.data = array; + this.offset = offset; + this.length = length; + } + + public void set(ByteArray o) { + set(o.data, o.offset, o.length); + } + + public void set(int offset, int length) { + this.offset = offset; + this.length = length; + } + + public void setLength(int length) { + this.length = length; + } + + public ByteArray copy() { + ByteArray copy; + if (data != null) { + copy = new ByteArray(length); + } else { + copy = new ByteArray(null); + } + copy.copyFrom(this); + return copy; + } + + //notice this will have a length header + public void exportData(ByteBuffer out) { + BytesUtil.writeByteArray(this.data, this.offset, this.length, out); + } + + public static ByteArray importData(ByteBuffer in) { + byte[] bytes = BytesUtil.readByteArray(in); + return new ByteArray(bytes); + } + + public void copyFrom(ByteArray other) { + if (other.data != null) { + System.arraycopy(other.array(), other.offset, data, offset, other.length); + } + this.length = other.length; + } + + public ByteBuffer asBuffer() { + if (data == null) + return null; + else if (offset == 0 && length == data.length) + return ByteBuffer.wrap(data); + else + return ByteBuffer.wrap(data, offset, length).slice(); + } + + public byte[] toBytes() { + return Bytes.copy(this.array(), this.offset(), this.length()); + } + + @Override + public int hashCode() { + if (data == null) { + return 0; + } else { + if (length <= Bytes.SIZEOF_LONG && length > 0) { + // to avoid hash collision of byte arrays those are converted from nearby integers/longs, which is the case for kylin dictionary + long value = BytesUtil.readLong(data, offset, length); + return (int) (value ^ (value >>> 32)); + } + return Bytes.hashCode(data, offset, length); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ByteArray o = (ByteArray) obj; + if (this.data == null && o.data == null) + return true; + else if (this.data == null || o.data == null) + return false; + else + return Bytes.equals(this.data, this.offset, this.length, o.data, o.offset, o.length); + } + + @Override + public int compareTo(ByteArray o) { + if (this.data == null && o.data == null) + return 0; + else if (this.data == null) + return -1; + else if (o.data == null) + return 1; + else + return Bytes.compareTo(this.data, this.offset, this.length, o.data, o.offset, o.length); + } + + public String toReadableText() { + if (data == null) { + return null; + } else { + return BytesUtil.toHex(data, offset, length); + } + } + + @Override + public String toString() { + if (data == null) + return null; + else + return Bytes.toStringBinary(data, offset, length); + } + + // ============================================================================ + + public static class ImmutableByteArray extends ByteArray { + + private static final long serialVersionUID = 1L; + + public ImmutableByteArray() { + super(); + } + + public ImmutableByteArray(byte[] data, int offset, int length) { + super(data, offset, length); + } + + public ImmutableByteArray(byte[] data) { + super(data); + } + + @Override + public void set(byte[] array) { + throw new UnsupportedOperationException(); + } + + @Override + public void set(byte[] array, int offset, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void set(ByteArray o) { + throw new UnsupportedOperationException(); + } + + @Override + public void setLength(int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFrom(ByteArray other) { + throw new UnsupportedOperationException(); + } + } + +}
http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java index 26342f5..699d569 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesSerializer.java @@ -1,35 +1,35 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.nio.ByteBuffer; - -/** - * @author yangli9 - * - */ -public interface BytesSerializer<T> { - - int SERIALIZE_BUFFER_SIZE = 65536; - - void serialize(T value, ByteBuffer out); - - T deserialize(ByteBuffer in); - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.nio.ByteBuffer; + +/** + * @author yangli9 + * + */ +public interface BytesSerializer<T> { + + int SERIALIZE_BUFFER_SIZE = 65536; + + void serialize(T value, ByteBuffer out); + + T deserialize(ByteBuffer in); + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java index 33d1d05..c644890 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesSplitter.java @@ -1,125 +1,125 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class BytesSplitter { - private static final Logger logger = LoggerFactory.getLogger(BytesSplitter.class); - - private static final int[] COMMON_DELIMS = new int[] { "\177".codePointAt(0), "|".codePointAt(0), "\t".codePointAt(0), ",".codePointAt(0) }; - - private SplittedBytes[] splitBuffers; - private int bufferSize; - - public SplittedBytes[] getSplitBuffers() { - return splitBuffers; - } - - public SplittedBytes getSplitBuffer(int index) { - return splitBuffers[index]; - } - - public int getBufferSize() { - return bufferSize; - } - - public BytesSplitter(int splitLen, int bytesLen) { - this.splitBuffers = new SplittedBytes[splitLen]; - for (int i = 0; i < splitLen; i++) { - this.splitBuffers[i] = new SplittedBytes(bytesLen); - } - this.bufferSize = 0; - } - - public int split(byte[] bytes, int byteLen, byte delimiter) { - this.bufferSize = 0; - int offset = 0; - int length = 0; - for (int i = 0; i < byteLen; i++) { - if (bytes[i] == delimiter) { - SplittedBytes split = this.splitBuffers[this.bufferSize++]; - if (length > split.value.length) { - length = split.value.length; - } - System.arraycopy(bytes, offset, split.value, 0, length); - split.length = length; - offset = i + 1; - length = 0; - } else { - length++; - } - } - SplittedBytes split = this.splitBuffers[this.bufferSize++]; - if (length > split.value.length) { - length = split.value.length; - } - System.arraycopy(bytes, offset, split.value, 0, length); - split.length = length; - - return bufferSize; - } - - public void setBuffers(byte[][] buffers) { - for (int i = 0; i < buffers.length; i++) { - splitBuffers[i].value = buffers[i]; - splitBuffers[i].length = buffers[i].length; - } - this.bufferSize = buffers.length; - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder(); - buf.append("["); - for (int i = 0; i < bufferSize; i++) { - if (i > 0) - buf.append(", "); - - buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length)); - } - return buf.toString(); - } - - public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) { - List<String> splitStrings = new ArrayList<String>(); - int splitOffset = 0; - int splitLength = 0; - for (int i = offset; i < bytes.length; i++) { - if (bytes[i] == delimiter) { - String str = Bytes.toString(bytes, splitOffset, splitLength); - splitStrings.add(str); - splitOffset = i + 1; - splitLength = 0; - } else { - splitLength++; - } - } - String str = Bytes.toString(bytes, splitOffset, splitLength); - splitStrings.add(str); - return splitStrings; - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + */ +public class BytesSplitter { + private static final Logger logger = LoggerFactory.getLogger(BytesSplitter.class); + + private static final int[] COMMON_DELIMS = new int[] { "\177".codePointAt(0), "|".codePointAt(0), "\t".codePointAt(0), ",".codePointAt(0) }; + + private SplittedBytes[] splitBuffers; + private int bufferSize; + + public SplittedBytes[] getSplitBuffers() { + return splitBuffers; + } + + public SplittedBytes getSplitBuffer(int index) { + return splitBuffers[index]; + } + + public int getBufferSize() { + return bufferSize; + } + + public BytesSplitter(int splitLen, int bytesLen) { + this.splitBuffers = new SplittedBytes[splitLen]; + for (int i = 0; i < splitLen; i++) { + this.splitBuffers[i] = new SplittedBytes(bytesLen); + } + this.bufferSize = 0; + } + + public int split(byte[] bytes, int byteLen, byte delimiter) { + this.bufferSize = 0; + int offset = 0; + int length = 0; + for (int i = 0; i < byteLen; i++) { + if (bytes[i] == delimiter) { + SplittedBytes split = this.splitBuffers[this.bufferSize++]; + if (length > split.value.length) { + length = split.value.length; + } + System.arraycopy(bytes, offset, split.value, 0, length); + split.length = length; + offset = i + 1; + length = 0; + } else { + length++; + } + } + SplittedBytes split = this.splitBuffers[this.bufferSize++]; + if (length > split.value.length) { + length = split.value.length; + } + System.arraycopy(bytes, offset, split.value, 0, length); + split.length = length; + + return bufferSize; + } + + public void setBuffers(byte[][] buffers) { + for (int i = 0; i < buffers.length; i++) { + splitBuffers[i].value = buffers[i]; + splitBuffers[i].length = buffers[i].length; + } + this.bufferSize = buffers.length; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("["); + for (int i = 0; i < bufferSize; i++) { + if (i > 0) + buf.append(", "); + + buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length)); + } + return buf.toString(); + } + + public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) { + List<String> splitStrings = new ArrayList<String>(); + int splitOffset = 0; + int splitLength = 0; + for (int i = offset; i < bytes.length; i++) { + if (bytes[i] == delimiter) { + String str = Bytes.toString(bytes, splitOffset, splitLength); + splitStrings.add(str); + splitOffset = i + 1; + splitLength = 0; + } else { + splitLength++; + } + } + String str = Bytes.toString(bytes, splitOffset, splitLength); + splitStrings.add(str); + return splitStrings; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java index aa25c22..f97b609 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java @@ -1,149 +1,149 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.apache.commons.io.FileUtils; - -/** - * @author yangli9 - */ -public class CliCommandExecutor { - - private String remoteHost; - private int port; - private String remoteUser; - private String remotePwd; - private int remoteTimeoutSeconds = 3600; - - public CliCommandExecutor() { - } - - public void setRunAtRemote(String host, int port, String user, String pwd) { - this.remoteHost = host; - this.port = port; - this.remoteUser = user; - this.remotePwd = pwd; - } - - public void setRunAtLocal() { - this.remoteHost = null; - this.remoteUser = null; - this.remotePwd = null; - } - - public void copyFile(String localFile, String destDir) throws IOException { - if (remoteHost == null) - copyNative(localFile, destDir); - else - copyRemote(localFile, destDir); - } - - private void copyNative(String localFile, String destDir) throws IOException { - File src = new File(localFile); - File dest = new File(destDir, src.getName()); - FileUtils.copyFile(src, dest); - } - - private void copyRemote(String localFile, String destDir) throws IOException { - SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd); - try { - ssh.scpFileToRemote(localFile, destDir); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e.getMessage(), e); - } - } - - public Pair<Integer, String> execute(String command) throws IOException { - return execute(command, new SoutLogger()); - } - - public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException { - Pair<Integer, String> r; - if (remoteHost == null) { - r = runNativeCommand(command, logAppender); - } else { - r = runRemoteCommand(command, logAppender); - } - - if (r.getFirst() != 0) - throw new IOException("OS command error exit with " + r.getFirst() // - + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") // - + " -- " + command + "\n" + r.getSecond()); - - return r; - } - - private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException { - SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd); - - SSHClientOutput sshOutput; - try { - sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender); - int exitCode = sshOutput.getExitCode(); - String output = sshOutput.getText(); - return Pair.newPair(exitCode, output); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e.getMessage(), e); - } - } - - private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException { - String[] cmd = new String[3]; - String osName = System.getProperty("os.name"); - if (osName.startsWith("Windows")) { - cmd[0] = "cmd.exe"; - cmd[1] = "/C"; - } else { - cmd[0] = "/bin/bash"; - cmd[1] = "-c"; - } - cmd[2] = command; - - ProcessBuilder builder = new ProcessBuilder(cmd); - builder.redirectErrorStream(true); - Process proc = builder.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); - String line; - StringBuilder result = new StringBuilder(); - while ((line = reader.readLine()) != null) { - result.append(line).append('\n'); - if (logAppender != null) { - logAppender.log(line); - } - } - - try { - int exitCode = proc.waitFor(); - return Pair.newPair(exitCode, result.toString()); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.apache.commons.io.FileUtils; + +/** + * @author yangli9 + */ +public class CliCommandExecutor { + + private String remoteHost; + private int port; + private String remoteUser; + private String remotePwd; + private int remoteTimeoutSeconds = 3600; + + public CliCommandExecutor() { + } + + public void setRunAtRemote(String host, int port, String user, String pwd) { + this.remoteHost = host; + this.port = port; + this.remoteUser = user; + this.remotePwd = pwd; + } + + public void setRunAtLocal() { + this.remoteHost = null; + this.remoteUser = null; + this.remotePwd = null; + } + + public void copyFile(String localFile, String destDir) throws IOException { + if (remoteHost == null) + copyNative(localFile, destDir); + else + copyRemote(localFile, destDir); + } + + private void copyNative(String localFile, String destDir) throws IOException { + File src = new File(localFile); + File dest = new File(destDir, src.getName()); + FileUtils.copyFile(src, dest); + } + + private void copyRemote(String localFile, String destDir) throws IOException { + SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd); + try { + ssh.scpFileToRemote(localFile, destDir); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e.getMessage(), e); + } + } + + public Pair<Integer, String> execute(String command) throws IOException { + return execute(command, new SoutLogger()); + } + + public Pair<Integer, String> execute(String command, Logger logAppender) throws IOException { + Pair<Integer, String> r; + if (remoteHost == null) { + r = runNativeCommand(command, logAppender); + } else { + r = runRemoteCommand(command, logAppender); + } + + if (r.getFirst() != 0) + throw new IOException("OS command error exit with " + r.getFirst() // + + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") // + + " -- " + command + "\n" + r.getSecond()); + + return r; + } + + private Pair<Integer, String> runRemoteCommand(String command, Logger logAppender) throws IOException { + SSHClient ssh = new SSHClient(remoteHost, port, remoteUser, remotePwd); + + SSHClientOutput sshOutput; + try { + sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender); + int exitCode = sshOutput.getExitCode(); + String output = sshOutput.getText(); + return Pair.newPair(exitCode, output); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e.getMessage(), e); + } + } + + private Pair<Integer, String> runNativeCommand(String command, Logger logAppender) throws IOException { + String[] cmd = new String[3]; + String osName = System.getProperty("os.name"); + if (osName.startsWith("Windows")) { + cmd[0] = "cmd.exe"; + cmd[1] = "/C"; + } else { + cmd[0] = "/bin/bash"; + cmd[1] = "-c"; + } + cmd[2] = command; + + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.redirectErrorStream(true); + Process proc = builder.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line; + StringBuilder result = new StringBuilder(); + while ((line = reader.readLine()) != null) { + result.append(line).append('\n'); + if (logAppender != null) { + logAppender.log(line); + } + } + + try { + int exitCode = proc.waitFor(); + return Pair.newPair(exitCode, result.toString()); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/MailService.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java index ec0eaac..5793967 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java @@ -79,7 +79,7 @@ public class MailService { if (!enabled) { logger.info("Email service is disabled; this mail will not be delivered: " + subject); - logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties"); + logger.info("To enable mail service, set 'kylin.job.notification-enabled=true' in kylin.properties"); return false; } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java b/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java index 8d728c8..f3ee411 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ShardingHash.java @@ -1,67 +1,67 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common.util; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; - -public class ShardingHash { - - static HashFunction hashFunc = Hashing.murmur3_128(); - - public static short getShard(int integerValue, int totalShards) { - if (totalShards <= 1) { - return 0; - } - long hash = hashFunc.hashInt(integerValue).asLong(); - return _getShard(hash, totalShards); - } - - public static short getShard(long longValue, int totalShards) { - if (totalShards <= 1) { - return 0; - } - long hash = hashFunc.hashLong(longValue).asLong(); - return _getShard(hash, totalShards); - } - - public static short getShard(byte[] byteValues, int offset, int length, int totalShards) { - if (totalShards <= 1) { - return 0; - } - - long hash = hashFunc.hashBytes(byteValues, offset, length).asLong(); - return _getShard(hash, totalShards); - } - - public static short normalize(short cuboidShardBase, short shardOffset, int totalShards) { - if (totalShards <= 1) { - return 0; - } - return (short) ((cuboidShardBase + shardOffset) % totalShards); - } - - private static short _getShard(long hash, int totalShard) { - long x = hash % totalShard; - if (x < 0) { - x += totalShard; - } - return (short) x; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; + +public class ShardingHash { + + static HashFunction hashFunc = Hashing.murmur3_128(); + + public static short getShard(int integerValue, int totalShards) { + if (totalShards <= 1) { + return 0; + } + long hash = hashFunc.hashInt(integerValue).asLong(); + return _getShard(hash, totalShards); + } + + public static short getShard(long longValue, int totalShards) { + if (totalShards <= 1) { + return 0; + } + long hash = hashFunc.hashLong(longValue).asLong(); + return _getShard(hash, totalShards); + } + + public static short getShard(byte[] byteValues, int offset, int length, int totalShards) { + if (totalShards <= 1) { + return 0; + } + + long hash = hashFunc.hashBytes(byteValues, offset, length).asLong(); + return _getShard(hash, totalShards); + } + + public static short normalize(short cuboidShardBase, short shardOffset, int totalShards) { + if (totalShards <= 1) { + return 0; + } + return (short) ((cuboidShardBase + shardOffset) % totalShards); + } + + private static short _getShard(long hash, int totalShard) { + long x = hash % totalShard; + if (x < 0) { + x += totalShard; + } + return (short) x; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/resources/kylin-backward-compatibility.properties ---------------------------------------------------------------------- diff --git a/core-common/src/main/resources/kylin-backward-compatibility.properties b/core-common/src/main/resources/kylin-backward-compatibility.properties index 70cc3f6..563ec26 100644 --- a/core-common/src/main/resources/kylin-backward-compatibility.properties +++ b/core-common/src/main/resources/kylin-backward-compatibility.properties @@ -1,183 +1,183 @@ -#### ENV ### -# -#deploy.env=kylin.env -#kylin.hdfs.working.dir=kylin.env.hdfs-working-dir -# -# -#### METADATA ### -# -#kylin.metadata.url=kylin.metadata.url -#kylin.realization.providers=kylin.metadata.realization-providers -#kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings -#kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types. -# -# -#### Dictionary ### -# -#kylin.table.snapshot.max_mb=kylin.snapshot.max-mb -#kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry -#kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb -#kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry -#kylin.dict.growing.enabled=kylin.dictionary.growing-enabled -#kylin.dict.append.entry.size=kylin.dictionary.append-entry-size -#kylin.dict.append.cache.size=kylin.dictionary.append-cache-size -# -# -#### CUBE ### -# -#kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio -#kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio -#kylin.cube.algorithm=kylin.cube.algorithm -#kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold -#kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit -#kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size -#kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination -#kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid -#kylin.cube.building.segment.max=kylin.cube.max-building-segments -# -# -#### JOB ### -# -#kylin.job.log.dir=kylin.job.log-dir -#kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir -#kylin.job.allow.empty.segment=kylin.job.allow-empty-segment -#kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs -#kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage -#kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision -#kylin.job.dependency.filterlist=kylin.job.dependency-filter-list -#kylin.job.retry=kylin.job.retry -#kylin.job.controller.lock=kylin.job.lock -#kylin.scheduler.=kylin.job.scheduler.provider. -#kylin.enable.scheduler=kylin.job.scheduler.default -# -#mail.enabled=kylin.job.notification-enabled -#mail.host=kylin.job.notification-mail-host -#mail.username=kylin.job.notification-mail-username -#mail.password=kylin.job.notification-mail-password -#mail.sender=kylin.job.notification-mail-sender -#kylin.job.admin.dls=kylin.job.notification-admin-emails -# -##for dev -#kylin.job.run.as.remote.cmd=kylin.job.use-remote-cli -#kylin.job.remote.cli.port=kylin.job.remote-cli-port -#kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname -#kylin.job.remote.cli.username=kylin.job.remote-cli-username -#kylin.job.remote.cli.password=kylin.job.remote-cli-password -# -# -#### ENGINE ### -# -#kylin.cube.engine.=kylin.engine.provider. -#kylin.default.cube.engine=kylin.engine.default -# -#kylin.job.mr.lib.dir=kylin.engine.mr.lib-dir -#kylin.job.mr.config.override.=kylin.engine.mr.config-override. -#kylin.job.jar=kylin.engine.mr.job-jar -#kylin.job.mapreduce.default.reduce.input.mb=kylin.engine.mr.reduce-input-mb -#kylin.job.mapreduce.default.reduce.count.ratio=kylin.engine.mr.reduce-count-ratio -#kylin.job.mapreduce.min.reducer.number=kylin.engine.mr.min-reducer-number -#kylin.job.mapreduce.max.reducer.number=kylin.engine.mr.max-reducer-number -#kylin.job.mapreduce.mapper.input.rows=kylin.engine.mr.mapper-input-rows -#kylin.job.uhc.reducer.count=kylin.engine.mr.uhc-reducer-count -#kylin.job.yarn.app.rest.check.interval.seconds=kylin.engine.mr.yarn-check-interval-seconds -# -#kylin.job.jar.spark=kylin.engine.spark.job-jar -#kylin.spark.home=kylin.engine.spark.spark-home -#kylin.spark.master=kylin.engine.spark.spark-master -# -##deprecated -#kylin.job.yarn.app.rest.check.status.url=kylin.engine.mr.yarn-check-status-url -# -# -#### SOURCE ### -# -#kylin.source.engine.=kylin.source.provider. -#kylin.hive.config.override.=kylin.source.hive.config-override. -#kylin.hive.keep.flat.table=kylin.source.hive.keep-flat-table -#kylin.job.hive.database.for.intermediatetable=kylin.source.hive.database-for-flat-table -#kylin.job.hive.intermediatetable.redistribute.enabled=kylin.source.hive.redistribute-flat-table -#kylin.hive.client=kylin.source.hive.client -#kylin.hive.beeline.params=kylin.source.hive.beeline-params -#kylin.hive.create.flat.table.method=kylin.source.hive.create-flat-table-method -# -##deprecated -#hive.url=kylin.source.hive.url -#hive.user=kylin.source.hive.user -#hive.password=kylin.source.hive.password -#hive.table.location.=kylin.source.hive.table-location. -# -# -#### STORAGE ### -# -#kylin.storage.url=kylin.storage.url -#kylin.storage.engine.=kylin.storage.provider. -#kylin.default.storage.engine=kylin.storage.default -# -#kylin.hbase.cluster.fs=kylin.storage.hbase.cluster-fs -#kylin.hbase.cluster.hdfs.config.file=kylin.storage.hbase.cluster-hdfs-config-file -#kylin.coprocessor.local.jar=kylin.storage.hbase.coprocessor-local-jar -#kylin.hbase.region.count.min=kylin.storage.hbase.min-region-count -#kylin.hbase.region.count.max=kylin.storage.hbase.max-region-count -#kylin.hbase.hfile.size.gb=kylin.storage.hbase.hfile-size-gb -#kylin.query.run.local.coprocessor=kylin.storage.hbase.run-local-coprocessor -#kylin.query.coprocessor.mem.gb=kylin.storage.hbase.coprocessor-mem-gb -#kylin.query.coprocessor.timeout.seconds=kylin.storage.hbase.coprocessor-timeout-seconds -#kylin.query.scan.fuzzykey.max=kylin.storage.hbase.max-fuzzykey-scan -#kylin.query.storage.visit.scanrange.max=kylin.storage.hbase.max-visit-scanrange -#kylin.query.storage.default.gtstorage=kylin.storage.hbase.gtstorage -#kylin.hbase.scan.cache_rows=kylin.storage.hbase.scan-cache-rows -#kylin.hbase.region.cut=kylin.storage.hbase.region-cut-gb -#kylin.hbase.scan.max_result_size=kylin.storage.hbase.max-scan-result-bytes -#kylin.hbase.default.compression.codec=kylin.storage.hbase.compression-codec -#kylin.hbase.default.encoding=kylin.storage.hbase.rowkey-encoding -#kylin.hbase.default.block.size=kylin.storage.hbase.block-size-bytes -#kylin.hbase.small.family.block.size=kylin.storage.hbase.small-family-block-size-bytes -#kylin.owner=kylin.storage.hbase.owner-tag -#kylin.query.endpoint.compression.result=kylin.storage.hbase.endpoint-compress-result -#kylin.query.hbase.hconnection.threads.max=kylin.storage.hbase.max-hconnection-threads -#kylin.query.hbase.hconnection.threads.core=kylin.storage.hbase.core-hconnection-threads -#kylin.query.hbase.hconnection.threads.alive.seconds=kylin.storage.hbase.hconnection-threads-alive-seconds -# -# -#### QUERY ### -# -#kylin.query.pushdown.limit.max=kylin.query.max-limit-pushdown -#kylin.query.scan.threshold=kylin.query.scan-threshold -#kylin.query.filter.derived_in.max=kylin.query.derived-filter-translation-threshold -#kylin.query.badquery.stacktrace.depth=kylin.query.badquery-stacktrace-depth -#kylin.query.badquery.history.num=kylin.query.badquery-history-number -#kylin.query.badquery.alerting.seconds=kylin.query.badquery-alerting-seconds -#kylin.query.badquery.detect.interval.seconds=kylin.query.badquery-detect-interval -#kylin.query.badquery.persistent.enable=kylin.query.badquery-persistent-enabled -#kylin.query.transformers=kylin.query.transformers -#kylin.query.cache.enabled=kylin.query.cache-enabled -#kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration -#kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count -#kylin.query.mem.budget=kylin.query.memory-budget -#kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function -#kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct -#kylin.query.security.enabled=kylin.query.security-enabled -#kylin.query.access.controller=kylin.query.access-controller -#kylin.query.udf.=kylin.query.udf. -# -# -#### SERVER ### -# -#kylin.init.tasks=kylin.server.init-tasks -#kylin.server.mode=kylin.server.mode -#kylin.cluster.name=kylin.server.cluster-name -#kylin.rest.servers=kylin.server.cluster-servers -#kylin.rest.workers.per.server=kylin.server.sequence-sql.workers-per-server -#kylin.query.sequence.expire.time=kylin.server.sequence-sql.expire-time -#kylin.query.metrics.enabled=kylin.server.query-metrics-enabled -#kylin.query.metrics.percentiles.intervals=kylin.server.query-metrics-percentiles-intervals -# -# -#### WEB ### -# -#kylin.rest.timezone=kylin.web.timezone -#crossdomain.enable=kylin.web.cross-domain-enabled +### ENV ### + +deploy.env=kylin.env +kylin.hdfs.working.dir=kylin.env.hdfs-working-dir + + +### METADATA ### + +kylin.metadata.url=kylin.metadata.url +kylin.realization.providers=kylin.metadata.realization-providers +kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings +kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types. + + +### Dictionary ### + +kylin.table.snapshot.max_mb=kylin.snapshot.max-mb +kylin.snapshot.cache.max.entry=kylin.snapshot.max-cache-entry +kylin.dictionary.forest.trie.size.max_mb=kylin.dictionary.forest-trie-max-mb +kylin.dict.cache.max.entry=kylin.dictionary.max-cache-entry +kylin.dict.growing.enabled=kylin.dictionary.growing-enabled +kylin.dict.append.entry.size=kylin.dictionary.append-entry-size +kylin.dict.append.cache.size=kylin.dictionary.append-cache-size + + +### CUBE ### + +kylin.job.cuboid.size.ratio=kylin.cube.size-estimate-ratio +kylin.job.cuboid.size.memhungry.ratio=kylin.cube.size-estimate-memhungry-ratio +kylin.cube.algorithm=kylin.cube.algorithm +kylin.cube.algorithm.auto.threshold=kylin.cube.algorithm.layer-or-inmem-threshold +kylin.cube.algorithm.auto.mapper.limit=kylin.cube.algorithm.inmem-split-limit +kylin.cube.aggrgroup.max.size=kylin.cube.aggrgroup.max-size +kylin.cube.aggrgroup.max.combination=kylin.cube.aggrgroup.max-combination +kylin.cube.aggrgroup.isMandatoryOnlyValid=kylin.cube.aggrgroup.is-mandatory-only-valid +kylin.cube.building.segment.max=kylin.cube.max-building-segments + + +### JOB ### + +kylin.job.log.dir=kylin.job.log-dir +kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir +kylin.job.allow.empty.segment=kylin.job.allow-empty-segment +kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs +kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage +kylin.job.cubing.inmem.sampling.hll.precision=kylin.job.sampling-hll-precision +kylin.job.dependency.filterlist=kylin.job.dependency-filter-list +kylin.job.retry=kylin.job.retry +kylin.job.controller.lock=kylin.job.lock +kylin.scheduler.=kylin.job.scheduler.provider. +kylin.enable.scheduler=kylin.job.scheduler.default + +mail.enabled=kylin.job.notification-enabled +mail.host=kylin.job.notification-mail-host +mail.username=kylin.job.notification-mail-username +mail.password=kylin.job.notification-mail-password +mail.sender=kylin.job.notification-mail-sender +kylin.job.admin.dls=kylin.job.notification-admin-emails + +#for dev +kylin.job.run.as.remote.cmd=kylin.job.use-remote-cli +kylin.job.remote.cli.port=kylin.job.remote-cli-port +kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname +kylin.job.remote.cli.username=kylin.job.remote-cli-username +kylin.job.remote.cli.password=kylin.job.remote-cli-password + + +### ENGINE ### + +kylin.cube.engine.=kylin.engine.provider. +kylin.default.cube.engine=kylin.engine.default + +kylin.job.mr.lib.dir=kylin.engine.mr.lib-dir +kylin.job.mr.config.override.=kylin.engine.mr.config-override. +kylin.job.jar=kylin.engine.mr.job-jar +kylin.job.mapreduce.default.reduce.input.mb=kylin.engine.mr.reduce-input-mb +kylin.job.mapreduce.default.reduce.count.ratio=kylin.engine.mr.reduce-count-ratio +kylin.job.mapreduce.min.reducer.number=kylin.engine.mr.min-reducer-number +kylin.job.mapreduce.max.reducer.number=kylin.engine.mr.max-reducer-number +kylin.job.mapreduce.mapper.input.rows=kylin.engine.mr.mapper-input-rows +kylin.job.uhc.reducer.count=kylin.engine.mr.uhc-reducer-count +kylin.job.yarn.app.rest.check.interval.seconds=kylin.engine.mr.yarn-check-interval-seconds + +kylin.job.jar.spark=kylin.engine.spark.job-jar +kylin.spark.home=kylin.engine.spark.spark-home +kylin.spark.master=kylin.engine.spark.spark-master + +#deprecated +kylin.job.yarn.app.rest.check.status.url=kylin.engine.mr.yarn-check-status-url + + +### SOURCE ### + +kylin.source.engine.=kylin.source.provider. +kylin.hive.config.override.=kylin.source.hive.config-override. +kylin.hive.keep.flat.table=kylin.source.hive.keep-flat-table +kylin.job.hive.database.for.intermediatetable=kylin.source.hive.database-for-flat-table +kylin.job.hive.intermediatetable.redistribute.enabled=kylin.source.hive.redistribute-flat-table +kylin.hive.client=kylin.source.hive.client +kylin.hive.beeline.params=kylin.source.hive.beeline-params +kylin.hive.create.flat.table.method=kylin.source.hive.create-flat-table-method + +#deprecated +hive.url=kylin.source.hive.url +hive.user=kylin.source.hive.user +hive.password=kylin.source.hive.password +hive.table.location.=kylin.source.hive.table-location. + + +### STORAGE ### + +kylin.storage.url=kylin.storage.url +kylin.storage.engine.=kylin.storage.provider. +kylin.default.storage.engine=kylin.storage.default + +kylin.hbase.cluster.fs=kylin.storage.hbase.cluster-fs +kylin.hbase.cluster.hdfs.config.file=kylin.storage.hbase.cluster-hdfs-config-file +kylin.coprocessor.local.jar=kylin.storage.hbase.coprocessor-local-jar +kylin.hbase.region.count.min=kylin.storage.hbase.min-region-count +kylin.hbase.region.count.max=kylin.storage.hbase.max-region-count +kylin.hbase.hfile.size.gb=kylin.storage.hbase.hfile-size-gb +kylin.query.run.local.coprocessor=kylin.storage.hbase.run-local-coprocessor +kylin.query.coprocessor.mem.gb=kylin.storage.hbase.coprocessor-mem-gb +kylin.query.coprocessor.timeout.seconds=kylin.storage.hbase.coprocessor-timeout-seconds +kylin.query.scan.fuzzykey.max=kylin.storage.hbase.max-fuzzykey-scan +kylin.query.storage.visit.scanrange.max=kylin.storage.hbase.max-visit-scanrange +kylin.query.storage.default.gtstorage=kylin.storage.hbase.gtstorage +kylin.hbase.scan.cache_rows=kylin.storage.hbase.scan-cache-rows +kylin.hbase.region.cut=kylin.storage.hbase.region-cut-gb +kylin.hbase.scan.max_result_size=kylin.storage.hbase.max-scan-result-bytes +kylin.hbase.default.compression.codec=kylin.storage.hbase.compression-codec +kylin.hbase.default.encoding=kylin.storage.hbase.rowkey-encoding +kylin.hbase.default.block.size=kylin.storage.hbase.block-size-bytes +kylin.hbase.small.family.block.size=kylin.storage.hbase.small-family-block-size-bytes +kylin.owner=kylin.storage.hbase.owner-tag +kylin.query.endpoint.compression.result=kylin.storage.hbase.endpoint-compress-result +kylin.query.hbase.hconnection.threads.max=kylin.storage.hbase.max-hconnection-threads +kylin.query.hbase.hconnection.threads.core=kylin.storage.hbase.core-hconnection-threads +kylin.query.hbase.hconnection.threads.alive.seconds=kylin.storage.hbase.hconnection-threads-alive-seconds + + +### QUERY ### + +kylin.query.pushdown.limit.max=kylin.query.max-limit-pushdown +kylin.query.scan.threshold=kylin.query.scan-threshold +kylin.query.filter.derived_in.max=kylin.query.derived-filter-translation-threshold +kylin.query.badquery.stacktrace.depth=kylin.query.badquery-stacktrace-depth +kylin.query.badquery.history.num=kylin.query.badquery-history-number +kylin.query.badquery.alerting.seconds=kylin.query.badquery-alerting-seconds +kylin.query.badquery.detect.interval.seconds=kylin.query.badquery-detect-interval +kylin.query.badquery.persistent.enable=kylin.query.badquery-persistent-enabled +kylin.query.transformers=kylin.query.transformers +kylin.query.cache.enabled=kylin.query.cache-enabled +kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration +kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count +kylin.query.mem.budget=kylin.query.memory-budget +kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function +kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct +kylin.query.security.enabled=kylin.query.security-enabled +kylin.query.access.controller=kylin.query.access-controller +kylin.query.udf.=kylin.query.udf. + + +### SERVER ### + +kylin.init.tasks=kylin.server.init-tasks +kylin.server.mode=kylin.server.mode +kylin.cluster.name=kylin.server.cluster-name +kylin.rest.servers=kylin.server.cluster-servers +kylin.rest.workers.per.server=kylin.server.sequence-sql.workers-per-server +kylin.query.sequence.expire.time=kylin.server.sequence-sql.expire-time +kylin.query.metrics.enabled=kylin.server.query-metrics-enabled +kylin.query.metrics.percentiles.intervals=kylin.server.query-metrics-percentiles-intervals + + +### WEB ### + +kylin.rest.timezone=kylin.web.timezone +crossdomain.enable=kylin.web.cross-domain-enabled ### TEST ### http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index 9257718..886d8ee 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -56,7 +56,7 @@ public class KylinConfigTest extends LocalFileMetadataTestCase { @Test public void testBackwardCompatibility() { KylinConfig config = KylinConfig.getInstanceFromEnv(); - final String oldk = "kylin.test.bcc.old.key"; + final String oldk = "kylin.test.bcc.new.key"; final String newk = "kylin.test.bcc.new.key"; assertNull(config.getOptional(oldk)); http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java b/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java index df06221..f0d5e4f 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/MailServiceTest.java @@ -51,7 +51,7 @@ public class MailServiceTest extends LocalFileMetadataTestCase { boolean sent = sendTestEmail(mailservice); assert sent; - // set mail.enabled=false, and run again, this time should be no mail delivered + // set kylin.job.notification-enabled=false, and run again, this time should be no mail delivered config.setMailEnabled(false); mailservice = new MailService(config); sent = sendTestEmail(mailservice); http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java b/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java index 4ddb06a..74012a4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/common/FuzzyValueCombination.java @@ -1,130 +1,130 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.common; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -public class FuzzyValueCombination { - - private static class Dim<K, V> { - K col; - Set<V> values; - } - - private static final Set SINGLE_NULL_SET = Sets.newHashSet(); - - static { - SINGLE_NULL_SET.add(null); - } - - public static <K, V> List<Map<K, V>> calculate(Map<K, Set<V>> fuzzyValues, long cap) { - Collections.emptyMap(); - Dim<K, V>[] dims = toDims(fuzzyValues); - // If a query has many IN clause and each IN clause has many values, then it will easily generate - // thousands of fuzzy keys. When there are lots of fuzzy keys, the scan performance is bottle necked - // on it. So simply choose to abandon all fuzzy keys in this case. - if (exceedCap(dims, cap)) { - return Lists.newArrayList(); - } else { - return combination(dims); - } - } - - @SuppressWarnings("unchecked") - private static <K, V> List<Map<K, V>> combination(Dim<K, V>[] dims) { - - List<Map<K, V>> result = Lists.newArrayList(); - - int emptyDims = 0; - for (Dim dim : dims) { - if (dim.values.isEmpty()) { - dim.values = SINGLE_NULL_SET; - emptyDims++; - } - } - if (emptyDims == dims.length) { - return result; - } - - Map<K, V> r = Maps.newHashMap(); - Iterator<V>[] iters = new Iterator[dims.length]; - int level = 0; - while (true) { - Dim<K, V> dim = dims[level]; - if (iters[level] == null) { - iters[level] = dim.values.iterator(); - } - - Iterator<V> it = iters[level]; - if (it.hasNext() == false) { - if (level == 0) - break; - r.remove(dim.col); - iters[level] = null; - level--; - continue; - } - - r.put(dim.col, it.next()); - if (level == dims.length - 1) { - result.add(new HashMap<K, V>(r)); - } else { - level++; - } - } - return result; - } - - private static <K, V> Dim<K, V>[] toDims(Map<K, Set<V>> fuzzyValues) { - Dim[] dims = new Dim[fuzzyValues.size()]; - int i = 0; - for (Entry<K, Set<V>> entry : fuzzyValues.entrySet()) { - dims[i] = new Dim<K, V>(); - dims[i].col = entry.getKey(); - dims[i].values = entry.getValue(); - if (dims[i].values == null) - dims[i].values = Collections.emptySet(); - i++; - } - return dims; - } - - private static boolean exceedCap(Dim[] dims, long cap) { - return combCount(dims) > cap; - } - - private static long combCount(Dim[] dims) { - long count = 1; - for (Dim dim : dims) { - count *= Math.max(dim.values.size(), 1); - } - return count; - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.common; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +public class FuzzyValueCombination { + + private static class Dim<K, V> { + K col; + Set<V> values; + } + + private static final Set SINGLE_NULL_SET = Sets.newHashSet(); + + static { + SINGLE_NULL_SET.add(null); + } + + public static <K, V> List<Map<K, V>> calculate(Map<K, Set<V>> fuzzyValues, long cap) { + Collections.emptyMap(); + Dim<K, V>[] dims = toDims(fuzzyValues); + // If a query has many IN clause and each IN clause has many values, then it will easily generate + // thousands of fuzzy keys. When there are lots of fuzzy keys, the scan performance is bottle necked + // on it. So simply choose to abandon all fuzzy keys in this case. + if (exceedCap(dims, cap)) { + return Lists.newArrayList(); + } else { + return combination(dims); + } + } + + @SuppressWarnings("unchecked") + private static <K, V> List<Map<K, V>> combination(Dim<K, V>[] dims) { + + List<Map<K, V>> result = Lists.newArrayList(); + + int emptyDims = 0; + for (Dim dim : dims) { + if (dim.values.isEmpty()) { + dim.values = SINGLE_NULL_SET; + emptyDims++; + } + } + if (emptyDims == dims.length) { + return result; + } + + Map<K, V> r = Maps.newHashMap(); + Iterator<V>[] iters = new Iterator[dims.length]; + int level = 0; + while (true) { + Dim<K, V> dim = dims[level]; + if (iters[level] == null) { + iters[level] = dim.values.iterator(); + } + + Iterator<V> it = iters[level]; + if (it.hasNext() == false) { + if (level == 0) + break; + r.remove(dim.col); + iters[level] = null; + level--; + continue; + } + + r.put(dim.col, it.next()); + if (level == dims.length - 1) { + result.add(new HashMap<K, V>(r)); + } else { + level++; + } + } + return result; + } + + private static <K, V> Dim<K, V>[] toDims(Map<K, Set<V>> fuzzyValues) { + Dim[] dims = new Dim[fuzzyValues.size()]; + int i = 0; + for (Entry<K, Set<V>> entry : fuzzyValues.entrySet()) { + dims[i] = new Dim<K, V>(); + dims[i].col = entry.getKey(); + dims[i].values = entry.getValue(); + if (dims[i].values == null) + dims[i].values = Collections.emptySet(); + i++; + } + return dims; + } + + private static boolean exceedCap(Dim[] dims, long cap) { + return combCount(dims) > cap; + } + + private static long combCount(Dim[] dims) { + long count = 1; + for (Dim dim : dims) { + count *= Math.max(dim.values.size(), 1); + } + return count; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java index d5918aa..05efb5e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidCLI.java @@ -1,143 +1,143 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.cuboid; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; -import java.util.TreeSet; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeDescManager; -import org.apache.kylin.cube.model.CubeDesc; - -/** - * @author yangli9 - */ -public class CuboidCLI { - - public static void main(String[] args) throws IOException { - CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); - - if ("test".equals(args[0])) { - CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]); - simulateCuboidGeneration(cubeDesc, true); - } - } - - public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) { - CuboidScheduler scheduler = new CuboidScheduler(cubeDesc); - long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); - Collection<Long> cuboidSet = new TreeSet<Long>(); - cuboidSet.add(baseCuboid); - LinkedList<Long> cuboidQueue = new LinkedList<Long>(); - cuboidQueue.push(baseCuboid); - while (!cuboidQueue.isEmpty()) { - long cuboid = cuboidQueue.pop(); - Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); - for (Long sc : spnanningCuboids) { - boolean notfound = cuboidSet.add(sc); - if (!notfound) { - throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid); - } - - cuboidQueue.push(sc); - - } - } - - if (validate) { - //only run this for test purpose, performance is bad when # of dims is large - TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc); - System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()]))); - if (enumCuboids.equals(cuboidSet) == false) { - throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet); - } - - //check all valid and invalid - for (long i = 0; i < baseCuboid; ++i) { - if (cuboidSet.contains(i)) { - if (!Cuboid.isValid(cubeDesc, i)) { - throw new RuntimeException(); - } - - if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) { - throw new RuntimeException(); - } - } else { - if (Cuboid.isValid(cubeDesc, i)) { - throw new RuntimeException(); - } - - long corrected = Cuboid.translateToValidCuboid(cubeDesc, i); - if (corrected == i) { - throw new RuntimeException(); - } - - if (!Cuboid.isValid(cubeDesc, corrected)) { - throw new RuntimeException(); - } - - if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) { - throw new RuntimeException(); - } - } - } - } - - return cuboidSet.size(); - - } - - public static TreeSet<Long> enumCalcCuboidCount(CubeDesc cube) { - long baseCuboid = Cuboid.getBaseCuboidId(cube); - TreeSet<Long> expectedCuboids = new TreeSet<Long>(); - for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) { - if (Cuboid.isValid(cube, cuboid)) { - expectedCuboids.add(cuboid); - } - } - return expectedCuboids; - } - - public static int[] calculateAllLevelCount(CubeDesc cube) { - int levels = cube.getBuildLevel(); - int[] allLevelCounts = new int[levels + 1]; - - CuboidScheduler scheduler = new CuboidScheduler(cube); - LinkedList<Long> nextQueue = new LinkedList<Long>(); - LinkedList<Long> currentQueue = new LinkedList<Long>(); - long baseCuboid = Cuboid.getBaseCuboidId(cube); - currentQueue.push(baseCuboid); - - for (int i = 0; i <= levels; i++) { - allLevelCounts[i] = currentQueue.size(); - while (!currentQueue.isEmpty()) { - long cuboid = currentQueue.pop(); - Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); - nextQueue.addAll(spnanningCuboids); - } - currentQueue = nextQueue; - nextQueue = new LinkedList<Long>(); - } - - return allLevelCounts; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.cuboid; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.TreeSet; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.model.CubeDesc; + +/** + * @author yangli9 + */ +public class CuboidCLI { + + public static void main(String[] args) throws IOException { + CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); + + if ("test".equals(args[0])) { + CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]); + simulateCuboidGeneration(cubeDesc, true); + } + } + + public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) { + CuboidScheduler scheduler = new CuboidScheduler(cubeDesc); + long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); + Collection<Long> cuboidSet = new TreeSet<Long>(); + cuboidSet.add(baseCuboid); + LinkedList<Long> cuboidQueue = new LinkedList<Long>(); + cuboidQueue.push(baseCuboid); + while (!cuboidQueue.isEmpty()) { + long cuboid = cuboidQueue.pop(); + Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); + for (Long sc : spnanningCuboids) { + boolean notfound = cuboidSet.add(sc); + if (!notfound) { + throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid); + } + + cuboidQueue.push(sc); + + } + } + + if (validate) { + //only run this for test purpose, performance is bad when # of dims is large + TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc); + System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()]))); + if (enumCuboids.equals(cuboidSet) == false) { + throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet); + } + + //check all valid and invalid + for (long i = 0; i < baseCuboid; ++i) { + if (cuboidSet.contains(i)) { + if (!Cuboid.isValid(cubeDesc, i)) { + throw new RuntimeException(); + } + + if (Cuboid.translateToValidCuboid(cubeDesc, i) != i) { + throw new RuntimeException(); + } + } else { + if (Cuboid.isValid(cubeDesc, i)) { + throw new RuntimeException(); + } + + long corrected = Cuboid.translateToValidCuboid(cubeDesc, i); + if (corrected == i) { + throw new RuntimeException(); + } + + if (!Cuboid.isValid(cubeDesc, corrected)) { + throw new RuntimeException(); + } + + if (Cuboid.translateToValidCuboid(cubeDesc, corrected) != corrected) { + throw new RuntimeException(); + } + } + } + } + + return cuboidSet.size(); + + } + + public static TreeSet<Long> enumCalcCuboidCount(CubeDesc cube) { + long baseCuboid = Cuboid.getBaseCuboidId(cube); + TreeSet<Long> expectedCuboids = new TreeSet<Long>(); + for (long cuboid = 0; cuboid <= baseCuboid; cuboid++) { + if (Cuboid.isValid(cube, cuboid)) { + expectedCuboids.add(cuboid); + } + } + return expectedCuboids; + } + + public static int[] calculateAllLevelCount(CubeDesc cube) { + int levels = cube.getBuildLevel(); + int[] allLevelCounts = new int[levels + 1]; + + CuboidScheduler scheduler = new CuboidScheduler(cube); + LinkedList<Long> nextQueue = new LinkedList<Long>(); + LinkedList<Long> currentQueue = new LinkedList<Long>(); + long baseCuboid = Cuboid.getBaseCuboidId(cube); + currentQueue.push(baseCuboid); + + for (int i = 0; i <= levels; i++) { + allLevelCounts[i] = currentQueue.size(); + while (!currentQueue.isEmpty()) { + long cuboid = currentQueue.pop(); + Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); + nextQueue.addAll(spnanningCuboids); + } + currentQueue = nextQueue; + nextQueue = new LinkedList<Long>(); + } + + return allLevelCounts; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java index ec0d39d..99dacc8 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowConstants.java @@ -1,47 +1,47 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube.kv; - -public class RowConstants { - - public static final int ROWKEY_COL_DEFAULT_LENGTH = 256; - - // row key lower bound - public static final byte ROWKEY_LOWER_BYTE = 0; - // row key upper bound - public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff; - - // row key cuboid id length - public static final int ROWKEY_CUBOIDID_LEN = 8; - // row key shard length - public static final int ROWKEY_SHARDID_LEN = 2; - - public static final int ROWKEY_SHARD_AND_CUBOID_LEN = ROWKEY_CUBOIDID_LEN + ROWKEY_SHARDID_LEN; - - public static final byte BYTE_ZERO = 0; - public static final byte BYTE_ONE = 1; - - // row value delimiter - public static final byte ROWVALUE_DELIMITER_BYTE = 7; - public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7); - public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 }; - - public static final int ROWKEY_BUFFER_SIZE = 65 * 256;// a little more than 64 dimensions * 256 bytes each - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube.kv; + +public class RowConstants { + + public static final int ROWKEY_COL_DEFAULT_LENGTH = 256; + + // row key lower bound + public static final byte ROWKEY_LOWER_BYTE = 0; + // row key upper bound + public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff; + + // row key cuboid id length + public static final int ROWKEY_CUBOIDID_LEN = 8; + // row key shard length + public static final int ROWKEY_SHARDID_LEN = 2; + + public static final int ROWKEY_SHARD_AND_CUBOID_LEN = ROWKEY_CUBOIDID_LEN + ROWKEY_SHARDID_LEN; + + public static final byte BYTE_ZERO = 0; + public static final byte BYTE_ONE = 1; + + // row value delimiter + public static final byte ROWVALUE_DELIMITER_BYTE = 7; + public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7); + public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 }; + + public static final int ROWKEY_BUFFER_SIZE = 65 * 256;// a little more than 64 dimensions * 256 bytes each + +}