incubator-hawq git commit: HAWQ-1585. Fixed Javadoc generation warnings and errors.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 23845381e -> 060ca66d2
HAWQ-1585. Fixed Javadoc generation warnings and errors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/060ca66d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/060ca66d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/060ca66d
Branch: refs/heads/master
Commit: 060ca66d288abcf24598f7fcfe4261ab900f18cb
Parents: 2384538
Author: Alex Diachenko
Authored: Wed Feb 7 13:04:00 2018 -0800
Committer: Alex Diachenko
Committed: Wed Feb 7 13:04:00 2018 -0800
--
.../apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java | 6 +-
.../apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java | 4 ++--
.../main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java | 2 +-
.../org/apache/hawq/pxf/service/rest/MetadataResource.java | 2 +-
4 files changed, 5 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/060ca66d/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
index 808c415..3a537b4 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
@@ -41,14 +41,11 @@ import
org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.*;
-import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
-import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
-import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.orc.Reader;
-import org.apache.hawq.pxf.api.Fragmenter;
import org.apache.hawq.pxf.api.Metadata;
import org.apache.hawq.pxf.api.Metadata.Field;
+import org.apache.hawq.pxf.api.Metadata.Item;
import org.apache.hawq.pxf.api.UnsupportedTypeException;
import org.apache.hawq.pxf.api.UserDataException;
import org.apache.hawq.pxf.api.utilities.EnumHawqType;
@@ -60,7 +57,6 @@ import
org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter;
import org.apache.hawq.pxf.plugins.hive.HiveTablePartition;
import
org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter.PXF_HIVE_INPUT_FORMATS;
import org.apache.hawq.pxf.plugins.hive.HiveUserData;
-import
org.apache.hawq.pxf.plugins.hive.utilities.HiveUtilities.PXF_HIVE_SERDES;
/**
* Class containing helper functions connecting
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/060ca66d/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java
--
diff --git
a/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java
b/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java
index c693fd4..914b7d9 100644
---
a/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java
+++
b/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPartitionFragmenter.java
@@ -292,7 +292,7 @@ public class JdbcPartitionFragmenter extends Fragmenter {
case DATE: {
SimpleDateFormat df = new SimpleDateFormat("-MM-dd");
//parse metadata of this fragment
-//validateï¼the length of metadata == 16 (long)
+//validate: the length of metadata == 16 (long)
byte[][] newb = ByteUtil.splitBytes(meta, 8);
Date fragStart = new Date(ByteUtil.toLong(newb[0]));
Date fragEnd = new Date(ByteUtil.toLong(newb[1]));
@@ -304,7 +304,7 @@ public class JdbcPartitionFragmenter extends Fragmenter {
break;
}
case INT: {
-//validateï¼the length of metadata ==8 ï¼int)
+//validate: the length of metadata == 8 (int)
byte[][] newb = ByteUtil.splitBytes(meta, 4);
int fragStart = ByteUtil.toInt(newb[0]);
int fragEnd = ByteUtil.toInt(newb[1]);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/060ca66d/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
--
diff --git
a/pxf/pxf-jdbc/src/main
incubator-hawq git commit: HAWQ-1575. Implemented readable Parquet profile for PXF.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 06d10c1f2 -> 041f0fb6d
HAWQ-1575. Implemented readable Parquet profile for PXF.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/041f0fb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/041f0fb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/041f0fb6
Branch: refs/heads/master
Commit: 041f0fb6da83b1db312b0a6e5da31471276b0dab
Parents: 06d10c1
Author: Alex Diachenko
Authored: Mon Feb 5 19:08:33 2018 -0800
Committer: Alex Diachenko
Committed: Mon Feb 5 19:08:33 2018 -0800
--
pxf/build.gradle| 2 +
pxf/gradle.properties | 3 +-
.../pxf/plugins/hdfs/ParquetDataFragmenter.java | 107 +++
.../pxf/plugins/hdfs/ParquetFileAccessor.java | 169 +
.../hawq/pxf/plugins/hdfs/ParquetResolver.java | 182 +++
.../hawq/pxf/plugins/hdfs/ParquetUserData.java | 42 +
.../plugins/hdfs/utilities/HdfsUtilities.java | 43 -
.../src/main/resources/pxf-profiles-default.xml | 9 +
8 files changed, 547 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/041f0fb6/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 20a9ff4..127de0d 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -216,6 +216,7 @@ project('pxf-service') {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
}
+runtime "org.apache.parquet:parquet-hadoop-bundle:$parquetVersion"
}
ospackage {
@@ -360,6 +361,7 @@ project('pxf-hdfs') {
compile "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
+compile "org.apache.parquet:parquet-hadoop-bundle:$parquetVersion"
}
ospackage {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/041f0fb6/pxf/gradle.properties
--
diff --git a/pxf/gradle.properties b/pxf/gradle.properties
index 06dcaba..a82f585 100644
--- a/pxf/gradle.properties
+++ b/pxf/gradle.properties
@@ -24,4 +24,5 @@ hbaseVersionJar=1.1.2
hbaseVersionRPM=1.1.2
tomcatVersion=7.0.62
pxfProtocolVersion=v15
-osFamily=el6
\ No newline at end of file
+osFamily=el6
+parquetVersion=1.9.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/041f0fb6/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetDataFragmenter.java
--
diff --git
a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetDataFragmenter.java
b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetDataFragmenter.java
new file mode 100644
index 000..811a8f6
--- /dev/null
+++
b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetDataFragmenter.java
@@ -0,0 +1,107 @@
+package org.apache.hawq.pxf.plugins.hdfs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hawq.pxf.api.Fragment;
+import org.apache.hawq.pxf.api.Fragmenter;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetInputFormat;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.hadoop.metadata.ParquetMet
incubator-hawq git commit: HAWQ-1538. Install internal profiles definition file in conf directory.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 46fac1851 -> e89dea51a
HAWQ-1538. Install internal profiles definition file in conf directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e89dea51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e89dea51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e89dea51
Branch: refs/heads/master
Commit: e89dea51a2fb36187183bb1e11f7c3bdd8ab4f91
Parents: 46fac18
Author: Oleksandr Diachenko
Authored: Mon Oct 9 17:18:25 2017 -0700
Committer: Oleksandr Diachenko
Committed: Mon Oct 9 17:18:25 2017 -0700
--
pxf/build.gradle| 5 ++-
.../src/main/resources/pxf-profiles-default.xml | 14 ++--
.../src/main/resources/pxf-profiles.xml | 36
3 files changed, 41 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e89dea51/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 0e2cf0c..2a99231 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -240,10 +240,9 @@ project('pxf-service') {
obsoletes('pxf-core')
obsoletes('pxf-api')
-from('src/main/resources/pxf-profiles-default.xml') {
+from('src/main/resources/pxf-profiles.xml') {
fileType CONFIG | NOREPLACE
into "/etc/pxf-${project.version}/conf"
-rename { 'pxf-profiles.xml' }
}
from('src/configs/pxf-site.xml') {
@@ -310,7 +309,7 @@ project('pxf-service') {
}
project.distTar {
-from('src/main/resources/pxf-profiles-default.xml') { into 'conf'
rename { 'pxf-profiles.xml' } }
+from('src/main/resources/pxf-profiles.xml') { into 'conf' }
from("src/main/resources") { into 'conf' include
'**/pxf-private*.classpath'}
from("src/main/resources/pxf-private${hddist}.classpath") { into
'conf' rename {'pxf-private.classpath'} }
from('src/main/resources/pxf-public.classpath') { into 'conf' }
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e89dea51/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
index df17969..a98d05e 100644
--- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
@@ -20,17 +20,9 @@ under the License.
-->
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e89dea51/pxf/pxf-service/src/main/resources/pxf-profiles.xml
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles.xml
b/pxf/pxf-service/src/main/resources/pxf-profiles.xml
new file mode 100644
index 000..796f49b
--- /dev/null
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
\ No newline at end of file
incubator-hawq git commit: HAWQ-1537. Sort files explicitly in PartitionedJsonParserNoSeekTest.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 9fd36e3e3 -> 46fac1851
HAWQ-1537. Sort files explicitly in PartitionedJsonParserNoSeekTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/46fac185
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/46fac185
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/46fac185
Branch: refs/heads/master
Commit: 46fac18511e82b5a3b4bba653c43a606ec202be5
Parents: 9fd36e3
Author: Oleksandr Diachenko
Authored: Mon Oct 9 15:46:54 2017 -0700
Committer: Oleksandr Diachenko
Committed: Mon Oct 9 15:46:54 2017 -0700
--
.../json/parser/PartitionedJsonParserNoSeekTest.java | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/46fac185/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserNoSeekTest.java
--
diff --git
a/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserNoSeekTest.java
b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserNoSeekTest.java
index cdc876b..5dab0ec 100644
---
a/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserNoSeekTest.java
+++
b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserNoSeekTest.java
@@ -28,6 +28,8 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -43,7 +45,7 @@ public class PartitionedJsonParserNoSeekTest {
File testsDir = new
File("src/test/resources/parser-tests/noseek");
File[] jsonFiles = testsDir.listFiles(new FilenameFilter() {
public boolean accept(File file, String s) {
- return s.endsWith(".json");
+ return s.endsWith(".json") &&
!s.contains("expected");
}
});
@@ -58,13 +60,13 @@ public class PartitionedJsonParserNoSeekTest {
try {
PartitionedJsonParser parser = new
PartitionedJsonParser(jsonInputStream);
- File[] jsonOjbectFiles =
jsonFile.getParentFile().listFiles(new FilenameFilter() {
+ File[] jsonObjectFiles =
jsonFile.getParentFile().listFiles(new FilenameFilter() {
public boolean accept(File file, String s) {
return s.contains(jsonFile.getName())
&& s.contains("expected");
}
});
-
- for (File jsonObjectFile : jsonOjbectFiles) {
+ Arrays.sort(jsonObjectFiles);
+ for (File jsonObjectFile : jsonObjectFiles) {
String expected =
trimWhitespaces(FileUtils.readFileToString(jsonObjectFile));
String result =
parser.nextObjectContainingMember("name");
assertNotNull(jsonFile.getName() + "/" +
jsonObjectFile.getName(), result);
@@ -80,4 +82,5 @@ public class PartitionedJsonParserNoSeekTest {
public String trimWhitespaces(String s) {
return s.replaceAll("[\\n\\t\\r \\t]+", " ").trim();
}
+
}
incubator-hawq git commit: HAWQ-1534. Change user to PXF_USER only for HAWQ database.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 634e0cdac -> c00b8d3ff
HAWQ-1534. Change user to PXF_USER only for HAWQ database.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c00b8d3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c00b8d3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c00b8d3f
Branch: refs/heads/master
Commit: c00b8d3ff4ea9ef543893a844bdb156cf2669d1a
Parents: 634e0cd
Author: Oleksandr Diachenko
Authored: Thu Oct 5 13:40:15 2017 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 5 13:40:15 2017 -0700
--
pxf/build.gradle| 10 --
pxf/gradle/profiles/gpdb.properties | 1 -
pxf/pxf-service/src/scripts/pxf-env.sh | 4 ++-
pxf/pxf-service/src/scripts/pxf-service | 48
4 files changed, 46 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c00b8d3f/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 6ac55b7..e27074e 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -273,7 +273,7 @@ project('pxf-service') {
filter(ReplaceTokens,
tokens: ['pxfLogDir' : databaseProperties.pxfLogDir,
'pxfRunDir' : databaseProperties.pxfRunDir,
- 'pxfDefaultUser':
databaseProperties.pxfDefaultUser])
+ 'pxfDefaultUser':
(databaseProperties.pxfDefaultUser == null ? "" :
databaseProperties.pxfDefaultUser)])
into "/etc/pxf-${project.version}/conf"
}
@@ -300,6 +300,9 @@ project('pxf-service') {
fileMode 0755
addParentDirs false
into "/opt/pxf-${project.version}"
+filter(ReplaceTokens,
+tokens: [
+'pxfDefaultUser':
(databaseProperties.pxfDefaultUser == null ? "" :
databaseProperties.pxfDefaultUser)])
}
link("/usr/lib/pxf-${project.version}/${project.name}.jar",
"${project.name}-${project.version}.jar")
@@ -618,6 +621,9 @@ task install(type: Copy, dependsOn: [subprojects.build,
tomcatGet]) {
into 'bin'
fileMode 0755
rename('pxf-service', 'pxf')
+filter(ReplaceTokens,
+tokens: [
+'pxfDefaultUser': (databaseProperties.pxfDefaultUser
== null ? "" : databaseProperties.pxfDefaultUser)])
}
from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' }
@@ -627,7 +633,7 @@ task install(type: Copy, dependsOn: [subprojects.build,
tomcatGet]) {
filter(ReplaceTokens,
tokens: ['pxfLogDir' : databaseProperties.pxfLogDir,
'pxfRunDir' : databaseProperties.pxfRunDir,
- 'pxfDefaultUser': databaseProperties.pxfDefaultUser])
into 'conf'
+ 'pxfDefaultUser': (databaseProperties.pxfDefaultUser
== null ? "" : databaseProperties.pxfDefaultUser)]) into 'conf'
}
from("pxf-service/src/configs/tomcat") { into 'tomcat-templates' }
from("pxf-service/src/configs/templates") { into 'conf-templates' }
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c00b8d3f/pxf/gradle/profiles/gpdb.properties
--
diff --git a/pxf/gradle/profiles/gpdb.properties
b/pxf/gradle/profiles/gpdb.properties
index 22b095f..90a8a60 100644
--- a/pxf/gradle/profiles/gpdb.properties
+++ b/pxf/gradle/profiles/gpdb.properties
@@ -15,6 +15,5 @@
# specific language governing permissions and limitations
# under the License.
-pxfDefaultUser=gpadmin
pxfLogDir=${PXF_HOME}/logs
pxfRunDir=${PXF_HOME}/run
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c00b8d3f/pxf/pxf-service/src/scripts/pxf-env.sh
--
diff --git a/pxf/pxf-service/src/scripts/pxf-env.sh
b/pxf/pxf-service/src/scripts/pxf-env.sh
index 284f08d..f33e61a 100644
--- a/pxf/pxf-service/src/scripts/pxf-env.sh
+++ b/pxf/pxf-service/src/scripts/pxf-env.sh
@@ -34,7 +34,9 @@ export PXF_LOGDIR=@pxfLogDir@
export PXF_RUNDIR=@pxfRunDir@
# Configured user
-export PXF_USER=${PXF_USER:-@pxfDefaultUser@}
+if [ ! -z '@pxfDefaultUser@' ]; then
+export PXF_USER=${PXF_USER:-@pxfDefaultUser@}
+fi
# Port
export PXF_PORT=${PXF_PORT:-51200}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c00b8d3f/pxf/pxf-service/src/scripts/pxf-service
--
diff --git a/pxf/pxf-service/s
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1531 [deleted] 2caabd4e3
incubator-hawq git commit: HAWQ-1531. Templetized PXF user and log directories.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 5f972f9be -> 634e0cdac
HAWQ-1531. Templetized PXF user and log directories.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/634e0cda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/634e0cda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/634e0cda
Branch: refs/heads/master
Commit: 634e0cdac3478600fe567f2fefd0ddaa8cf3db50
Parents: 5f972f9
Author: Oleksandr Diachenko
Authored: Fri Sep 29 12:33:50 2017 -0700
Committer: Oleksandr Diachenko
Committed: Fri Sep 29 12:33:50 2017 -0700
--
pxf/Makefile| 5 +
pxf/README.md | 9 ++
pxf/build.gradle| 19 ++-
pxf/gradle.properties | 2 +-
pxf/gradle/profiles/gpdb.properties | 20 +++
pxf/gradle/profiles/hawq.properties | 20 +++
.../templates/pxf-private.classpath.template| 80 ++
.../src/configs/tomcat/bin/setenv.sh| 4 +-
.../src/main/resources/pxf-log4j.properties | 2 +-
pxf/pxf-service/src/scripts/pxf-env.sh | 18 ++-
pxf/pxf-service/src/scripts/pxf-service | 152 +--
11 files changed, 306 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/634e0cda/pxf/Makefile
--
diff --git a/pxf/Makefile b/pxf/Makefile
index da83cc8..3ac38b1 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -42,6 +42,10 @@ ifneq "$(PXF_VERSION)" ""
BUILD_PARAMS+= -Pversion="$(PXF_VERSION)"
endif
+ifneq "$(DATABASE)" ""
+BUILD_PARAMS+= -Ddatabase="$(DATABASE)"
+endif
+
help:
@echo
@echo "help it is then"
@@ -51,6 +55,7 @@ help:
@echo " - - LICENSE= - add license info to created
RPMs"
@echo " - - VENDOR= - add vendor name to created RPMs"
@echo " - - PLUGINS_EXCLUDE_RPM=plugin1, plugin2,... - do not
build and RPM for given comma-separated list of plugins"
+ @echo " - - DATABASE=build pxf for specific database(HAWQ is
default, other databases are defined under gradle/profiles)"
@echo " - tomcat - builds tomcat rpm from downloaded tarball"
@echo " - - LICENSE and VENDOR parameters can be used as well"
@echo " - install - setup PXF along with tomcat in the configured
deployPath"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/634e0cda/pxf/README.md
--
diff --git a/pxf/README.md b/pxf/README.md
index 540e761..e51d83c 100644
--- a/pxf/README.md
+++ b/pxf/README.md
@@ -35,4 +35,13 @@ Building
./gradlew clean build [buildRpm] [distTar]
For all available tasks run: ./gradlew tasks
+
+Building for a specific database
+
+
+PXF could be built for a diffent databases, currently HAWQ and Greenplum are
supported.
+Configuration for target databases are stored in **gradle/profiles**.
+HAWQ is a default database. To build it for Greenplum:
+
+./gradlew clean build [buildRpm] [distTar] -Ddatabase="gpdb"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/634e0cda/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 52e4020..6ac55b7 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -19,6 +19,13 @@
import org.apache.tools.ant.filters.ReplaceTokens
+// Get database property, use HAWQ as a default database
+def database = System.getProperty("database", "hawq");
+def databaseFileName = "gradle/profiles/" + database + ".properties";
+assert file(databaseFileName).exists()
+def databaseProperties = new Properties()
+file(databaseFileName).withInputStream { databaseProperties.load(it) }
+
buildscript {
repositories {
// mavenCentral without https:
@@ -263,6 +270,10 @@ project('pxf-service') {
from('src/scripts/pxf-env.sh') {
fileMode 0755
fileType NOREPLACE
+filter(ReplaceTokens,
+tokens: ['pxfLogDir' : databaseProperties.pxfLogDir,
+ 'pxfRunDir' : databaseProperties.pxfRunDir,
+ 'pxfDefaultUser':
databaseProperties.pxfDefaultUser])
into "/etc/pxf-${project.version}/conf"
}
@@ -612,8 +623,14 @@ task install(type: Copy, dependsOn: [subprojects.build,
tomcatGet]) {
from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' }
from("pxf-service/src/main/resources") { into 'conf' }
from("pxf
incubator-hawq git commit: HAWQ-1531. Templetized PXF user and log directories.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1531 [created] 2caabd4e3
HAWQ-1531. Templetized PXF user and log directories.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2caabd4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2caabd4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2caabd4e
Branch: refs/heads/HAWQ-1531
Commit: 2caabd4e33833ae926659490dcd18c3d2a064c42
Parents: b282aef
Author: Alex Diachenko
Authored: Tue Sep 26 11:23:01 2017 -0700
Committer: Oleksandr Diachenko
Committed: Fri Sep 29 12:23:27 2017 -0700
--
pxf/Makefile| 5 +
pxf/README.md | 9 ++
pxf/build.gradle| 19 ++-
pxf/gradle.properties | 2 +-
pxf/gradle/profiles/gpdb.properties | 20 +++
pxf/gradle/profiles/hawq.properties | 20 +++
.../templates/pxf-private.classpath.template| 80 ++
.../src/configs/tomcat/bin/setenv.sh| 4 +-
.../src/main/resources/pxf-log4j.properties | 2 +-
pxf/pxf-service/src/scripts/pxf-env.sh | 18 ++-
pxf/pxf-service/src/scripts/pxf-service | 152 +--
11 files changed, 306 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2caabd4e/pxf/Makefile
--
diff --git a/pxf/Makefile b/pxf/Makefile
index da83cc8..3ac38b1 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -42,6 +42,10 @@ ifneq "$(PXF_VERSION)" ""
BUILD_PARAMS+= -Pversion="$(PXF_VERSION)"
endif
+ifneq "$(DATABASE)" ""
+BUILD_PARAMS+= -Ddatabase="$(DATABASE)"
+endif
+
help:
@echo
@echo "help it is then"
@@ -51,6 +55,7 @@ help:
@echo " - - LICENSE= - add license info to created
RPMs"
@echo " - - VENDOR= - add vendor name to created RPMs"
@echo " - - PLUGINS_EXCLUDE_RPM=plugin1, plugin2,... - do not
build and RPM for given comma-separated list of plugins"
+ @echo " - - DATABASE=build pxf for specific database(HAWQ is
default, other databases are defined under gradle/profiles)"
@echo " - tomcat - builds tomcat rpm from downloaded tarball"
@echo " - - LICENSE and VENDOR parameters can be used as well"
@echo " - install - setup PXF along with tomcat in the configured
deployPath"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2caabd4e/pxf/README.md
--
diff --git a/pxf/README.md b/pxf/README.md
index 540e761..e51d83c 100644
--- a/pxf/README.md
+++ b/pxf/README.md
@@ -35,4 +35,13 @@ Building
./gradlew clean build [buildRpm] [distTar]
For all available tasks run: ./gradlew tasks
+
+Building for a specific database
+
+
+PXF could be built for a diffent databases, currently HAWQ and Greenplum are
supported.
+Configuration for target databases are stored in **gradle/profiles**.
+HAWQ is a default database. To build it for Greenplum:
+
+./gradlew clean build [buildRpm] [distTar] -Ddatabase="gpdb"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2caabd4e/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 52e4020..6ac55b7 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -19,6 +19,13 @@
import org.apache.tools.ant.filters.ReplaceTokens
+// Get database property, use HAWQ as a default database
+def database = System.getProperty("database", "hawq");
+def databaseFileName = "gradle/profiles/" + database + ".properties";
+assert file(databaseFileName).exists()
+def databaseProperties = new Properties()
+file(databaseFileName).withInputStream { databaseProperties.load(it) }
+
buildscript {
repositories {
// mavenCentral without https:
@@ -263,6 +270,10 @@ project('pxf-service') {
from('src/scripts/pxf-env.sh') {
fileMode 0755
fileType NOREPLACE
+filter(ReplaceTokens,
+tokens: ['pxfLogDir' : databaseProperties.pxfLogDir,
+ 'pxfRunDir' : databaseProperties.pxfRunDir,
+ 'pxfDefaultUser':
databaseProperties.pxfDefaultUser])
into "/etc/pxf-${project.version}/conf"
}
@@ -612,8 +623,14 @@ task install(type: Copy, dependsOn: [subprojects.build,
tomcatGet]) {
from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' }
from("pxf-service/src/main/resources") { into 'conf' }
from("pxf-s
incubator-hawq git commit: HAWQ-1526. Added support for functions in WHERE clause for PXF tables.
Repository: incubator-hawq
Updated Branches:
refs/heads/master be4af7785 -> b282aef2e
HAWQ-1526. Added support for functions in WHERE clause for PXF tables.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b282aef2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b282aef2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b282aef2
Branch: refs/heads/master
Commit: b282aef2e1a6105ef5fa1db340ad7e11aff72ba1
Parents: be4af77
Author: Oleksandr Diachenko
Authored: Fri Sep 22 11:26:49 2017 -0700
Committer: Oleksandr Diachenko
Committed: Fri Sep 22 11:26:49 2017 -0700
--
src/backend/access/external/pxffilters.c| 110 ++
src/backend/access/external/pxfheaders.c| 8 +-
.../access/external/test/pxffilters_test.c | 114 +--
.../access/external/test/pxfheaders_test.c | 35 ++
src/include/access/pxffilters.h | 2 +-
5 files changed, 237 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b282aef2/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index f502e4e..7c1db56 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -44,6 +44,7 @@ static void scalar_const_to_str(Const *constval, StringInfo
buf);
static void list_const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
static void enrich_trivial_expression(List *expressionItems);
+static List* get_attrs_from_expr(Expr *expr, bool* expressionIsSupported);
/*
* All supported HAWQ operators, and their respective HDFS operator code.
@@ -851,12 +852,51 @@ append_attr_from_var(Var* var, List* attrs)
return attrs;
}
+/*
+ * append_attr_from_func_args
+ *
+ * extracts all columns from FuncExpr into attrs
+ * assigns false to expressionIsSupported if at least one of items is not
supported
+ */
+static List*
+append_attr_from_func_args(FuncExpr *expr, List* attrs, bool*
expressionIsSupported) {
+ if (!expressionIsSupported) {
+ return NIL;
+ }
+ ListCell *lc = NULL;
+ foreach (lc, expr->args)
+ {
+ Node *node = (Node *) lfirst(lc);
+ if (IsA(node, FuncExpr)) {
+ attrs = append_attr_from_func_args((FuncExpr *) node,
attrs, expressionIsSupported);
+ } else if (IsA(node, Var)) {
+ attrs = append_attr_from_var((Var *) node, attrs);
+ } else if (IsA(node, OpExpr)) {
+ attrs = get_attrs_from_expr((OpExpr *) node,
expressionIsSupported);
+ } else {
+ *expressionIsSupported = false;
+ return NIL;
+ }
+ }
+
+ return attrs;
+
+}
+
+/*
+ * get_attrs_from_expr
+ *
+ * extracts and returns list of all columns from Expr
+ * assigns false to expressionIsSupported if at least one of items is not
supported
+ */
static List*
-get_attrs_from_expr(Expr *expr)
+get_attrs_from_expr(Expr *expr, bool* expressionIsSupported)
{
- Node*leftop = NULL;
- Node*rightop= NULL;
- List*attrs = NIL;
+ Node *leftop = NULL;
+ Node *rightop = NULL;
+ List *attrs = NIL;
+
+ *expressionIsSupported = true;
if ((!expr))
return attrs;
@@ -870,30 +910,46 @@ get_attrs_from_expr(Expr *expr)
ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) expr;
leftop = (Node *) linitial(saop->args);
rightop = (Node *) lsecond(saop->args);
+ } else {
+ // If expression type is not known, report that it's not
supported
+ *expressionIsSupported = false;
+ return NIL;
}
- //Process left operand
- //For most of datatypes column is represented by Var node
- if (IsA(leftop, Var))
+ // We support following combinations of operands:
+ // Var, Const
+ // Relabel, Const
+ // FuncExpr, Const
+ // Const, Var
+ // Const, Relabel
+ // Const, FuncExpr
+ // For most of datatypes column is represented by Var node
+ // For varchar column is represented by RelabelType node
+ if (IsA(leftop, Var) && IsA(rightop, Const))
{
attrs = append_attr_from_var((Var *) leftop, attrs);
- }
- //For varchar column is represented by RelabelType node
- if (IsA(leftop, RelabelType))
+ } else if (IsA(leftop, RelabelType) && IsA(rightop, Const))
incubator-hawq git commit: HAWQ-1198. Fix to filter out irrelevant fragments while accessing Hive.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 9f76faa70 -> 10f085f9a
HAWQ-1198. Fix to filter out irrelevant fragments while accessing Hive.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/10f085f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/10f085f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/10f085f9
Branch: refs/heads/master
Commit: 10f085f9aa2e4b00d775648cfbaf6059483b3dfc
Parents: 9f76faa
Author: Shubham Sharma
Authored: Thu Sep 14 16:24:27 2017 -0700
Committer: Oleksandr Diachenko
Committed: Thu Sep 14 16:24:27 2017 -0700
--
.../pxf/plugins/hive/HiveDataFragmenter.java| 16 ++-
.../plugins/hive/HiveDataFragmenterTest.java| 111 +++
2 files changed, 118 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/10f085f9/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
index 6e193c2..c24b552 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
@@ -402,17 +402,11 @@ public class HiveDataFragmenter extends Fragmenter {
// Let's look first at the filter
BasicFilter bFilter = (BasicFilter) filter;
-// In case this is not an "equality filter", we ignore this filter (no
-// add to filter list)
-if (!(bFilter.getOperation() == FilterParser.Operation.HDOP_EQ)) {
-LOG.debug("Filter operator is not EQ, ignore this filter for hive
: "
-+ filter);
-return false;
-}
-
// Extract column name and value
int filterColumnIndex = bFilter.getColumn().index();
-String filterValue = bFilter.getConstant().constant().toString();
+// Avoids NullPointerException in case of operations like HDOP_IS_NULL,
+// HDOP_IS_NOT_NULL where no constant value is passed as part of query
+String filterValue = bFilter.getConstant()!= null ?
bFilter.getConstant().constant().toString() : "";
ColumnDescriptor filterColumn = inputData.getColumn(filterColumnIndex);
String filterColumnName = filterColumn.columnName();
@@ -453,6 +447,10 @@ public class HiveDataFragmenter extends Fragmenter {
case HDOP_NE:
filtersString.append(HIVE_API_NE);
break;
+default:
+// Set filter string to blank in case of unimplemented
operations
+filtersString.setLength(0);
+return false;
}
filtersString.append(HIVE_API_DQUOTE);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/10f085f9/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java
--
diff --git
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java
index f93f7ba..2c28500 100755
---
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java
+++
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenterTest.java
@@ -20,7 +20,11 @@ package org.apache.hawq.pxf.plugins.hive;
*/
+import org.apache.hawq.pxf.api.FilterParser;
import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.BasicFilter;
+import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
+import static org.apache.hawq.pxf.api.FilterParser.Operation.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -39,6 +43,10 @@ import
org.powermock.core.classloader.annotations.PrepareForTest;
import
org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.*;
+
@RunWith(PowerMockRunner.class)
@PrepareForTest({HiveDataFragmenter.class}) // Enables mocking 'new' calls
@SuppressStaticInitializationFor({"org.apache.hadoop.mapred.JobConf",
@@ -88,6 +96,109 @@ public class HiveDataFragmenterTest {
}
}
+@Test
+public void testBuildSingleFilter() throws Exception {
+prepareConstruction();
+fragmenter = new HiveDataFragmenter(inputData);
+Col
incubator-hawq git commit: HAWQ-1503. Fixed ftp link causing build failure on centos-6 docker.
Repository: incubator-hawq Updated Branches: refs/heads/master 6424efa5c -> 278b3d3c0 HAWQ-1503. Fixed ftp link causing build failure on centos-6 docker. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/278b3d3c Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/278b3d3c Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/278b3d3c Branch: refs/heads/master Commit: 278b3d3c06f6e22528164af3579a6291fd4216a0 Parents: 6424efa Author: Shubham Sharma Authored: Sat Jul 15 15:30:48 2017 -0700 Committer: Oleksandr Diachenko Committed: Sat Jul 15 15:30:48 2017 -0700 -- contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/278b3d3c/contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile -- diff --git a/contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile b/contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile index 9fb8476..7905723 100644 --- a/contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile +++ b/contrib/hawq-docker/centos6-docker/hawq-dev/Dockerfile @@ -35,7 +35,7 @@ RUN yum install -y epel-release && \ # update gcc RUN wget -O /etc/yum.repos.d/slc6-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc6-devtoolset.repo && \ - rpm --import http://ftp.scientificlinux.org/linux/scientific/5x/x86_64/RPM-GPG-KEYs/RPM-GPG-KEY-cern && \ + rpm --import http://linuxsoft.cern.ch/cern/slc68/x86_64/RPM-GPG-KEY-cern && \ yum install -y devtoolset-2-gcc devtoolset-2-binutils devtoolset-2-gcc-c++ && \ echo "source /opt/rh/devtoolset-2/enable" >> ~/.bashrc && \ source ~/.bashrc
incubator-hawq git commit: HAWQ-1492. Enable packaging for PXF JDBC plugin.
Repository: incubator-hawq
Updated Branches:
refs/heads/master f157d59fa -> 8a9a4d977
HAWQ-1492. Enable packaging for PXF JDBC plugin.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8a9a4d97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8a9a4d97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8a9a4d97
Branch: refs/heads/master
Commit: 8a9a4d9778e51aa33e70d423e6931dd92e4e0aae
Parents: f157d59
Author: Lav Jain
Authored: Thu Jun 29 18:02:16 2017 -0700
Committer: Oleksandr Diachenko
Committed: Thu Jun 29 18:02:16 2017 -0700
--
pxf/build.gradle | 1 +
pxf/gradle.properties | 2 +-
.../src/main/resources/pxf-private.classpath | 1 +
.../main/resources/pxf-privatebigtop.classpath| 1 +
.../src/main/resources/pxf-privatehdp.classpath | 1 +
.../src/main/resources/pxf-privatephd.classpath | 3 ++-
.../src/main/resources/pxf-profiles-default.xml | 18 --
7 files changed, 19 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 213b2dd..6f10dbb 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -319,6 +319,7 @@ project('pxf') {
requires(versionedPackageName('pxf-hive'), project.version, GREATER |
EQUAL)
requires(versionedPackageName('pxf-hbase'), project.version, GREATER |
EQUAL)
requires(versionedPackageName('pxf-json'), project.version, GREATER |
EQUAL)
+requires(versionedPackageName('pxf-jdbc'), project.version, GREATER |
EQUAL)
preInstall file('src/scripts/pre-install.sh')
postInstall file('src/scripts/post-install.sh')
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/gradle.properties
--
diff --git a/pxf/gradle.properties b/pxf/gradle.properties
index 8544e89..82190ea 100644
--- a/pxf/gradle.properties
+++ b/pxf/gradle.properties
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-version=3.2.1.0
+version=3.3.0.0
license=ASL 2.0
vendor=Apache HAWQ Incubating
hadoopVersion=2.7.1
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/pxf-service/src/main/resources/pxf-private.classpath
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-private.classpath
b/pxf/pxf-service/src/main/resources/pxf-private.classpath
index 17a1716..7987ac9 100644
--- a/pxf/pxf-service/src/main/resources/pxf-private.classpath
+++ b/pxf/pxf-service/src/main/resources/pxf-private.classpath
@@ -36,6 +36,7 @@ pxf/pxf-hbase-*[0-9].jar
pxf/pxf-hdfs-*[0-9].jar
pxf/pxf-hive-*[0-9].jar
pxf/pxf-json-*[0-9].jar
+pxf/pxf-jdbc-*[0-9].jar
# Hadoop Libraries
hadoop/share/hadoop/hdfs/hadoop-hdfs-*[0-9].jar
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/pxf-service/src/main/resources/pxf-privatebigtop.classpath
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-privatebigtop.classpath
b/pxf/pxf-service/src/main/resources/pxf-privatebigtop.classpath
index 8eff55c..e949dcc 100644
--- a/pxf/pxf-service/src/main/resources/pxf-privatebigtop.classpath
+++ b/pxf/pxf-service/src/main/resources/pxf-privatebigtop.classpath
@@ -60,3 +60,4 @@
/usr/lib/pxf/pxf-hdfs-*[0-9].jar
/usr/lib/pxf/pxf-hive-*[0-9].jar
/usr/lib/pxf/pxf-json-*[0-9].jar
+/usr/lib/pxf/pxf-jdbc-*[0-9].jar
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath
b/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath
index 5f3a743..f1e789b 100644
--- a/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath
+++ b/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath
@@ -64,3 +64,4 @@
/usr/lib/pxf/pxf-hdfs-*[0-9].jar
/usr/lib/pxf/pxf-hive-*[0-9].jar
/usr/lib/pxf/pxf-json-*[0-9].jar
+/usr/lib/pxf/pxf-jdbc-*[0-9].jar
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8a9a4d97/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath
b/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath
index 7523d7d..a380033 100644
--- a/pxf/pxf-service/src/main/resources/p
incubator-hawq git commit: HAWQ-1446: Introduce vectorized profile for ORC.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 339806f3a -> 29a160839
HAWQ-1446: Introduce vectorized profile for ORC.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/29a16083
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/29a16083
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/29a16083
Branch: refs/heads/master
Commit: 29a160839949a1a08244962c5255933f714af46c
Parents: 339806f
Author: Oleksandr Diachenko
Authored: Wed Jun 21 19:58:08 2017 -0700
Committer: Oleksandr Diachenko
Committed: Wed Jun 21 19:58:08 2017 -0700
--
.../hawq/pxf/api/ReadVectorizedResolver.java| 39 ++
.../org/apache/hawq/pxf/api/StatsAccessor.java | 2 +-
.../hawq/pxf/api/utilities/Utilities.java | 22 +-
.../hawq/pxf/api/utilities/UtilitiesTest.java | 29 ++
.../pxf/plugins/hive/HiveDataFragmenter.java| 5 +-
.../plugins/hive/HiveORCVectorizedAccessor.java | 106 ++
.../plugins/hive/HiveORCVectorizedResolver.java | 367 +++
.../plugins/hive/utilities/ProfileFactory.java | 15 +-
.../hawq/pxf/service/BridgeOutputBuilder.java | 13 +
.../org/apache/hawq/pxf/service/ReadBridge.java | 2 +-
.../hawq/pxf/service/ReadVectorizedBridge.java | 102 ++
.../hawq/pxf/service/rest/BridgeResource.java | 3 +
.../src/main/resources/pxf-profiles-default.xml | 11 +
13 files changed, 707 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a16083/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/ReadVectorizedResolver.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/ReadVectorizedResolver.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/ReadVectorizedResolver.java
new file mode 100644
index 000..55f8df5
--- /dev/null
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/ReadVectorizedResolver.java
@@ -0,0 +1,39 @@
+package org.apache.hawq.pxf.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+/**
+ *
+ * Interface that defines deserialization batch of records at once.
+ *
+ */
+public interface ReadVectorizedResolver {
+
+/**
+ * Returns resolved list of tuples
+ *
+ * @param batch unresolved batch
+ * @return list of tuples
+ */
+public List> getFieldsForBatch(OneRow batch);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a16083/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
index d256e77..ec65bd8 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
@@ -29,7 +29,7 @@ public interface StatsAccessor extends ReadAccessor {
/**
* Method which reads needed statistics for current split
- * @throws Exception if retrieving the stats failed
+ * @throws Exception when unable to retrieve statistics
*/
public void retrieveStats() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a16083/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index ed8ad28..175a6e1 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFacto
incubator-hawq git commit: HAWQ-1486. Catch error out on NULL condition for churl_init_download.
Repository: incubator-hawq Updated Branches: refs/heads/master ee200369c -> b53484511 HAWQ-1486. Catch error out on NULL condition for churl_init_download. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b5348451 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b5348451 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b5348451 Branch: refs/heads/master Commit: b534845110f7cd0f54c17aba08c6ce951e703e19 Parents: ee20036 Author: John Gaskin Authored: Thu Jun 15 12:17:26 2017 -0700 Committer: Oleksandr Diachenko Committed: Thu Jun 15 12:17:26 2017 -0700 -- src/backend/access/external/pxfutils.c | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b5348451/src/backend/access/external/pxfutils.c -- diff --git a/src/backend/access/external/pxfutils.c b/src/backend/access/external/pxfutils.c index fb6b799..2f0171e 100644 --- a/src/backend/access/external/pxfutils.c +++ b/src/backend/access/external/pxfutils.c @@ -108,6 +108,8 @@ static void process_request(ClientContext* client_context, char *uri) print_http_headers(client_context->http_headers); client_context->handle = churl_init_download(uri, client_context->http_headers); + if (client_context->handle == NULL) + elog(ERROR, "Unsuccessful connection to uri: %s", uri); memset(buffer, 0, RAW_BUF_SIZE); resetStringInfo(&(client_context->the_rest_buf));
incubator-hawq git commit: HAWQ-1461. Improve partition parameters validation for PXF-JDBC plugin.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 099557973 -> b7f9f36f5
HAWQ-1461. Improve partition parameters validation for PXF-JDBC plugin.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b7f9f36f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b7f9f36f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b7f9f36f
Branch: refs/heads/master
Commit: b7f9f36f5ff2f1473960803ab52bce2886d45abf
Parents: 0995579
Author: Lav Jain
Authored: Fri Jun 9 15:34:35 2017 -0700
Committer: Oleksandr Diachenko
Committed: Fri Jun 9 15:34:35 2017 -0700
--
pxf/build.gradle| 2 +-
.../org/apache/hawq/pxf/api/StatsAccessor.java | 2 +
.../hawq/pxf/api/utilities/Utilities.java | 4 +-
pxf/pxf-jdbc/README.md | 2 -
.../plugins/jdbc/JdbcPartitionFragmenter.java | 40 +++--
.../jdbc/JdbcPartitionFragmenterTest.java | 173 ---
6 files changed, 139 insertions(+), 84 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b7f9f36f/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 3fd9992..80d30df 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -520,7 +520,7 @@ def tomcatTargetDir = "tomcat/build"
task tomcatGet << {
apply plugin: 'de.undercouch.download'
-
+
def TarGzSuffix = ".tar.gz"
def tomcatTar = "${tomcatName}${TarGzSuffix}"
def tomcatUrl =
"http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcatVersion}/bin/${tomcatTar}";
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b7f9f36f/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
index 7ecdf52..d256e77 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
@@ -29,11 +29,13 @@ public interface StatsAccessor extends ReadAccessor {
/**
* Method which reads needed statistics for current split
+ * @throws Exception if retrieving the stats failed
*/
public void retrieveStats() throws Exception;
/**
* Returns next tuple based on statistics information without actual
reading of data
+ * @return next row without reading it from disk
*/
public OneRow emitAggObject();
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b7f9f36f/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index 916c2b5..ed8ad28 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -164,7 +164,7 @@ public class Utilities {
* @param inputData input data which has protocol information
* @return fragment metadata
* @throws IllegalArgumentException if fragment metadata information
wasn't found in input data
- * @throws Exception
+ * @throws Exception if unable to parse the fragment
*/
public static FragmentMetadata parseFragmentMetadata(InputData inputData)
throws Exception {
byte[] serializedLocation = inputData.getFragmentMetadata();
@@ -198,7 +198,7 @@ public class Utilities {
/**
* Based on accessor information determines whether to use AggBridge
*
- * @param protData
+ * @param inputData input data
* @return true if AggBridge is applicable for current context
*/
public static boolean useAggBridge(InputData inputData) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b7f9f36f/pxf/pxf-jdbc/README.md
--
diff --git a/pxf/pxf-jdbc/README.md b/pxf/pxf-jdbc/README.md
index e8c4bc0..c158f32 100644
--- a/pxf/pxf-jdbc/README.md
+++ b/pxf/pxf-jdbc/README.md
@@ -105,8 +105,6 @@ The `PARTITION_BY` parameter can be null, and there will be
only one fragment.
The `RANGE` parameter indicates the range of data to be queried , it can be
split by colon(':').
The range is left-closed, ie: `>= start_value AND < end_value` .
- If the `column_type` is `int`, the `end_value` can be empty.
- If the `column_type` is `enum`,the parameter `RANGE` can be empty.
T
incubator-hawq git commit: HAWQ-1434. Remove forced upper casing on table name.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 740e04dfe -> a3a5acbc9
HAWQ-1434. Remove forced upper casing on table name.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a3a5acbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a3a5acbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a3a5acbc
Branch: refs/heads/master
Commit: a3a5acbc9094ec3fb913088617f06747dff0a8b0
Parents: 740e04d
Author: Michael André Pearce
Authored: Tue May 9 11:09:46 2017 -0700
Committer: Oleksandr Diachenko
Committed: Tue May 9 11:13:28 2017 -0700
--
.../src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java | 2 --
1 file changed, 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a3a5acbc/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
--
diff --git
a/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
b/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
index 53848f1..2e03264 100644
---
a/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
+++
b/pxf/pxf-jdbc/src/main/java/org/apache/hawq/pxf/plugins/jdbc/JdbcPlugin.java
@@ -75,8 +75,6 @@ public class JdbcPlugin extends Plugin {
tblName = input.getDataSource();
if (tblName == null) {
throw new UserDataException("TABLE_NAME must be set as
DataSource.");
-} else {
-tblName = tblName.toUpperCase();
}
}
incubator-hawq git commit: HAWQ-1431. Do not use StatsAccessor when column specified in SELECT clause.
Repository: incubator-hawq
Updated Branches:
refs/heads/master c7a76523d -> d6dd24fd3
HAWQ-1431. Do not use StatsAccessor when column specified in SELECT clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d6dd24fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d6dd24fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d6dd24fd
Branch: refs/heads/master
Commit: d6dd24fd31dd3cb6a7c9235f68d1c5dc32f11206
Parents: c7a7652
Author: Oleksandr Diachenko
Authored: Thu Apr 13 17:01:39 2017 -0700
Committer: Oleksandr Diachenko
Committed: Thu Apr 13 17:01:39 2017 -0700
--
.../hawq/pxf/api/utilities/InputData.java | 26
.../hawq/pxf/api/utilities/Utilities.java | 14 +--
.../hawq/pxf/api/utilities/UtilitiesTest.java | 7 ++
.../pxf/plugins/hive/HiveORCAccessorTest.java | 1 +
.../pxf/service/utilities/ProtocolData.java | 1 +
5 files changed, 42 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 959cda6..7bf57ed 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -75,6 +75,16 @@ public class InputData {
protected ColumnDescriptor recordkeyColumn;
/**
+ * Number of attributes projected in query.
+ *
+ * Example:
+ * SELECT col1, col2, col3... : number of attributes projected - 3
+ * SELECT col1, col2, col3... WHERE col4=a : number of attributes
projected - 4
+ * SELECT *... : number of attributes projected - 0
+ */
+protected int numAttrsProjected;
+
+/**
* Constructs an empty InputData
*/
public InputData() {
@@ -369,4 +379,20 @@ public class InputData {
this.fragmentIndex = fragmentIndex;
}
+/**
+ * Returns number of attributes projected in a query
+ * @return number of attributes projected
+ */
+public int getNumAttrsProjected() {
+return numAttrsProjected;
+}
+
+/**
+ * Sets number of attributes projected
+ * @param numAttrsProjected number of attrivutes projected
+ */
+public void setNumAttrsProjected(int numAttrsProjected) {
+this.numAttrsProjected = numAttrsProjected;
+}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index 6dd0044..916c2b5 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -204,6 +204,9 @@ public class Utilities {
public static boolean useAggBridge(InputData inputData) {
boolean isStatsAccessor = false;
try {
+if (inputData == null || inputData.getAccessor() == null) {
+throw new IllegalArgumentException("Missing accessor
information");
+}
isStatsAccessor =
ArrayUtils.contains(Class.forName(inputData.getAccessor()).getInterfaces(),
StatsAccessor.class);
} catch (ClassNotFoundException e) {
LOG.error("Unable to load accessor class: " + e.getMessage());
@@ -213,6 +216,7 @@ public class Utilities {
return (inputData != null) && !inputData.hasFilter()
&& (inputData.getAggType() != null)
&& inputData.getAggType().isOptimizationSupported()
+&& inputData.getNumAttrsProjected() == 0
&& isStatsAccessor;
}
@@ -224,14 +228,10 @@ public class Utilities {
* @return true if this accessor should use statistic information
*/
public static boolean useStats(ReadAccessor accessor, InputData inputData)
{
-if (accessor instanceof StatsAccessor) {
-/* Make sure filter is not present and aggregate operation
supports optimization */
-if (inputData != null && !inputData.hasFilter()
-&& inputData.getAggType() != null
-&& inputData.getAggType().isOptimizationSupported()) {
+if (accessor instanceof StatsAccessor && useAggBridge(inputData)) {
incubator-hawq git commit: HAWQ-1429. Do not use AggBridge when WHERE clause specified.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 0899c5343 -> 2d32de10f
HAWQ-1429. Do not use AggBridge when WHERE clause specified.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2d32de10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2d32de10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2d32de10
Branch: refs/heads/master
Commit: 2d32de10f132f1870ad7785640d9a57f711db89a
Parents: 0899c53
Author: Oleksandr Diachenko
Authored: Wed Apr 12 14:43:03 2017 -0700
Committer: Oleksandr Diachenko
Committed: Wed Apr 12 14:44:07 2017 -0700
--
.../java/org/apache/hawq/pxf/api/utilities/Utilities.java | 5 -
.../org/apache/hawq/pxf/api/utilities/UtilitiesTest.java | 10 ++
2 files changed, 14 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2d32de10/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index 29d9c52..6dd0044 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -209,7 +209,9 @@ public class Utilities {
LOG.error("Unable to load accessor class: " + e.getMessage());
return false;
}
-return (inputData != null) && (inputData.getAggType() != null)
+/* Make sure filter is not present, aggregate operation supports
optimization and accessor implements StatsAccessor interface */
+return (inputData != null) && !inputData.hasFilter()
+&& (inputData.getAggType() != null)
&& inputData.getAggType().isOptimizationSupported()
&& isStatsAccessor;
}
@@ -223,6 +225,7 @@ public class Utilities {
*/
public static boolean useStats(ReadAccessor accessor, InputData inputData)
{
if (accessor instanceof StatsAccessor) {
+/* Make sure filter is not present and aggregate operation
supports optimization */
if (inputData != null && !inputData.hasFilter()
&& inputData.getAggType() != null
&& inputData.getAggType().isOptimizationSupported()) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2d32de10/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
--
diff --git
a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
index 6fe896a..12ba3d1 100644
---
a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
+++
b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
@@ -194,6 +194,12 @@ public class UtilitiesTest {
when(metaData.getAccessor()).thenReturn(UtilitiesTest.class.getName());
when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT);
assertFalse(Utilities.useAggBridge(metaData));
+
+//Do not use AggBridge when input data has filter
+
when(metaData.getAccessor()).thenReturn(StatsAccessorImpl.class.getName());
+when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT);
+when(metaData.hasFilter()).thenReturn(true);
+assertFalse(Utilities.useAggBridge(metaData));
}
@Test
@@ -204,5 +210,9 @@ public class UtilitiesTest {
assertTrue(Utilities.useStats(accessor, metaData));
ReadAccessor nonStatusAccessor = new NonStatsAccessorImpl();
assertFalse(Utilities.useStats(nonStatusAccessor, metaData));
+
+//Do not use stats when input data has filter
+when(metaData.hasFilter()).thenReturn(true);
+assertFalse(Utilities.useStats(accessor, metaData));
}
}
incubator-hawq git commit: HAWQ-1404. PXF to leverage file-level stats of ORC file and emit records for COUNT(*).
Repository: incubator-hawq
Updated Branches:
refs/heads/master be0547200 -> f5ffddf26
HAWQ-1404. PXF to leverage file-level stats of ORC file and emit records for
COUNT(*).
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f5ffddf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f5ffddf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f5ffddf2
Branch: refs/heads/master
Commit: f5ffddf26ddc990d616c905f883c383e7f1c8542
Parents: be05472
Author: Oleksandr Diachenko
Authored: Tue Apr 4 02:32:28 2017 -0700
Committer: Oleksandr Diachenko
Committed: Tue Apr 4 02:32:28 2017 -0700
--
.../org/apache/hawq/pxf/api/StatsAccessor.java | 40 +
.../pxf/api/utilities/EnumAggregationType.java | 50 +++
.../pxf/api/utilities/FragmentMetadata.java | 86 ++
.../hawq/pxf/api/utilities/InputData.java | 34 +++
.../hawq/pxf/api/utilities/Utilities.java | 80 +
.../hawq/pxf/api/utilities/UtilitiesTest.java | 91 +++
.../plugins/hdfs/HdfsAtomicDataAccessor.java| 2 +-
.../hdfs/HdfsSplittableDataAccessor.java| 2 +-
.../plugins/hdfs/utilities/HdfsUtilities.java | 26 +-
.../hdfs/utilities/HdfsUtilitiesTest.java | 21 +
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 77 +++-
.../plugins/hive/utilities/HiveUtilities.java | 27 +-
.../pxf/plugins/hive/HiveORCAccessorTest.java | 13 +++
.../org/apache/hawq/pxf/service/AggBridge.java | 95
.../hawq/pxf/service/rest/BridgeResource.java | 5 +-
.../pxf/service/utilities/ProtocolData.java | 13 +++
src/bin/gpfusion/gpbridgeapi.c | 1 +
17 files changed, 632 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f5ffddf2/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
new file mode 100644
index 000..7ecdf52
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hawq.pxf.api;
+
+import org.apache.hawq.pxf.api.OneRow;
+
+/**
+ * Interface of accessor which can leverage statistic information for
aggregate queries
+ *
+ */
+public interface StatsAccessor extends ReadAccessor {
+
+/**
+ * Method which reads needed statistics for current split
+ */
+public void retrieveStats() throws Exception;
+
+/**
+ * Returns next tuple based on statistics information without actual
reading of data
+ */
+public OneRow emitAggObject();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f5ffddf2/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
new file mode 100644
index 000..5557c49
--- /dev/null
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * soft
[1/2] incubator-hawq git commit: [#140722851] PXF to use file level stats for count(*) without condition.
Repository: incubator-hawq
Updated Branches:
refs/heads/140722851 [created] 478f0a891
[#140722851] PXF to use file level stats for count(*) without condition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/73a20485
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/73a20485
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/73a20485
Branch: refs/heads/140722851
Commit: 73a204854f6400d31f5b26237ed8df9dfb087d91
Parents: 55d9e85
Author: Oleksandr Diachenko
Authored: Wed Mar 8 13:45:13 2017 -0800
Committer: Oleksandr Diachenko
Committed: Fri Mar 17 15:18:25 2017 -0700
--
.../pxf/api/utilities/EnumAggregationType.java | 51 +++
.../pxf/api/utilities/FragmentMetadata.java | 59 +
.../hawq/pxf/api/utilities/InputData.java | 9 ++
.../hawq/pxf/api/utilities/Utilities.java | 25 ++
.../plugins/hdfs/HdfsAtomicDataAccessor.java| 2 +-
.../hdfs/HdfsSplittableDataAccessor.java| 2 +-
.../plugins/hdfs/utilities/HdfsUtilities.java | 26 +-
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 69 ++-
.../plugins/hive/utilities/HiveUtilities.java | 23 +
pxf/pxf-service/src/main/.DS_Store | Bin 0 -> 6148 bytes
.../org/apache/hawq/pxf/service/AggBridge.java | 86 +++
.../hawq/pxf/service/rest/BridgeResource.java | 4 +-
.../pxf/service/utilities/ProtocolData.java | 5 ++
13 files changed, 333 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/73a20485/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
new file mode 100644
index 000..ee38f18
--- /dev/null
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hawq.pxf.api.utilities;
+
+public enum EnumAggregationType {
+
+COUNT("count", true),
+UNKNOWN("unknown", false);
+
+private String aggOperationCode;
+private boolean optimizationSupported;
+
+EnumAggregationType(String aggOperationCode, boolean
optimizationSupported) {
+this.aggOperationCode = aggOperationCode;
+this.optimizationSupported = optimizationSupported;
+}
+
+public String getAggOperationCode() {
+return this.aggOperationCode;
+}
+
+public boolean isOptimizationSupported() {
+return this.optimizationSupported;
+}
+
+public static EnumAggregationType getAggregationType(String
aggOperationCode) {
+for (EnumAggregationType at : values()) {
+if (at.getAggOperationCode().equals(aggOperationCode)) {
+return at;
+}
+}
+return EnumAggregationType.UNKNOWN;
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/73a20485/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
new file mode 100644
index 000..7a82065
--- /dev/null
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/140722851 [deleted] 478f0a891
[2/2] incubator-hawq git commit: [#140722851] Applied code-review feedback.
[#140722851] Applied code-review feedback.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/478f0a89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/478f0a89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/478f0a89
Branch: refs/heads/140722851
Commit: 478f0a89163554a835bf3963b4a7e2df3341fe34
Parents: 73a2048
Author: Oleksandr Diachenko
Authored: Mon Mar 27 17:11:47 2017 -0700
Committer: Oleksandr Diachenko
Committed: Mon Mar 27 17:11:47 2017 -0700
--
.../org/apache/hawq/pxf/api/StatsAccessor.java | 40 ++
.../pxf/api/utilities/EnumAggregationType.java | 2 +-
.../pxf/api/utilities/FragmentMetadata.java | 17 +++-
.../hawq/pxf/api/utilities/Utilities.java | 44 ++--
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 42 ++-
.../org/apache/hawq/pxf/service/AggBridge.java | 5 +++
.../pxf/service/utilities/ProtocolData.java | 1 +
7 files changed, 127 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/478f0a89/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
new file mode 100644
index 000..724448b
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/StatsAccessor.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hawq.pxf.api;
+
+import org.apache.hawq.pxf.api.OneRow;
+
+/**
+ * Interface of accessor which can leverage statistic information for
aggregate queries
+ *
+ */
+public interface StatsAccessor {
+
+/**
+ * Method which reads needed statistics for current split
+ */
+public void retrieveStats() throws Exception;
+
+/**
+ * Returns next tuple based on statistics information without actual
reading of data
+ */
+public OneRow emitAggObject();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/478f0a89/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
index ee38f18..69716c6 100644
---
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/EnumAggregationType.java
@@ -27,7 +27,7 @@ public enum EnumAggregationType {
private String aggOperationCode;
private boolean optimizationSupported;
-EnumAggregationType(String aggOperationCode, boolean
optimizationSupported) {
+private EnumAggregationType(String aggOperationCode, boolean
optimizationSupported) {
this.aggOperationCode = aggOperationCode;
this.optimizationSupported = optimizationSupported;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/478f0a89/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
index 7a82065..7f266ec 100644
---
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
+++
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/FragmentMetadata.java
@@ -19,9 +19,12 @@
package org.apache.hawq.pxf.api.utilities;
+/**
+ * Class which holds metadata of a file split and locality information.
+ *
+ */
public class FragmentMetadata {
-
private long start;
private long end;
private
incubator-hawq git commit: HAWQ-1414. Bump up PXF version to 3.2.1.0.
Repository: incubator-hawq Updated Branches: refs/heads/master 9a4ff843f -> 34cea88d1 HAWQ-1414. Bump up PXF version to 3.2.1.0. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/34cea88d Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/34cea88d Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/34cea88d Branch: refs/heads/master Commit: 34cea88d1e649a88ec86bee2b49425d3c8248181 Parents: 9a4ff84 Author: Oleksandr Diachenko Authored: Mon Mar 27 16:54:29 2017 -0700 Committer: Oleksandr Diachenko Committed: Mon Mar 27 16:54:29 2017 -0700 -- contrib/hawq-ambari-plugin/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/34cea88d/contrib/hawq-ambari-plugin/build.properties -- diff --git a/contrib/hawq-ambari-plugin/build.properties b/contrib/hawq-ambari-plugin/build.properties index aeb4259..1cd6645 100644 --- a/contrib/hawq-ambari-plugin/build.properties +++ b/contrib/hawq-ambari-plugin/build.properties @@ -1,6 +1,6 @@ hawq.release.version=2.2.0 hawq.common.services.version=2.0.0 -pxf.release.version=3.2.0 +pxf.release.version=3.2.1 pxf.common.services.version=3.0.0 hawq.repo.prefix=hawq hawq.addons.repo.prefix=hawq-add-ons
incubator-hawq git commit: HAWQ-1414. Bump up PXF version to 3.2.1.0.
Repository: incubator-hawq Updated Branches: refs/heads/master 4921e92c4 -> 9a4ff843f HAWQ-1414. Bump up PXF version to 3.2.1.0. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9a4ff843 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9a4ff843 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9a4ff843 Branch: refs/heads/master Commit: 9a4ff843f91171aff8c0e8311ff920674aa39cc0 Parents: 4921e92 Author: Oleksandr Diachenko Authored: Mon Mar 27 14:00:31 2017 -0700 Committer: Oleksandr Diachenko Committed: Mon Mar 27 14:00:31 2017 -0700 -- pxf/gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9a4ff843/pxf/gradle.properties -- diff --git a/pxf/gradle.properties b/pxf/gradle.properties index ba8ad2e..8544e89 100644 --- a/pxf/gradle.properties +++ b/pxf/gradle.properties @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -version=3.2.0.0 +version=3.2.1.0 license=ASL 2.0 vendor=Apache HAWQ Incubating hadoopVersion=2.7.1
incubator-hawq git commit: HAWQ-1402. Updated comments for Hive-related profiles.
Repository: incubator-hawq Updated Branches: refs/heads/master 477af283e -> e4ac99c7a HAWQ-1402. Updated comments for Hive-related profiles. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e4ac99c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e4ac99c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e4ac99c7 Branch: refs/heads/master Commit: e4ac99c7a77f3b0cf2956a51e7fbc153339d08f1 Parents: 477af28 Author: Oleksandr Diachenko Authored: Thu Mar 23 11:45:52 2017 -0700 Committer: Oleksandr Diachenko Committed: Thu Mar 23 11:46:03 2017 -0700 -- pxf/pxf-service/src/main/resources/pxf-profiles-default.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e4ac99c7/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml -- diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml index 53f15bb..f076ead 100644 --- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml +++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml @@ -46,6 +46,7 @@ under the License. This profile is suitable for using when connecting to Hive. Supports GPDBWritable output format, as specified in FORMAT header parameter. +It auto-detects actual file storage format and uses an optimized profile (HiveRC, HiveText, HiveORC) if applicable. org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter @@ -59,7 +60,6 @@ under the License. HiveRC This profile is suitable only for Hive tables stored in RC files and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe. -It is much faster than the general purpose Hive profile. DELIMITER parameter is mandatory. Supports both GPDBWritable and TEXT output formats, as specified in FORMAT header parameter. Primary optimized for TEXT output format. @@ -75,7 +75,6 @@ under the License. HiveText This profile is suitable only for Hive tables stored as Text files. -It is much faster than the general purpose Hive profile. DELIMITER parameter is mandatory. Supports both GPDBWritable and TEXT output formats, as specified in FORMAT header parameter. Primary optimized for TEXT output format. @@ -92,7 +91,6 @@ under the License. HiveORC This profile is suitable only for Hive tables stored in ORC files and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe. -It is much faster than the general purpose Hive profile. Supports GPDBWritable output format, as specified in FORMAT header parameter.
incubator-hawq git commit: HAWQ-1395. PXF build script to support excluding building RPMs for some plugins.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 19a7b22b6 -> efaf24ea4
HAWQ-1395. PXF build script to support excluding building RPMs for some plugins.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/efaf24ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/efaf24ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/efaf24ea
Branch: refs/heads/master
Commit: efaf24ea40c49c033da17cb6129a3af2ebe2ef29
Parents: 19a7b22
Author: Oleksandr Diachenko
Authored: Tue Mar 21 15:25:24 2017 -0700
Committer: Oleksandr Diachenko
Committed: Tue Mar 21 15:27:00 2017 -0700
--
pxf/Makefile | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/efaf24ea/pxf/Makefile
--
diff --git a/pxf/Makefile b/pxf/Makefile
index 9095ccb..30137fe 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -44,19 +44,20 @@ endif
help:
@echo
- @echo"help it is then"
+ @echo "help it is then"
@echo "Possible targets"
@echo " - all (clean, build, unittest, jar, tar, rpm)"
@echo " - - HD= - set classpath to match hadoop
distribution. default phd"
@echo " - - LICENSE= - add license info to created
RPMs"
@echo " - - VENDOR= - add vendor name to created RPMs"
+ @echo " - - PLUGINS_EXCLUDE_RPM=plugin1, plugin2,... - do not
build and RPM for given comma-separated list of plugins"
@echo " - tomcat - builds tomcat rpm from downloaded tarball"
@echo " - - LICENSE and VENDOR parameters can be used as well"
@echo " - deploy - setup PXF along with tomcat in the configured
deployPath"
@echo " - doc - creates aggregate javadoc under docs"
-all:
- ./gradlew clean release $(BUILD_PARAMS)
+all:
+ ./gradlew clean release $(BUILD_PARAMS) $$(echo $$(for i in $$(echo
${PLUGINS_EXCLUDE_RPM} | sed 's/,/ /g'); do echo -x $$i:buildRpm ; done))
unittest:
./gradlew test
incubator-hawq git commit: HAWQ-1314. Added upgrade for function pxf_get_item_fields.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 9a86ed8c9 -> 103373b7f
HAWQ-1314. Added upgrade for function pxf_get_item_fields.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/103373b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/103373b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/103373b7
Branch: refs/heads/master
Commit: 103373b7f7cfabef181bd624a6ceb4d6e5095954
Parents: 9a86ed8
Author: Oleksandr Diachenko
Authored: Tue Feb 14 18:20:02 2017 -0800
Committer: Oleksandr Diachenko
Committed: Tue Feb 14 18:20:16 2017 -0800
--
tools/bin/hawqupgrade | 48 +++---
1 file changed, 45 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/103373b7/tools/bin/hawqupgrade
--
diff --git a/tools/bin/hawqupgrade b/tools/bin/hawqupgrade
index a71f4d6..0352018 100755
--- a/tools/bin/hawqupgrade
+++ b/tools/bin/hawqupgrade
@@ -89,12 +89,52 @@ def register_func(options, args):
except pg.DatabaseError as e:
logger.error('%s', e)
logger.info('Hawq Upgrade Failed.')
-return 1
+sys.exit(1)
except pg.InternalError:
logger.error('Fail to connect to database, this script can only be run
when database is up.')
logger.info('Hawq Upgrade Failed.')
-return 1
-logger.info('Hawq Upgrade Succeed.')
+sys.exit(1)
+
+def pxf_get_item_fields_func(options, args):
+# connect db
+try:
+url = dbconn.DbURL(hostname=options.host, port=options.port,
+ dbname="template1", username=options.user)
+utility_conn = pg.connect(dbname=url.pgdb, host=url.pghost,
port=url.pgport,
+ user=url.pguser, passwd=url.pgpass, opt='-c
gp_session_role=utility')
+
+
+# update function for template1
+update_func_query = """
+SET allow_system_table_mods = 'dml';
+UPDATE pg_proc
+SET proallargtypes = '{25,25,25,25,25,25,25}', proargmodes =
'{i,i,o,o,o,o,o}', proargnames =
'{profile,pattern,path,itemname,fieldname,fieldtype,sourcefieldtype}'
+WHERE proname = 'pxf_get_item_fields';
+"""
+
+utility_conn.query(update_func_query)
+logger.info('Function pxf_get_item_fields successfully updated into
database template1.')
+ # pxf_get_item_fields function for other databases
+query = """select datname from pg_database;"""
+dbs = utility_conn.query(query).dictresult()
+for cur_db in dbs:
+if cur_db['datname'] not in ['template1', 'template0', 'hcatalog']:
+url = dbconn.DbURL(hostname=options.host, port=options.port,
+ dbname=cur_db['datname'], username=options.user)
+existed_db_conn = pg.connect(dbname=cur_db['datname'],
host=url.pghost,
+ port=url.pgport, user=url.pguser,
passwd=url.pgpass)
+existed_db_conn.query(update_func_query);
+logger.info('Function pxf_get_item_fields successfully updated
in database %s.',cur_db['datname'])
+existed_db_conn.close()
+utility_conn.close()
+except pg.DatabaseError as e:
+logger.error('%s', e)
+logger.info('Hawq Upgrade Failed.')
+sys.exit(1)
+except pg.InternalError:
+logger.error('Fail to connect to database, this script can only be run
when database is up.')
+logger.info('Hawq Upgrade Failed.')
+sys.exit(1)
if __name__ == '__main__':
parser = option_parser()
@@ -103,3 +143,5 @@ if __name__ == '__main__':
parser.print_help(sys.stderr)
sys.exit(1)
register_func(options, args)
+pxf_get_item_fields_func(options, args)
+logger.info('Hawq Upgrade Succeed.')
\ No newline at end of file
incubator-hawq git commit: HAWQ-1308. Fixed Javadoc warnings.
Repository: incubator-hawq
Updated Branches:
refs/heads/master f6452d273 -> 7d02472b8
HAWQ-1308. Fixed Javadoc warnings.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7d02472b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7d02472b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7d02472b
Branch: refs/heads/master
Commit: 7d02472b82e841254d63215c9acdfc51a2ecd7e3
Parents: f6452d2
Author: Oleksandr Diachenko
Authored: Thu Feb 2 14:43:45 2017 -0800
Committer: Oleksandr Diachenko
Committed: Thu Feb 2 14:44:19 2017 -0800
--
.../java/org/apache/hawq/pxf/api/OutputFormat.java | 3 ++-
.../pxf/plugins/hive/utilities/HiveUtilities.java| 15 ---
2 files changed, 10 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7d02472b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 89c4b30..4d2d806 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -44,7 +44,8 @@ public enum OutputFormat {
/**
* Looks up output format for given class name if it exists.
- *
+ *
+ * @param className class name implementing certain output format
* @throws UnsupportedTypeException if output format with given class
wasn't found
* @return an output format with given class name
*/
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7d02472b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
index 37f4ac2..3328c9f 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
@@ -455,11 +455,11 @@ public class HiveUtilities {
/**
* The method which serializes fragment-related attributes, needed for
reading and resolution to string
*
- * @param fragmenterClassName
- * @param partData
- * @param filterInFragmenter
+ * @param fragmenterClassName fragmenter class name
+ * @param partData partition data
+ * @param filterInFragmenter whether filtering was done in fragmenter
* @return serialized representation of fragment-related attributes
- * @throws Exception
+ * @throws Exception when error occurred during serialization
*/
@SuppressWarnings("unchecked")
public static byte[] makeUserData(String fragmenterClassName,
HiveTablePartition partData, boolean filterInFragmenter) throws Exception {
@@ -494,7 +494,7 @@ public class HiveUtilities {
* @param input input data
* @param supportedSerdes list of allowed serdes in current context
* @return instance of HiveUserData class
- * @throws UserDataException
+ * @throws UserDataException when incorrect number of tokens in Hive user
data received
*/
public static HiveUserData parseHiveUserData(InputData input,
PXF_HIVE_SERDES... supportedSerdes) throws UserDataException{
String userData = new String(input.getFragmentUserData());
@@ -610,10 +610,11 @@ public class HiveUtilities {
/**
* Creates an instance of a given serde type
*
- * @param serdeType
- * @param allowedSerdes
+ * @param serdeType SerDe type
+ * @param allowedSerdes allowed serdes in current context
* @return instance of a given serde
* @throws UnsupportedTypeException if given serde is not allowed in
current context
+ * @throws Exception if other error occurred during creation of SerDe
instance
*/
@SuppressWarnings("deprecation")
public static SerDe createDeserializer(PXF_HIVE_SERDES serdeType,
PXF_HIVE_SERDES... allowedSerdes) throws Exception{
incubator-hawq git commit: HAWQ-1306. Removed links in Javadoc referencing pxf-service from pxf-api.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 524e2e501 -> 42c1cc131
HAWQ-1306. Removed links in Javadoc referencing pxf-service from pxf-api.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/42c1cc13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/42c1cc13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/42c1cc13
Branch: refs/heads/master
Commit: 42c1cc1310c5eadf69c1882ed0f532a057bc1505
Parents: 524e2e5
Author: Oleksandr Diachenko
Authored: Wed Feb 1 16:49:23 2017 -0800
Committer: Oleksandr Diachenko
Committed: Wed Feb 1 16:49:23 2017 -0800
--
.../src/main/java/org/apache/hawq/pxf/api/OutputFormat.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/42c1cc13/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 565db13..89c4b30 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -21,7 +21,7 @@ package org.apache.hawq.pxf.api;
/**
- * PXF supported output formats: {@link org.apache.hawq.pxf.service.io.Text}
and {@link org.apache.hawq.pxf.service.io.GPDBWritable}
+ * PXF supported output formats, enum which contains serializations classes
*/
public enum OutputFormat {
TEXT("org.apache.hawq.pxf.service.io.Text"),
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1177 [deleted] 65e1ed10a
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1228 [deleted] 607184c36
[2/2] incubator-hawq git commit: HAWQ-1228. Use profile based on file format in HCatalog integration(HiveRC, HiveText profiles).
HAWQ-1228. Use profile based on file format in HCatalog integration(HiveRC,
HiveText profiles).
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6fa1ced2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6fa1ced2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6fa1ced2
Branch: refs/heads/master
Commit: 6fa1ced20e8bb2820b73e6904f77c4b4a1ed6de2
Parents: aac8868
Author: Oleksandr Diachenko
Authored: Mon Jan 30 23:38:06 2017 -0800
Committer: Oleksandr Diachenko
Committed: Mon Jan 30 23:38:50 2017 -0800
--
pxf/gradle.properties | 2 +-
.../java/org/apache/hawq/pxf/api/Metadata.java | 51 +++-
.../org/apache/hawq/pxf/api/OutputFormat.java | 37 ++-
.../hawq/pxf/api/utilities/InputData.java | 1 +
.../org/apache/hawq/pxf/api/MetadataTest.java | 2 +-
.../hawq/pxf/plugins/hive/HiveAccessor.java | 14 +-
.../plugins/hive/HiveColumnarSerdeResolver.java | 60 ++---
.../pxf/plugins/hive/HiveDataFragmenter.java| 28 +-
.../plugins/hive/HiveInputFormatFragmenter.java | 41 ---
.../pxf/plugins/hive/HiveLineBreakAccessor.java | 10 +-
.../pxf/plugins/hive/HiveMetadataFetcher.java | 85 +++---
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 9 +-
.../pxf/plugins/hive/HiveORCSerdeResolver.java | 44 +---
.../pxf/plugins/hive/HiveRCFileAccessor.java| 10 +-
.../hawq/pxf/plugins/hive/HiveResolver.java | 107
.../plugins/hive/HiveStringPassResolver.java| 39 ++-
.../hawq/pxf/plugins/hive/HiveUserData.java | 135 ++
.../hive/utilities/EnumHiveToHawqType.java | 31 ++-
.../plugins/hive/utilities/HiveUtilities.java | 263 +++
.../plugins/hive/utilities/ProfileFactory.java | 61 +
.../plugins/hive/HiveMetadataFetcherTest.java | 3 +
.../pxf/plugins/hive/HiveORCAccessorTest.java | 9 +-
.../hive/utilities/HiveUtilitiesTest.java | 53
.../hive/utilities/ProfileFactoryTest.java | 65 +
.../hawq/pxf/service/BridgeOutputBuilder.java | 8 +-
.../pxf/service/MetadataResponseFormatter.java | 3 +-
.../apache/hawq/pxf/service/ProfileFactory.java | 45
.../hawq/pxf/service/rest/MetadataResource.java | 9 +-
.../hawq/pxf/service/rest/VersionResource.java | 2 +-
.../pxf/service/utilities/ProtocolData.java | 22 +-
.../src/main/resources/pxf-profiles-default.xml | 14 +-
.../service/MetadataResponseFormatterTest.java | 16 +-
src/backend/access/external/fileam.c| 3 +
src/backend/access/external/pxfheaders.c| 21 +-
.../access/external/test/pxfheaders_test.c | 18 ++
src/backend/catalog/external/externalmd.c | 137 +++---
src/bin/gpfusion/gpbridgeapi.c | 6 +-
src/include/access/hd_work_mgr.h| 2 +
src/include/access/pxfheaders.h | 1 +
src/include/access/pxfuriparser.h | 2 +-
src/include/catalog/external/itemmd.h | 5 +
src/include/catalog/pg_exttable.h | 14 +-
.../regress/data/hcatalog/single_table.json | 2 +-
.../data/hcatalog/single_table_text.json| 1 +
src/test/regress/input/json_load.source | 12 +-
src/test/regress/json_utils.c | 24 +-
src/test/regress/output/json_load.source| 35 ++-
47 files changed, 1109 insertions(+), 453 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/gradle.properties
--
diff --git a/pxf/gradle.properties b/pxf/gradle.properties
index b003c56..2af17ef 100644
--- a/pxf/gradle.properties
+++ b/pxf/gradle.properties
@@ -23,5 +23,5 @@ hiveVersion=1.2.1
hbaseVersionJar=1.1.2
hbaseVersionRPM=1.1.2
tomcatVersion=7.0.62
-pxfProtocolVersion=v14
+pxfProtocolVersion=v15
osFamily=el6
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
index 9e1c137..bb22d41 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
@@ -22,6 +22,8 @@ package org.apache.hawq.pxf.api;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hawq.pxf.api.utilities.EnumHawqType;
import org.apache.commons.lang.StringUtils;
@@ -68,14 +70,16 @@ public class Metadata {
}
/**
- * Class representing item field - name, type, source type, modifiers.
+
[1/2] incubator-hawq git commit: HAWQ-1228. Use profile based on file format in HCatalog integration(HiveRC, HiveText profiles).
Repository: incubator-hawq
Updated Branches:
refs/heads/master aac8868f8 -> 6fa1ced20
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
--
diff --git
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
index b736bba..05ef001 100644
---
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
+++
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
@@ -23,10 +23,18 @@ package org.apache.hawq.pxf.plugins.hive.utilities;
import static org.junit.Assert.*;
import java.util.Arrays;
+import java.util.Collections;
import com.google.common.base.Joiner;
+
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
+import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe;
+import org.apache.hadoop.hive.serde2.*;
import org.junit.Test;
import org.apache.hawq.pxf.api.Metadata;
import org.apache.hawq.pxf.api.UnsupportedTypeException;
@@ -399,4 +407,49 @@ public class HiveUtilitiesTest {
assertEquals(errorMsg, e.getMessage());
}
}
+
+@Test
+public void createDeserializer() throws Exception {
+SerDe serde =
HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE,
HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE);
+assertTrue(serde instanceof OrcSerde);
+
+serde =
HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE,
HiveUtilities.PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE);
+assertTrue(serde instanceof
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe);
+
+serde =
HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE,
HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE);
+assertTrue(serde instanceof
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe);
+
+try {
+serde =
HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE,
HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE);
+fail("shouldn't be able to create deserializer with not allowed
serde");
+} catch (UnsupportedTypeException e) {
+assertTrue(e.getMessage().equals("Unsupported Hive Serde: " +
HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE.name()));
+}
+}
+
+@Test
+public void getDelimiterCode() {
+
+//Default delimiter code should be 44(comma)
+Integer delimiterCode = HiveUtilities.getDelimiterCode(null);
+char defaultDelim = ',';
+assertTrue(delimiterCode == (int) defaultDelim);
+
+//Some serdes use FIELD_DELIM key
+char expectedDelim = '%';
+StorageDescriptor sd = new StorageDescriptor();
+SerDeInfo si = new SerDeInfo();
+si.setParameters(Collections.singletonMap(serdeConstants.FIELD_DELIM,
String.valueOf(expectedDelim)));
+sd.setSerdeInfo(si);
+delimiterCode = HiveUtilities.getDelimiterCode(sd);
+assertTrue(delimiterCode == (int) expectedDelim);
+
+//Some serdes use SERIALIZATION_FORMAT key
+sd = new StorageDescriptor();
+si = new SerDeInfo();
+
si.setParameters(Collections.singletonMap(serdeConstants.SERIALIZATION_FORMAT,
String.valueOf((int)expectedDelim)));
+sd.setSerdeInfo(si);
+delimiterCode = HiveUtilities.getDelimiterCode(sd);
+assertTrue(delimiterCode == (int) expectedDelim);
+}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
--
diff --git
a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
new file mode 100644
index 000..d588d35
--- /dev/null
+++
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
@@ -0,0 +1,65 @@
+package org.apache.hawq.pxf.plugins.hive.utilities;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may no
[4/4] incubator-hawq git commit: HAWQ-1228. Fixed complex types for HiveText.
HAWQ-1228. Fixed complex types for HiveText.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/607184c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/607184c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/607184c3
Branch: refs/heads/HAWQ-1228
Commit: 607184c365d872ce9b567ee2366122aac903c4a7
Parents: 8ba2e16
Author: Oleksandr Diachenko
Authored: Mon Jan 23 18:32:21 2017 -0800
Committer: Oleksandr Diachenko
Committed: Mon Jan 23 18:32:21 2017 -0800
--
.../java/org/apache/hawq/pxf/api/Metadata.java | 18 +++-
.../org/apache/hawq/pxf/api/MetadataTest.java | 2 +-
.../plugins/hive/HiveColumnarSerdeResolver.java | 5 --
.../pxf/plugins/hive/HiveDataFragmenter.java| 25 +++---
.../pxf/plugins/hive/HiveMetadataFetcher.java | 63 +++--
.../hawq/pxf/plugins/hive/HiveResolver.java | 66 +++---
.../plugins/hive/HiveStringPassResolver.java| 13 +--
.../hawq/pxf/plugins/hive/HiveUserData.java | 20 -
.../hive/utilities/EnumHiveToHawqType.java | 31 +--
.../plugins/hive/utilities/HiveUtilities.java | 94 ++--
.../pxf/plugins/hive/HiveORCAccessorTest.java | 2 +-
.../pxf/service/MetadataResponseFormatter.java | 3 +-
.../apache/hawq/pxf/service/ProfileFactory.java | 9 +-
.../service/MetadataResponseFormatterTest.java | 16 ++--
src/backend/catalog/external/externalmd.c | 65 +-
15 files changed, 268 insertions(+), 164 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/607184c3/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
index a920e4f..8701813 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
@@ -78,6 +78,7 @@ public class Metadata {
public static class Field {
private String name;
private EnumHawqType type; // field type which PXF exposes
+private boolean isComplexType; // whether source field's type is
complex
private String sourceType; // field type PXF reads from
private String[] modifiers; // type modifiers, optional field
@@ -93,12 +94,17 @@ public class Metadata {
this.sourceType = sourceType;
}
-public Field(String name, EnumHawqType type, String sourceType,
-String[] modifiers) {
+public Field(String name, EnumHawqType type, String sourceType,
String[] modifiers) {
this(name, type, sourceType);
this.modifiers = modifiers;
}
+public Field(String name, EnumHawqType type, boolean isComplexType,
String sourceType, String[] modifiers) {
+this(name, type, sourceType);
+this.modifiers = modifiers;
+this.isComplexType = isComplexType;
+}
+
public String getName() {
return name;
}
@@ -114,6 +120,14 @@ public class Metadata {
public String[] getModifiers() {
return modifiers;
}
+
+public boolean isComplexType() {
+return isComplexType;
+}
+
+public void setComplexType(boolean isComplexType) {
+this.isComplexType = isComplexType;
+}
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/607184c3/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
--
diff --git
a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
index 327a15b..9244ba2 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
@@ -32,7 +32,7 @@ public class MetadataTest {
@Test
public void createFieldEmptyNameType() {
try {
-Metadata.Field field = new Metadata.Field(null, null, null, null);
+Metadata.Field field = new Metadata.Field(null, null, false, null,
null);
fail("Empty name, type and source type shouldn't be allowed.");
} catch (IllegalArgumentException e) {
assertEquals("Field name, type and source type cannot be empty",
e.getMessage());
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/607184c3/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
--
dif
[3/4] incubator-hawq git commit: HAWQ-1228. Support custom delimiter, flat serialization of complex types in HiveText profile.
HAWQ-1228. Support custom delimiter, flat serialization of complex types in
HiveText profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8ba2e161
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8ba2e161
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8ba2e161
Branch: refs/heads/HAWQ-1228
Commit: 8ba2e16180064b92d7b1d8a987a63a0ba6aff704
Parents: 8fe2703
Author: Oleksandr Diachenko
Authored: Thu Jan 19 17:20:20 2017 -0800
Committer: Oleksandr Diachenko
Committed: Thu Jan 19 17:20:20 2017 -0800
--
.../java/org/apache/hawq/pxf/api/Metadata.java | 14 ++--
.../hawq/pxf/api/utilities/InputData.java | 2 ++
.../plugins/hive/HiveColumnarSerdeResolver.java | 1 -
.../pxf/plugins/hive/HiveMetadataFetcher.java | 20 +++-
.../hawq/pxf/plugins/hive/HiveResolver.java | 6 ++--
.../plugins/hive/HiveStringPassResolver.java| 18 ---
.../hawq/pxf/plugins/hive/HiveUserData.java | 34 +---
.../plugins/hive/utilities/HiveUtilities.java | 9 --
.../pxf/plugins/hive/HiveORCAccessorTest.java | 2 +-
.../apache/hawq/pxf/service/ProfileFactory.java | 1 +
src/backend/catalog/external/externalmd.c | 26 +++
src/include/catalog/external/itemmd.h | 2 ++
12 files changed, 106 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8ba2e161/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
index 7e3b92e..a920e4f 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
@@ -22,6 +22,7 @@ package org.apache.hawq.pxf.api;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.hawq.pxf.api.utilities.EnumHawqType;
@@ -124,18 +125,25 @@ public class Metadata {
* Item's fields
*/
private List fields;
-
-
private Set outputFormats;
+private Map outputParameters;
public Set getOutputFormats() {
return outputFormats;
}
-public void setFormats(Set outputFormats) {
+public void setOutputFormats(Set outputFormats) {
this.outputFormats = outputFormats;
}
+public Map getOutputParameters() {
+return outputParameters;
+}
+
+public void setOutputParameters(Map outputParameters) {
+this.outputParameters = outputParameters;
+}
+
/**
* Constructs an item's Metadata.
*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8ba2e161/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 5afedca..0ecb9eb 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -31,6 +31,8 @@ import java.util.*;
*/
public class InputData {
+public static final String DELIMITER_KEY = "DELIMITER";
+
public static final int INVALID_SPLIT_IDX = -1;
private static final Log LOG = LogFactory.getLog(InputData.class);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8ba2e161/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 8fd37e0..157f723 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -59,7 +59,6 @@ import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
*/
public class HiveColumnarSerdeResolver extends HiveResolver {
private static final Log LOG =
LogFactory.getLog(HiveColumnarSerdeResolver.class);
-//private ColumnarSerDeBase deserializer;
private boolean firstColumn;
private StringBuilder builder;
private StringBuilder parts;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8ba2e161/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.j
[2/4] incubator-hawq git commit: HAWQ-1228. Hardcoded comma as delimiter.
HAWQ-1228. Hardcoded comma as delimiter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8fe27032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8fe27032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8fe27032
Branch: refs/heads/HAWQ-1228
Commit: 8fe27032f1faf786544a5109e854d319f6e6a1c4
Parents: 0574e75
Author: Oleksandr Diachenko
Authored: Fri Jan 6 18:39:32 2017 -0800
Committer: Oleksandr Diachenko
Committed: Fri Jan 6 18:39:32 2017 -0800
--
.../org/apache/hawq/pxf/api/OutputFormat.java | 2 +-
.../plugins/hive/HiveColumnarSerdeResolver.java | 2 +-
.../pxf/plugins/hive/HiveMetadataFetcher.java | 2 +-
.../pxf/plugins/hive/HiveORCSerdeResolver.java | 22
.../hawq/pxf/plugins/hive/HiveResolver.java | 3 ---
.../plugins/hive/HiveStringPassResolver.java| 2 +-
src/backend/catalog/external/externalmd.c | 2 +-
src/bin/gpfusion/gpbridgeapi.c | 4 ++--
src/include/access/hd_work_mgr.h| 2 ++
9 files changed, 9 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 82a747f..230f9ff 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -23,4 +23,4 @@ package org.apache.hawq.pxf.api;
/**
* PXF supported output formats: {@link #TEXT} and {@link #BINARY}
*/
-public enum OutputFormat {TEXT, BINARY, UNKNOWN}
+public enum OutputFormat {TEXT, BINARY}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 2bf39ff..8fd37e0 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -241,6 +241,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver
{
@Override
void parseDelimiterChar(InputData input) {
-delimiter = 1;
+delimiter = 44; //,
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index 3d04bc1..76e8e18 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -162,7 +162,7 @@ public class HiveMetadataFetcher extends MetadataFetcher {
}
private OutputFormat getOutputFormat(String inputFormat) {
-OutputFormat outputFormat = OutputFormat.UNKNOWN;
+OutputFormat outputFormat = null;
try {
InputFormat fformat =
HiveDataFragmenter.makeInputFormat(inputFormat, jobConf);
String profile = ProfileFactory.get(fformat);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
index 381c407..0c7a417 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
@@ -44,7 +44,6 @@ import java.util.*;
*/
public class HiveORCSerdeResolver extends HiveResolver {
private static final Log LOG =
LogFactory.getLog(HiveORCSerdeResolver.class);
-private OrcSerde deserializer;
private HiveUtilities.PXF_HIVE_SERDES serdeType;
public HiveORCSerdeResolver(InputData input) throws Exception {
@@ -63,27 +62,6 @@ public class HiveORCSe
[1/4] incubator-hawq git commit: HAWQ-1228. Initial commit.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1228 [created] 607184c36
HAWQ-1228. Initial commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0574e75f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0574e75f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0574e75f
Branch: refs/heads/HAWQ-1228
Commit: 0574e75fa972f6ccddd1f55a98972223b3860759
Parents: 25c87ec
Author: Oleksandr Diachenko
Authored: Wed Dec 28 14:03:50 2016 -0800
Committer: Oleksandr Diachenko
Committed: Fri Jan 6 11:55:19 2017 -0800
--
.../java/org/apache/hawq/pxf/api/Metadata.java | 12 ++
.../org/apache/hawq/pxf/api/OutputFormat.java | 2 +-
.../hawq/pxf/plugins/hive/HiveAccessor.java | 14 +--
.../plugins/hive/HiveColumnarSerdeResolver.java | 62 +-
.../pxf/plugins/hive/HiveDataFragmenter.java| 1 -
.../plugins/hive/HiveInputFormatFragmenter.java | 40 ---
.../pxf/plugins/hive/HiveLineBreakAccessor.java | 10 +-
.../pxf/plugins/hive/HiveMetadataFetcher.java | 41 ++-
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 11 +-
.../pxf/plugins/hive/HiveORCSerdeResolver.java | 17 ++-
.../pxf/plugins/hive/HiveRCFileAccessor.java| 10 +-
.../hawq/pxf/plugins/hive/HiveResolver.java | 38 +++
.../plugins/hive/HiveStringPassResolver.java| 42 +--
.../hawq/pxf/plugins/hive/HiveUserData.java | 71
.../plugins/hive/utilities/HiveUtilities.java | 113 +++
.../pxf/plugins/hive/HiveORCAccessorTest.java | 9 +-
.../apache/hawq/pxf/service/ProfileFactory.java | 20 ++--
.../src/main/resources/pxf-profiles-default.xml | 4 +
src/backend/access/external/fileam.c| 4 +
src/backend/catalog/external/externalmd.c | 57 --
src/bin/gpfusion/gpbridgeapi.c | 43 ++-
src/include/access/extprotocol.h| 1 +
src/include/access/fileam.h | 1 +
src/include/catalog/external/itemmd.h | 3 +
src/include/catalog/pg_exttable.h | 10 +-
25 files changed, 428 insertions(+), 208 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0574e75f/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
index 9e1c137..7e3b92e 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
@@ -22,6 +22,7 @@ package org.apache.hawq.pxf.api;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import org.apache.hawq.pxf.api.utilities.EnumHawqType;
import org.apache.commons.lang.StringUtils;
@@ -124,6 +125,17 @@ public class Metadata {
*/
private List fields;
+
+private Set outputFormats;
+
+public Set getOutputFormats() {
+return outputFormats;
+}
+
+public void setFormats(Set outputFormats) {
+this.outputFormats = outputFormats;
+}
+
/**
* Constructs an item's Metadata.
*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0574e75f/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 230f9ff..82a747f 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -23,4 +23,4 @@ package org.apache.hawq.pxf.api;
/**
* PXF supported output formats: {@link #TEXT} and {@link #BINARY}
*/
-public enum OutputFormat {TEXT, BINARY}
+public enum OutputFormat {TEXT, BINARY, UNKNOWN}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0574e75f/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
index ef9f76e..ea3accb 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
@@ -28,6 +28,7 @@ import org.apache.hawq.pxf.api.UnsupportedTypeException;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.I
[1/2] incubator-hawq git commit: HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 2a077062d -> 25c87ec2b
HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/25c87ec2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/25c87ec2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/25c87ec2
Branch: refs/heads/master
Commit: 25c87ec2b2bee27bb242363de693ae7a564f19d9
Parents: 0ee9d96
Author: Oleksandr Diachenko
Authored: Tue Dec 27 13:59:05 2016 -0800
Committer: Oleksandr Diachenko
Committed: Tue Dec 27 13:59:18 2016 -0800
--
src/backend/access/external/hd_work_mgr.c | 21 +-
src/backend/access/external/pxfheaders.c| 14 +-
src/backend/access/external/pxfmasterapi.c | 9 +
src/backend/access/external/pxfuriparser.c | 156 +-
.../access/external/test/hd_work_mgr_test.c | 112
.../access/external/test/pxfuriparser_test.c| 558 ++-
src/bin/gpfusion/gpbridgeapi.c | 11 +
src/include/access/pxfmasterapi.h | 1 +
src/include/access/pxfuriparser.h | 3 +
9 files changed, 841 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25c87ec2/src/backend/access/external/hd_work_mgr.c
--
diff --git a/src/backend/access/external/hd_work_mgr.c
b/src/backend/access/external/hd_work_mgr.c
index 6829de5..520b5de 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -53,6 +53,7 @@ typedef struct sAllocatedDataFragment
char *source_name; /* source name */
char *fragment_md; /* fragment meta data */
char *user_data; /* additional user data */
+ char *profile; /* recommended profile to work with fragment */
} AllocatedDataFragment;
/*
@@ -713,6 +714,7 @@ create_allocated_fragment(DataFragment *fragment)
allocated->source_name = pstrdup(fragment->source_name);
allocated->fragment_md = (fragment->fragment_md) ?
pstrdup(fragment->fragment_md) : NULL;
allocated->user_data = (fragment->user_data) ?
pstrdup(fragment->user_data) : NULL;
+ allocated->profile = (fragment->profile) ? pstrdup(fragment->profile) :
NULL;
return allocated;
}
@@ -782,12 +784,22 @@ make_allocation_output_string(List *segment_fragments)
appendStringInfo(&fragment_str, "%d", frag->index);
appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->fragment_md)
+ {
appendStringInfo(&fragment_str, "%s",
frag->fragment_md);
+ }
+
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->user_data)
{
- appendStringInfoChar(&fragment_str,
SEGWORK_IN_PAIR_DELIM);
appendStringInfo(&fragment_str, "%s", frag->user_data);
}
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+ if (frag->profile)
+ {
+ appendStringInfo(&fragment_str, "%s", frag->profile);
+ }
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+
fragment_size = strlen(fragment_str.data);
appendStringInfo(&segwork, "%d", fragment_size);
@@ -817,6 +829,8 @@ free_allocated_frags(List *segment_fragments)
pfree(frag->fragment_md);
if (frag->user_data)
pfree(frag->user_data);
+ if (frag->profile)
+ pfree(frag->profile);
pfree(frag);
}
list_free(segment_fragments);
@@ -856,6 +870,11 @@ print_fragment_list(List *fragments)
{
appendStringInfo(&log_str, "user data: %s\n",
frag->user_data);
}
+
+ if (frag->profile)
+ {
+ appendStringInfo(&log_str, "profile: %s\n",
frag->profile);
+ }
}
elog(FRAGDEBUG, "%s", log_str.data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25c87ec2/src/backend/access/external/pxfheaders.c
--
diff --git a/src/backend/access/external/pxfheaders.c
b/src/backend/access/external/pxfheaders.c
index fe7cd6a..2381044 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -27,12 +27,12 @@
#include "catalog/pg_exttable.h"
#include "access/pxfheaders.h"
#include "access/pxffilters.h"
+#
[2/2] incubator-hawq git commit: HAWQ-1178. Enhance Fragmenter api to return profile name.
HAWQ-1178. Enhance Fragmenter api to return profile name.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0ee9d96b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0ee9d96b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0ee9d96b
Branch: refs/heads/master
Commit: 0ee9d96b5157843e74ec104f02f731d96a0666e8
Parents: 2a07706
Author: Oleksandr Diachenko
Authored: Tue Dec 27 13:58:35 2016 -0800
Committer: Oleksandr Diachenko
Committed: Tue Dec 27 13:59:18 2016 -0800
--
pxf/build.gradle| 6 +
.../java/org/apache/hawq/pxf/api/Fragment.java | 22 +++
.../hawq/pxf/api/utilities/InputData.java | 11 ++
.../pxf/plugins/hive/HiveDataFragmenter.java| 101 +++---
.../plugins/hive/HiveInputFormatFragmenter.java | 71 --
.../pxf/plugins/hive/HiveTablePartition.java| 56
.../plugins/hive/utilities/HiveUtilities.java | 134 +++
.../apache/hawq/pxf/service/ProfileFactory.java | 45 +++
.../pxf/service/utilities/ProtocolData.java | 1 -
9 files changed, 294 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0ee9d96b/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index aed48b1..385bf08 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -193,7 +193,12 @@ project('pxf-service') {
providedCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-auth:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-annotations:$hadoopVersion"
+providedCompile
"org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
providedCompile "org.apache.tomcat:tomcat-catalina:$tomcatVersion"
+providedCompile("org.apache.hive:hive-exec:$hiveVersion") {
+exclude module: 'calcite-core'
+exclude module: 'calcite-avatica'
+}
}
ospackage {
@@ -342,6 +347,7 @@ project('pxf-hdfs') {
project('pxf-hive') {
dependencies {
compile(project(':pxf-hdfs'))
+compile(project(':pxf-service'))
compile("org.apache.hive:hive-exec:$hiveVersion") {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0ee9d96b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
index 130a91d..ff6f131 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
@@ -51,6 +51,11 @@ public class Fragment {
private byte[] userData;
/**
+ * Profile name, recommended for reading given Fragment.
+ */
+private String profile;
+
+/**
* Constructs a Fragment.
*
* @param sourceName the resource uri (File path+name, table name, etc.)
@@ -83,6 +88,15 @@ public class Fragment {
this.userData = userData;
}
+public Fragment(String sourceName,
+String[] hosts,
+byte[] metadata,
+byte[] userData,
+String profile) {
+this(sourceName, hosts, metadata, userData);
+this.profile = profile;
+}
+
public String getSourceName() {
return sourceName;
}
@@ -118,4 +132,12 @@ public class Fragment {
public void setUserData(byte[] userData) {
this.userData = userData;
}
+
+public String getProfile() {
+return profile;
+}
+
+public void setProfile(String profile) {
+this.profile = profile;
+}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0ee9d96b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 891dba8..5afedca 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -43,6 +43,7 @@ public class InputData {
protected boolean filterStringValid;
protected String filterString;
protected String dataSource;
+protected String profile;
protected String accessor;
protected String resolver;
protected String fragmenter;
@@ -94,6 +9
[1/2] incubator-hawq git commit: Revert "HAWQ-1178. Enhance Fragmenter api to return profile name."
Repository: incubator-hawq
Updated Branches:
refs/heads/master fbef55d49 -> 90a69266e
Revert "HAWQ-1178. Enhance Fragmenter api to return profile name."
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2f8392e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2f8392e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2f8392e6
Branch: refs/heads/master
Commit: 2f8392e627cde75b9ababd74de6c8be125a3c595
Parents: fbef55d
Author: Oleksandr Diachenko
Authored: Thu Dec 22 01:55:28 2016 -0800
Committer: Oleksandr Diachenko
Committed: Thu Dec 22 01:55:28 2016 -0800
--
pxf/build.gradle| 6 -
.../java/org/apache/hawq/pxf/api/Fragment.java | 22
.../hawq/pxf/api/utilities/InputData.java | 11 --
.../pxf/plugins/hive/HiveDataFragmenter.java| 101 ---
.../plugins/hive/HiveInputFormatFragmenter.java | 71 ++
.../pxf/plugins/hive/HiveTablePartition.java| 56
.../plugins/hive/utilities/HiveUtilities.java | 129 ---
.../apache/hawq/pxf/service/ProfileFactory.java | 45 ---
.../pxf/service/utilities/ProtocolData.java | 1 +
9 files changed, 153 insertions(+), 289 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 385bf08..aed48b1 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -193,12 +193,7 @@ project('pxf-service') {
providedCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-auth:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-annotations:$hadoopVersion"
-providedCompile
"org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
providedCompile "org.apache.tomcat:tomcat-catalina:$tomcatVersion"
-providedCompile("org.apache.hive:hive-exec:$hiveVersion") {
-exclude module: 'calcite-core'
-exclude module: 'calcite-avatica'
-}
}
ospackage {
@@ -347,7 +342,6 @@ project('pxf-hdfs') {
project('pxf-hive') {
dependencies {
compile(project(':pxf-hdfs'))
-compile(project(':pxf-service'))
compile("org.apache.hive:hive-exec:$hiveVersion") {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
index ff6f131..130a91d 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
@@ -51,11 +51,6 @@ public class Fragment {
private byte[] userData;
/**
- * Profile name, recommended for reading given Fragment.
- */
-private String profile;
-
-/**
* Constructs a Fragment.
*
* @param sourceName the resource uri (File path+name, table name, etc.)
@@ -88,15 +83,6 @@ public class Fragment {
this.userData = userData;
}
-public Fragment(String sourceName,
-String[] hosts,
-byte[] metadata,
-byte[] userData,
-String profile) {
-this(sourceName, hosts, metadata, userData);
-this.profile = profile;
-}
-
public String getSourceName() {
return sourceName;
}
@@ -132,12 +118,4 @@ public class Fragment {
public void setUserData(byte[] userData) {
this.userData = userData;
}
-
-public String getProfile() {
-return profile;
-}
-
-public void setProfile(String profile) {
-this.profile = profile;
-}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 5afedca..891dba8 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -43,7 +43,6 @@ public class InputData {
protected boolean filterStringValid;
protected String filterString;
protected String dataSource;
-protected String profile;
pro
[2/2] incubator-hawq git commit: Revert "HAWQ-1179. Call Bridge api with profile value read from Fragmenter call."
Revert "HAWQ-1179. Call Bridge api with profile value read from Fragmenter
call."
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/90a69266
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/90a69266
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/90a69266
Branch: refs/heads/master
Commit: 90a69266eb7af6b61fa32414e11299ae535c98cc
Parents: 2f8392e
Author: Oleksandr Diachenko
Authored: Thu Dec 22 01:55:59 2016 -0800
Committer: Oleksandr Diachenko
Committed: Thu Dec 22 01:55:59 2016 -0800
--
src/backend/access/external/hd_work_mgr.c | 21 +-
src/backend/access/external/pxfheaders.c| 14 +-
src/backend/access/external/pxfmasterapi.c | 9 -
src/backend/access/external/pxfuriparser.c | 156 +-
.../access/external/test/hd_work_mgr_test.c | 112
.../access/external/test/pxfuriparser_test.c| 558 +--
src/bin/gpfusion/gpbridgeapi.c | 11 -
src/include/access/pxfmasterapi.h | 1 -
src/include/access/pxfuriparser.h | 3 -
9 files changed, 44 insertions(+), 841 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/hd_work_mgr.c
--
diff --git a/src/backend/access/external/hd_work_mgr.c
b/src/backend/access/external/hd_work_mgr.c
index 520b5de..6829de5 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -53,7 +53,6 @@ typedef struct sAllocatedDataFragment
char *source_name; /* source name */
char *fragment_md; /* fragment meta data */
char *user_data; /* additional user data */
- char *profile; /* recommended profile to work with fragment */
} AllocatedDataFragment;
/*
@@ -714,7 +713,6 @@ create_allocated_fragment(DataFragment *fragment)
allocated->source_name = pstrdup(fragment->source_name);
allocated->fragment_md = (fragment->fragment_md) ?
pstrdup(fragment->fragment_md) : NULL;
allocated->user_data = (fragment->user_data) ?
pstrdup(fragment->user_data) : NULL;
- allocated->profile = (fragment->profile) ? pstrdup(fragment->profile) :
NULL;
return allocated;
}
@@ -784,22 +782,12 @@ make_allocation_output_string(List *segment_fragments)
appendStringInfo(&fragment_str, "%d", frag->index);
appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->fragment_md)
- {
appendStringInfo(&fragment_str, "%s",
frag->fragment_md);
- }
-
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->user_data)
{
+ appendStringInfoChar(&fragment_str,
SEGWORK_IN_PAIR_DELIM);
appendStringInfo(&fragment_str, "%s", frag->user_data);
}
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
- if (frag->profile)
- {
- appendStringInfo(&fragment_str, "%s", frag->profile);
- }
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
-
fragment_size = strlen(fragment_str.data);
appendStringInfo(&segwork, "%d", fragment_size);
@@ -829,8 +817,6 @@ free_allocated_frags(List *segment_fragments)
pfree(frag->fragment_md);
if (frag->user_data)
pfree(frag->user_data);
- if (frag->profile)
- pfree(frag->profile);
pfree(frag);
}
list_free(segment_fragments);
@@ -870,11 +856,6 @@ print_fragment_list(List *fragments)
{
appendStringInfo(&log_str, "user data: %s\n",
frag->user_data);
}
-
- if (frag->profile)
- {
- appendStringInfo(&log_str, "profile: %s\n",
frag->profile);
- }
}
elog(FRAGDEBUG, "%s", log_str.data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/pxfheaders.c
--
diff --git a/src/backend/access/external/pxfheaders.c
b/src/backend/access/external/pxfheaders.c
index 2381044..fe7cd6a 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -27,12 +27,12 @@
#include "catalog/pg_exttable.h"
#include "access/pxfheaders.h"
#include "access/pxffilters.h"
-#include "utils/formatting.h"
#include "utils/guc.h"
static void add_alignmen
[1/2] incubator-hawq git commit: HAWQ-1178. Enhance Fragmenter api to return profile name.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 6a6fa2755 -> fbef55d49
HAWQ-1178. Enhance Fragmenter api to return profile name.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9cd42ada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9cd42ada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9cd42ada
Branch: refs/heads/master
Commit: 9cd42ada65f20b6181c26e03215e9fe81de88d49
Parents: 6a6fa27
Author: Oleksandr Diachenko
Authored: Wed Dec 21 22:33:06 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Dec 21 22:33:06 2016 -0800
--
pxf/build.gradle| 6 +
.../java/org/apache/hawq/pxf/api/Fragment.java | 22
.../hawq/pxf/api/utilities/InputData.java | 11 ++
.../pxf/plugins/hive/HiveDataFragmenter.java| 101 +++
.../plugins/hive/HiveInputFormatFragmenter.java | 71 --
.../pxf/plugins/hive/HiveTablePartition.java| 56
.../plugins/hive/utilities/HiveUtilities.java | 129 +++
.../apache/hawq/pxf/service/ProfileFactory.java | 45 +++
.../pxf/service/utilities/ProtocolData.java | 1 -
9 files changed, 289 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9cd42ada/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index aed48b1..385bf08 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -193,7 +193,12 @@ project('pxf-service') {
providedCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-auth:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-annotations:$hadoopVersion"
+providedCompile
"org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
providedCompile "org.apache.tomcat:tomcat-catalina:$tomcatVersion"
+providedCompile("org.apache.hive:hive-exec:$hiveVersion") {
+exclude module: 'calcite-core'
+exclude module: 'calcite-avatica'
+}
}
ospackage {
@@ -342,6 +347,7 @@ project('pxf-hdfs') {
project('pxf-hive') {
dependencies {
compile(project(':pxf-hdfs'))
+compile(project(':pxf-service'))
compile("org.apache.hive:hive-exec:$hiveVersion") {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9cd42ada/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
index 130a91d..ff6f131 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
@@ -51,6 +51,11 @@ public class Fragment {
private byte[] userData;
/**
+ * Profile name, recommended for reading given Fragment.
+ */
+private String profile;
+
+/**
* Constructs a Fragment.
*
* @param sourceName the resource uri (File path+name, table name, etc.)
@@ -83,6 +88,15 @@ public class Fragment {
this.userData = userData;
}
+public Fragment(String sourceName,
+String[] hosts,
+byte[] metadata,
+byte[] userData,
+String profile) {
+this(sourceName, hosts, metadata, userData);
+this.profile = profile;
+}
+
public String getSourceName() {
return sourceName;
}
@@ -118,4 +132,12 @@ public class Fragment {
public void setUserData(byte[] userData) {
this.userData = userData;
}
+
+public String getProfile() {
+return profile;
+}
+
+public void setProfile(String profile) {
+this.profile = profile;
+}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9cd42ada/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 891dba8..5afedca 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -43,6 +43,7 @@ public class InputData {
protected boolean filterStringValid;
protected String filterString;
protected String dataSource;
+protected String profile;
protected St
[2/2] incubator-hawq git commit: HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fbef55d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fbef55d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fbef55d4
Branch: refs/heads/master
Commit: fbef55d49ccce734d44434a6985845a2892d37cc
Parents: 9cd42ad
Author: Oleksandr Diachenko
Authored: Wed Dec 21 22:33:48 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Dec 21 22:33:48 2016 -0800
--
src/backend/access/external/hd_work_mgr.c | 21 +-
src/backend/access/external/pxfheaders.c| 14 +-
src/backend/access/external/pxfmasterapi.c | 9 +
src/backend/access/external/pxfuriparser.c | 156 +-
.../access/external/test/hd_work_mgr_test.c | 112
.../access/external/test/pxfuriparser_test.c| 558 ++-
src/bin/gpfusion/gpbridgeapi.c | 11 +
src/include/access/pxfmasterapi.h | 1 +
src/include/access/pxfuriparser.h | 3 +
9 files changed, 841 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fbef55d4/src/backend/access/external/hd_work_mgr.c
--
diff --git a/src/backend/access/external/hd_work_mgr.c
b/src/backend/access/external/hd_work_mgr.c
index 6829de5..520b5de 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -53,6 +53,7 @@ typedef struct sAllocatedDataFragment
char *source_name; /* source name */
char *fragment_md; /* fragment meta data */
char *user_data; /* additional user data */
+ char *profile; /* recommended profile to work with fragment */
} AllocatedDataFragment;
/*
@@ -713,6 +714,7 @@ create_allocated_fragment(DataFragment *fragment)
allocated->source_name = pstrdup(fragment->source_name);
allocated->fragment_md = (fragment->fragment_md) ?
pstrdup(fragment->fragment_md) : NULL;
allocated->user_data = (fragment->user_data) ?
pstrdup(fragment->user_data) : NULL;
+ allocated->profile = (fragment->profile) ? pstrdup(fragment->profile) :
NULL;
return allocated;
}
@@ -782,12 +784,22 @@ make_allocation_output_string(List *segment_fragments)
appendStringInfo(&fragment_str, "%d", frag->index);
appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->fragment_md)
+ {
appendStringInfo(&fragment_str, "%s",
frag->fragment_md);
+ }
+
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->user_data)
{
- appendStringInfoChar(&fragment_str,
SEGWORK_IN_PAIR_DELIM);
appendStringInfo(&fragment_str, "%s", frag->user_data);
}
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+ if (frag->profile)
+ {
+ appendStringInfo(&fragment_str, "%s", frag->profile);
+ }
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+
fragment_size = strlen(fragment_str.data);
appendStringInfo(&segwork, "%d", fragment_size);
@@ -817,6 +829,8 @@ free_allocated_frags(List *segment_fragments)
pfree(frag->fragment_md);
if (frag->user_data)
pfree(frag->user_data);
+ if (frag->profile)
+ pfree(frag->profile);
pfree(frag);
}
list_free(segment_fragments);
@@ -856,6 +870,11 @@ print_fragment_list(List *fragments)
{
appendStringInfo(&log_str, "user data: %s\n",
frag->user_data);
}
+
+ if (frag->profile)
+ {
+ appendStringInfo(&log_str, "profile: %s\n",
frag->profile);
+ }
}
elog(FRAGDEBUG, "%s", log_str.data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fbef55d4/src/backend/access/external/pxfheaders.c
--
diff --git a/src/backend/access/external/pxfheaders.c
b/src/backend/access/external/pxfheaders.c
index fe7cd6a..2381044 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -27,12 +27,12 @@
#include "catalog/pg_exttable.h"
#include "access/pxfheaders.h"
#include "access/pxffilters.h"
+#include "utils/formatting.h"
#include "utils/guc.h"
static void add_alignment_size_htt
[2/2] incubator-hawq git commit: HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
HAWQ-1179. Call Bridge api with profile value read from Fragmenter call.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/65e1ed10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/65e1ed10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/65e1ed10
Branch: refs/heads/HAWQ-1177
Commit: 65e1ed10a0399711d7548f5038697c0d624456c6
Parents: 7ec8536
Author: Oleksandr Diachenko
Authored: Wed Dec 21 15:55:02 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Dec 21 15:55:02 2016 -0800
--
src/backend/access/external/hd_work_mgr.c | 21 +-
src/backend/access/external/pxfheaders.c| 14 +-
src/backend/access/external/pxfmasterapi.c | 9 +
src/backend/access/external/pxfuriparser.c | 156 +-
.../access/external/test/hd_work_mgr_test.c | 112
.../access/external/test/pxfuriparser_test.c| 558 ++-
src/bin/gpfusion/gpbridgeapi.c | 11 +
src/include/access/pxfmasterapi.h | 1 +
src/include/access/pxfuriparser.h | 3 +
9 files changed, 841 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65e1ed10/src/backend/access/external/hd_work_mgr.c
--
diff --git a/src/backend/access/external/hd_work_mgr.c
b/src/backend/access/external/hd_work_mgr.c
index 6829de5..520b5de 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -53,6 +53,7 @@ typedef struct sAllocatedDataFragment
char *source_name; /* source name */
char *fragment_md; /* fragment meta data */
char *user_data; /* additional user data */
+ char *profile; /* recommended profile to work with fragment */
} AllocatedDataFragment;
/*
@@ -713,6 +714,7 @@ create_allocated_fragment(DataFragment *fragment)
allocated->source_name = pstrdup(fragment->source_name);
allocated->fragment_md = (fragment->fragment_md) ?
pstrdup(fragment->fragment_md) : NULL;
allocated->user_data = (fragment->user_data) ?
pstrdup(fragment->user_data) : NULL;
+ allocated->profile = (fragment->profile) ? pstrdup(fragment->profile) :
NULL;
return allocated;
}
@@ -782,12 +784,22 @@ make_allocation_output_string(List *segment_fragments)
appendStringInfo(&fragment_str, "%d", frag->index);
appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->fragment_md)
+ {
appendStringInfo(&fragment_str, "%s",
frag->fragment_md);
+ }
+
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->user_data)
{
- appendStringInfoChar(&fragment_str,
SEGWORK_IN_PAIR_DELIM);
appendStringInfo(&fragment_str, "%s", frag->user_data);
}
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+ if (frag->profile)
+ {
+ appendStringInfo(&fragment_str, "%s", frag->profile);
+ }
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
+
fragment_size = strlen(fragment_str.data);
appendStringInfo(&segwork, "%d", fragment_size);
@@ -817,6 +829,8 @@ free_allocated_frags(List *segment_fragments)
pfree(frag->fragment_md);
if (frag->user_data)
pfree(frag->user_data);
+ if (frag->profile)
+ pfree(frag->profile);
pfree(frag);
}
list_free(segment_fragments);
@@ -856,6 +870,11 @@ print_fragment_list(List *fragments)
{
appendStringInfo(&log_str, "user data: %s\n",
frag->user_data);
}
+
+ if (frag->profile)
+ {
+ appendStringInfo(&log_str, "profile: %s\n",
frag->profile);
+ }
}
elog(FRAGDEBUG, "%s", log_str.data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65e1ed10/src/backend/access/external/pxfheaders.c
--
diff --git a/src/backend/access/external/pxfheaders.c
b/src/backend/access/external/pxfheaders.c
index fe7cd6a..2381044 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -27,12 +27,12 @@
#include "catalog/pg_exttable.h"
#include "access/pxfheaders.h"
#include "access/pxffilters.h"
+#include "utils/formatting.h"
#include "utils/guc.h"
static void add_alignment_size_
[1/2] incubator-hawq git commit: HAWQ-1178. Enhance Fragmenter api to return profile name.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1177 [created] 65e1ed10a
HAWQ-1178. Enhance Fragmenter api to return profile name.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7ec8536a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7ec8536a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7ec8536a
Branch: refs/heads/HAWQ-1177
Commit: 7ec8536ac83d160dfb947eb92a219668d8306f7f
Parents: 9a25e2c
Author: Oleksandr Diachenko
Authored: Wed Dec 21 15:54:32 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Dec 21 15:54:32 2016 -0800
--
pxf/build.gradle| 6 +
.../java/org/apache/hawq/pxf/api/Fragment.java | 22
.../hawq/pxf/api/utilities/InputData.java | 11 ++
.../pxf/plugins/hive/HiveDataFragmenter.java| 101 +++
.../plugins/hive/HiveInputFormatFragmenter.java | 71 --
.../pxf/plugins/hive/HiveTablePartition.java| 56
.../plugins/hive/utilities/HiveUtilities.java | 129 +++
.../apache/hawq/pxf/service/ProfileFactory.java | 45 +++
.../pxf/service/utilities/ProtocolData.java | 1 -
9 files changed, 289 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7ec8536a/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index aed48b1..385bf08 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -193,7 +193,12 @@ project('pxf-service') {
providedCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-auth:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-annotations:$hadoopVersion"
+providedCompile
"org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
providedCompile "org.apache.tomcat:tomcat-catalina:$tomcatVersion"
+providedCompile("org.apache.hive:hive-exec:$hiveVersion") {
+exclude module: 'calcite-core'
+exclude module: 'calcite-avatica'
+}
}
ospackage {
@@ -342,6 +347,7 @@ project('pxf-hdfs') {
project('pxf-hive') {
dependencies {
compile(project(':pxf-hdfs'))
+compile(project(':pxf-service'))
compile("org.apache.hive:hive-exec:$hiveVersion") {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7ec8536a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
index 130a91d..ff6f131 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
@@ -51,6 +51,11 @@ public class Fragment {
private byte[] userData;
/**
+ * Profile name, recommended for reading given Fragment.
+ */
+private String profile;
+
+/**
* Constructs a Fragment.
*
* @param sourceName the resource uri (File path+name, table name, etc.)
@@ -83,6 +88,15 @@ public class Fragment {
this.userData = userData;
}
+public Fragment(String sourceName,
+String[] hosts,
+byte[] metadata,
+byte[] userData,
+String profile) {
+this(sourceName, hosts, metadata, userData);
+this.profile = profile;
+}
+
public String getSourceName() {
return sourceName;
}
@@ -118,4 +132,12 @@ public class Fragment {
public void setUserData(byte[] userData) {
this.userData = userData;
}
+
+public String getProfile() {
+return profile;
+}
+
+public void setProfile(String profile) {
+this.profile = profile;
+}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7ec8536a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 891dba8..5afedca 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -43,6 +43,7 @@ public class InputData {
protected boolean filterStringValid;
protected String filterString;
protected String dataSource;
+protected String profile;
protected
incubator-hawq git commit: HAWQ-1191. Decomissioned DELIMITER from HiveOrc profile.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 52333bb00 -> ca5d8de05
HAWQ-1191. Decomissioned DELIMITER from HiveOrc profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca5d8de0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca5d8de0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca5d8de0
Branch: refs/heads/master
Commit: ca5d8de050f62ec5929471f1ab5d0a1420a363b7
Parents: 52333bb
Author: Oleksandr Diachenko
Authored: Thu Dec 8 11:33:18 2016 -0800
Committer: Oleksandr Diachenko
Committed: Thu Dec 8 11:33:18 2016 -0800
--
.../org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java | 2 --
.../java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java| 6 ++
pxf/pxf-service/src/main/resources/pxf-profiles-default.xml| 1 -
3 files changed, 2 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca5d8de0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
index c789402..7673713 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
@@ -61,7 +61,6 @@ public class HiveORCSerdeResolver extends HiveResolver {
throw new UnsupportedTypeException("Unsupported Hive Serde: " +
serdeEnumStr);
}
partitionKeys = toks[HiveInputFormatFragmenter.TOK_KEYS];
-parseDelimiterChar(input);
collectionDelim = input.getUserProperty("COLLECTION_DELIM") == null ?
COLLECTION_DELIM
: input.getUserProperty("COLLECTION_DELIM");
mapkeyDelim = input.getUserProperty("MAPKEY_DELIM") == null ?
MAPKEY_DELIM
@@ -126,5 +125,4 @@ public class HiveORCSerdeResolver extends HiveResolver {
deserializer.initialize(new JobConf(new Configuration(),
HiveORCSerdeResolver.class), serdeProperties);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca5d8de0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
index d5e6e96..3837f78 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
@@ -79,7 +79,7 @@ public class HiveResolver extends Plugin implements
ReadResolver {
private String serdeName;
private String propsString;
String partitionKeys;
-char delimiter;
+protected char delimiter;
String nullChar = "\\N";
private Configuration conf;
private String hiveDefaultPartName;
@@ -263,8 +263,8 @@ public class HiveResolver extends Plugin implements
ReadResolver {
"Unsupported partition type: " + type);
}
addOneFieldToRecord(partitionFields, convertedType,
convertedValue);
-numberOfPartitions = partitionFields.size();
}
+numberOfPartitions = partitionFields.size();
}
/*
@@ -281,7 +281,6 @@ public class HiveResolver extends Plugin implements
ReadResolver {
String type = levelKey[1];
String val = levelKey[2];
parts.append(delimiter);
-
if (isDefaultPartition(type, val)) {
parts.append(nullChar);
} else {
@@ -619,7 +618,6 @@ public class HiveResolver extends Plugin implements
ReadResolver {
DataType gpdbWritableType, Object val) {
record.add(new OneField(gpdbWritableType.getOID(), val));
}
-
/*
* Gets the delimiter character from the URL, verify and store it. Must be
a
* single ascii character (same restriction as Hawq's). If a hex
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca5d8de0/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
--
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
index 71e6845..1edb6d5 100644
--- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
@@ -83,
incubator-hawq git commit: HAWQ-1196. Added partitons support to HiveOrc profile.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 01373b943 -> 52333bb00
HAWQ-1196. Added partitons support to HiveOrc profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/52333bb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/52333bb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/52333bb0
Branch: refs/heads/master
Commit: 52333bb0099b34c8c354f78c10a235dc377560cf
Parents: 01373b9
Author: Oleksandr Diachenko
Authored: Wed Dec 7 15:16:11 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Dec 7 15:16:11 2016 -0800
--
.../pxf/plugins/hive/HiveColumnarSerdeResolver.java | 5 ++---
.../hawq/pxf/plugins/hive/HiveORCSerdeResolver.java | 13 -
.../apache/hawq/pxf/plugins/hive/HiveResolver.java | 16 +---
3 files changed, 19 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52333bb0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 987fc0f..362ac0d 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -61,7 +61,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver {
private boolean firstColumn;
private StringBuilder builder;
private StringBuilder parts;
-private int numberOfPartitions;
private HiveInputFormatFragmenter.PXF_HIVE_SERDES serdeType;
public HiveColumnarSerdeResolver(InputData input) throws Exception {
@@ -88,7 +87,7 @@ public class HiveColumnarSerdeResolver extends HiveResolver {
@Override
void initPartitionFields() {
-numberOfPartitions = initPartitionFields(parts);
+initPartitionFields(parts);
}
/**
@@ -118,7 +117,7 @@ public class HiveColumnarSerdeResolver extends HiveResolver
{
@Override
void initSerde(InputData input) throws Exception {
Properties serdeProperties = new Properties();
-int numberOfDataColumns = input.getColumns() - numberOfPartitions;
+int numberOfDataColumns = input.getColumns() - getNumberOfPartitions();
LOG.debug("Serde number of columns is " + numberOfDataColumns);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52333bb0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
index 9e32633..c789402 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
@@ -44,8 +44,6 @@ import java.util.*;
public class HiveORCSerdeResolver extends HiveResolver {
private static final Log LOG =
LogFactory.getLog(HiveORCSerdeResolver.class);
private OrcSerde deserializer;
-private StringBuilder parts;
-private int numberOfPartitions;
private HiveInputFormatFragmenter.PXF_HIVE_SERDES serdeType;
public HiveORCSerdeResolver(InputData input) throws Exception {
@@ -70,12 +68,6 @@ public class HiveORCSerdeResolver extends HiveResolver {
: input.getUserProperty("MAPKEY_DELIM");
}
-@Override
-void initPartitionFields() {
-parts = new StringBuilder();
-numberOfPartitions = initPartitionFields(parts);
-}
-
/**
* getFields returns a singleton list of OneField item.
* OneField item contains two fields: an integer representing the VARCHAR
type and a Java
@@ -89,6 +81,9 @@ public class HiveORCSerdeResolver extends HiveResolver {
StructObjectInspector soi = (StructObjectInspector)
deserializer.getObjectInspector();
List record = traverseStruct(tuple, soi, false);
+//Add partition fields if any
+record.addAll(getPartitionFields());
+
return record;
}
@@ -102,7 +97,7 @@ public class HiveORCSerdeResolver extends HiveResolver {
@Override
void initSerde(InputData input) throws Exception {
Properties serdeProperties = new Properties();
-int numberOfDataColumns = input.getColumns() - numberOfPartitions;
+int numberOfDataColumns = input.getColumns() -
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1114_ [deleted] 6763adbf3
[2/2] incubator-hawq git commit: HAWQ-1115. Implement filter-push down for IN on PXF service side.
HAWQ-1115. Implement filter-push down for IN on PXF service side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/713fc02b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/713fc02b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/713fc02b
Branch: refs/heads/master
Commit: 713fc02bc900d373729c3df611326982739b0483
Parents: 36f034f
Author: Oleksandr Diachenko
Authored: Wed Nov 23 16:08:15 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Nov 23 16:08:15 2016 -0800
--
.../org/apache/hawq/pxf/api/FilterParser.java | 56 ++--
.../org/apache/hawq/pxf/api/io/DataType.java| 19 +++
.../apache/hawq/pxf/api/FilterParserTest.java | 22
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 9
.../pxf/plugins/hive/HiveORCAccessorTest.java | 15 ++
.../plugins/hive/HiveORCSearchArgumentTest.java | 42 ---
6 files changed, 151 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/713fc02b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index e362eed..ff0d972 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -25,6 +25,8 @@ import org.apache.hawq.pxf.api.io.DataType;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Stack;
/**
@@ -65,11 +67,13 @@ public class FilterParser {
private Stack operandsStack;
private FilterBuilder filterBuilder;
public static final char COL_OP = 'a';
-public static final char CONST_OP = 'c';
+public static final char SCALAR_CONST_OP = 'c';
+public static final char LIST_CONST_OP = 'm';
public static final char CONST_LEN = 's';
public static final char CONST_DATA = 'd';
public static final char COMP_OP = 'o';
public static final char LOG_OP = 'l';
+
public static final String DEFAULT_CHARSET = "UTF-8";
/** Supported operations by the parser. */
@@ -83,7 +87,8 @@ public class FilterParser {
HDOP_NE,
HDOP_LIKE,
HDOP_IS_NULL,
-HDOP_IS_NOT_NULL
+HDOP_IS_NOT_NULL,
+HDOP_IN
}
/**
@@ -213,8 +218,11 @@ public class FilterParser {
case COL_OP:
operandsStack.push(new
ColumnIndex(safeToInt(parseNumber(;
break;
-case CONST_OP:
-operandsStack.push(new Constant(parseParameter()));
+case SCALAR_CONST_OP:
+operandsStack.push(new Constant(parseScalarParameter()));
+break;
+case LIST_CONST_OP:
+operandsStack.push(new Constant(parseListParameter()));
break;
case COMP_OP:
opNumber = safeToInt(parseNumber());
@@ -354,6 +362,10 @@ public class FilterParser {
}
private Object convertDataType(byte[] byteData, int start, int end,
DataType dataType) throws Exception {
+
+if (byteData.length < end)
+throw new FilterStringSyntaxException("filter string is shorter
than expected");
+
String data = new String(byteData, start, end-start, DEFAULT_CHARSET);
try {
switch (dataType) {
@@ -391,7 +403,7 @@ public class FilterParser {
/**
* Parses either a number or a string.
*/
-private Object parseParameter() throws Exception {
+private Object parseScalarParameter() throws Exception {
if (index == filterByteArr.length) {
throw new FilterStringSyntaxException("argument should follow at "
+ index);
}
@@ -418,6 +430,40 @@ public class FilterParser {
return data;
}
+private Object parseListParameter() throws Exception {
+if (index == filterByteArr.length) {
+throw new FilterStringSyntaxException("argument should follow at "
+ index);
+}
+
+DataType dataType = DataType.get(parseConstDataType());
+List data = new ArrayList();
+if (dataType == DataType.UNSUPPORTED_TYPE) {
+throw new FilterStringSyntaxException("invalid DataType OID at " +
(index - 1));
+}
+
+if (dataType.getTypeElem() == null) {
+throw new FilterStringSyntaxException("expected non-scalar
datatype, but got datatype with oid = " + dataType.getOID());
+}
+
+whi
[1/2] incubator-hawq git commit: HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 3354de0b1 -> 713fc02bc
HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/36f034fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/36f034fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/36f034fe
Branch: refs/heads/master
Commit: 36f034fec2effec8f4f2154bfad3015db9ce530a
Parents: 3354de0
Author: Oleksandr Diachenko
Authored: Wed Nov 23 16:07:46 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Nov 23 16:07:46 2016 -0800
--
src/backend/access/external/pxffilters.c| 533 +--
.../access/external/test/pxffilters_test.c | 351 ++--
src/include/access/pxffilters.h | 37 +-
src/include/catalog/pg_type.h | 1 +
4 files changed, 823 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/36f034fe/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 6699b84..f502e4e 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -35,8 +35,13 @@ static List* pxf_make_expression_items_list(List *quals,
Node *parent, int *logi
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
+static bool scalar_array_op_expr_to_pxffilter(ScalarArrayOpExpr *expr,
PxfFilterDesc *filter);
+static bool var_to_pxffilter(Var *var, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
-static void const_to_str(Const *constval, StringInfo buf);
+static bool supported_operator_type_op_expr(Oid type, PxfFilterDesc *filter);
+static bool supported_operator_type_scalar_array_op_expr(Oid type,
PxfFilterDesc *filter, bool useOr);
+static void scalar_const_to_str(Const *constval, StringInfo buf);
+static void list_const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
static void enrich_trivial_expression(List *expressionItems);
@@ -46,7 +51,7 @@ static void enrich_trivial_expression(List *expressionItems);
* down system catalog operators.
* see pg_operator.h
*/
-dbop_pxfop_map pxf_supported_opr[] =
+dbop_pxfop_map pxf_supported_opr_op_expr[] =
{
/* int2 */
{Int2EqualOperator /* int2eq */, PXFOP_EQ},
@@ -130,7 +135,7 @@ dbop_pxfop_map pxf_supported_opr[] =
{1869 /* int82ne */, PXFOP_NE},
/* date */
- {DateEqualOperator /* eq */, PXFOP_EQ},
+ {DateEqualOperator /* date_eq */, PXFOP_EQ},
{1095 /* date_lt */, PXFOP_LT},
{1097 /* date_gt */, PXFOP_GT},
{1096 /* date_le */, PXFOP_LE},
@@ -159,7 +164,16 @@ dbop_pxfop_map pxf_supported_opr[] =
{1060 /* bpchargt */, PXFOP_GT},
{1059 /* bpcharle */, PXFOP_LE},
{1061 /* bpcharge */, PXFOP_GE},
- {1057 /* bpcharne */, PXFOP_NE}
+ {1057 /* bpcharne */, PXFOP_NE},
+
+ /* boolean */
+ {BooleanEqualOperator /* booleq */, PXFOP_EQ},
+ {58 /* boollt */, PXFOP_LT},
+ {59 /* boolgt */, PXFOP_GT},
+ {1694 /* boolle */, PXFOP_LE},
+ {1695 /* boolge */, PXFOP_GE},
+ {85 /* boolne */, PXFOP_NE}
+
/* bytea */
// TODO: uncomment once HAWQ-1085 is done
@@ -172,6 +186,59 @@ dbop_pxfop_map pxf_supported_opr[] =
};
+
+dbop_pxfop_array_map pxf_supported_opr_scalar_array_op_expr[] =
+{
+ /* int2 */
+ {Int2EqualOperator /* int2eq */, PXFOP_IN, true},
+
+ /* int4 */
+ {Int4EqualOperator /* int4eq */, PXFOP_IN, true},
+
+ /* int8 */
+ {Int8EqualOperator /* int8eq */, PXFOP_IN, true},
+
+ /* text */
+ {TextEqualOperator /* texteq */, PXFOP_IN, true},
+
+ /* int2 to int4 */
+ {Int24EqualOperator /* int24eq */, PXFOP_IN, true},
+
+ /* int4 to int2 */
+ {Int42EqualOperator /* int42eq */, PXFOP_IN, true},
+
+ /* int8 to int4 */
+ {Int84EqualOperator /* int84eq */, PXFOP_IN, true},
+
+ /* int4 to int8 */
+ {Int48EqualOperator /* int48eq */, PXFOP_IN, true},
+
+ /* int2 to int8 */
+ {Int28EqualOperator /* int28eq */, PXFOP_IN, true},
+
+ /* int8 to int2 */
+ {Int82EqualOperator /* int82eq */, PXFOP_IN, true},
+
+ /* date */
+ {DateEqualOperator /* date_eq */, PXFOP_IN, true},
+
+ /* float8 */
+ {Float8EqualOperator /* float8eq */, PXFOP_IN, true},
+
+ /* float48 */
+ {1120 /* float48eq */, PXFOP_
[1/2] incubator-hawq git commit: HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1114_ [created] 6763adbf3
HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ed3376ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ed3376ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ed3376ae
Branch: refs/heads/HAWQ-1114_
Commit: ed3376ae30bea8d1c9df45ccd10334356259005d
Parents: 3354de0
Author: Oleksandr Diachenko
Authored: Wed Nov 23 15:10:09 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Nov 23 15:10:09 2016 -0800
--
src/backend/access/external/pxffilters.c| 533 +--
.../access/external/test/pxffilters_test.c | 351 ++--
src/include/access/pxffilters.h | 37 +-
src/include/catalog/pg_type.h | 1 +
4 files changed, 823 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ed3376ae/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 6699b84..f502e4e 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -35,8 +35,13 @@ static List* pxf_make_expression_items_list(List *quals,
Node *parent, int *logi
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
+static bool scalar_array_op_expr_to_pxffilter(ScalarArrayOpExpr *expr,
PxfFilterDesc *filter);
+static bool var_to_pxffilter(Var *var, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
-static void const_to_str(Const *constval, StringInfo buf);
+static bool supported_operator_type_op_expr(Oid type, PxfFilterDesc *filter);
+static bool supported_operator_type_scalar_array_op_expr(Oid type,
PxfFilterDesc *filter, bool useOr);
+static void scalar_const_to_str(Const *constval, StringInfo buf);
+static void list_const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
static void enrich_trivial_expression(List *expressionItems);
@@ -46,7 +51,7 @@ static void enrich_trivial_expression(List *expressionItems);
* down system catalog operators.
* see pg_operator.h
*/
-dbop_pxfop_map pxf_supported_opr[] =
+dbop_pxfop_map pxf_supported_opr_op_expr[] =
{
/* int2 */
{Int2EqualOperator /* int2eq */, PXFOP_EQ},
@@ -130,7 +135,7 @@ dbop_pxfop_map pxf_supported_opr[] =
{1869 /* int82ne */, PXFOP_NE},
/* date */
- {DateEqualOperator /* eq */, PXFOP_EQ},
+ {DateEqualOperator /* date_eq */, PXFOP_EQ},
{1095 /* date_lt */, PXFOP_LT},
{1097 /* date_gt */, PXFOP_GT},
{1096 /* date_le */, PXFOP_LE},
@@ -159,7 +164,16 @@ dbop_pxfop_map pxf_supported_opr[] =
{1060 /* bpchargt */, PXFOP_GT},
{1059 /* bpcharle */, PXFOP_LE},
{1061 /* bpcharge */, PXFOP_GE},
- {1057 /* bpcharne */, PXFOP_NE}
+ {1057 /* bpcharne */, PXFOP_NE},
+
+ /* boolean */
+ {BooleanEqualOperator /* booleq */, PXFOP_EQ},
+ {58 /* boollt */, PXFOP_LT},
+ {59 /* boolgt */, PXFOP_GT},
+ {1694 /* boolle */, PXFOP_LE},
+ {1695 /* boolge */, PXFOP_GE},
+ {85 /* boolne */, PXFOP_NE}
+
/* bytea */
// TODO: uncomment once HAWQ-1085 is done
@@ -172,6 +186,59 @@ dbop_pxfop_map pxf_supported_opr[] =
};
+
+dbop_pxfop_array_map pxf_supported_opr_scalar_array_op_expr[] =
+{
+ /* int2 */
+ {Int2EqualOperator /* int2eq */, PXFOP_IN, true},
+
+ /* int4 */
+ {Int4EqualOperator /* int4eq */, PXFOP_IN, true},
+
+ /* int8 */
+ {Int8EqualOperator /* int8eq */, PXFOP_IN, true},
+
+ /* text */
+ {TextEqualOperator /* texteq */, PXFOP_IN, true},
+
+ /* int2 to int4 */
+ {Int24EqualOperator /* int24eq */, PXFOP_IN, true},
+
+ /* int4 to int2 */
+ {Int42EqualOperator /* int42eq */, PXFOP_IN, true},
+
+ /* int8 to int4 */
+ {Int84EqualOperator /* int84eq */, PXFOP_IN, true},
+
+ /* int4 to int8 */
+ {Int48EqualOperator /* int48eq */, PXFOP_IN, true},
+
+ /* int2 to int8 */
+ {Int28EqualOperator /* int28eq */, PXFOP_IN, true},
+
+ /* int8 to int2 */
+ {Int82EqualOperator /* int82eq */, PXFOP_IN, true},
+
+ /* date */
+ {DateEqualOperator /* date_eq */, PXFOP_IN, true},
+
+ /* float8 */
+ {Float8EqualOperator /* float8eq */, PXFOP_IN, true},
+
+ /* float48 */
+ {1120 /* float48eq */, P
[2/2] incubator-hawq git commit: HAWQ-1115. Implement filter-push down for IN on PXF service side.
HAWQ-1115. Implement filter-push down for IN on PXF service side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6763adbf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6763adbf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6763adbf
Branch: refs/heads/HAWQ-1114_
Commit: 6763adbf31c428b548ec8f278c2f3acc1db288ab
Parents: ed3376a
Author: Oleksandr Diachenko
Authored: Wed Nov 23 15:10:31 2016 -0800
Committer: Oleksandr Diachenko
Committed: Wed Nov 23 15:10:31 2016 -0800
--
.../org/apache/hawq/pxf/api/FilterParser.java | 56 ++--
.../org/apache/hawq/pxf/api/io/DataType.java| 19 +++
.../apache/hawq/pxf/api/FilterParserTest.java | 22
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 9
.../pxf/plugins/hive/HiveORCAccessorTest.java | 15 ++
.../plugins/hive/HiveORCSearchArgumentTest.java | 42 ---
6 files changed, 151 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6763adbf/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index e362eed..ff0d972 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -25,6 +25,8 @@ import org.apache.hawq.pxf.api.io.DataType;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Stack;
/**
@@ -65,11 +67,13 @@ public class FilterParser {
private Stack operandsStack;
private FilterBuilder filterBuilder;
public static final char COL_OP = 'a';
-public static final char CONST_OP = 'c';
+public static final char SCALAR_CONST_OP = 'c';
+public static final char LIST_CONST_OP = 'm';
public static final char CONST_LEN = 's';
public static final char CONST_DATA = 'd';
public static final char COMP_OP = 'o';
public static final char LOG_OP = 'l';
+
public static final String DEFAULT_CHARSET = "UTF-8";
/** Supported operations by the parser. */
@@ -83,7 +87,8 @@ public class FilterParser {
HDOP_NE,
HDOP_LIKE,
HDOP_IS_NULL,
-HDOP_IS_NOT_NULL
+HDOP_IS_NOT_NULL,
+HDOP_IN
}
/**
@@ -213,8 +218,11 @@ public class FilterParser {
case COL_OP:
operandsStack.push(new
ColumnIndex(safeToInt(parseNumber(;
break;
-case CONST_OP:
-operandsStack.push(new Constant(parseParameter()));
+case SCALAR_CONST_OP:
+operandsStack.push(new Constant(parseScalarParameter()));
+break;
+case LIST_CONST_OP:
+operandsStack.push(new Constant(parseListParameter()));
break;
case COMP_OP:
opNumber = safeToInt(parseNumber());
@@ -354,6 +362,10 @@ public class FilterParser {
}
private Object convertDataType(byte[] byteData, int start, int end,
DataType dataType) throws Exception {
+
+if (byteData.length < end)
+throw new FilterStringSyntaxException("filter string is shorter
than expected");
+
String data = new String(byteData, start, end-start, DEFAULT_CHARSET);
try {
switch (dataType) {
@@ -391,7 +403,7 @@ public class FilterParser {
/**
* Parses either a number or a string.
*/
-private Object parseParameter() throws Exception {
+private Object parseScalarParameter() throws Exception {
if (index == filterByteArr.length) {
throw new FilterStringSyntaxException("argument should follow at "
+ index);
}
@@ -418,6 +430,40 @@ public class FilterParser {
return data;
}
+private Object parseListParameter() throws Exception {
+if (index == filterByteArr.length) {
+throw new FilterStringSyntaxException("argument should follow at "
+ index);
+}
+
+DataType dataType = DataType.get(parseConstDataType());
+List data = new ArrayList();
+if (dataType == DataType.UNSUPPORTED_TYPE) {
+throw new FilterStringSyntaxException("invalid DataType OID at " +
(index - 1));
+}
+
+if (dataType.getTypeElem() == null) {
+throw new FilterStringSyntaxException("expected non-scalar
datatype, but got datatype with oid = " + dataType.getOID());
+}
+
+
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1114_ [deleted] 3988fa9d6
incubator-hawq git commit: HAWQ-1169. HAWQ Bridge incorrectly pushes T_NullTest predicates with byte arrays.
Repository: incubator-hawq
Updated Branches:
refs/heads/master cb6811182 -> 2a0dd7dd3
HAWQ-1169. HAWQ Bridge incorrectly pushes T_NullTest predicates with byte
arrays.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2a0dd7dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2a0dd7dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2a0dd7dd
Branch: refs/heads/master
Commit: 2a0dd7dd337348a6571c31313e6914cd8a91c0a6
Parents: cb68111
Author: Oleksandr Diachenko
Authored: Tue Nov 22 16:37:00 2016 -0800
Committer: Oleksandr Diachenko
Committed: Tue Nov 22 16:37:00 2016 -0800
--
src/backend/access/external/pxffilters.c | 13 ++---
src/backend/access/external/test/pxffilters_test.c | 3 +--
2 files changed, 11 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a0dd7dd/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 3961b48..6699b84 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -184,7 +184,6 @@ Oid pxf_supported_types[] =
VARCHAROID,
BPCHAROID,
CHAROID,
- BYTEAOID,
BOOLOID,
DATEOID,
TIMESTAMPOID
@@ -434,15 +433,23 @@ pxf_serialize_filter_list(List *expressionItems)
{
elog(DEBUG1, "pxf_serialize_filter_list: node
tag %d (T_NullTest)", tag);
NullTest *expr = (NullTest *) node;
+ Var *var = (Var *) expr->arg;
+
+ //TODO: add check for supported operation
+ if (!supported_filter_type(var->vartype))
+ {
+ elog(DEBUG1, "Query will not be
optimized to use filter push-down.");
+ return NULL;
+ }
/* filter expression for T_NullTest will not
have any constant value */
if (expr->nulltesttype == IS_NULL)
{
- appendStringInfo(resbuf, "%c%d%c%d",
PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE,
PXFOP_IS_NULL);
+ appendStringInfo(resbuf, "%c%d%c%d",
PXF_ATTR_CODE, var->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NULL);
}
else if (expr->nulltesttype == IS_NOT_NULL)
{
- appendStringInfo(resbuf, "%c%d%c%d",
PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE,
PXFOP_IS_NOTNULL);
+ appendStringInfo(resbuf, "%c%d%c%d",
PXF_ATTR_CODE, var->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NOTNULL);
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a0dd7dd/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index e618563..0e45eee 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -43,7 +43,6 @@ test__supported_filter_type(void **state)
VARCHAROID,
BPCHAROID,
CHAROID,
- BYTEAOID,
BOOLOID,
DATEOID,
CIRCLEOID /* unsupported type */
@@ -65,7 +64,7 @@ test__supported_filter_type(void **state)
/* go over pxf_supported_types array */
int nargs = sizeof(pxf_supported_types) / sizeof(Oid);
- assert_int_equal(nargs, 14);
+ assert_int_equal(nargs, 13);
for (i = 0; i < nargs; ++i)
{
assert_true(supported_filter_type(pxf_supported_types[i]));
[1/2] incubator-hawq git commit: HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1114_ [created] 3988fa9d6
HAWQ-1114. Implement filter-push down for IN on HAWQ bridge side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f5369009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f5369009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f5369009
Branch: refs/heads/HAWQ-1114_
Commit: f5369009872e61bbbc83dfc296671f79f0094826
Parents: cb68111
Author: Oleksandr Diachenko
Authored: Tue Nov 22 15:36:49 2016 -0800
Committer: Oleksandr Diachenko
Committed: Tue Nov 22 15:36:49 2016 -0800
--
src/backend/access/external/pxffilters.c| 527 +--
.../access/external/test/pxffilters_test.c | 351 ++--
src/include/access/pxffilters.h | 37 +-
src/include/catalog/pg_type.h | 1 +
4 files changed, 817 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f5369009/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 3961b48..86f11e6 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -35,8 +35,13 @@ static List* pxf_make_expression_items_list(List *quals,
Node *parent, int *logi
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
+static bool scalar_array_op_expr_to_pxffilter(ScalarArrayOpExpr *expr,
PxfFilterDesc *filter);
+static bool var_to_pxffilter(Var *var, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
-static void const_to_str(Const *constval, StringInfo buf);
+static bool supported_operator_type_op_expr(Oid type, PxfFilterDesc *filter);
+static bool supported_operator_type_scalar_array_op_expr(Oid type,
PxfFilterDesc *filter, bool useOr);
+static void scalar_const_to_str(Const *constval, StringInfo buf);
+static void list_const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
static void enrich_trivial_expression(List *expressionItems);
@@ -46,7 +51,7 @@ static void enrich_trivial_expression(List *expressionItems);
* down system catalog operators.
* see pg_operator.h
*/
-dbop_pxfop_map pxf_supported_opr[] =
+dbop_pxfop_map pxf_supported_opr_op_expr[] =
{
/* int2 */
{Int2EqualOperator /* int2eq */, PXFOP_EQ},
@@ -130,7 +135,7 @@ dbop_pxfop_map pxf_supported_opr[] =
{1869 /* int82ne */, PXFOP_NE},
/* date */
- {DateEqualOperator /* eq */, PXFOP_EQ},
+ {DateEqualOperator /* date_eq */, PXFOP_EQ},
{1095 /* date_lt */, PXFOP_LT},
{1097 /* date_gt */, PXFOP_GT},
{1096 /* date_le */, PXFOP_LE},
@@ -159,7 +164,16 @@ dbop_pxfop_map pxf_supported_opr[] =
{1060 /* bpchargt */, PXFOP_GT},
{1059 /* bpcharle */, PXFOP_LE},
{1061 /* bpcharge */, PXFOP_GE},
- {1057 /* bpcharne */, PXFOP_NE}
+ {1057 /* bpcharne */, PXFOP_NE},
+
+ /* boolean */
+ {BooleanEqualOperator /* booleq */, PXFOP_EQ},
+ {58 /* boollt */, PXFOP_LT},
+ {59 /* boolgt */, PXFOP_GT},
+ {1694 /* boolle */, PXFOP_LE},
+ {1695 /* boolge */, PXFOP_GE},
+ {85 /* boolne */, PXFOP_NE}
+
/* bytea */
// TODO: uncomment once HAWQ-1085 is done
@@ -172,6 +186,59 @@ dbop_pxfop_map pxf_supported_opr[] =
};
+
+dbop_pxfop_array_map pxf_supported_opr_scalar_array_op_expr[] =
+{
+ /* int2 */
+ {Int2EqualOperator /* int2eq */, PXFOP_IN, true},
+
+ /* int4 */
+ {Int4EqualOperator /* int4eq */, PXFOP_IN, true},
+
+ /* int8 */
+ {Int8EqualOperator /* int8eq */, PXFOP_IN, true},
+
+ /* text */
+ {TextEqualOperator /* texteq */, PXFOP_IN, true},
+
+ /* int2 to int4 */
+ {Int24EqualOperator /* int24eq */, PXFOP_IN, true},
+
+ /* int4 to int2 */
+ {Int42EqualOperator /* int42eq */, PXFOP_IN, true},
+
+ /* int8 to int4 */
+ {Int84EqualOperator /* int84eq */, PXFOP_IN, true},
+
+ /* int4 to int8 */
+ {Int48EqualOperator /* int48eq */, PXFOP_IN, true},
+
+ /* int2 to int8 */
+ {Int28EqualOperator /* int28eq */, PXFOP_IN, true},
+
+ /* int8 to int2 */
+ {Int82EqualOperator /* int82eq */, PXFOP_IN, true},
+
+ /* date */
+ {DateEqualOperator /* date_eq */, PXFOP_IN, true},
+
+ /* float8 */
+ {Float8EqualOperator /* float8eq */, PXFOP_IN, true},
+
+ /* float48 */
+ {1120 /* float48eq */, P
[2/2] incubator-hawq git commit: HAWQ-1115. Implement filter-push down for IN on PXF service side.
HAWQ-1115. Implement filter-push down for IN on PXF service side.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3988fa9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3988fa9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3988fa9d
Branch: refs/heads/HAWQ-1114_
Commit: 3988fa9d650bfce07c9951abfe52f093f148b3ee
Parents: f536900
Author: Oleksandr Diachenko
Authored: Tue Nov 22 15:37:15 2016 -0800
Committer: Oleksandr Diachenko
Committed: Tue Nov 22 15:37:15 2016 -0800
--
.../org/apache/hawq/pxf/api/FilterParser.java | 56 ++--
.../org/apache/hawq/pxf/api/io/DataType.java| 19 +++
.../apache/hawq/pxf/api/FilterParserTest.java | 22
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 9
.../pxf/plugins/hive/HiveORCAccessorTest.java | 15 ++
.../plugins/hive/HiveORCSearchArgumentTest.java | 42 ---
6 files changed, 151 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3988fa9d/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index e362eed..ff0d972 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -25,6 +25,8 @@ import org.apache.hawq.pxf.api.io.DataType;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Stack;
/**
@@ -65,11 +67,13 @@ public class FilterParser {
private Stack operandsStack;
private FilterBuilder filterBuilder;
public static final char COL_OP = 'a';
-public static final char CONST_OP = 'c';
+public static final char SCALAR_CONST_OP = 'c';
+public static final char LIST_CONST_OP = 'm';
public static final char CONST_LEN = 's';
public static final char CONST_DATA = 'd';
public static final char COMP_OP = 'o';
public static final char LOG_OP = 'l';
+
public static final String DEFAULT_CHARSET = "UTF-8";
/** Supported operations by the parser. */
@@ -83,7 +87,8 @@ public class FilterParser {
HDOP_NE,
HDOP_LIKE,
HDOP_IS_NULL,
-HDOP_IS_NOT_NULL
+HDOP_IS_NOT_NULL,
+HDOP_IN
}
/**
@@ -213,8 +218,11 @@ public class FilterParser {
case COL_OP:
operandsStack.push(new
ColumnIndex(safeToInt(parseNumber(;
break;
-case CONST_OP:
-operandsStack.push(new Constant(parseParameter()));
+case SCALAR_CONST_OP:
+operandsStack.push(new Constant(parseScalarParameter()));
+break;
+case LIST_CONST_OP:
+operandsStack.push(new Constant(parseListParameter()));
break;
case COMP_OP:
opNumber = safeToInt(parseNumber());
@@ -354,6 +362,10 @@ public class FilterParser {
}
private Object convertDataType(byte[] byteData, int start, int end,
DataType dataType) throws Exception {
+
+if (byteData.length < end)
+throw new FilterStringSyntaxException("filter string is shorter
than expected");
+
String data = new String(byteData, start, end-start, DEFAULT_CHARSET);
try {
switch (dataType) {
@@ -391,7 +403,7 @@ public class FilterParser {
/**
* Parses either a number or a string.
*/
-private Object parseParameter() throws Exception {
+private Object parseScalarParameter() throws Exception {
if (index == filterByteArr.length) {
throw new FilterStringSyntaxException("argument should follow at "
+ index);
}
@@ -418,6 +430,40 @@ public class FilterParser {
return data;
}
+private Object parseListParameter() throws Exception {
+if (index == filterByteArr.length) {
+throw new FilterStringSyntaxException("argument should follow at "
+ index);
+}
+
+DataType dataType = DataType.get(parseConstDataType());
+List data = new ArrayList();
+if (dataType == DataType.UNSUPPORTED_TYPE) {
+throw new FilterStringSyntaxException("invalid DataType OID at " +
(index - 1));
+}
+
+if (dataType.getTypeElem() == null) {
+throw new FilterStringSyntaxException("expected non-scalar
datatype, but got datatype with oid = " + dataType.getOID());
+}
+
+
incubator-hawq git commit: HAWQ-1130. Make HCatalog integration work with non-superusers.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 38b447d7e -> 475dbb5cb
HAWQ-1130. Make HCatalog integration work with non-superusers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/475dbb5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/475dbb5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/475dbb5c
Branch: refs/heads/master
Commit: 475dbb5cb264177199a9722ca2f0b7f2c8c6743b
Parents: 38b447d
Author: Oleksandr Diachenko
Authored: Mon Nov 21 11:14:01 2016 -0800
Committer: Oleksandr Diachenko
Committed: Mon Nov 21 11:14:01 2016 -0800
--
src/backend/access/transam/varsup.c | 143 ++-
1 file changed, 103 insertions(+), 40 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/475dbb5c/src/backend/access/transam/varsup.c
--
diff --git a/src/backend/access/transam/varsup.c
b/src/backend/access/transam/varsup.c
index fadefd5..73a4769 100644
--- a/src/backend/access/transam/varsup.c
+++ b/src/backend/access/transam/varsup.c
@@ -16,6 +16,7 @@
#include "access/clog.h"
#include "access/subtrans.h"
#include "access/transam.h"
+#include "catalog/catquery.h"
#include "executor/spi.h"
#include "miscadmin.h"
#include "postmaster/autovacuum.h"
@@ -408,6 +409,9 @@ GetNewExternalObjectId(void)
/*
* must perform check on External Oid range on
* initial access of NextExternalOid
+*
+* It's needed for upgrade scenario from old version
+* of HAWQ which doesn't support dedicated oid pool for HCatalog objects
*/
if (!IsExternalOidInitialized)
{
@@ -416,6 +420,11 @@ GetNewExternalObjectId(void)
ResetExternalObjectId();
}
+ /*
+* This check is needed for upgrade from old HAWQ versions, which don't
support
+* oid pool for HCatalog objects.
+* In current implementation max oid will be always less than
FirstExternalObjectId.
+*/
if (!IsValidExternalOidRange)
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
@@ -474,74 +483,128 @@ ResetExternalObjectId(void)
/*
* master_highest_used_oid
- * Query the database to find the highest used Oid by
+ * Uses CAQL and SPI to find the highest used Oid among user and
catalog tables
+ *
+ * Uses CAQL to query catalog tables
+ * Uses SPI to query user tables, because CAQL supports tables
from CatCoreRelation array only
* 1) Find all the relations that has Oids
* 2) Find max oid from those relations
*/
Oid
master_highest_used_oid(void)
{
+ Oid oidMaxCatalog = InvalidOid;
+ Oid oidMaxUser = InvalidOid;
Oid oidMax = InvalidOid;
+ Oid currentOid;
+ Form_pg_class classForm;
+ cqContext *pcqOuterCtx;
+ cqContext *pcqInnerCtx;
+ HeapTuple outerTuple;
+ HeapTuple innerTuple;
+ /* number of user tables having oids*/
+ int userTablesNum = 0;
+ int ret;
+
+ pcqOuterCtx = caql_beginscan(NULL, cql("SELECT * FROM pg_class WHERE
relhasoids = :1", BoolGetDatum(true)));
- if (SPI_OK_CONNECT != SPI_connect())
+ outerTuple = caql_getnext(pcqOuterCtx);
+
+ if (!HeapTupleIsValid(outerTuple))
{
- ereport(ERROR, (errcode(ERRCODE_CDB_INTERNAL_ERROR),
- errmsg("Unable to connect to execute internal
query for HCatalog.")));
+ caql_endscan(pcqOuterCtx);
+ elog(DEBUG1, "Unable to get list of tables having oids");
+ return oidMax;
}
- int ret = SPI_execute("SELECT relname FROM pg_class where
relhasoids=true", true, 0);
+ /* construct query to get max oid from all tables with oids */
+ StringInfo sqlstrCatalog = makeStringInfo();
+ StringInfo sqlstrUser = makeStringInfo();
+ appendStringInfo(sqlstrUser, "SELECT max(oid) FROM (");
+ while (HeapTupleIsValid(outerTuple))
+ {
+ classForm = (Form_pg_class) GETSTRUCT(outerTuple);
- int rows = SPI_processed;
+ /* use CAQL for accessing catalog tables*/
+ if (classForm->relnamespace == PG_CATALOG_NAMESPACE)
+ {
+ appendStringInfo(sqlstrCatalog,
+ "SELECT oid FROM %s WHERE oid > :1
ORDER BY oid",
+ classForm->relname.data);
- char *tableNames[rows];
+ pcqInnerCtx = caql_beginscan(NULL,
+ cql1(sqlstrCatalog->data, __FILE__,
__LI
incubator-hawq git commit: HAWQ-1103. Send constant datatype and length in filter to PXF.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 402e08da6 -> 55f0ff7e4
HAWQ-1103. Send constant datatype and length in filter to PXF.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/55f0ff7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/55f0ff7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/55f0ff7e
Branch: refs/heads/master
Commit: 55f0ff7e4e03f5d90dc963f287a78a426f66495d
Parents: 402e08d
Author: Oleksandr Diachenko
Authored: Mon Oct 24 12:36:29 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Oct 24 12:36:48 2016 -0700
--
src/backend/access/external/pxffilters.c| 58 +++-
.../access/external/test/pxffilters_test.c | 33 ++-
src/include/access/pxffilters.h | 9 ++-
3 files changed, 68 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55f0ff7e/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 56aea55..f06b07f 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,7 +31,7 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_expression_items_list(List *quals, Node *parent, bool
*logicalOpsNum);
+static List* pxf_make_expression_items_list(List *quals, Node *parent, int
*logicalOpsNum);
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
@@ -193,7 +193,6 @@ Oid pxf_supported_types[] =
static void
pxf_free_expression_items_list(List *expressionItems, bool freeBoolExprNodes)
{
- ListCell*lc = NULL;
ExpressionItem *expressionItem = NULL;
int previousLength;
@@ -228,7 +227,7 @@ pxf_free_expression_items_list(List *expressionItems, bool
freeBoolExprNodes)
*
*/
static List *
-pxf_make_expression_items_list(List *quals, Node *parent, bool *logicalOpsNum)
+pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum)
{
ExpressionItem *expressionItem = NULL;
List*result = NIL;
@@ -330,7 +329,27 @@ pxf_free_filter(PxfFilterDesc* filter)
*
* Yields the following serialized string:
*
- * a0c1o2a0c5o1o7a2c"third"o5o7
+ * a0c23s1d1o2a1c23s1d5o1a2c25s5dthirdo5l0l0
+ *
+ * Where:
+ *
+ * a0 - first column of table
+ * c23- scalar constant with type oid 23(INT4)
+ * s1 - size of constant in bytes
+ * d1 - serialized constant value
+ * o2 - greater than operation
+ * a1 - second column of table
+ * c23- scalar constant with type oid 23(INT4)
+ * s1 - size of constant in bytes
+ * d5 - serialized constant value
+ * o1 - less than operation
+ * a2 - third column of table
+ * c25- scalar constant with type oid 25(TEXT)
+ * s5 - size of constant in bytes
+ * dthird - serialized constant value
+ * o5 - equals operation
+ * l0 - AND operator
+ * l0 - AND operator
*
*/
static char *
@@ -369,14 +388,18 @@ pxf_serialize_filter_list(List *expressionItems)
PxfOperatorCode o = filter->op;
if (pxfoperand_is_attr(l) &&
pxfoperand_is_const(r))
{
- appendStringInfo(resbuf,
"%c%d%c%s",
+ appendStringInfo(resbuf,
"%c%d%c%d%c%lu%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
-
PXF_CONST_CODE, (r.conststr)->data);
+
PXF_CONST_CODE, r.consttype,
+
PXF_SIZE_BYTES, strlen(r.conststr->data),
+
PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
- appendStringInfo(resbuf,
"%c%s%c%d",
-
PXF_CONST_CODE, (l.cons
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1103 [deleted] b2dd7de0c
[3/5] incubator-hawq git commit: HAWQ-1103. Added test cases with string containing a quote.
HAWQ-1103. Added test cases with string containing a quote.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2497b6f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2497b6f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2497b6f3
Branch: refs/heads/HAWQ-1103
Commit: 2497b6f327944e1fbdc5434dddacab0fcacfa7f9
Parents: d9f77da
Author: Oleksandr Diachenko
Authored: Fri Oct 14 11:45:02 2016 -0700
Committer: Oleksandr Diachenko
Committed: Fri Oct 14 11:45:02 2016 -0700
--
src/backend/access/external/pxffilters.c | 4 ++--
src/backend/access/external/test/pxffilters_test.c | 7 +--
src/include/access/pxffilters.h| 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2497b6f3/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index ded9f9a..ef7b16f 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -372,14 +372,14 @@ pxf_serialize_filter_list(List *expressionItems)
appendStringInfo(resbuf,
"%c%d%c%d%c%d%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
PXF_CONST_CODE, r.consttype,
-
PXF_LENGTH_TO_READ, strlen(r.conststr->data),
+
PXF_SIZE_BYTES, strlen(r.conststr->data),
PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
appendStringInfo(resbuf,
"%c%d%c%d%c%s%c%d",
PXF_CONST_CODE, l.consttype,
-
PXF_LENGTH_TO_READ, strlen(l.conststr->data),
+
PXF_SIZE_BYTES, strlen(l.conststr->data),
PXF_CONST_DATA, (l.conststr)->data,
PXF_ATTR_CODE, r.attnum - 1); /* Java attrs are 0-based */
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2497b6f3/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 6db7ae7..3679587 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -517,20 +517,23 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"George Orwell", TEXTOID, TextEqualOperator);
ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"Winston", TEXTOID, TextEqualOperator);
ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"Eric-%", TEXTOID, 1209);
+ ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID,
"\"Ugly\" string with quotes", TEXTOID, TextEqualOperator);
expressionItems = lappend(expressionItems, expressionItem1);
expressionItems = lappend(expressionItems, expressionItem2);
expressionItems = lappend(expressionItems, expressionItem3);
expressionItems = lappend(expressionItems, expressionItem4);
+ expressionItems = lappend(expressionItems, expressionItem5);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with
quoteso5");
pfree(result);
+ int trivia
[4/5] incubator-hawq git commit: HAWQ-1103. Fixed warnings.
HAWQ-1103. Fixed warnings.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/35442228
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/35442228
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/35442228
Branch: refs/heads/HAWQ-1103
Commit: 354422287af2cb1af0e9465821d8ec9f6062fa10
Parents: 2497b6f
Author: Oleksandr Diachenko
Authored: Tue Oct 18 14:44:49 2016 -0700
Committer: Oleksandr Diachenko
Committed: Tue Oct 18 14:44:49 2016 -0700
--
src/backend/access/external/pxffilters.c | 15 +--
1 file changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/35442228/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index ef7b16f..0d13074 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,7 +31,7 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_expression_items_list(List *quals, Node *parent, bool
*logicalOpsNum);
+static List* pxf_make_expression_items_list(List *quals, Node *parent, int
*logicalOpsNum);
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
@@ -193,7 +193,6 @@ Oid pxf_supported_types[] =
static void
pxf_free_expression_items_list(List *expressionItems, bool freeBoolExprNodes)
{
- ListCell*lc = NULL;
ExpressionItem *expressionItem = NULL;
int previousLength;
@@ -228,7 +227,7 @@ pxf_free_expression_items_list(List *expressionItems, bool
freeBoolExprNodes)
*
*/
static List *
-pxf_make_expression_items_list(List *quals, Node *parent, bool *logicalOpsNum)
+pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum)
{
ExpressionItem *expressionItem = NULL;
List*result = NIL;
@@ -369,7 +368,7 @@ pxf_serialize_filter_list(List *expressionItems)
PxfOperatorCode o = filter->op;
if (pxfoperand_is_attr(l) &&
pxfoperand_is_const(r))
{
- appendStringInfo(resbuf,
"%c%d%c%d%c%d%c%s",
+ appendStringInfo(resbuf,
"%c%d%c%d%c%lu%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
PXF_CONST_CODE, r.consttype,
PXF_SIZE_BYTES, strlen(r.conststr->data),
@@ -377,7 +376,7 @@ pxf_serialize_filter_list(List *expressionItems)
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
- appendStringInfo(resbuf,
"%c%d%c%d%c%s%c%d",
+ appendStringInfo(resbuf,
"%c%d%c%lu%c%s%c%d",
PXF_CONST_CODE, l.consttype,
PXF_SIZE_BYTES, strlen(l.conststr->data),
PXF_CONST_DATA, (l.conststr)->data,
@@ -410,6 +409,10 @@ pxf_serialize_filter_list(List *expressionItems)
appendStringInfo(resbuf, "%c%d",
PXF_LOGICAL_OPERATOR_CODE, boolType);
break;
}
+ default:
+ {
+ elog(DEBUG5, "Skipping tag: %d", tag);
+ }
}
}
@@ -726,7 +729,7 @@ void enrich_trivial_expression(List *expressionItems) {
andExpr->boolop = AND_EXPR;
- andExpressionItem->node = andExpr;
+ andExpressionItem->node = (Node *) andExpr;
andExpressionItem->parent = NULL;
andExpressionItem->processed = false;
[1/5] incubator-hawq git commit: HAWQ-1103. Draft implementation without taking care of different charsets.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1103 [created] b2dd7de0c
HAWQ-1103. Draft implementation without taking care of different charsets.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b318fa0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b318fa0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b318fa0e
Branch: refs/heads/HAWQ-1103
Commit: b318fa0e7d038084ded2bc99c6e631aa26700b21
Parents: 7f3658d
Author: Oleksandr Diachenko
Authored: Thu Oct 13 16:00:03 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 16:00:03 2016 -0700
--
src/backend/access/external/pxffilters.c | 25 +++--
src/include/access/pxffilters.h | 9 ++---
2 files changed, 21 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b318fa0e/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 56aea55..ded9f9a 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -369,14 +369,18 @@ pxf_serialize_filter_list(List *expressionItems)
PxfOperatorCode o = filter->op;
if (pxfoperand_is_attr(l) &&
pxfoperand_is_const(r))
{
- appendStringInfo(resbuf,
"%c%d%c%s",
+ appendStringInfo(resbuf,
"%c%d%c%d%c%d%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
-
PXF_CONST_CODE, (r.conststr)->data);
+
PXF_CONST_CODE, r.consttype,
+
PXF_LENGTH_TO_READ, strlen(r.conststr->data),
+
PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
- appendStringInfo(resbuf,
"%c%s%c%d",
-
PXF_CONST_CODE, (l.conststr)->data,
+ appendStringInfo(resbuf,
"%c%d%c%d%c%s%c%d",
+
PXF_CONST_CODE, l.consttype,
+
PXF_LENGTH_TO_READ, strlen(l.conststr->data),
+
PXF_CONST_DATA, (l.conststr)->data,
PXF_ATTR_CODE, r.attnum - 1); /* Java attrs are 0-based */
}
else
@@ -468,6 +472,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
{
filter->l.opcode = PXF_ATTR_CODE;
filter->l.attnum = ((Var *) leftop)->varattno;
+ filter->l.consttype = InvalidOid;
if (filter->l.attnum <= InvalidAttrNumber)
return false; /* system attr not supported */
@@ -476,6 +481,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->r.conststr = makeStringInfo();
initStringInfo(filter->r.conststr);
const_to_str((Const *)rightop, filter->r.conststr);
+ filter->r.consttype = ((Const *)rightop)->consttype;
}
else if (IsA(leftop, Const) && IsA(rightop, Var))
{
@@ -484,9 +490,11 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->l.conststr = makeStringInfo();
initStringInfo(filter->l.conststr);
const_to_str((Const *)leftop, filter->l.conststr);
+ filter->l.consttype = ((Const *)leftop)->consttype;
filter->r.opcode = PXF_ATTR_CODE;
filter->r.attnum = ((Var *) rightop)->varattno;
+ filter->r.consttype = InvalidOid;
[5/5] incubator-hawq git commit: HAWQ-1103. Added comments and test case for empty string.
HAWQ-1103. Added comments and test case for empty string. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b2dd7de0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b2dd7de0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b2dd7de0 Branch: refs/heads/HAWQ-1103 Commit: b2dd7de0c332922f647309f58e6e4a3630d33418 Parents: 3544222 Author: Oleksandr Diachenko Authored: Fri Oct 21 15:09:39 2016 -0700 Committer: Oleksandr Diachenko Committed: Fri Oct 21 15:09:39 2016 -0700 -- src/backend/access/external/pxffilters.c| 22 +++- .../access/external/test/pxffilters_test.c | 4 +++- 2 files changed, 24 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b2dd7de0/src/backend/access/external/pxffilters.c -- diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c index 0d13074..f06b07f 100644 --- a/src/backend/access/external/pxffilters.c +++ b/src/backend/access/external/pxffilters.c @@ -329,7 +329,27 @@ pxf_free_filter(PxfFilterDesc* filter) * * Yields the following serialized string: * - * a0c1o2a0c5o1o7a2c"third"o5o7 + * a0c23s1d1o2a1c23s1d5o1a2c25s5dthirdo5l0l0 + * + * Where: + * + * a0 - first column of table + * c23- scalar constant with type oid 23(INT4) + * s1 - size of constant in bytes + * d1 - serialized constant value + * o2 - greater than operation + * a1 - second column of table + * c23- scalar constant with type oid 23(INT4) + * s1 - size of constant in bytes + * d5 - serialized constant value + * o1 - less than operation + * a2 - third column of table + * c25- scalar constant with type oid 25(TEXT) + * s5 - size of constant in bytes + * dthird - serialized constant value + * o5 - equals operation + * l0 - AND operator + * l0 - AND operator * */ static char * http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b2dd7de0/src/backend/access/external/test/pxffilters_test.c -- diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c index 3679587..fa1db94 100644 --- a/src/backend/access/external/test/pxffilters_test.c +++ b/src/backend/access/external/test/pxffilters_test.c @@ -518,6 +518,7 @@ test__pxf_serialize_filter_list__manyFilters(void **state) ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID, "Winston", TEXTOID, TextEqualOperator); ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID, "Eric-%", TEXTOID, 1209); ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID, "\"Ugly\" string with quotes", TEXTOID, TextEqualOperator); + ExpressionItem* expressionItem6 = build_expression_item(6, TEXTOID, "", TEXTOID, TextEqualOperator); expressionItems = lappend(expressionItems, expressionItem1); @@ -525,9 +526,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state) expressionItems = lappend(expressionItems, expressionItem3); expressionItems = lappend(expressionItems, expressionItem4); expressionItems = lappend(expressionItems, expressionItem5); + expressionItems = lappend(expressionItems, expressionItem6); result = pxf_serialize_filter_list(expressionItems); - assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5"); + assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5"); pfree(result); int trivialExpressionItems = expressionItems->length;
[2/5] incubator-hawq git commit: HAWQ-1103. Updated unit-tests.
HAWQ-1103. Updated unit-tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d9f77da1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d9f77da1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d9f77da1
Branch: refs/heads/HAWQ-1103
Commit: d9f77da1db31e260092fbce603686f92540848f3
Parents: b318fa0
Author: Oleksandr Diachenko
Authored: Thu Oct 13 16:45:33 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 16:45:33 2016 -0700
--
.../access/external/test/pxffilters_test.c | 26 ++--
1 file changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d9f77da1/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 76be330..6db7ae7 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -183,19 +183,19 @@ test__const_to_str__int(void **state)
void
test__const_to_str__text(void **state)
{
- verify__const_to_str(false, "that", TEXTOID, "\\\"that\\\"");
- verify__const_to_str(false, "joke", VARCHAROID, "\\\"joke\\\"");
- verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\"");
- verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\"");
- verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\"");
- verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\"");
+ verify__const_to_str(false, "that", TEXTOID, "that");
+ verify__const_to_str(false, "joke", VARCHAROID, "joke");
+ verify__const_to_str(false, "isn't", BPCHAROID, "isn't");
+ verify__const_to_str(false, "funny", CHAROID, "funny");
+ verify__const_to_str(false, "anymore", BYTEAOID, "anymore");
+ verify__const_to_str(false, "iamdate", DATEOID, "iamdate");
}
void
test__const_to_str__boolean(void **state)
{
- verify__const_to_str(false, "t", BOOLOID, "\"true\"");
- verify__const_to_str(false, "f", BOOLOID, "\"false\"");
+ verify__const_to_str(false, "t", BOOLOID, "true");
+ verify__const_to_str(false, "f", BOOLOID, "false");
}
void
@@ -499,7 +499,7 @@ void test__pxf_serialize_filter_list__oneFilter(void
**state) {
expressionItems = lappend(expressionItems, filterExpressionItem);
char* result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5");
+ assert_string_equal(result, "a0c25s4d1984o5");
pxf_free_expression_items_list(expressionItems, true);
expressionItems = NIL;
@@ -514,9 +514,9 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
List* expressionItems = NIL;
ExpressionItem* expressionItem1 = build_expression_item(1, TEXTOID,
"1984", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"\"George Orwell\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"\"Winston\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"\"Eric-%\"", TEXTOID, 1209);
+ ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"George Orwell", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"Winston", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"Eric-%", TEXTOID, 1209);
expressionItems = lappend(expressionItems, expressionItem1);
@@ -525,7 +525,7 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
expressionItems = lappend(expressionItems, expressionItem4);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5a1c\\\"\"George
Orwell\"\\\"o5a2c\\\"\"Winston\"\\\"o5a3c\\\"\"Eric-%\"\\\"o7");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7");
pfree(result);
enrich_trivial_expression(expressionItems);
incubator-hawq git commit: HAWQ-1057. Skip SARG_PUSHDOWN parameter if any of filters isn't supported.
Repository: incubator-hawq
Updated Branches:
refs/heads/master c8ef3a0ac -> 6e921df29
HAWQ-1057. Skip SARG_PUSHDOWN parameter if any of filters isn't supported.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6e921df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6e921df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6e921df2
Branch: refs/heads/master
Commit: 6e921df29d25d846b5d4916994c807b8b619662a
Parents: c8ef3a0
Author: Oleksandr Diachenko
Authored: Tue Oct 18 15:57:02 2016 -0700
Committer: Oleksandr Diachenko
Committed: Tue Oct 18 15:57:20 2016 -0700
--
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 37 +++-
1 file changed, 28 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e921df2/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
index ab2f96e..1cea9c7 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
@@ -20,6 +20,8 @@ package org.apache.hawq.pxf.plugins.hive;
*/
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -41,6 +43,8 @@ import static
org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter.PXF_HIV
*/
public class HiveORCAccessor extends HiveAccessor {
+private static final Log LOG = LogFactory.getLog(HiveORCAccessor.class);
+
private final String READ_COLUMN_IDS_CONF_STR =
"hive.io.file.readcolumn.ids";
private final String READ_ALL_COLUMNS = "hive.io.file.read.all.columns";
private final String READ_COLUMN_NAMES_CONF_STR =
"hive.io.file.readcolumn.names";
@@ -106,18 +110,23 @@ public class HiveORCAccessor extends HiveAccessor {
* need special case logic to make sure to still wrap the filter in a
* startAnd() & end() block
*/
-if (filter instanceof LogicalFilter)
-buildExpression(filterBuilder, Arrays.asList(filter));
+if (filter instanceof LogicalFilter) {
+if (!buildExpression(filterBuilder, Arrays.asList(filter))) {
+return;
+}
+}
else {
filterBuilder.startAnd();
-buildArgument(filterBuilder, filter);
+if(!buildArgument(filterBuilder, filter)) {
+return;
+}
filterBuilder.end();
}
SearchArgument sarg = filterBuilder.build();
jobConf.set(SARG_PUSHDOWN, sarg.toKryo());
}
-private void buildExpression(SearchArgument.Builder builder, List
filterList) {
+private boolean buildExpression(SearchArgument.Builder builder,
List filterList) {
for (Object f : filterList) {
if (f instanceof LogicalFilter) {
switch(((LogicalFilter) f).getOperator()) {
@@ -131,15 +140,21 @@ public class HiveORCAccessor extends HiveAccessor {
builder.startNot();
break;
}
-buildExpression(builder, ((LogicalFilter) f).getFilterList());
-builder.end();
+if (buildExpression(builder, ((LogicalFilter)
f).getFilterList())) {
+builder.end();
+} else {
+return false;
+}
} else {
-buildArgument(builder, f);
+if (!buildArgument(builder, f)) {
+return false;
+}
}
}
+return true;
}
-private void buildArgument(SearchArgument.Builder builder, Object
filterObj) {
+private boolean buildArgument(SearchArgument.Builder builder, Object
filterObj) {
/* The below functions will not be compatible and requires update
with Hive 2.0 APIs */
BasicFilter filter = (BasicFilter) filterObj;
int filterColumnIndex = filter.getColumn().index();
@@ -166,8 +181,12 @@ public class HiveORCAccessor extends HiveAccessor {
case HDOP_NE:
builder.startNot().equals(filterColumnName, filterValue).end();
break;
+default: {
+LOG.debug("Filter push-down is not supported for " +
filter.getOperation(
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-964 [deleted] a14076c2e
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1103 [deleted] 2497b6f32
[2/3] incubator-hawq git commit: HAWQ-1103. Updated unit-tests.
HAWQ-1103. Updated unit-tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d9f77da1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d9f77da1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d9f77da1
Branch: refs/heads/HAWQ-1103
Commit: d9f77da1db31e260092fbce603686f92540848f3
Parents: b318fa0
Author: Oleksandr Diachenko
Authored: Thu Oct 13 16:45:33 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 16:45:33 2016 -0700
--
.../access/external/test/pxffilters_test.c | 26 ++--
1 file changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d9f77da1/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 76be330..6db7ae7 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -183,19 +183,19 @@ test__const_to_str__int(void **state)
void
test__const_to_str__text(void **state)
{
- verify__const_to_str(false, "that", TEXTOID, "\\\"that\\\"");
- verify__const_to_str(false, "joke", VARCHAROID, "\\\"joke\\\"");
- verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\"");
- verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\"");
- verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\"");
- verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\"");
+ verify__const_to_str(false, "that", TEXTOID, "that");
+ verify__const_to_str(false, "joke", VARCHAROID, "joke");
+ verify__const_to_str(false, "isn't", BPCHAROID, "isn't");
+ verify__const_to_str(false, "funny", CHAROID, "funny");
+ verify__const_to_str(false, "anymore", BYTEAOID, "anymore");
+ verify__const_to_str(false, "iamdate", DATEOID, "iamdate");
}
void
test__const_to_str__boolean(void **state)
{
- verify__const_to_str(false, "t", BOOLOID, "\"true\"");
- verify__const_to_str(false, "f", BOOLOID, "\"false\"");
+ verify__const_to_str(false, "t", BOOLOID, "true");
+ verify__const_to_str(false, "f", BOOLOID, "false");
}
void
@@ -499,7 +499,7 @@ void test__pxf_serialize_filter_list__oneFilter(void
**state) {
expressionItems = lappend(expressionItems, filterExpressionItem);
char* result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5");
+ assert_string_equal(result, "a0c25s4d1984o5");
pxf_free_expression_items_list(expressionItems, true);
expressionItems = NIL;
@@ -514,9 +514,9 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
List* expressionItems = NIL;
ExpressionItem* expressionItem1 = build_expression_item(1, TEXTOID,
"1984", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"\"George Orwell\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"\"Winston\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"\"Eric-%\"", TEXTOID, 1209);
+ ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"George Orwell", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"Winston", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"Eric-%", TEXTOID, 1209);
expressionItems = lappend(expressionItems, expressionItem1);
@@ -525,7 +525,7 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
expressionItems = lappend(expressionItems, expressionItem4);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5a1c\\\"\"George
Orwell\"\\\"o5a2c\\\"\"Winston\"\\\"o5a3c\\\"\"Eric-%\"\\\"o7");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7");
pfree(result);
enrich_trivial_expression(expressionItems);
[3/3] incubator-hawq git commit: HAWQ-1103. Added test cases with string containing a quote.
HAWQ-1103. Added test cases with string containing a quote.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2497b6f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2497b6f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2497b6f3
Branch: refs/heads/HAWQ-1103
Commit: 2497b6f327944e1fbdc5434dddacab0fcacfa7f9
Parents: d9f77da
Author: Oleksandr Diachenko
Authored: Fri Oct 14 11:45:02 2016 -0700
Committer: Oleksandr Diachenko
Committed: Fri Oct 14 11:45:02 2016 -0700
--
src/backend/access/external/pxffilters.c | 4 ++--
src/backend/access/external/test/pxffilters_test.c | 7 +--
src/include/access/pxffilters.h| 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2497b6f3/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index ded9f9a..ef7b16f 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -372,14 +372,14 @@ pxf_serialize_filter_list(List *expressionItems)
appendStringInfo(resbuf,
"%c%d%c%d%c%d%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
PXF_CONST_CODE, r.consttype,
-
PXF_LENGTH_TO_READ, strlen(r.conststr->data),
+
PXF_SIZE_BYTES, strlen(r.conststr->data),
PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
appendStringInfo(resbuf,
"%c%d%c%d%c%s%c%d",
PXF_CONST_CODE, l.consttype,
-
PXF_LENGTH_TO_READ, strlen(l.conststr->data),
+
PXF_SIZE_BYTES, strlen(l.conststr->data),
PXF_CONST_DATA, (l.conststr)->data,
PXF_ATTR_CODE, r.attnum - 1); /* Java attrs are 0-based */
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2497b6f3/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 6db7ae7..3679587 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -517,20 +517,23 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID,
"George Orwell", TEXTOID, TextEqualOperator);
ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID,
"Winston", TEXTOID, TextEqualOperator);
ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID,
"Eric-%", TEXTOID, 1209);
+ ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID,
"\"Ugly\" string with quotes", TEXTOID, TextEqualOperator);
expressionItems = lappend(expressionItems, expressionItem1);
expressionItems = lappend(expressionItems, expressionItem2);
expressionItems = lappend(expressionItems, expressionItem3);
expressionItems = lappend(expressionItems, expressionItem4);
+ expressionItems = lappend(expressionItems, expressionItem5);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge
Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with
quoteso5");
pfree(result);
+ int trivia
[1/3] incubator-hawq git commit: HAWQ-1103. Draft implementation without taking care of different charsets.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1103 [created] 2497b6f32
HAWQ-1103. Draft implementation without taking care of different charsets.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b318fa0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b318fa0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b318fa0e
Branch: refs/heads/HAWQ-1103
Commit: b318fa0e7d038084ded2bc99c6e631aa26700b21
Parents: 7f3658d
Author: Oleksandr Diachenko
Authored: Thu Oct 13 16:00:03 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 16:00:03 2016 -0700
--
src/backend/access/external/pxffilters.c | 25 +++--
src/include/access/pxffilters.h | 9 ++---
2 files changed, 21 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b318fa0e/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 56aea55..ded9f9a 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -369,14 +369,18 @@ pxf_serialize_filter_list(List *expressionItems)
PxfOperatorCode o = filter->op;
if (pxfoperand_is_attr(l) &&
pxfoperand_is_const(r))
{
- appendStringInfo(resbuf,
"%c%d%c%s",
+ appendStringInfo(resbuf,
"%c%d%c%d%c%d%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
-
PXF_CONST_CODE, (r.conststr)->data);
+
PXF_CONST_CODE, r.consttype,
+
PXF_LENGTH_TO_READ, strlen(r.conststr->data),
+
PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) &&
pxfoperand_is_attr(r))
{
- appendStringInfo(resbuf,
"%c%s%c%d",
-
PXF_CONST_CODE, (l.conststr)->data,
+ appendStringInfo(resbuf,
"%c%d%c%d%c%s%c%d",
+
PXF_CONST_CODE, l.consttype,
+
PXF_LENGTH_TO_READ, strlen(l.conststr->data),
+
PXF_CONST_DATA, (l.conststr)->data,
PXF_ATTR_CODE, r.attnum - 1); /* Java attrs are 0-based */
}
else
@@ -468,6 +472,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
{
filter->l.opcode = PXF_ATTR_CODE;
filter->l.attnum = ((Var *) leftop)->varattno;
+ filter->l.consttype = InvalidOid;
if (filter->l.attnum <= InvalidAttrNumber)
return false; /* system attr not supported */
@@ -476,6 +481,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->r.conststr = makeStringInfo();
initStringInfo(filter->r.conststr);
const_to_str((Const *)rightop, filter->r.conststr);
+ filter->r.consttype = ((Const *)rightop)->consttype;
}
else if (IsA(leftop, Const) && IsA(rightop, Var))
{
@@ -484,9 +490,11 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->l.conststr = makeStringInfo();
initStringInfo(filter->l.conststr);
const_to_str((Const *)leftop, filter->l.conststr);
+ filter->l.consttype = ((Const *)leftop)->consttype;
filter->r.opcode = PXF_ATTR_CODE;
filter->r.attnum = ((Var *) rightop)->varattno;
+ filter->r.consttype = InvalidOid;
incubator-hawq git commit: HAWQ-1048. Support OR, NOT logical operators in the HAWQ/PXF Bridge.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 2cc152eb0 -> 7f3658d3a
HAWQ-1048. Support OR, NOT logical operators in the HAWQ/PXF Bridge.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7f3658d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7f3658d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7f3658d3
Branch: refs/heads/master
Commit: 7f3658d3aa5933028835b753df8fb6cfc4934856
Parents: 2cc152e
Author: Oleksandr Diachenko
Authored: Thu Oct 13 14:19:12 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 14:19:29 2016 -0700
--
src/backend/access/external/pxffilters.c| 328 ---
.../access/external/test/pxffilters_test.c | 106 +++---
src/include/access/pxffilters.h | 10 +-
3 files changed, 260 insertions(+), 184 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7f3658d3/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 354c719..56aea55 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,14 +31,14 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_filter_list(List* quals);
+static List* pxf_make_expression_items_list(List *quals, Node *parent, bool
*logicalOpsNum);
static void pxf_free_filter(PxfFilterDesc* filter);
-static void pxf_free_filter_list(List *filters);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
static void const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
+static void enrich_trivial_expression(List *expressionItems);
/*
* All supported HAWQ operators, and their respective HFDS operator code.
@@ -135,7 +135,40 @@ dbop_pxfop_map pxf_supported_opr[] =
{1097 /* date_gt */, PXFOP_GT},
{1096 /* date_le */, PXFOP_LE},
{1098 /* date_ge */, PXFOP_GE},
- {1094 /* date_ne */, PXFOP_NE}
+ {1094 /* date_ne */, PXFOP_NE},
+
+ /* float8 */
+ {Float8EqualOperator /* float8eq */, PXFOP_EQ},
+ {672 /* float8lt */, PXFOP_LT},
+ {674 /* float8gt */, PXFOP_GT},
+ {673 /* float8le */, PXFOP_LE},
+ {675 /* float8ge */, PXFOP_GE},
+ {671 /* float8ne */, PXFOP_NE},
+
+ /* float48 */
+ {1120 /* float48eq */, PXFOP_EQ},
+ {1122 /* float48lt */, PXFOP_LT},
+ {1123 /* float48gt */, PXFOP_GT},
+ {1124 /* float48le */, PXFOP_LE},
+ {1125 /* float48ge */, PXFOP_GE},
+ {1121 /* float48ne */, PXFOP_NE},
+
+ /* bpchar */
+ {BPCharEqualOperator /* bpchareq */, PXFOP_EQ},
+ {1058 /* bpcharlt */, PXFOP_LT},
+ {1060 /* bpchargt */, PXFOP_GT},
+ {1059 /* bpcharle */, PXFOP_LE},
+ {1061 /* bpcharge */, PXFOP_GE},
+ {1057 /* bpcharne */, PXFOP_NE}
+
+ /* bytea */
+ // TODO: uncomment once HAWQ-1085 is done
+ //,{ByteaEqualOperator /* byteaeq */, PXFOP_EQ},
+ //{1957 /* bytealt */, PXFOP_LT},
+ //{1959 /* byteagt */, PXFOP_GT},
+ //{1958 /* byteale */, PXFOP_LE},
+ //{1960 /* byteage */, PXFOP_GE},
+ //{1956 /* byteane */, PXFOP_NE}
};
@@ -153,72 +186,102 @@ Oid pxf_supported_types[] =
CHAROID,
BYTEAOID,
BOOLOID,
- DATEOID
+ DATEOID,
+ TIMESTAMPOID
};
+static void
+pxf_free_expression_items_list(List *expressionItems, bool freeBoolExprNodes)
+{
+ ListCell*lc = NULL;
+ ExpressionItem *expressionItem = NULL;
+ int previousLength;
+
+ while (list_length(expressionItems) > 0)
+ {
+ expressionItem = (ExpressionItem *)
lfirst(list_head(expressionItems));
+ if (freeBoolExprNodes && nodeTag(expressionItem->node) ==
T_BoolExpr)
+ {
+ pfree((BoolExpr *)expressionItem->node);
+ }
+ pfree(expressionItem);
+
+ /* to avoid freeing already freed items - delete all
occurrences of current expression*/
+ previousLength = expressionItems->length + 1;
+ while (expressionItems != NULL && previousLength >
expressionItems->length)
+ {
+ previousLength = expressionItems->length;
+ expressionItems = list_delete_ptr(expressionItems,
expressionItem);
+ }
+ }
+}
+
/*
- * pxf_make_filter_list
+ * pxf_make_expression_items_list
*
* Gi
incubator-hawq git commit: HAWQ-1084. Fixed memory allocation for table printing.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 9a3f8b94d -> cb050d153
HAWQ-1084. Fixed memory allocation for table printing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cb050d15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cb050d15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cb050d15
Branch: refs/heads/master
Commit: cb050d153f3b5cf97c63170ae6906845ce2284a0
Parents: 9a3f8b9
Author: Oleksandr Diachenko
Authored: Thu Oct 13 11:02:42 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 13 11:02:42 2016 -0700
--
src/bin/psql/describe.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cb050d15/src/bin/psql/describe.c
--
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 85012b2..0ef6c6a 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -4316,10 +4316,10 @@ describePxfTable(const char *profile, const char
*pattern, bool verbose)
if (verbose)
{
sourcefieldtype = PQgetvalue(res, i, 4);
- total_fields = PQgetvalue(res, i, 5);
+ total_fields = atoi(PQgetvalue(res, i, 5));
} else
{
- total_fields = PQgetvalue(res, i, 4);
+ total_fields = atoi(PQgetvalue(res, i, 4));
}
/* First row for current table */
[2/2] incubator-hawq git commit: Merge branch 'HAWQ-1048' into HAWQ-964
Merge branch 'HAWQ-1048' into HAWQ-964 Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a14076c2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a14076c2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a14076c2 Branch: refs/heads/HAWQ-964 Commit: a14076c2eb925c92cba8d3be1e26e6b6a8ce2ea9 Parents: 73273c2 071c300 Author: Oleksandr Diachenko Authored: Fri Oct 7 17:43:34 2016 -0700 Committer: Oleksandr Diachenko Committed: Fri Oct 7 17:43:34 2016 -0700 -- src/backend/access/external/hd_work_mgr.c | 2 +- src/backend/access/external/pxffilters.c | 64 -- 2 files changed, 52 insertions(+), 14 deletions(-) --
[1/2] incubator-hawq git commit: HAWQ-1048. Fixed filter builder for ORCA/default optimizer.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-964 73273c2cc -> a14076c2e
HAWQ-1048. Fixed filter builder for ORCA/default optimizer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/071c3008
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/071c3008
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/071c3008
Branch: refs/heads/HAWQ-964
Commit: 071c300808f4f35c254f9b66e8e4ddde8e8483d4
Parents: 2ebc98a
Author: Oleksandr Diachenko
Authored: Fri Oct 7 17:41:17 2016 -0700
Committer: Oleksandr Diachenko
Committed: Fri Oct 7 17:42:13 2016 -0700
--
src/backend/access/external/hd_work_mgr.c | 2 +-
src/backend/access/external/pxffilters.c | 64 --
2 files changed, 52 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/071c3008/src/backend/access/external/hd_work_mgr.c
--
diff --git a/src/backend/access/external/hd_work_mgr.c
b/src/backend/access/external/hd_work_mgr.c
index e7314f7..6829de5 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -158,7 +158,7 @@ char** map_hddata_2gp_segments(char* uri, int total_segs,
int working_segs, Rela
inputData.headers = client_context.http_headers;
inputData.gphduri = hadoop_uri;
inputData.rel = relation;
- inputData.filterstr = NULL; /* We do not supply filter data to the HTTP
header */
+ inputData.filterstr = serializePxfFilterQuals(quals); /* We do supply
filter data to the HTTP header */
generate_delegation_token(&inputData);
build_http_header(&inputData);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/071c3008/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 154babf..fda341d 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,13 +31,14 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_expression_items_list(List *quals, Node *parent);
+static List* pxf_make_expression_items_list(List *quals, Node *parent, bool
*logicalOpsNum);
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
static void const_to_str(Const *constval, StringInfo buf);
static List* append_attr_from_var(Var* var, List* attrs);
+static void enrichTrivialExpression(List *expressionItems);
/*
* All supported HAWQ operators, and their respective HFDS operator code.
@@ -150,15 +151,16 @@ dbop_pxfop_map pxf_supported_opr[] =
{1060 /* bpchargt */, PXFOP_GT},
{1059 /* bpcharle */, PXFOP_LE},
{1061 /* bpcharge */, PXFOP_GE},
- {1057 /* bpcharne */, PXFOP_NE},
+ {1057 /* bpcharne */, PXFOP_NE}
/* bytea */
- {ByteaEqualOperator /* byteaeq */, PXFOP_EQ},
- {1957 /* bytealt */, PXFOP_LT},
- {1959 /* byteagt */, PXFOP_GT},
- {1958 /* byteale */, PXFOP_LE},
- {1960 /* byteage */, PXFOP_GE},
- {1956 /* byteane */, PXFOP_NE}
+ // TODO: uncomment ocne HAWQ-1085 is done
+ //,{ByteaEqualOperator /* byteaeq */, PXFOP_EQ},
+ //{1957 /* bytealt */, PXFOP_LT},
+ //{1959 /* byteagt */, PXFOP_GT},
+ //{1958 /* byteale */, PXFOP_LE},
+ //{1960 /* byteage */, PXFOP_GE},
+ //{1956 /* byteane */, PXFOP_NE}
};
@@ -181,7 +183,7 @@ Oid pxf_supported_types[] =
};
static void
-pxf_free_filter_list(List *expressionItems)
+pxf_free_filter_list(List *expressionItems, bool isTrivialExpression)
{
ListCell*lc = NULL;
ExpressionItem *expressionItem = NULL;
@@ -190,6 +192,10 @@ pxf_free_filter_list(List *expressionItems)
while (list_length(expressionItems) > 0)
{
expressionItem = (ExpressionItem *)
lfirst(list_head(expressionItems));
+ if (isTrivialExpression)
+ {
+ pfree((BoolExpr *)expressionItem->node);
+ }
pfree(expressionItem);
/* to avoid freeing already freed items - delete all
occurrences of current expression*/
@@ -214,7 +220,7 @@ pxf_free_filter_list(List *expressionItems)
*
*/
static List *
-pxf_make_expression_items_list(List *quals, Node *parent)
+pxf_make_expression_items_list(List *quals, Node *parent, bool *logicalOpsNum)
{
ExpressionItem *expressionItem =
[3/3] incubator-hawq git commit: Merge branch 'HAWQ-1048' into HAWQ-964
Merge branch 'HAWQ-1048' into HAWQ-964 Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/73273c2c Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/73273c2c Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/73273c2c Branch: refs/heads/HAWQ-964 Commit: 73273c2cc286f556b3e6dd8f88e93469c343887c Parents: 509aef1 2ebc98a Author: Oleksandr Diachenko Authored: Thu Oct 6 14:57:18 2016 -0700 Committer: Oleksandr Diachenko Committed: Thu Oct 6 14:57:18 2016 -0700 -- src/backend/access/external/hd_work_mgr.c | 2 +- src/backend/access/external/pxffilters.c | 6 -- 2 files changed, 1 insertion(+), 7 deletions(-) --
[1/3] incubator-hawq git commit: HAWQ-1048. Do not send filter string on fragmenter call.
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-964 509aef120 -> 73273c2cc HAWQ-1048. Do not send filter string on fragmenter call. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bbfde70d Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bbfde70d Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bbfde70d Branch: refs/heads/HAWQ-964 Commit: bbfde70d90ceece472a4c5df17761b0369b4936d Parents: dffc760 Author: Oleksandr Diachenko Authored: Thu Oct 6 14:53:18 2016 -0700 Committer: Oleksandr Diachenko Committed: Thu Oct 6 14:53:18 2016 -0700 -- src/backend/access/external/hd_work_mgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bbfde70d/src/backend/access/external/hd_work_mgr.c -- diff --git a/src/backend/access/external/hd_work_mgr.c b/src/backend/access/external/hd_work_mgr.c index 6829de5..e7314f7 100644 --- a/src/backend/access/external/hd_work_mgr.c +++ b/src/backend/access/external/hd_work_mgr.c @@ -158,7 +158,7 @@ char** map_hddata_2gp_segments(char* uri, int total_segs, int working_segs, Rela inputData.headers = client_context.http_headers; inputData.gphduri = hadoop_uri; inputData.rel = relation; - inputData.filterstr = serializePxfFilterQuals(quals); /* We do supply filter data to the HTTP header */ + inputData.filterstr = NULL; /* We do not supply filter data to the HTTP header */ generate_delegation_token(&inputData); build_http_header(&inputData);
[2/3] incubator-hawq git commit: HAWQ-1048. Do not send filter string on fragmenter call.
HAWQ-1048. Do not send filter string on fragmenter call.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2ebc98a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2ebc98a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2ebc98a1
Branch: refs/heads/HAWQ-964
Commit: 2ebc98a191f4086a10f79b1dc00b4bb6934a7e5f
Parents: bbfde70
Author: Oleksandr Diachenko
Authored: Thu Oct 6 14:55:13 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 6 14:55:13 2016 -0700
--
src/backend/access/external/pxffilters.c | 6 --
1 file changed, 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2ebc98a1/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 2ada44e..154babf 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -322,8 +322,6 @@ static char *
pxf_serialize_filter_list(List *expressionItems)
{
- printf("Serializing filter list 1\n");
-
StringInfo resbuf;
ListCell*lc = NULL;
@@ -333,8 +331,6 @@ pxf_serialize_filter_list(List *expressionItems)
resbuf = makeStringInfo();
initStringInfo(resbuf);
- printf("Serializing filter list 2\n");
-
/*
* Iterate through the expression items in the list and serialize them
one after the other.
*/
@@ -398,8 +394,6 @@ pxf_serialize_filter_list(List *expressionItems)
}
}
- printf("Serializing filter list 30\n");
-
if (resbuf->len == 0)
{
pfree(resbuf->data);
incubator-hawq git commit: HAWQ-1083. Do not use CURLOPT_RESOLVE when call curl.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 0abbac2d8 -> 90bd061ba
HAWQ-1083. Do not use CURLOPT_RESOLVE when call curl.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/90bd061b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/90bd061b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/90bd061b
Branch: refs/heads/master
Commit: 90bd061ba899a80d856db7b305727c920a894af0
Parents: 0abbac2
Author: Oleksandr Diachenko
Authored: Thu Oct 6 12:00:36 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Oct 6 12:00:36 2016 -0700
--
src/backend/access/external/libchurl.c | 35 +++
src/backend/access/external/pxfutils.c | 92 +
src/include/access/libchurl.h | 1 -
src/include/access/pxfutils.h | 8 ++-
4 files changed, 123 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90bd061b/src/backend/access/external/libchurl.c
--
diff --git a/src/backend/access/external/libchurl.c
b/src/backend/access/external/libchurl.c
index 2d4aab4..964b3b0 100644
--- a/src/backend/access/external/libchurl.c
+++ b/src/backend/access/external/libchurl.c
@@ -21,6 +21,7 @@
#include "lib/stringinfo.h"
#include "utils/guc.h"
#include "miscadmin.h"
+#include "access/pxfutils.h"
/* include libcurl without typecheck.
* This allows wrapping curl_easy_setopt to be wrapped
@@ -310,17 +311,6 @@ CHURL_HANDLE churl_init(const char* url, CHURL_HEADERS
headers)
create_curl_handle(context);
clear_error_buffer(context);
- /* needed to resolve localhost */
- if (strstr(url, LocalhostIpV4) != NULL) {
- struct curl_slist *resolve_hosts = NULL;
- char *pxf_host_entry = (char *)
palloc0(strlen(pxf_service_address) + strlen(LocalhostIpV4Entry) + 1);
- strcat(pxf_host_entry, pxf_service_address);
- strcat(pxf_host_entry, LocalhostIpV4Entry);
- resolve_hosts = curl_slist_append(NULL, pxf_host_entry);
- set_curl_option(context, CURLOPT_RESOLVE, resolve_hosts);
- pfree(pxf_host_entry);
- }
-
set_curl_option(context, CURLOPT_URL, url);
set_curl_option(context, CURLOPT_VERBOSE, (const void*)FALSE);
set_curl_option(context, CURLOPT_ERRORBUFFER,
context->curl_error_buffer);
@@ -491,6 +481,29 @@ void create_curl_handle(churl_context* context)
void set_curl_option(churl_context* context, CURLoption option, const void*
data)
{
int curl_error;
+
+ if (option == CURLOPT_URL)
+ {
+ const char* url = (char* )data;
+ /* needed to resolve localhost */
+ if (strstr(url, LocalhostIpV4) != NULL) {
+ //get loopback interface ip address
+ char* loopback_addr = get_loopback_ip_addr();
+ elog(DEBUG1, "Loopback interface IP address: %s",
loopback_addr);
+ char* replaced_url = replace_string(url, LocalhostIpV4,
loopback_addr);
+ elog(DEBUG1, "Replaced url: %s", replaced_url);
+ if (CURLE_OK != (curl_error =
curl_easy_setopt(context->curl_handle, option, replaced_url)))
+ elog(ERROR, "internal error: curl_easy_setopt
%d error (%d - %s)",
+option, curl_error,
curl_easy_strerror(curl_error));
+
+ //release memory
+ pfree(replaced_url);
+ pfree(loopback_addr);
+ return;
+ }
+ }
+
+
if (CURLE_OK != (curl_error = curl_easy_setopt(context->curl_handle,
option, data)))
elog(ERROR, "internal error: curl_easy_setopt %d error (%d -
%s)",
option, curl_error, curl_easy_strerror(curl_error));
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90bd061b/src/backend/access/external/pxfutils.c
--
diff --git a/src/backend/access/external/pxfutils.c
b/src/backend/access/external/pxfutils.c
index 1b53a4a..fec3223 100644
--- a/src/backend/access/external/pxfutils.c
+++ b/src/backend/access/external/pxfutils.c
@@ -24,6 +24,10 @@
#include "commands/dbcommands.h"
#include "miscadmin.h"
#include "utils/builtins.h"
+#include
+#include
+#include
+#include
/* Wrapper for libchurl */
static void process_request(ClientContext* client_context, char *uri);
@@ -133,3 +137,91 @@ static void process_request(ClientContext* client_context,
char *uri)
}
+
+/*
+ * Finds ip address of any available
[08/18] incubator-hawq git commit: HAWQ-1048. Return NULL instead of empty string.
HAWQ-1048. Return NULL instead of empty string.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0ff70373
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0ff70373
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0ff70373
Branch: refs/heads/HAWQ-964
Commit: 0ff70373b3b700028892c72fd9e1fb516bbcaa25
Parents: 7a9cc88
Author: Oleksandr Diachenko
Authored: Wed Sep 21 18:19:18 2016 -0700
Committer: Oleksandr Diachenko
Committed: Wed Sep 21 18:19:18 2016 -0700
--
src/backend/access/external/pxffilters.c | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0ff70373/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 0d1e1fa..8c3864d 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -317,6 +317,12 @@ pxf_serialize_filter_list(List *expressionItems)
}
}
+ if (resbuf->len == 0)
+ {
+ pfree(resbuf->data);
+ return NULL;
+ }
+
return resbuf->data;
}
[17/18] incubator-hawq git commit: HAWQ-964. Remove old HBaseLogic, typed Partition Comparison
HAWQ-964. Remove old HBaseLogic, typed Partition Comparison
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/55db7ecd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/55db7ecd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/55db7ecd
Branch: refs/heads/HAWQ-964
Commit: 55db7ecd0c3a7ffc365a1c850b5500053d92e4f4
Parents: 8f73c2b
Author: Kavinder Dhaliwal
Authored: Tue Oct 4 15:17:49 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Tue Oct 4 15:17:49 2016 -0700
--
.../pxf/plugins/hbase/HBaseFilterBuilder.java | 13 +---
.../hawq/pxf/plugins/hive/HiveAccessor.java | 68 +++-
2 files changed, 68 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55db7ecd/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
--
diff --git
a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
index 29c8686..5ec0652 100644
---
a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
+++
b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
@@ -240,18 +240,7 @@ public class HBaseFilterBuilder implements
FilterParser.FilterBuilder {
* Currently, 1, 2 can occur, since no parenthesis are used.
*/
private Filter handleCompoundOperations(FilterParser.LogicalOperation
opId, Filter left, Filter right) {
-FilterList result;
-
-if (left instanceof FilterList) {
-result = (FilterList) left;
-result.addFilter(right);
-
-return result;
-}
-
-result = new FilterList(logicalOperatorsMap.get(opId), new Filter[]
{left, right});
-
-return result;
+return new FilterList(logicalOperatorsMap.get(opId), new Filter[]
{left, right});
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55db7ecd/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
index b2b3e4b..8cff706 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
@@ -19,9 +19,12 @@ package org.apache.hawq.pxf.plugins.hive;
* under the License.
*/
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hawq.pxf.api.BasicFilter;
import org.apache.hawq.pxf.api.FilterParser;
import org.apache.hawq.pxf.api.LogicalFilter;
+import org.apache.hawq.pxf.api.UnsupportedTypeException;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.plugins.hdfs.HdfsSplittableDataAccessor;
@@ -33,10 +36,16 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import java.io.IOException;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
+import static org.apache.hawq.pxf.api.io.DataType.*;
+import static org.apache.hawq.pxf.api.io.DataType.BPCHAR;
+import static org.apache.hawq.pxf.api.io.DataType.BYTEA;
+
/**
* Accessor for Hive tables. The accessor will open and read a split belonging
* to a Hive table. Opening a split means creating the corresponding
InputFormat
@@ -247,7 +256,16 @@ public class HiveAccessor extends
HdfsSplittableDataAccessor {
* the filter field matches a partition field, but the values
do
* not match
*/
-return filterValue.equals(partition.val);
+boolean keepPartition =
filterValue.equals(partition.val);
+
+/*
+ * If the string comparison fails then we should check
the comparison of
+ * the two operands as typed values
+ */
+if (!keepPartition &&
!partition.val.equals("__HIVE_DEFAULT_PARTITION__")){
+keepPartition = testFilterByType(filterValue,
partition);
+}
+return keepPartition;
}
}
@@ -261,6 +279,54 @@ public class HiveAccessor extends
HdfsSplitta
[05/18] incubator-hawq git commit: HAWQ-964. Support for OR and NOT Logical Operators
HAWQ-964. Support for OR and NOT Logical Operators
Signed-off-by: Leslie Chang
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/726be6cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/726be6cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/726be6cc
Branch: refs/heads/HAWQ-964
Commit: 726be6cced0edd67fd0a0308d0483ee54b8ac116
Parents: e6c7fda
Author: Kavinder Dhaliwal
Authored: Thu Sep 15 10:56:20 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Tue Sep 20 09:45:44 2016 -0700
--
.../org/apache/hawq/pxf/api/FilterParser.java | 71 ++--
.../apache/hawq/pxf/api/FilterParserTest.java | 115 +++
.../pxf/plugins/hbase/HBaseFilterBuilder.java | 52 +
.../hawq/pxf/plugins/hive/HiveAccessor.java | 5 +-
.../pxf/plugins/hive/HiveDataFragmenter.java| 3 +-
.../pxf/plugins/hive/HiveFilterBuilder.java | 80 ++---
.../hawq/pxf/plugins/hive/HiveORCAccessor.java | 39 +--
.../pxf/plugins/hive/HiveFilterBuilderTest.java | 46 +---
.../hive/HiveORCSearchArgumentExample.java | 84 ++
9 files changed, 370 insertions(+), 125 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/726be6cc/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index 00fbf2b..22c76a6 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -52,6 +52,7 @@ public class FilterParser {
private FilterBuilder filterBuilder;
private static Map operatorTranslationMap =
initOperatorTransMap();
+private static Map
logicalOperationTranslationMap = initLogicalOperatorTransMap();
/** Supported operations by the parser. */
public enum Operation {
@@ -65,6 +66,12 @@ public class FilterParser {
HDOP_LIKE
}
+public enum LogicalOperation {
+HDOP_AND,
+HDOP_OR,
+HDOP_NOT
+}
+
/**
* Interface a user of FilterParser should implement.
* This is used to let the user build filter expressions in the manner she
sees fit.
@@ -81,6 +88,8 @@ public class FilterParser {
* @throws Exception if building the filter failed
*/
public Object build(Operation operation, Object left, Object right)
throws Exception;
+public Object build(LogicalOperation operation, Object left, Object
right) throws Exception;
+public Object build(LogicalOperation operation, Object filter) throws
Exception;
}
/** Represents a column index. */
@@ -110,42 +119,6 @@ public class FilterParser {
}
/**
- * Basic filter provided for cases where the target storage system does
not provide it own filter
- * For example: Hbase storage provides its own filter but for a Writable
based record in a
- * SequenceFile there is no filter provided and so we need to have a
default
- */
-static public class BasicFilter {
-private Operation oper;
-private ColumnIndex column;
-private Constant constant;
-
-/**
- * Constructs a BasicFilter.
- *
- * @param oper the parse operation to perform
- * @param column the column index
- * @param constant the constant object
- */
-public BasicFilter(Operation oper, ColumnIndex column, Constant
constant) {
-this.oper = oper;
-this.column = column;
-this.constant = constant;
-}
-
-public Operation getOperation() {
-return oper;
-}
-
-public ColumnIndex getColumn() {
-return column;
-}
-
-public Constant getConstant() {
-return constant;
-}
-}
-
-/**
* Thrown when a filter's parsing exception occurs.
*/
@SuppressWarnings("serial")
@@ -220,6 +193,24 @@ public class FilterParser {
// Store result on stack
operandsStack.push(result);
break;
+case 'l':
+LogicalOperation logicalOperation =
logicalOperationTranslationMap.get(safeToInt(parseNumber()));
+
+if (logicalOperation == null) {
+throw new FilterStringSyntaxException("unknown op
ending at " + index);
+}
+
+if (logicalOperation == LogicalOperation.HDOP_NOT) {
+Object exp = operandsStack.pop();
+
[15/18] incubator-hawq git commit: HAWQ-1048. Updated log level to DEBUG1.
HAWQ-1048. Updated log level to DEBUG1.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/38cb4b00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/38cb4b00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/38cb4b00
Branch: refs/heads/HAWQ-964
Commit: 38cb4b0040122992837c9655d52241b6b9757529
Parents: d563ab5
Author: Oleksandr Diachenko
Authored: Mon Sep 26 12:23:32 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Sep 26 12:23:32 2016 -0700
--
src/backend/access/external/pxffilters.c | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38cb4b00/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index af6d2ea..4c28bc1 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -266,7 +266,7 @@ pxf_make_expression_items_list(List *quals, Node *parent)
break;
}
default:
- elog(DEBUG5, "pxf_make_expression_items_list:
unsupported node tag %d", tag);
+ elog(DEBUG1, "pxf_make_expression_items_list:
unsupported node tag %d", tag);
break;
}
}
@@ -343,7 +343,7 @@ pxf_serialize_filter_list(List *expressionItems)
{
case T_OpExpr:
{
- elog(DEBUG5, "pxf_serialize_filter_list: node
tag %d (T_OpExpr)", tag);
+ elog(DEBUG1, "pxf_serialize_filter_list: node
tag %d (T_OpExpr)", tag);
PxfFilterDesc *filter = (PxfFilterDesc *)
palloc0(sizeof(PxfFilterDesc));
OpExpr *expr = (OpExpr *) node;
if (opexpr_to_pxffilter(expr, filter))
@@ -386,7 +386,7 @@ pxf_serialize_filter_list(List *expressionItems)
{
BoolExpr *expr = (BoolExpr *) node;
BoolExprType boolType = expr->boolop;
- elog(DEBUG5, "pxf_serialize_filter_list: node
tag %d (T_BoolExpr), bool node type %d", tag, boolType);
+ elog(DEBUG1, "pxf_serialize_filter_list: node
tag %d (T_BoolExpr), bool node type %d", tag, boolType);
appendStringInfo(resbuf, "%c%d",
PXF_LOGICAL_OPERATOR_CODE, boolType);
break;
}
@@ -430,12 +430,12 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
/* only binary oprs supported currently */
if (!rightop)
{
- elog(DEBUG5, "opexpr_to_pxffilter: unary op! leftop_type: %d,
op: %d",
+ elog(DEBUG1, "opexpr_to_pxffilter: unary op! leftop_type: %d,
op: %d",
leftop_type, expr->opno);
return false;
}
- elog(DEBUG5, "opexpr_to_gphdfilter: leftop (expr type: %d, arg type:
%d), "
+ elog(DEBUG1, "opexpr_to_gphdfilter: leftop (expr type: %d, arg type:
%d), "
"rightop_type (expr type: %d, arg type %d), op: %d",
leftop_type, nodeTag(leftop),
rightop_type, nodeTag(rightop),
@@ -674,7 +674,7 @@ char *serializePxfFilterQuals(List *quals)
}
- elog(DEBUG2, "serializePxfFilterQuals: filter result: %s", (result ==
NULL) ? "null" : result);
+ elog(DEBUG1, "serializePxfFilterQuals: filter result: %s", (result ==
NULL) ? "null" : result);
return result;
}
[14/18] incubator-hawq git commit: HAWQ-1048. Changed logic to support more than one logical operator in list.
HAWQ-1048. Changed logic to support more than one logical operator in list.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d563ab5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d563ab5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d563ab5b
Branch: refs/heads/HAWQ-964
Commit: d563ab5b184b07b86406659ebe0abec7e4915d81
Parents: 9225016
Author: Oleksandr Diachenko
Authored: Fri Sep 23 18:35:41 2016 -0700
Committer: Oleksandr Diachenko
Committed: Fri Sep 23 18:35:41 2016 -0700
--
src/backend/access/external/pxffilters.c | 64 ---
src/include/access/pxffilters.h | 8
2 files changed, 65 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d563ab5b/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index e945062..af6d2ea 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,7 +31,7 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_expression_items_list(List *quals);
+static List* pxf_make_expression_items_list(List *quals, Node *parent);
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
@@ -180,6 +180,28 @@ Oid pxf_supported_types[] =
TIMESTAMPOID
};
+static void
+pxf_free_filter_list(List *expressionItems)
+{
+ ListCell*lc = NULL;
+ ExpressionItem *expressionItem = NULL;
+ int previousLength;
+
+ while (list_length(expressionItems) > 0)
+ {
+ expressionItem = (ExpressionItem *)
lfirst(list_head(expressionItems));
+ pfree(expressionItem);
+
+ /* to avoid freeing already freed items - delete all
occurrences of current expression*/
+ previousLength = expressionItems->length + 1;
+ while (expressionItems != NULL && previousLength >
expressionItems->length)
+ {
+ previousLength = expressionItems->length;
+ expressionItems = list_delete_ptr(expressionItems,
expressionItem);
+ }
+ }
+}
+
/*
* pxf_make_expression_items_list
*
@@ -189,12 +211,15 @@ Oid pxf_supported_types[] =
*
* Basically this function just transforms expression tree to Reversed Polish
Notation list.
*
+ *
*/
static List *
-pxf_make_expression_items_list(List *quals)
+pxf_make_expression_items_list(List *quals, Node *parent)
{
+ ExpressionItem *expressionItem = NULL;
List*result = NIL;
ListCell*lc = NULL;
+ ListCell*ilc = NULL;
if (list_length(quals) == 0)
return NIL;
@@ -203,20 +228,41 @@ pxf_make_expression_items_list(List *quals)
{
Node *node = (Node *) lfirst(lc);
NodeTag tag = nodeTag(node);
+ expressionItem = (ExpressionItem *)
palloc0(sizeof(ExpressionItem));
+ expressionItem->node = node;
+ expressionItem->parent = parent;
+ expressionItem->processed = false;
switch (tag)
{
case T_OpExpr:
{
- result = lappend(result, node);
+ result = lappend(result, expressionItem);
break;
}
case T_BoolExpr:
{
BoolExpr*expr = (BoolExpr *) node;
- List *inner_result =
pxf_make_expression_items_list(expr->args);
+ List *inner_result =
pxf_make_expression_items_list(expr->args, node);
result = list_concat(result, inner_result);
- result = lappend(result, node);
+
+ int childNodesNum = 0;
+
+ /* Find number of child nodes on first level*/
+ foreach (ilc, inner_result)
+ {
+ ExpressionItem *ei = (ExpressionItem *)
lfirst(ilc);
+ if (!ei->processed && ei->parent ==
node)
+ {
+ ei->processed = true;
+
[12/18] incubator-hawq git commit: HAWQ-1048. Added filter-push down support for bpchar, bytea.
HAWQ-1048. Added filter-push down support for bpchar, bytea.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d1873951
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d1873951
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d1873951
Branch: refs/heads/HAWQ-964
Commit: d1873951dbca3928fef948044a948d5477ae7fe9
Parents: 98bc759
Author: Oleksandr Diachenko
Authored: Thu Sep 22 16:45:57 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Sep 22 16:45:57 2016 -0700
--
src/backend/access/external/pxffilters.c | 32 +++
1 file changed, 28 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1873951/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 20956bf..17e184a 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -142,7 +142,24 @@ dbop_pxfop_map pxf_supported_opr[] =
{674 /* float8gt */, PXFOP_GT},
{673 /* float8le */, PXFOP_LE},
{675 /* float8ge */, PXFOP_GE},
- {671 /* float8ne */, PXFOP_NE}
+ {671 /* float8ne */, PXFOP_NE},
+
+ /* bpchar */
+ {BPCharEqualOperator /* bpchareq */, PXFOP_EQ},
+ {1058 /* bpcharlt */, PXFOP_LT},
+ {1060 /* bpchargt */, PXFOP_GT},
+ {1059 /* bpcharle */, PXFOP_LE},
+ {1061 /* bpcharge */, PXFOP_GE},
+ {1057 /* bpcharne */, PXFOP_NE},
+
+ /* bytea */
+ {ByteaEqualOperator /* byteaeq */, PXFOP_EQ},
+ {1957 /* bytealt */, PXFOP_LT},
+ {1959 /* byteagt */, PXFOP_GT},
+ {1958 /* byteale */, PXFOP_LE},
+ {1960 /* byteage */, PXFOP_GE},
+ {1956 /* byteane */, PXFOP_NE}
+
};
Oid pxf_supported_types[] =
@@ -159,7 +176,8 @@ Oid pxf_supported_types[] =
CHAROID,
BYTEAOID,
BOOLOID,
- DATEOID
+ DATEOID,
+ TIMESTAMPOID
};
/*
@@ -310,7 +328,7 @@ pxf_serialize_filter_list(List *expressionItems)
pxf_free_filter(filter);
} else {
/* if at least one expression item is
not supported, whole filter doesn't make sense*/
- elog(INFO, "Query will not be optimized
to use filter push-down.");
+ elog(DEBUG1, "Query will not be
optimized to use filter push-down.");
pfree(filter);
pfree(resbuf->data);
return NULL;
@@ -411,6 +429,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
}
else
{
+ elog(DEBUG1, "opexpr_to_pxffilter: expression is not a
Var+Const");
return false;
}
@@ -426,9 +445,10 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
}
}
+ elog(DEBUG1, "opexpr_to_pxffilter: operator is not supported, operator
code: %d", expr->opno);
+
/* NOTE: if more validation needed, add it before the operators test
* or alternatively change it to use a false flag and return true below
*/
-
return false;
}
@@ -510,6 +530,9 @@ supported_filter_type(Oid type)
if (type == pxf_supported_types[i])
return true;
}
+
+ elog(DEBUG1, "supported_filter_type: filter pushdown is not supported
for datatype oid: %d", type);
+
return false;
}
@@ -555,6 +578,7 @@ const_to_str(Const *constval, StringInfo buf)
case CHAROID:
case BYTEAOID:
case DATEOID:
+ case TIMESTAMPOID:
appendStringInfo(buf, "\\\"%s\\\"", extval);
break;
[13/18] incubator-hawq git commit: Merge branch 'master' into HAWQ-1048
Merge branch 'master' into HAWQ-1048 Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9225016b Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9225016b Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9225016b Branch: refs/heads/HAWQ-964 Commit: 9225016bdff957afb6f4d3d21c65d2daaa454b05 Parents: d187395 dc12e94 Author: Oleksandr Diachenko Authored: Thu Sep 22 16:46:46 2016 -0700 Committer: Oleksandr Diachenko Committed: Thu Sep 22 16:46:46 2016 -0700 -- .travis.yml |4 +- LICENSE | 39 +- licenses/LICENSE-json.txt | 131 -- licenses/LICENSE-pexpect-4.2.txt| 20 + licenses/LICENSE-plperl.txt | 135 -- licenses/LICENSE-ppport.txt | 135 ++ licenses/LICENSE-ptyprocess-0.5.1.txt | 16 + src/backend/access/external/pxffilters.c| 13 +- src/backend/access/external/pxfheaders.c|9 +- src/include/catalog/JSON.pm | 2223 -- src/include/catalog/JSON/PP.pm | 2191 - src/include/catalog/JSON/PP/Boolean.pm | 26 - src/include/catalog/JSON/PP5005.pm | 146 -- src/include/catalog/JSON/PP56.pm| 198 -- src/include/catalog/JSON/PP58.pm| 93 - .../ManagementTool/test_hawq_register.cpp | 104 - .../feature/ManagementTool/test_hawq_register.h | 85 +- .../test_hawq_register_usage2_case1.cpp | 207 ++ .../test_hawq_register_usage2_case2.cpp | 77 + .../test_hawq_register_usage2_case3.cpp | 77 + .../ManagementTool/usage2case1/bucket0_tpl.yml | 31 + .../usage2case1/error_encoding_tpl.yml | 21 + .../usage2case1/includedirectory.yml| 21 + .../usage2case1/larger_eof_tpl.yml | 21 + .../wrong_distributed_policy_tpl.yml| 21 + .../ManagementTool/usage2case2/t_tpl_1.yml | 31 + .../ManagementTool/usage2case2/t_tpl_2.yml | 33 + .../ManagementTool/usage2case2/t_tpl_3.yml | 21 + .../ManagementTool/usage2case2/t_tpl_4.yml | 23 + .../ManagementTool/usage2case2/t_tpl_new_1.yml | 43 + .../ManagementTool/usage2case2/t_tpl_new_2.yml | 45 + .../ManagementTool/usage2case2/t_tpl_new_3.yml | 23 + .../ManagementTool/usage2case2/t_tpl_new_4.yml | 25 + .../ManagementTool/usage2case3/t_tpl_old_1.yml | 31 + .../ManagementTool/usage2case3/t_tpl_old_2.yml | 33 + .../ManagementTool/usage2case3/t_tpl_old_3.yml | 21 + .../ManagementTool/usage2case3/t_tpl_old_4.yml | 23 + tools/bin/Makefile | 12 - tools/bin/hawqregister | 1003 +--- tools/bin/lib/.gitignore|7 - 40 files changed, 1765 insertions(+), 5653 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9225016b/src/backend/access/external/pxffilters.c --
[18/18] incubator-hawq git commit: Merge branch 'HAWQ-1048' into HAWQ-964
Merge branch 'HAWQ-1048' into HAWQ-964 Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/509aef12 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/509aef12 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/509aef12 Branch: refs/heads/HAWQ-964 Commit: 509aef120a33261555234cdfb57f8ca32d9daef3 Parents: 55db7ec dffc760 Author: Oleksandr Diachenko Authored: Wed Oct 5 11:28:49 2016 -0700 Committer: Oleksandr Diachenko Committed: Wed Oct 5 11:28:49 2016 -0700 -- .travis.yml |4 +- LICENSE | 39 +- licenses/LICENSE-json.txt | 131 -- licenses/LICENSE-pexpect-4.2.txt| 20 + licenses/LICENSE-plperl.txt | 135 -- licenses/LICENSE-ppport.txt | 135 ++ licenses/LICENSE-ptyprocess-0.5.1.txt | 16 + src/backend/access/external/pxffilters.c| 295 ++- src/backend/access/external/pxfheaders.c|9 +- .../access/external/test/pxffilters_test.c | 20 +- .../access/external/test/pxfheaders_test.c |9 + .../utils/mmgr/test/memaccounting_test.c|2 + src/include/access/pxffilters.h | 10 +- src/include/catalog/JSON.pm | 2223 -- src/include/catalog/JSON/PP.pm | 2191 - src/include/catalog/JSON/PP/Boolean.pm | 26 - src/include/catalog/JSON/PP5005.pm | 146 -- src/include/catalog/JSON/PP56.pm| 198 -- src/include/catalog/JSON/PP58.pm| 93 - .../feature/ManagementTool/test_hawq_register.h | 31 + .../test_hawq_register_usage2_case1.cpp | 155 ++ .../test_hawq_register_usage2_case2.cpp | 85 +- .../test_hawq_register_usage2_case3.cpp | 85 +- .../ManagementTool/usage2case1/bucket0_tpl.yml | 31 + .../usage2case1/error_encoding_tpl.yml | 21 + .../usage2case1/includedirectory.yml| 21 + .../usage2case1/larger_eof_tpl.yml | 21 + .../wrong_distributed_policy_tpl.yml| 21 + tools/bin/Makefile | 12 - tools/bin/hawqregister | 123 +- tools/bin/lib/.gitignore|7 - 31 files changed, 918 insertions(+), 5397 deletions(-) --
[11/18] incubator-hawq git commit: HAWQ-1048. Added filter-push down support for float8.
HAWQ-1048. Added filter-push down support for float8.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/98bc759f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/98bc759f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/98bc759f
Branch: refs/heads/HAWQ-964
Commit: 98bc759fe34363121049fd2f08ed6d72679082b9
Parents: 694ad03
Author: Oleksandr Diachenko
Authored: Thu Sep 22 13:34:20 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Sep 22 13:34:20 2016 -0700
--
src/backend/access/external/pxffilters.c | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/98bc759f/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 847f120..20956bf 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -134,8 +134,15 @@ dbop_pxfop_map pxf_supported_opr[] =
{1097 /* date_gt */, PXFOP_GT},
{1096 /* date_le */, PXFOP_LE},
{1098 /* date_ge */, PXFOP_GE},
- {1094 /* date_ne */, PXFOP_NE}
-
+ {1094 /* date_ne */, PXFOP_NE},
+
+ /* float8 */
+ {Float8EqualOperator /* float8eq */, PXFOP_EQ},
+ {672 /* float8lt */, PXFOP_LT},
+ {674 /* float8gt */, PXFOP_GT},
+ {673 /* float8le */, PXFOP_LE},
+ {675 /* float8ge */, PXFOP_GE},
+ {671 /* float8ne */, PXFOP_NE}
};
Oid pxf_supported_types[] =
[02/18] incubator-hawq git commit: HAWQ-1048. Added free logic.
HAWQ-1048. Added free logic.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7ef7e0fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7ef7e0fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7ef7e0fb
Branch: refs/heads/HAWQ-964
Commit: 7ef7e0fbfe332d28755175045b1835f5c1117e25
Parents: cd186f6
Author: Oleksandr Diachenko
Authored: Mon Sep 19 14:05:22 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Sep 19 14:05:22 2016 -0700
--
src/backend/access/external/pxffilters.c | 31 +++
1 file changed, 3 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7ef7e0fb/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 76e83b4..bcd781c 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -33,7 +33,6 @@
static List* pxf_make_expression_items_list(List *quals);
static void pxf_free_filter(PxfFilterDesc* filter);
-static void pxf_free_filter_list(List *filters);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
static bool supported_filter_type(Oid type);
@@ -226,30 +225,6 @@ pxf_free_filter(PxfFilterDesc* filter)
}
/*
- * pxf_free_filter_list
- *
- * free all memory associated with the filters once no longer needed.
- * alternatively we could have allocated them in a shorter lifespan
- * memory context, however explicitly freeing them is easier and makes
- * more sense.
- */
-static void
-pxf_free_filter_list(List *filters)
-{
- ListCell*lc = NULL;
- PxfFilterDesc *filter = NULL;
-
- if (list_length(filters) == 0)
- return;
-
- foreach (lc, filters)
- {
- filter = (PxfFilterDesc *) lfirst(lc);
- pxf_free_filter(filter);
- }
-}
-
-/*
* pxf_serialize_filter_list
*
* Given a list of implicitly ANDed PxfFilterDesc objects, produce a
@@ -328,9 +303,10 @@ pxf_serialize_filter_list(List *expressionItems)
"filter_list. Found a non const+attr filter")));
}
appendStringInfo(resbuf, "%c%d",
PXF_OPERATOR_CODE, o);
- }
- else
+ pxf_free_filter(filter);
+ } else{
pfree(filter);
+ }
break;
}
case T_BoolExpr:
@@ -621,7 +597,6 @@ char *serializePxfFilterQuals(List *quals)
List *expressionItems = pxf_make_expression_items_list(quals);
result = pxf_serialize_filter_list(expressionItems);
- //pxf_free_filter_list(expressionItems);
}
elog(DEBUG2, "serializePxfFilterQuals: filter result: %s", (result ==
NULL) ? "null" : result);
[06/18] incubator-hawq git commit: HAWQ-964. Update unittests and remove @Test from example code
HAWQ-964. Update unittests and remove @Test from example code
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f3668dcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f3668dcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f3668dcc
Branch: refs/heads/HAWQ-964
Commit: f3668dcca2d5862247ba4d5e8e17e7bd2778aba8
Parents: 545f8aa
Author: Kavinder Dhaliwal
Authored: Mon Sep 19 14:51:10 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Tue Sep 20 09:45:44 2016 -0700
--
.../apache/hawq/pxf/api/FilterParserTest.java | 71 ++--
.../hive/HiveORCSearchArgumentExample.java | 1 -
2 files changed, 35 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f3668dcc/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
--
diff --git
a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
index 83bb2dc..a129a4b 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
@@ -288,40 +288,65 @@ public class FilterParserTest {
@Test
public void parseLogicalAndOperator() throws Exception {
-filter = "l0";
-Object op = "filter with 1 AND operator";
+filter = "a1c0o5a2c3o2l0";
+
+Object firstOp = "first operation HDOP_EQ";
+Object secondOp = "second operation HDOP_GT";
+Object lastOp = "filter with 2 operations connected by AND";
+
+when(filterBuilder.build(eq(Operation.HDOP_EQ),
+any(),
+any())).thenReturn(firstOp);
+
+when(filterBuilder.build(eq(Operation.HDOP_GT),
+any(),
+any())).thenReturn(secondOp);
when(filterBuilder.build(eq(LogicalOperation.HDOP_AND),
any(),
-any())).thenReturn(op);
+any())).thenReturn(lastOp);
Object result = filterParser.parse(filter);
-assertEquals(op, result);
+assertEquals(lastOp, result);
}
@Test
public void parseLogicalOrOperator() throws Exception {
-filter = "l1";
+filter = "a1c0o5a2c3o2l1";
+
+Object firstOp = "first operation HDOP_EQ";
+Object secondOp = "second operation HDOP_GT";
+Object lastOp = "filter with 1 OR operator";
-Object op = "filter with 1 OR operator";
+when(filterBuilder.build(eq(Operation.HDOP_EQ),
+any(),
+any())).thenReturn(firstOp);
+
+when(filterBuilder.build(eq(Operation.HDOP_GT),
+any(),
+any())).thenReturn(secondOp);
when(filterBuilder.build(eq(LogicalOperation.HDOP_OR),
any(),
-any())).thenReturn(op);
+any())).thenReturn(lastOp);
Object result = filterParser.parse(filter);
-assertEquals(op, result);
+assertEquals(lastOp, result);
}
@Test
public void parseLogicalNotOperator() throws Exception {
-filter = "l2";
+filter = "a1c0o5l2";
+Object firstOp = "first operation HDOP_EQ";
Object op = "filter with NOT operator";
-when(filterBuilder.build(eq(LogicalOperation.HDOP_NOT),
+when(filterBuilder.build(eq(Operation.HDOP_EQ),
any(),
+any())).thenReturn(firstOp);
+
+when(filterBuilder.build(eq(LogicalOperation.HDOP_NOT),
any())).thenReturn(op);
Object result = filterParser.parse(filter);
@@ -344,31 +369,6 @@ public class FilterParserTest {
}
@Test
-public void parseLogicalOperatorWithExpressions() throws Exception {
-filter = "a1c\"first\"o5a2c2o2l0";
-Object firstOp = "first operation HDOP_EQ";
-Object secondOp = "second operation HDOP_GT";
-Object lastOp = "filter with 2 operations connected by AND";
-
-when(filterBuilder.build(eq(Operation.HDOP_EQ),
-any(),
-any())).thenReturn(firstOp);
-
-
-when(filterBuilder.build(eq(Operation.HDOP_GT),
-any(),
-any())).thenReturn(secondOp);
-
-when(filterBuilder.build(eq(LogicalOperation.HDOP_AND),
-any(),
-any())).thenReturn(lastOp);
-
-
-Object result = filterParser.parse(filter);
-assertEquals(lastOp, result);
-}
-
-@Test
public void parseLogicalOperatorNotExpression() throws Exception {
filter = "a1c\"first\"o5a2c2o2l0l2";
Object f
[03/18] incubator-hawq git commit: HAWQ-1048. Updated comments.
HAWQ-1048. Updated comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7a9cc88d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7a9cc88d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7a9cc88d
Branch: refs/heads/HAWQ-964
Commit: 7a9cc88df6b53bbe6cb9b6e5735f08da1a651a42
Parents: 7ef7e0f
Author: Oleksandr Diachenko
Authored: Mon Sep 19 17:01:25 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Sep 19 17:01:25 2016 -0700
--
src/backend/access/external/pxffilters.c | 35 ---
src/include/access/pxffilters.h | 7 --
2 files changed, 10 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7a9cc88d/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index bcd781c..0d1e1fa 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -159,10 +159,11 @@ Oid pxf_supported_types[] =
* pxf_make_expression_items_list
*
* Given a scan node qual list, find the filters that are eligible to be used
- * by PXF, construct a PxfFilterDesc list that describes the filter
information,
+ * by PXF, construct an expressions list, which consists of OpExpr or BoolExpr
nodes
* and return it to the caller.
*
- * Caller is responsible for pfreeing the returned PxfFilterDesc List.
+ * Basically this function just transforms expression tree to Reversed Polish
Notation list.
+ *
*/
static List *
pxf_make_expression_items_list(List *quals)
@@ -227,7 +228,7 @@ pxf_free_filter(PxfFilterDesc* filter)
/*
* pxf_serialize_filter_list
*
- * Given a list of implicitly ANDed PxfFilterDesc objects, produce a
+ * Takes expression items list in RPN notation, produce a
* serialized string representation in order to communicate this list
* over the wire.
*
@@ -239,9 +240,7 @@ pxf_free_filter(PxfFilterDesc* filter)
*
* Example filter list:
*
- * Column(0) > 1
- * Column(0) < 5
- * Column(2) == "third"
+ * Column(0) > 1 AND Column(0) < 5 AND Column(2) == "third"
*
* Yields the following serialized string:
*
@@ -261,10 +260,7 @@ pxf_serialize_filter_list(List *expressionItems)
initStringInfo(resbuf);
/*
-* Iterate through the filters in the list and serialize them one after
-* the other. We use buffer copying because it's clear. Considering the
-* typical small number of memcpy's this generates overall, there's no
-* point in optimizing, better keep it clear.
+* Iterate through the expression items in the list and serialize them
one after the other.
*/
foreach (lc, expressionItems)
{
@@ -275,6 +271,7 @@ pxf_serialize_filter_list(List *expressionItems)
{
case T_OpExpr:
{
+ elog(DEBUG5, "pxf_serialize_filter_list: node
tag %d (T_OpExpr)", tag);
PxfFilterDesc *filter = (PxfFilterDesc *)
palloc0(sizeof(PxfFilterDesc));
OpExpr *expr = (OpExpr *) node;
if (opexpr_to_pxffilter(expr, filter))
@@ -296,7 +293,7 @@ pxf_serialize_filter_list(List *expressionItems)
}
else
{
- /* pxf_make_filter_list()
should have never let this happen */
+ /* opexpr_to_pxffilter() should
have never let this happen */
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("internal error in pxffilters.c:pxf_serialize_"
@@ -313,20 +310,8 @@ pxf_serialize_filter_list(List *expressionItems)
{
BoolExpr *expr = (BoolExpr *) node;
BoolExprType boolType = expr->boolop;
- PxfOperatorCode pxfOperandCode;
- switch (boolType)
- {
- case AND_EXPR:
- pxfOperandCode = PXFLOP_AND;
- break;
- case OR_EXPR:
- pxfOperandCode = PXFLOP_OR;
-
[04/18] incubator-hawq git commit: HAWQ-964. Add New Filter Classes
HAWQ-964. Add New Filter Classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/545f8aa0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/545f8aa0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/545f8aa0
Branch: refs/heads/HAWQ-964
Commit: 545f8aa0ab28f2a1112fb43d232e9db2b2cc5a80
Parents: 726be6c
Author: Kavinder Dhaliwal
Authored: Mon Sep 19 14:25:08 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Tue Sep 20 09:45:44 2016 -0700
--
.../org/apache/hawq/pxf/api/BasicFilter.java| 37
.../org/apache/hawq/pxf/api/LogicalFilter.java | 30
.../plugins/hbase/HBaseFilterBuilderTest.java | 16 +
3 files changed, 83 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/545f8aa0/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/BasicFilter.java
--
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/BasicFilter.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/BasicFilter.java
new file mode 100644
index 000..a35a9dd
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/BasicFilter.java
@@ -0,0 +1,37 @@
+package org.apache.hawq.pxf.api;
+
+/**
+ * Basic filter provided for cases where the target storage system does not
provide it own filter
+ * For example: Hbase storage provides its own filter but for a Writable based
record in a
+ * SequenceFile there is no filter provided and so we need to have a default
+ */
+public class BasicFilter {
+private FilterParser.Operation oper;
+private FilterParser.ColumnIndex column;
+private FilterParser.Constant constant;
+
+/**
+ * Constructs a BasicFilter.
+ *
+ * @param oper the parse operation to perform
+ * @param column the column index
+ * @param constant the constant object
+ */
+public BasicFilter(FilterParser.Operation oper, FilterParser.ColumnIndex
column, FilterParser.Constant constant) {
+this.oper = oper;
+this.column = column;
+this.constant = constant;
+}
+
+public FilterParser.Operation getOperation() {
+return oper;
+}
+
+public FilterParser.ColumnIndex getColumn() {
+return column;
+}
+
+public FilterParser.Constant getConstant() {
+return constant;
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/545f8aa0/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/LogicalFilter.java
--
diff --git
a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/LogicalFilter.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/LogicalFilter.java
new file mode 100644
index 000..d7e570e
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/LogicalFilter.java
@@ -0,0 +1,30 @@
+package org.apache.hawq.pxf.api;
+
+
+import java.util.List;
+
+public class LogicalFilter {
+private FilterParser.LogicalOperation operator;
+private List filterList;
+
+public LogicalFilter(FilterParser.LogicalOperation operator, List
result) {
+this.operator = operator;
+this.filterList = result;
+}
+
+public FilterParser.LogicalOperation getOperator() {
+return operator;
+}
+
+public void setOperator(FilterParser.LogicalOperation operator) {
+this.operator = operator;
+}
+
+public List getFilterList() {
+return filterList;
+}
+
+public void setFilterList(List filterList) {
+this.filterList = filterList;
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/545f8aa0/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
--
diff --git
a/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
new file mode 100644
index 000..9bc5474
--- /dev/null
+++
b/pxf/pxf-hbase/src/test/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilderTest.java
@@ -0,0 +1,16 @@
+package org.apache.hawq.pxf.plugins.hbase;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class HBaseFilterBuilderTest {
+
+@Test
+public void parseNOTExpressionIgnored() throws Exception {
+String filter = "a1c2o1a1c2o2l0l2";
+HBaseFilterBuilder builder = new HBaseFilterBuilder(null);
+assertNull(builder.getFilterObject(filter));
+}
+
+}
\ No newline at end of file
[10/18] incubator-hawq git commit: HAWQ-1048. Discard whole filter if at least one operator is not supported.
HAWQ-1048. Discard whole filter if at least one operator is not supported.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/694ad038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/694ad038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/694ad038
Branch: refs/heads/HAWQ-964
Commit: 694ad0383cce829057636365a87b92056bb52e03
Parents: 0ff7037
Author: Oleksandr Diachenko
Authored: Thu Sep 22 11:43:19 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Sep 22 11:43:19 2016 -0700
--
src/backend/access/external/pxffilters.c | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/694ad038/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 8c3864d..847f120 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -301,8 +301,12 @@ pxf_serialize_filter_list(List *expressionItems)
}
appendStringInfo(resbuf, "%c%d",
PXF_OPERATOR_CODE, o);
pxf_free_filter(filter);
- } else{
+ } else {
+ /* if at least one expression item is
not supported, whole filter doesn't make sense*/
+ elog(INFO, "Query will not be optimized
to use filter push-down.");
pfree(filter);
+ pfree(resbuf->data);
+ return NULL;
}
break;
}
[01/18] incubator-hawq git commit: HAWQ-1048. Draft implementation.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-964 [created] 509aef120
HAWQ-1048. Draft implementation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cd186f6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cd186f6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cd186f6f
Branch: refs/heads/HAWQ-964
Commit: cd186f6fb3ab6bc74833ef5184da03f113ee7995
Parents: 981c0a9
Author: Oleksandr Diachenko
Authored: Fri Sep 16 19:00:50 2016 -0700
Committer: Oleksandr Diachenko
Committed: Fri Sep 16 19:00:50 2016 -0700
--
src/backend/access/external/pxffilters.c | 152 --
src/include/access/pxffilters.h | 9 +-
2 files changed, 80 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cd186f6f/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 6767735..76e83b4 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,7 +31,7 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_filter_list(List* quals);
+static List* pxf_make_expression_items_list(List *quals);
static void pxf_free_filter(PxfFilterDesc* filter);
static void pxf_free_filter_list(List *filters);
static char* pxf_serialize_filter_list(List *filters);
@@ -157,7 +157,7 @@ Oid pxf_supported_types[] =
};
/*
- * pxf_make_filter_list
+ * pxf_make_expression_items_list
*
* Given a scan node qual list, find the filters that are eligible to be used
* by PXF, construct a PxfFilterDesc list that describes the filter
information,
@@ -166,7 +166,7 @@ Oid pxf_supported_types[] =
* Caller is responsible for pfreeing the returned PxfFilterDesc List.
*/
static List *
-pxf_make_filter_list(List *quals)
+pxf_make_expression_items_list(List *quals)
{
List*result = NIL;
ListCell*lc = NULL;
@@ -174,10 +174,6 @@ pxf_make_filter_list(List *quals)
if (list_length(quals) == 0)
return NIL;
- /*
-* Iterate over all implicitly ANDed qualifiers and add the ones
-* that are supported for push-down into the result filter list.
-*/
foreach (lc, quals)
{
Node *node = (Node *) lfirst(lc);
@@ -187,38 +183,19 @@ pxf_make_filter_list(List *quals)
{
case T_OpExpr:
{
- OpExpr *expr = (OpExpr *)
node;
- PxfFilterDesc *filter;
-
- filter = (PxfFilterDesc *)
palloc0(sizeof(PxfFilterDesc));
- elog(DEBUG5, "pxf_make_filter_list: node tag %d
(T_OpExpr)", tag);
-
- if (opexpr_to_pxffilter(expr, filter))
- result = lappend(result, filter);
- else
- pfree(filter);
-
+ result = lappend(result, node);
break;
}
case T_BoolExpr:
{
BoolExpr*expr = (BoolExpr *) node;
- BoolExprType boolType = expr->boolop;
- elog(DEBUG5, "pxf_make_filter_list: node tag %d
(T_BoolExpr), bool node type %d %s",
- tag, boolType,
boolType==AND_EXPR ? "(AND_EXPR)" : "");
-
- /* only AND_EXPR is supported */
- if (expr->boolop == AND_EXPR)
- {
- List *inner_result =
pxf_make_filter_list(expr->args);
- elog(DEBUG5, "pxf_make_filter_list:
inner result size %d", list_length(inner_result));
- result = list_concat(result,
inner_result);
- }
+ List *inner_result =
pxf_make_expression_items_list(expr->args);
+ result = list_concat(result, inner_result);
+ result = lappend(result, node);
break;
}
default:
- /* expression not supported. ignore */
- elog(DEBUG5, "pxf_make_filter_list: unsupported
node tag %d", tag);
+
[09/18] incubator-hawq git commit: HAWQ-964. Fix issues where only one filter is present
HAWQ-964. Fix issues where only one filter is present
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8f73c2b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8f73c2b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8f73c2b9
Branch: refs/heads/HAWQ-964
Commit: 8f73c2b9295e93be68958b0c6b35a53143e5f7a7
Parents: 93052b9
Author: Kavinder Dhaliwal
Authored: Thu Sep 22 11:34:11 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Thu Sep 22 11:34:11 2016 -0700
--
.../apache/hawq/pxf/plugins/hive/HiveAccessor.java| 2 +-
.../apache/hawq/pxf/plugins/hive/HiveORCAccessor.java | 14 +-
2 files changed, 14 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8f73c2b9/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
index 2f1c26e..b2b3e4b 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
@@ -276,7 +276,7 @@ public class HiveAccessor extends
HdfsSplittableDataAccessor {
Object filter, InputData input) {
// Let's look first at the filter and escape if there are any OR or
NOT ops
if (!testForUnsupportedOperators(Arrays.asList(filter)))
-return false;
+return true;
return testForPartitionEquality(partitionFields,
Arrays.asList(filter), input);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8f73c2b9/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
index 195382a..ab2f96e 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessor.java
@@ -100,7 +100,19 @@ public class HiveORCAccessor extends HiveAccessor {
HiveFilterBuilder eval = new HiveFilterBuilder(inputData);
Object filter = eval.getFilterObject(filterStr);
SearchArgument.Builder filterBuilder =
SearchArgumentFactory.newBuilder();
-buildExpression(filterBuilder, Arrays.asList(filter));
+
+/*
+ * If there is only a single filter it will be of type Basic Filter
+ * need special case logic to make sure to still wrap the filter in a
+ * startAnd() & end() block
+ */
+if (filter instanceof LogicalFilter)
+buildExpression(filterBuilder, Arrays.asList(filter));
+else {
+filterBuilder.startAnd();
+buildArgument(filterBuilder, filter);
+filterBuilder.end();
+}
SearchArgument sarg = filterBuilder.build();
jobConf.set(SARG_PUSHDOWN, sarg.toKryo());
}
[16/18] incubator-hawq git commit: HAWQ-1048. Fixed unit-tests.
HAWQ-1048. Fixed unit-tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/dffc760d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/dffc760d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/dffc760d
Branch: refs/heads/HAWQ-964
Commit: dffc760de7c1ed9ec1a40e5e94d16be202e75b97
Parents: 38cb4b0
Author: Oleksandr Diachenko
Authored: Mon Oct 3 13:17:10 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Oct 3 13:17:10 2016 -0700
--
src/backend/access/external/pxffilters.c| 7 +++
.../access/external/test/pxffilters_test.c | 20 +++-
.../access/external/test/pxfheaders_test.c | 9 +
.../utils/mmgr/test/memaccounting_test.c| 2 ++
4 files changed, 37 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dffc760d/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 4c28bc1..2ada44e 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -321,6 +321,9 @@ pxf_free_filter(PxfFilterDesc* filter)
static char *
pxf_serialize_filter_list(List *expressionItems)
{
+
+ printf("Serializing filter list 1\n");
+
StringInfo resbuf;
ListCell*lc = NULL;
@@ -330,6 +333,8 @@ pxf_serialize_filter_list(List *expressionItems)
resbuf = makeStringInfo();
initStringInfo(resbuf);
+ printf("Serializing filter list 2\n");
+
/*
* Iterate through the expression items in the list and serialize them
one after the other.
*/
@@ -393,6 +398,8 @@ pxf_serialize_filter_list(List *expressionItems)
}
}
+ printf("Serializing filter list 30\n");
+
if (resbuf->len == 0)
{
pfree(resbuf->data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dffc760d/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 65b1aef..c1c54cd 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -25,6 +25,9 @@
#include "c.h"
#include "../pxffilters.c"
+void run__const_to_str(Const* input, StringInfo result, char* expected);
+void run__const_to_str__negative(Const* input, StringInfo result, char* value);
+
void
test__supported_filter_type(void **state)
{
@@ -62,7 +65,7 @@ test__supported_filter_type(void **state)
/* go over pxf_supported_types array */
int nargs = sizeof(pxf_supported_types) / sizeof(Oid);
- assert_int_equal(nargs, 13);
+ assert_int_equal(nargs, 14);
for (i = 0; i < nargs; ++i)
{
assert_true(supported_filter_type(pxf_supported_types[i]));
@@ -475,14 +478,29 @@ test__opexpr_to_pxffilter__unsupportedOpNot(void **state)
void
test__pxf_serialize_filter_list__oneFilter(void **state)
{
+
+ printf("Entered one filter test 1\n");
+
+int c = 1, d = 1, f = 1;
+
+ for ( c = 1 ; c <= 32767 ; c++ )
+ for ( d = 1 ; d <= 32767 ; d++ )
+ for ( f = 1 ; f <= 10 ; f++ )
+ {}
+
List* filter_list = NIL;
+ printf("Entered one filter test 2\n");
+
PxfFilterDesc* filter = build_filter(
PXF_ATTR_CODE, 1, NULL,
PXF_CONST_CODE, 0, "1984",
PXFOP_GT);
+
+ printf("Entered one filter test 3\n");
filter_list = lappend(filter_list, filter);
+
char* result = pxf_serialize_filter_list(filter_list);
assert_string_equal(result, "a0c1984o2");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dffc760d/src/backend/access/external/test/pxfheaders_test.c
--
diff --git a/src/backend/access/external/test/pxfheaders_test.c
b/src/backend/access/external/test/pxfheaders_test.c
index 7c79b5a..454ecdc 100644
--- a/src/backend/access/external/test/pxfheaders_test.c
+++ b/src/backend/access/external/test/pxfheaders_test.c
@@ -32,6 +32,15 @@ static extvar_t *mock_extvar = NULL;
static char *old_pxf_remote_service_login = NULL;
static char *old_pxf_remote_service_secret = NULL;
+void expect_churl_headers(const char *key, const char *value);
+void expect_churl_headers_alignment();
+void store_gucs();
+void setup_gphd_uri();
+void setup_input_data();
+void setup_external_vars();
+void
[07/18] incubator-hawq git commit: HAWQ-964. Ignore Expressions with OR/NOT from partition filtering
HAWQ-964. Ignore Expressions with OR/NOT from partition filtering
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/93052b93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/93052b93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/93052b93
Branch: refs/heads/HAWQ-964
Commit: 93052b93098de971eaaa0ed2c5da03372e201509
Parents: f3668dc
Author: Kavinder Dhaliwal
Authored: Tue Sep 20 15:35:08 2016 -0700
Committer: Kavinder Dhaliwal
Committed: Tue Sep 20 15:35:08 2016 -0700
--
.../hawq/pxf/plugins/hive/HiveAccessor.java | 86 +---
1 file changed, 57 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/93052b93/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
index 20a1b9f..2f1c26e 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
@@ -21,6 +21,7 @@ package org.apache.hawq.pxf.plugins.hive;
import org.apache.hawq.pxf.api.BasicFilter;
import org.apache.hawq.pxf.api.FilterParser;
+import org.apache.hawq.pxf.api.LogicalFilter;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.plugins.hdfs.HdfsSplittableDataAccessor;
@@ -32,6 +33,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@@ -206,51 +208,77 @@ public class HiveAccessor extends
HdfsSplittableDataAccessor {
return testOneFilter(partitionFields, filter, inputData);
}
-/*
- * We are testing one filter against all the partition fields. The filter
- * has the form "fieldA = valueA". The partitions have the form
- * partitionOne=valueOne/partitionTwo=ValueTwo/partitionThree=valueThree 1.
- * For a filter to match one of the partitions, lets say partitionA for
- * example, we need: fieldA = partittionOne and valueA = valueOne. If this
- * condition occurs, we return true. 2. If fieldA does not match any one of
- * the partition fields we also return true, it means we ignore this filter
- * because it is not on a partition field. 3. If fieldA = partittionOne and
- * valueA != valueOne, then we return false.
- */
-private boolean testOneFilter(List partitionFields,
- Object filter, InputData input) {
-// Let's look first at the filter
-BasicFilter bFilter = (BasicFilter) filter;
+private boolean testForUnsupportedOperators(List filterList) {
+boolean nonAndOp = true;
+for (Object filter : filterList) {
+if (filter instanceof LogicalFilter) {
+if (((LogicalFilter) filter).getOperator() !=
FilterParser.LogicalOperation.HDOP_AND)
+return false;
+if (((LogicalFilter) filter).getFilterList() != null)
+nonAndOp = testForUnsupportedOperators(((LogicalFilter)
filter).getFilterList());
+}
+}
+return nonAndOp;
+}
-boolean isFilterOperationEqual = (bFilter.getOperation() ==
FilterParser.Operation.HDOP_EQ);
-if (!isFilterOperationEqual) /*
+private boolean testForPartitionEquality(List
partitionFields, List filterList, InputData input) {
+boolean partitionAllowed = true;
+for (Object filter : filterList) {
+if (filter instanceof BasicFilter) {
+BasicFilter bFilter = (BasicFilter) filter;
+boolean isFilterOperationEqual = (bFilter.getOperation() ==
FilterParser.Operation.HDOP_EQ);
+if (!isFilterOperationEqual) /*
* in case this is not an "equality
filter"
* we ignore it here - in partition
* filtering
*/{
-return true;
-}
+return true;
+}
-int filterColumnIndex = bFilter.getColumn().index();
-String filterValue = bFilter.getConstant().constant().toString();
-ColumnDescriptor filterColumn = input.getColumn(filterColumnIndex);
-String filterColumnName = filterColumn.columnName();
+int filterColumnIndex = bFilter.getColumn().index
incubator-hawq git commit: HAWQ-1080. Fixed unit-tests on local environment.
Repository: incubator-hawq
Updated Branches:
refs/heads/master b33e4d5db -> 0abbac2d8
HAWQ-1080. Fixed unit-tests on local environment.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0abbac2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0abbac2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0abbac2d
Branch: refs/heads/master
Commit: 0abbac2d8a140e79751917c62545088d7b28233a
Parents: b33e4d5
Author: Oleksandr Diachenko
Authored: Tue Oct 4 16:45:06 2016 -0700
Committer: Oleksandr Diachenko
Committed: Tue Oct 4 16:45:06 2016 -0700
--
src/backend/access/external/test/pxffilters_test.c | 3 +++
src/backend/access/external/test/pxfheaders_test.c | 9 +
src/backend/utils/mmgr/test/memaccounting_test.c | 2 ++
3 files changed, 14 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0abbac2d/src/backend/access/external/test/pxffilters_test.c
--
diff --git a/src/backend/access/external/test/pxffilters_test.c
b/src/backend/access/external/test/pxffilters_test.c
index 65b1aef..7dc3aec 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -25,6 +25,9 @@
#include "c.h"
#include "../pxffilters.c"
+void run__const_to_str(Const* input, StringInfo result, char* expected);
+void run__const_to_str__negative(Const* input, StringInfo result, char* value);
+
void
test__supported_filter_type(void **state)
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0abbac2d/src/backend/access/external/test/pxfheaders_test.c
--
diff --git a/src/backend/access/external/test/pxfheaders_test.c
b/src/backend/access/external/test/pxfheaders_test.c
index 7c79b5a..454ecdc 100644
--- a/src/backend/access/external/test/pxfheaders_test.c
+++ b/src/backend/access/external/test/pxfheaders_test.c
@@ -32,6 +32,15 @@ static extvar_t *mock_extvar = NULL;
static char *old_pxf_remote_service_login = NULL;
static char *old_pxf_remote_service_secret = NULL;
+void expect_churl_headers(const char *key, const char *value);
+void expect_churl_headers_alignment();
+void store_gucs();
+void setup_gphd_uri();
+void setup_input_data();
+void setup_external_vars();
+void expect_external_vars();
+void restore_gucs();
+
void
test__build_http_header__remote_login_is_null(void **state)
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0abbac2d/src/backend/utils/mmgr/test/memaccounting_test.c
--
diff --git a/src/backend/utils/mmgr/test/memaccounting_test.c
b/src/backend/utils/mmgr/test/memaccounting_test.c
index a946040..841d171 100644
--- a/src/backend/utils/mmgr/test/memaccounting_test.c
+++ b/src/backend/utils/mmgr/test/memaccounting_test.c
@@ -40,6 +40,8 @@
#define AllocPointerGetChunk(ptr) \
((StandardChunkHeader *)(((char
*)(ptr)) - ALLOC_CHUNKHDRSZ))
+void write_stderr_mock(const char *fmt,...);
+
static StringInfoData outputBuffer;
/* We will capture write_stderr output using write_stderr_mock */
incubator-hawq git commit: HAWQ-1079. Fixed issue with symlink.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 9c97bccfd -> 9b7f90b74
HAWQ-1079. Fixed issue with symlink.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9b7f90b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9b7f90b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9b7f90b7
Branch: refs/heads/master
Commit: 9b7f90b744850ff83769c3a46e6d4daeb109cc68
Parents: 9c97bcc
Author: Oleksandr Diachenko
Authored: Mon Sep 26 21:17:41 2016 -0700
Committer: Oleksandr Diachenko
Committed: Mon Sep 26 21:19:00 2016 -0700
--
pxf/build.gradle| 6 +--
pxf/pxf-service/src/scripts/post-install.sh | 40 ---
pxf/pxf-service/src/scripts/pre-install.sh | 49
pxf/pxf/src/scripts/post-install.sh | 40 +++
pxf/pxf/src/scripts/pre-install.sh | 49
5 files changed, 92 insertions(+), 92 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9b7f90b7/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index a35b35a..a66c586 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -213,9 +213,6 @@ project('pxf-service') {
obsoletes('pxf-core')
obsoletes('pxf-api')
-preInstall file('src/scripts/pre-install.sh')
-postInstall file('src/scripts/post-install.sh')
-
from('src/main/resources/pxf-profiles-default.xml') {
fileType CONFIG | NOREPLACE
into "/etc/pxf-${project.version}/conf"
@@ -302,6 +299,9 @@ project('pxf') {
requires(versionedPackageName('pxf-hbase'), project.version, GREATER |
EQUAL)
requires(versionedPackageName('pxf-json'), project.version, GREATER |
EQUAL)
+preInstall file('src/scripts/pre-install.sh')
+postInstall file('src/scripts/post-install.sh')
+
link('/usr/lib/pxf', "/usr/lib/pxf-${project.version}")
link('/etc/pxf', "/etc/pxf-${project.version}")
link('/opt/pxf', "/opt/pxf-${project.version}")
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9b7f90b7/pxf/pxf-service/src/scripts/post-install.sh
--
diff --git a/pxf/pxf-service/src/scripts/post-install.sh
b/pxf/pxf-service/src/scripts/post-install.sh
deleted file mode 100644
index 799959e..000
--- a/pxf/pxf-service/src/scripts/post-install.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/sh
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Post installation script
-
-env_script=/etc/pxf/conf/pxf-env.sh
-user=pxf
-group=pxf
-
-# load pxf-env.sh script
-if [ ! -f $env_script ]; then
- echo WARNING: failed to find $env_script
-else
- source $env_script
-fi
-
-# create log directory with necessary permission
-mkdir -p $PXF_LOGDIR
-chown $user:$group -R $PXF_LOGDIR
-chmod 755 $PXF_LOGDIR
-
-# create run directory with necessary permission
-mkdir -p $PXF_RUNDIR
-chown $user:$group -R $PXF_RUNDIR
-chmod 755 $PXF_RUNDIR
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9b7f90b7/pxf/pxf-service/src/scripts/pre-install.sh
--
diff --git a/pxf/pxf-service/src/scripts/pre-install.sh
b/pxf/pxf-service/src/scripts/pre-install.sh
deleted file mode 100644
index 964ae6a..000
--- a/pxf/pxf-service/src/scripts/pre-install.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/sh
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1048 [deleted] 694ad0383
incubator-hawq git commit: HAWQ-1048. Discard whole filter if at least one operator is not supported.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-1048 0ff70373b -> 694ad0383
HAWQ-1048. Discard whole filter if at least one operator is not supported.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/694ad038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/694ad038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/694ad038
Branch: refs/heads/HAWQ-1048
Commit: 694ad0383cce829057636365a87b92056bb52e03
Parents: 0ff7037
Author: Oleksandr Diachenko
Authored: Thu Sep 22 11:43:19 2016 -0700
Committer: Oleksandr Diachenko
Committed: Thu Sep 22 11:43:19 2016 -0700
--
src/backend/access/external/pxffilters.c | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/694ad038/src/backend/access/external/pxffilters.c
--
diff --git a/src/backend/access/external/pxffilters.c
b/src/backend/access/external/pxffilters.c
index 8c3864d..847f120 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -301,8 +301,12 @@ pxf_serialize_filter_list(List *expressionItems)
}
appendStringInfo(resbuf, "%c%d",
PXF_OPERATOR_CODE, o);
pxf_free_filter(filter);
- } else{
+ } else {
+ /* if at least one expression item is
not supported, whole filter doesn't make sense*/
+ elog(INFO, "Query will not be optimized
to use filter push-down.");
pfree(filter);
+ pfree(resbuf->data);
+ return NULL;
}
break;
}
