incubator-hawq git commit: HAWQ-1443. Implement Ranger lookup for HAWQ with Kerberos enabled.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 9775424ba -> 6768af3e0


HAWQ-1443. Implement Ranger lookup for HAWQ with Kerberos enabled.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6768af3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6768af3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6768af3e

Branch: refs/heads/master
Commit: 6768af3e033737ba6386c76ef0c8b201c0706938
Parents: 9775424
Author: hubertzhang <hubertzh...@apache.org>
Authored: Sat May 27 10:06:11 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Sat May 27 10:06:11 2017 +0800

--
 .../main/java/org/apache/hawq/ranger/service/HawqClient.java   | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6768af3e/ranger-plugin/admin-plugin/src/main/java/org/apache/hawq/ranger/service/HawqClient.java
--
diff --git 
a/ranger-plugin/admin-plugin/src/main/java/org/apache/hawq/ranger/service/HawqClient.java
 
b/ranger-plugin/admin-plugin/src/main/java/org/apache/hawq/ranger/service/HawqClient.java
index 7e05a82..f8a252b 100644
--- 
a/ranger-plugin/admin-plugin/src/main/java/org/apache/hawq/ranger/service/HawqClient.java
+++ 
b/ranger-plugin/admin-plugin/src/main/java/org/apache/hawq/ranger/service/HawqClient.java
@@ -72,7 +72,7 @@ public class HawqClient extends BaseClient {
 private static final String DEFAULT_DATABASE = "postgres";
 private static final String JDBC_DRIVER_CLASS = "org.postgresql.Driver";
 
-private boolean isKerberosAuth;
+private boolean isKerberosAuth = false;
 private Connection con;
 private Map<String, String> connectionProperties;
 
@@ -94,7 +94,9 @@ public class HawqClient extends BaseClient {
 }
 
 public void initHawq() throws Exception {
-   isKerberosAuth = 
connectionProperties.get(AUTHENTICATION).equals(KERBEROS);
+   if(connectionProperties.containsKey(AUTHENTICATION)) {
+   isKerberosAuth = 
connectionProperties.get(AUTHENTICATION).equals(KERBEROS);
+   }
if (isKerberosAuth) {
LOG.info("Secured Mode: JDBC Connection done with 
preAuthenticated Subject");

incubator-hawq git commit: HAWQ-1396. Add more test cases for querying external table via PXF with Ranger enable.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 2d9bc0306 -> 72928dab2


HAWQ-1396. Add more test cases for querying external table via PXF with Ranger 
enable.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/72928dab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/72928dab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/72928dab

Branch: refs/heads/master
Commit: 72928dab2dfc386118ae95e4016f3b549e347de1
Parents: 2d9bc03
Author: interma <inte...@outlook.com>
Authored: Wed Apr 5 13:47:40 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Apr 13 17:38:29 2017 +0800

--
 src/test/feature/.gitignore |   2 +
 .../feature/Ranger/ans/manual1_success.ans  |  10 +
 .../feature/Ranger/ans/normal1_success.ans  |  10 -
 src/test/feature/Ranger/ans/pxf1_fail.ans   |   2 +-
 src/test/feature/Ranger/ans/pxf2_fail.ans   |   9 +
 src/test/feature/Ranger/ans/pxf2_success.ans|   8 +
 src/test/feature/Ranger/ans/pxf3_success.ans|  10 +
 src/test/feature/Ranger/ans/pxf4_fail.ans   |   9 +
 src/test/feature/Ranger/ans/pxf4_success.ans|   8 +
 src/test/feature/Ranger/ans/pxf5_success.ans|  13 +
 src/test/feature/Ranger/ans/pxf6_fail.ans   |   8 +
 src/test/feature/Ranger/ans/pxf6_success.ans|   8 +
 src/test/feature/Ranger/ans/pxf7_success.ans|  14 +
 src/test/feature/Ranger/ans/pxf8_fail.ans   |   9 +
 src/test/feature/Ranger/ans/pxf8_success.ans|   8 +
 src/test/feature/Ranger/ans/pxf9_success.ans|  12 +
 .../Ranger/data/testhive_externaltable.sql  |   5 +
 src/test/feature/Ranger/policy/1/1.json |   1 -
 src/test/feature/Ranger/policy_helper.cpp   | 204 +
 src/test/feature/Ranger/policy_helper.h | 110 +++
 src/test/feature/Ranger/pxfpolicy/1/3.json  |   2 +-
 src/test/feature/Ranger/rangerpolicy.py |  56 ++--
 src/test/feature/Ranger/rangerrest.py   |   1 +
 src/test/feature/Ranger/sql/manual/1.sql|   2 +-
 src/test/feature/Ranger/sql/pxf/2.sql   |   4 +
 src/test/feature/Ranger/sql/pxf/3.sql   |   4 +
 src/test/feature/Ranger/sql/pxf/4.sql   |   4 +
 src/test/feature/Ranger/sql/pxf/5.sql   |   3 +
 src/test/feature/Ranger/sql/pxf/6.sql   |   4 +
 src/test/feature/Ranger/sql/pxf/7.sql   |   3 +
 src/test/feature/Ranger/sql/pxf/8.sql   |   4 +
 src/test/feature/Ranger/sql/pxf/9.sql   |   3 +
 src/test/feature/Ranger/test_policyhelper.cpp   |  92 ++
 src/test/feature/Ranger/test_ranger.cpp | 301 ---
 src/test/feature/Ranger/test_ranger.h   |   7 +-
 src/test/feature/sanity_tests.txt   |   4 +-
 36 files changed, 873 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/.gitignore
--
diff --git a/src/test/feature/.gitignore b/src/test/feature/.gitignore
index fec82bd..10fc2f6 100644
--- a/src/test/feature/.gitignore
+++ b/src/test/feature/.gitignore
@@ -23,3 +23,5 @@ testlib/ans/template.ans
 testlib/sql/template.sql
 utility/ans/copytest.csv
 utility/ans/onek.data
+
+*.pyc

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/manual1_success.ans
--
diff --git a/src/test/feature/Ranger/ans/manual1_success.ans 
b/src/test/feature/Ranger/ans/manual1_success.ans
new file mode 100644
index 000..73f878b
--- /dev/null
+++ b/src/test/feature/Ranger/ans/manual1_success.ans
@@ -0,0 +1,10 @@
+-- start_ignore
+-- end_ignore
+set session role=usermanual1;
+SET
+select count(*) from information_schema.view_table_usage;
+0
+select count(*) from hawq_toolkit.hawq_table_indexes;
+0
+select count(*) from pg_catalog.pg_compression;
+4

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/normal1_success.ans
--
diff --git a/src/test/feature/Ranger/ans/normal1_success.ans 
b/src/test/feature/Ranger/ans/normal1_success.ans
deleted file mode 100644
index 125e649..000
--- a/src/test/feature/Ranger/ans/normal1_success.ans
+++ /dev/null
@@ -1,10 +0,0 @@
--- start_ignore
--- end_ignore
-set session role=usertest1;
-SET
-select count(*) from information_schema.view_table_usage;
-0
-select count(*) from hawq_toolkit.hawq_table_indexes;
-0
-select count(*) from pg_catalog.pg_compression;
-4

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/an

incubator-hawq git commit: HAWQ-1430. Update ranger related log level to avoid log flood

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 2d32de10f -> 2d9bc0306


HAWQ-1430. Update ranger related log level to avoid log flood


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2d9bc030
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2d9bc030
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2d9bc030

Branch: refs/heads/master
Commit: 2d9bc0306ea38836d360dcb5c30b4d93d3791234
Parents: 2d32de1
Author: stanlyxiang <stanly.sxi...@gmail.com>
Authored: Wed Apr 12 17:17:53 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Apr 13 17:35:20 2017 +0800

--
 src/backend/catalog/aclchk.c   |  4 ++--
 src/backend/libpq/rangerrest.c | 20 ++--
 src/backend/tcop/postgres.c|  6 +++---
 src/include/utils/rangerrest.h |  2 ++
 4 files changed, 17 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2d9bc030/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 77c70fa..207ce12 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2746,7 +2746,7 @@ List *pg_rangercheck_batch(List *arg_list)
List *aclresults = NIL;
List *requestargs = NIL;
ListCell *arg;
-   elog(DEBUG3, "ranger acl batch check, acl list length: %d\n", 
arg_list->length);
+   elog(RANGER_LOG, "ranger acl batch check, acl list length: %d\n", 
arg_list->length);
foreach(arg, arg_list)
{
RangerPrivilegeArgs *arg_ptr = (RangerPrivilegeArgs *) 
lfirst(arg);
@@ -2816,7 +2816,7 @@ pg_rangercheck(AclObjectKind objkind, Oid object_oid, Oid 
roleid,
List* actions = getActionName(mask);
bool isAll = (how == ACLMASK_ALL) ? true: false;
 
-   elog(DEBUG3, "ranger acl check kind: %d, object name: %s, object 
oid:%d, role: %s, mask: %u\n",
+   elog(RANGER_LOG, "ranger acl check kind: %d, object name: %s, object 
oid:%d, role: %s, mask: %u\n",
objkind, objectname, object_oid, rolename, mask);
 
List *resultargs = NIL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2d9bc030/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index c920575..92852bd 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -80,7 +80,7 @@ static int parse_ranger_response(char* buffer, List 
*result_list)
if (buffer == NULL || strlen(buffer) == 0)
return -1;
 
-   elog(DEBUG3, "parse ranger restful response content : %s", buffer);
+   elog(RANGER_LOG, "parse ranger restful response content : %s", buffer);
 
struct json_object *response = json_tokener_parse(buffer);
if (response == NULL) 
@@ -97,7 +97,7 @@ static int parse_ranger_response(char* buffer, List 
*result_list)
}
 
int arraylen = json_object_array_length(accessObj);
-   elog(DEBUG3, "parse ranger response result array length: %d",arraylen);
+   elog(RANGER_LOG, "parse ranger response result array length: 
%d",arraylen);
for (int i=0; i< arraylen; i++){
struct json_object *jvalue = NULL;
struct json_object *jallow = NULL;
@@ -120,7 +120,7 @@ static int parse_ranger_response(char* buffer, List 
*result_list)
const char *privilege_str = json_object_get_string(jprivilege);
uint32 resource_sign = string_hash(resource_str, 
strlen(resource_str));
uint32 privilege_sign = string_hash(privilege_str, 
strlen(privilege_str));
-   elog(DEBUG3, "ranger response access sign, resource_str: %s, 
privilege_str: %s",
+   elog(RANGER_LOG, "ranger response access sign, resource_str: 
%s, privilege_str: %s",
resource_str, privilege_str);
 
ListCell *result;
@@ -215,7 +215,7 @@ static json_object *create_ranger_request_json(List 
*request_list, List *result_
AclObjectKind kind = arg_ptr->kind;
char* object = arg_ptr->object;
Assert(user != NULL && object != NULL);
-   elog(DEBUG3, "build json for ranger restful request, user:%s, 
kind:%s, object:%s",
+   elog(RANGER_LOG, "build json for ranger restful request, 
user:%s, kind:%s, object:%s",
user, AclObjectKindStr[kind], object);
 
 

incubator-hawq git commit: HAWQ-1359. Add additional policy for select view.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 68013fac1 -> 597576433


HAWQ-1359. Add additional policy for select view.

Signed-off-by: Chuling Wang <wangchunlin...@126.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/59757643
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/59757643
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/59757643

Branch: refs/heads/master
Commit: 59757643376f1f4c765ffce4a6d8cac66010cfae
Parents: 68013fa
Author: Hubert Zhang <hubertzh...@apache.org>
Authored: Fri Mar 3 17:20:23 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Fri Mar 3 17:35:19 2017 +0800

--
 src/test/feature/Ranger/policy/43/3.json | 2 +-
 src/test/feature/Ranger/sql/init_file| 1 +
 src/test/feature/gtest-parallel  | 8 
 3 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/59757643/src/test/feature/Ranger/policy/43/3.json
--
diff --git a/src/test/feature/Ranger/policy/43/3.json 
b/src/test/feature/Ranger/policy/43/3.json
index 7762ff7..0318f3d 100644
--- a/src/test/feature/Ranger/policy/43/3.json
+++ b/src/test/feature/Ranger/policy/43/3.json
@@ -1 +1 @@
-{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy43-3", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"select"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": 
["usertest43", "usersuper43"]}], "resources": {"database": {"isExcludes": 
false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": 
{"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": 
{"isExcludes": false, "isRecursive": false, "values": ["a"]}}, "service": 
"hawq", "version": 1}
\ No newline at end of file
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy43-3", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"select"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": 
["usertest34", "usersuper34","usertest43", "usersuper43"]}], "resources": 
{"database": {"isExcludes": false, "isRecursive": false, "values": 
["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": 
false, "values": ["public"]}, "table": {"isExcludes": false, "isRecursive": 
false, "values": ["a"]}}, "service": "hawq", "version": 1}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/59757643/src/test/feature/Ranger/sql/init_file
--
diff --git a/src/test/feature/Ranger/sql/init_file 
b/src/test/feature/Ranger/sql/init_file
index 6c5dbe7..9647404 100644
--- a/src/test/feature/Ranger/sql/init_file
+++ b/src/test/feature/Ranger/sql/init_file
@@ -15,6 +15,7 @@
 -- start_matchignore
 m/You are now connected to database*/
 m/Settings:  default_hash_table_bucket_number=.*/
+m/Settings:  optimizer=.*/
 m/[O|o]ptimizer status:.*/
 s/public\|a\|table\|.*\|append only/public\|a\|table\|gpadmin\|append only/
 s/public\|aa\|table\|.*\|append only/public\|aa\|table\|gpadmin\|append only/

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/59757643/src/test/feature/gtest-parallel
--
diff --git a/src/test/feature/gtest-parallel b/src/test/feature/gtest-parallel
index 19c0aed..757c001 100755
--- a/src/test/feature/gtest-parallel
+++ b/src/test/feature/gtest-parallel
@@ -381,13 +381,14 @@ for test_binary in binaries:
   
   if options.gtest_filter :
 tests = do_gtest_filter(list_command, command, options.gtest_filter)
+parallel_id = 0
 if options.workers > 1:
   for test in tests:
 test_map[test[2]] = 'PARALLEL'
+  parallel_id = parallel_id + len(tests)
 else:
   for test in tests:
 test_map[test[2]] = 'SERIAL'
-parallel_id = len(tests)
 
 if options.failed:
   # The first element of each entry is the runtime of the most recent
@@ -468,9 +469,9 @@ def start_daemon(func):
   t.start()
   return t
 
-
 #run test in serail way
 job_id = parallel_id
+
 while True:
   job = None
   if job_id < len(tests):
@@ -482,13 +483,12 @@ while True:
 break
   times.record_test_time(test_binary, test, run_job(job))
 
-
+job_id = 0
 #run test in parallel way
 workers = [start_daemon(worker) for i in range(options.workers)]
 
 [t.join() for t in workers]
 
-
 logger.end()
 times.write_to_file(save_file)
 if options.print_test_times:

incubator-hawq git commit: HAWQ-1359. Fix bugs in policy tests for HAWQ with Ranger enabled and add test for HAWQ-1367

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1470d5302 -> 68013fac1


HAWQ-1359. Fix bugs in policy tests for HAWQ with Ranger enabled and add test 
for HAWQ-1367


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/68013fac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/68013fac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/68013fac

Branch: refs/heads/master
Commit: 68013fac13a757489d7a7ebe361065826ea5d66a
Parents: 1470d53
Author: Chunling Wang 
Authored: Fri Mar 3 11:23:32 2017 +0800
Committer: Chunling Wang 
Committed: Fri Mar 3 11:23:32 2017 +0800

--
 src/test/feature/Ranger/ans/adminfirst41.ans |  8 ++--
 src/test/feature/Ranger/ans/adminsecond41.ans| 16 ++--
 src/test/feature/Ranger/ans/normal35_success.ans |  2 +-
 src/test/feature/Ranger/ans/normal41_fail.ans|  8 
 src/test/feature/Ranger/ans/normal41_success.ans |  8 ++--
 src/test/feature/Ranger/ans/super63_fail.ans |  6 +++---
 src/test/feature/Ranger/sql/admin/41.sql |  2 +-
 src/test/feature/Ranger/sql/init_file|  4 
 src/test/feature/Ranger/sql/normal/41.sql|  2 +-
 src/test/feature/Ranger/sql/sqlsplitinput.sql|  2 +-
 src/test/feature/Ranger/test_ranger.cpp  | 12 ++--
 src/test/feature/gtest-parallel  | 13 +++--
 12 files changed, 34 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/adminfirst41.ans
--
diff --git a/src/test/feature/Ranger/ans/adminfirst41.ans 
b/src/test/feature/Ranger/ans/adminfirst41.ans
index d28e95d..ce0b96d 100644
--- a/src/test/feature/Ranger/ans/adminfirst41.ans
+++ b/src/test/feature/Ranger/ans/adminfirst41.ans
@@ -1,8 +1,4 @@
 -- start_ignore
 -- end_ignore
-select * from a order by i;
-1
-1
-1
-1
-1
+select * from pg_database, a order by oid, i limit 1;
+template1|10|6|t|t|-1|0|881|1663|16385|||1

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/adminsecond41.ans
--
diff --git a/src/test/feature/Ranger/ans/adminsecond41.ans 
b/src/test/feature/Ranger/ans/adminsecond41.ans
index 2930484..ce0b96d 100644
--- a/src/test/feature/Ranger/ans/adminsecond41.ans
+++ b/src/test/feature/Ranger/ans/adminsecond41.ans
@@ -1,16 +1,4 @@
 -- start_ignore
 -- end_ignore
-select * from a order by i;
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-2
+select * from pg_database, a order by oid, i limit 1;
+template1|10|6|t|t|-1|0|881|1663|16385|||1

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/normal35_success.ans
--
diff --git a/src/test/feature/Ranger/ans/normal35_success.ans 
b/src/test/feature/Ranger/ans/normal35_success.ans
index 0957278..cf37292 100644
--- a/src/test/feature/Ranger/ans/normal35_success.ans
+++ b/src/test/feature/Ranger/ans/normal35_success.ans
@@ -3,6 +3,6 @@
 set session role=usertest35;
 SET
 create table aa as select * from a order by i;
-psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING:  skipping "aa" --- error 
returned: permission denied for relation(s): aa
+psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING:  skipping "aa" --- error 
returned: permission denied for relation(s): public.aa
 psql:/tmp/TestHawqRanger_BasicTest.sql:4: INFO:  ANALYZE completed. Success: 
0, Failure: 1 (aa)
 SELECT 2

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68013fac/src/test/feature/Ranger/ans/normal41_fail.ans
--
diff --git a/src/test/feature/Ranger/ans/normal41_fail.ans 
b/src/test/feature/Ranger/ans/normal41_fail.ans
index 8ac9d96..095b94f 100644
--- a/src/test/feature/Ranger/ans/normal41_fail.ans
+++ b/src/test/feature/Ranger/ans/normal41_fail.ans
@@ -2,10 +2,10 @@
 -- end_ignore
 set session role=usertest41;
 SET
-select * from a order by i;
+select * from pg_database, a order by oid, i limit 1;
 psql:/tmp/TestHawqRanger_BasicTest.sql:4: WARNING:  usage privilege of 
namespace public is required.
-LINE 1: select * from a order by i;
+LINE 1: select * from pg_database, a order by oid, i limit 1;
   ^
 psql:/tmp/TestHawqRanger_BasicTest.sql:4: ERROR:  relation "a" does not exist
-LINE 1: select * from a order by i;
-  ^
+LINE 1: select * from pg_database, a order by oid, i limit 1;
+   ^

incubator-hawq git commit: HAWQ-1359. Fix rat-check issues.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 120892679 -> aece96726


HAWQ-1359. Fix rat-check issues.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aece9672
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aece9672
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aece9672

Branch: refs/heads/master
Commit: aece96726ae479677bcb2af0d41d6f962da3460a
Parents: 1208926
Author: hubertzhang <hubertzh...@apache.org>
Authored: Thu Mar 2 10:19:05 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Mar 2 10:19:05 2017 +0800

--
 src/test/feature/Ranger/sql/init_file |  14 ++
 src/test/feature/Ranger/sql/sqlsplit.cpp  |   2 +-
 src/test/feature/Ranger/sql/sqlsplitinput | 142 -
 src/test/feature/Ranger/sql/sqlsplitinput.sql | 142 +
 4 files changed, 157 insertions(+), 143 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aece9672/src/test/feature/Ranger/sql/init_file
--
diff --git a/src/test/feature/Ranger/sql/init_file 
b/src/test/feature/Ranger/sql/init_file
index 3c1fd76..5f4c68d 100644
--- a/src/test/feature/Ranger/sql/init_file
+++ b/src/test/feature/Ranger/sql/init_file
@@ -1,3 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 -- start_matchignore
 m/You are now connected to database*/
 s/public\|a\|table\|.*\|append only/public\|a\|table\|gpadmin\|append only/

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aece9672/src/test/feature/Ranger/sql/sqlsplit.cpp
--
diff --git a/src/test/feature/Ranger/sql/sqlsplit.cpp 
b/src/test/feature/Ranger/sql/sqlsplit.cpp
index 61ea00e..785c297 100644
--- a/src/test/feature/Ranger/sql/sqlsplit.cpp
+++ b/src/test/feature/Ranger/sql/sqlsplit.cpp
@@ -23,7 +23,7 @@
 
 int main() {
 FILE *fin, *fout, *foutsuper;
-fin = fopen("sqlsplitinput", "r");
+fin = fopen("sqlsplitinput.sql", "r");
 char line[1000];
 int i = 0;
 int isnormalquery = 0;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aece9672/src/test/feature/Ranger/sql/sqlsplitinput
--
diff --git a/src/test/feature/Ranger/sql/sqlsplitinput 
b/src/test/feature/Ranger/sql/sqlsplitinput
deleted file mode 100644
index 564b0a0..000
--- a/src/test/feature/Ranger/sql/sqlsplitinput
+++ /dev/null
@@ -1,142 +0,0 @@
-1
-create table a(i int);
-0
-create language plpythonu;
-0
-CREATE OR REPLACE FUNCTION f4() RETURNS TEXT AS $$ plpy.execute("select * from 
a order by i") $$ LANGUAGE plpythonu VOLATILE;
-1
-select * from f4();
-0
-drop function f4();
-0
-drop language plpythonu;
-1
-CREATE OR REPLACE FUNCTION normalize_si(text) RETURNS text AS $$ BEGIN RETURN 
substring($1, 9, 2) || substring($1, 7, 2) || substring($1, 5, 2) || 
substring($1, 1, 4); END; $$LANGUAGE 'plpgsql' IMMUTABLE;
-1
-CREATE OR REPLACE FUNCTION si_lt(text, text) RETURNS boolean AS $$ BEGIN 
RETURN normalize_si($1) < normalize_si($2); END; $$ LANGUAGE 'plpgsql' 
IMMUTABLE;
-1
-CREATE OPERATOR <# ( PROCEDURE=si_lt,LEFTARG=text, RIGHTARG=text);
-1
-CREATE OR REPLACE FUNCTION si_same(text, text) RETURNS int AS $$ BEGIN IF 
normalize_si($1) < normalize_si($2) THEN RETURN -1; END IF; END; $$ LANGUAGE 
'plpgsql' IMMUTABLE;
-0
-CREATE OPERATOR CLASS sva_special_ops FOR TYPE text USING btree AS OPERATOR 1 
<#, FUNCTION 1 si_same(text, text);
-0
-drop OPERATOR CLASS sva_special_ops USING btree;
-1
-drop OPERATOR <# (text,text) CASCADE;
-1
-drop FUNCTION si_same(text, text);
-1
-drop FUNCTION si_lt(text, text);
-1
-drop FUNCTION normalize_si(text);
-0
-CREATE RESOURCE QUEUE myqueue WITH (PARENT='pg_root', ACTIVE_STATEMENTS=20, 
MEMORY_LIMIT_CLUSTER=50%, CORE_LIMIT_CLUSTER=50%);   
-0
-DROP RESOURCE QUEUE myqueu

[1/6] incubator-hawq git commit: HAWQ-1359. Add policy test for HAWQ with Ranger enabled.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master ee79ec2fc -> 120892679


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/lib/psql.h
--
diff --git a/src/test/feature/lib/psql.h b/src/test/feature/lib/psql.h
index 7fe94c1..215a43b 100644
--- a/src/test/feature/lib/psql.h
+++ b/src/test/feature/lib/psql.h
@@ -69,7 +69,7 @@ class PSQL {
   virtual ~PSQL(){};
 
   PSQL& runSQLCommand(const std::string& sql_cmd);
-  PSQL& runSQLFile(const std::string& sql_file);
+  PSQL& runSQLFile(const std::string& sql_file, bool printTupleOnly = false);
   const PSQLQueryResult& getQueryResult(const std::string& sql);
 
   PSQL& setHost(const std::string& host);
@@ -96,7 +96,7 @@ class PSQL {
 
   const std::string _getPSQLBaseCommand() const;
   const std::string _getPSQLQueryCommand(const std::string& query) const;
-  const std::string _getPSQLFileCommand(const std::string& file) const;
+  const std::string _getPSQLFileCommand(const std::string& file, bool 
printTupleOnly = false) const;
 
   std::string _dbname;
   std::string _host;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/lib/sql_util.cpp
--
diff --git a/src/test/feature/lib/sql_util.cpp 
b/src/test/feature/lib/sql_util.cpp
index a19abea..b52c4f0 100644
--- a/src/test/feature/lib/sql_util.cpp
+++ b/src/test/feature/lib/sql_util.cpp
@@ -135,7 +135,9 @@ void SQLUtility::query(const string , const string 
) {
 
 void SQLUtility::execSQLFile(const string ,
  const string ,
- const string ) {
+ const string ,
+bool 
usingDefaultSchema,
+bool printTupleOnly) {
   FilePath fp;
 
   // do precheck for sqlFile & ansFile
@@ -152,12 +154,12 @@ void SQLUtility::execSQLFile(const string ,
 ASSERT_TRUE(false) << ansFileAbsPath << " is invalid";
 
   // generate new sql file with set search_path added at the begining
-  const string newSqlFile = generateSQLFile(sqlFile);
+  const string newSqlFile = generateSQLFile(sqlFile, usingDefaultSchema);
 
   // outFile is located in the same folder with ansFile
   string outFileAbsPath = fp.path + "/" + fp.fileBaseName + ".out";
   conn->setOutputFile(outFileAbsPath);
-  EXPECT_EQ(0, conn->runSQLFile(newSqlFile).getLastStatus());
+  EXPECT_EQ(0, conn->runSQLFile(newSqlFile, printTupleOnly).getLastStatus());
   conn->resetOutput();
 
   // initFile if any
@@ -197,19 +199,18 @@ bool SQLUtility::execSQLFile(const string ) {
   FilePath fp = splitFilePath(sqlFile);
   if (fp.fileBaseName.empty())
 return false;
-
   // outFile is located in the same folder with ansFile
   string outFileAbsPath = "/tmp/" + fp.fileBaseName + ".out";
 
   // generate new sql file with set search_path added at the begining
-  const string newSqlFile = generateSQLFile(sqlFile);
+  const string newSqlFile = generateSQLFile(sqlFile, false);
 
   // run sql file and store its result in output file
   conn->setOutputFile(outFileAbsPath);
   return conn->runSQLFile(newSqlFile).getLastStatus() == 0 ? true : false;
 }
 
-const string SQLUtility::generateSQLFile(const string ) {
+const string SQLUtility::generateSQLFile(const string , bool 
usingDefaultSchema) {
   const string originSqlFile = testRootPath + "/" + sqlFile;
   const string newSqlFile = "/tmp/" + string(test_info->test_case_name()) + 
"_" + test_info->name() + ".sql";
   std::fstream in;
@@ -223,7 +224,9 @@ const string SQLUtility::generateSQLFile(const string 
) {
 EXPECT_TRUE(false) << "Error opening file " << newSqlFile;
   }
   out << "-- start_ignore" << std::endl;
-  out << "SET SEARCH_PATH=" + schemaName + ";" << std::endl;
+  if (!usingDefaultSchema) {
+ out << "SET SEARCH_PATH=" + schemaName + ";" << std::endl;
+  }
   if (sql_util_mode ==  MODE_DATABASE) {
 out << "\\c " << databaseName << std::endl;
   }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/lib/sql_util.h
--
diff --git a/src/test/feature/lib/sql_util.h b/src/test/feature/lib/sql_util.h
index 03544a0..3bdce10 100644
--- a/src/test/feature/lib/sql_util.h
+++ b/src/test/feature/lib/sql_util.h
@@ -31,6 +31,7 @@
 #define HAWQ_USER (getenv("PGUSER") ? getenv("PGUSER") : "")
 #define HAWQ_PASSWORD (getenv("PGPASSWORD") ? getenv("PGPASSWORD") : "")
 #define HAWQ_DEFAULT_SCHEMA ("public")
+#define RANGER_HOST (getenv("RANGERHOST") ? getenv("RANGERHOST") : "localhost")
 
 namespace hawq {
 namespace test {
@@ -88,7 +89,9 @@ class SQLUtility {
   // @param ansFile The given ansFile which is relative path to test root dir
   // @param initFile The given initFile (used by gpdiff.pl) which is relative 

[3/6] incubator-hawq git commit: HAWQ-1359. Add policy test for HAWQ with Ranger enabled.

[hubertzhang]

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/Ranger/policy/39/2.json
--
diff --git a/src/test/feature/Ranger/policy/39/2.json 
b/src/test/feature/Ranger/policy/39/2.json
new file mode 100644
index 000..f21e2e5
--- /dev/null
+++ b/src/test/feature/Ranger/policy/39/2.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy39-2", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"insert"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": 
["usertest39", "usersuper39"]}], "resources": {"database": {"isExcludes": 
false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": 
{"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": 
{"isExcludes": false, "isRecursive": false, "values": ["a"]}}, "service": 
"hawq", "version": 1}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/Ranger/policy/4/1.json
--
diff --git a/src/test/feature/Ranger/policy/4/1.json 
b/src/test/feature/Ranger/policy/4/1.json
new file mode 100644
index 000..f4fc273
--- /dev/null
+++ b/src/test/feature/Ranger/policy/4/1.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy4-1", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"usage-schema"}], "conditions": [], "delegateAdmin": true, "groups": null, 
"users": ["usertest4", "usersuper4"]}], "resources": {"database": 
{"isExcludes": false, "isRecursive": false, "values": 
["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": 
false, "values": ["public"]}, "table": {"isExcludes": false, "isRecursive": 
false, "values": ["*"]}}, "service": "hawq", "version": 1}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/Ranger/policy/4/2.json
--
diff --git a/src/test/feature/Ranger/policy/4/2.json 
b/src/test/feature/Ranger/policy/4/2.json
new file mode 100644
index 000..33b40e4
--- /dev/null
+++ b/src/test/feature/Ranger/policy/4/2.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy4-2", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"select"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": 
["usertest4", "usersuper4"]}], "resources": {"database": {"isExcludes": false, 
"isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": 
{"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": 
{"isExcludes": false, "isRecursive": false, "values": ["a"]}}, "service": 
"hawq", "version": 1}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/Ranger/policy/4/3.json
--
diff --git a/src/test/feature/Ranger/policy/4/3.json 
b/src/test/feature/Ranger/policy/4/3.json
new file mode 100644
index 000..cf5fe38
--- /dev/null
+++ b/src/test/feature/Ranger/policy/4/3.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy4-3", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"execute"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": 
["usertest4", "usersuper4"]}], "resources": {"database": {"isExcludes": false, 
"isRecursive": false, "values": ["hawq_feature_test_db"]}, "function": 
{"isExcludes": false, "isRecursive": false, "values": ["f4"]}, "schema": 
{"isExcludes": false, "isRecursive": false, "values": ["public"]}}, "service": 
"hawq", "version": 1}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12089267/src/test/feature/Ranger/policy/40/1.json
--
diff --git a/src/test/feature/Ranger/policy/40/1.json 
b/src/test/feature/Ranger/policy/40/1.json
new file mode 100644
index 000..217d364
--- /dev/null
+++ b/src/test/feature/Ranger/policy/40/1.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], 
"description": "no description", "isAuditEnabled": true, "isEnabled": true, 
"name": "policy40-1", "policyItems": [{"accesses": [{"isAllowed": true, "type": 
"usage"}, {"isAllowed": true, "type": "update"}], "conditions": [], 
"delegateAdmin": true, "groups": null, "users": ["usertest40", 

[6/6] incubator-hawq git commit: HAWQ-1359. Add policy test for HAWQ with Ranger enabled.

[hubertzhang]

HAWQ-1359. Add policy test for HAWQ with Ranger enabled.

Signed-off-by: Chuling Wang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/12089267
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/12089267
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/12089267

Branch: refs/heads/master
Commit: 1208926791cc3b30cc64e243e57edf86a9794af1
Parents: ee79ec2
Author: Hubert Zhang 
Authored: Thu Mar 2 09:53:39 2017 +0800
Committer: Chuling Wang 
Committed: Thu Mar 2 09:53:39 2017 +0800

--
 .../feature/ExternalSource/test_errortbl.cpp|   8 +
 src/test/feature/Ranger/ans/adminfirst1.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst10.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst11.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst12.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst13.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst14.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst15.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst16.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst17.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst18.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst19.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst2.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst20.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst21.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst22.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst23.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst24.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst25.ans|   5 +
 src/test/feature/Ranger/ans/adminfirst26.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst27.ans|  29 
 src/test/feature/Ranger/ans/adminfirst28.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst29.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst3.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst30.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst31.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst32.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst33.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst34.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst35.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst36.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst37.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst38.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst39.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst4.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst40.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst41.ans|   8 +
 src/test/feature/Ranger/ans/adminfirst42.ans|   6 +
 src/test/feature/Ranger/ans/adminfirst43.ans|   8 +
 src/test/feature/Ranger/ans/adminfirst44.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst45.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst46.ans|   6 +
 src/test/feature/Ranger/ans/adminfirst47.ans|   5 +
 src/test/feature/Ranger/ans/adminfirst48.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst49.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst5.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst50.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst51.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst52.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst53.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst54.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst55.ans|   9 ++
 src/test/feature/Ranger/ans/adminfirst56.ans|   5 +
 src/test/feature/Ranger/ans/adminfirst57.ans|   8 +
 src/test/feature/Ranger/ans/adminfirst58.ans|   9 ++
 src/test/feature/Ranger/ans/adminfirst59.ans|   8 +
 src/test/feature/Ranger/ans/adminfirst6.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst60.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst61.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst62.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst63.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst64.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst65.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst66.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst67.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst68.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst69.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst7.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst70.ans|   4 +
 src/test/feature/Ranger/ans/adminfirst71.ans|   5 +
 src/test/feature/Ranger/ans/adminfirst8.ans |   4 +
 src/test/feature/Ranger/ans/adminfirst9.ans |   4 +
 src/test/feature/Ranger/ans/adminsecond1.ans|   4 +
 src/test/feature/Ranger/ans/adminsecond10.ans   |   4 +
 

incubator-hawq git commit: HAWQ-1360. Check privilege of sequence pass the wrong type to RPS.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master c7717d438 -> 8f4d0f522


HAWQ-1360. Check privilege of sequence pass the wrong type to RPS.

Signed-off-by: Chuling Wang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8f4d0f52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8f4d0f52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8f4d0f52

Branch: refs/heads/master
Commit: 8f4d0f522dbf837bc9a5708b5270dd3c00a0d299
Parents: c7717d4
Author: Hubert Zhang 
Authored: Tue Feb 28 10:59:40 2017 +0800
Committer: Chuling Wang 
Committed: Tue Feb 28 10:59:40 2017 +0800

--
 src/backend/catalog/aclchk.c | 71 +++
 src/include/utils/acl.h  |  1 -
 2 files changed, 20 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8f4d0f52/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index b968a2f..b361beb 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -82,6 +82,8 @@ static AclMode restrict_and_check_grant(bool is_grant, 
AclMode avail_goptions,
 static AclMode pg_aclmask(AclObjectKind objkind, Oid table_oid, Oid roleid,
   AclMode mask, AclMaskHow how);
 
+static bool is_sequence(Oid object_oid);
+
 
 #ifdef ACLDEBUG
 static void
@@ -2359,54 +2361,6 @@ char *getClassNameFromOid(Oid object_oid)
   return tname.data;
 }
 
-char *getSequenceNameFromOid(Oid object_oid)
-{
-  StringInfoData tname;
-  initStringInfo();
-
-  Assert(OidIsValid(object_oid));
-  char* seq_name = caql_getcstring(
-  NULL,
-  cql("SELECT relname FROM pg_class "
-" WHERE oid = :1",
-ObjectIdGetDatum(object_oid)));
-  if (seq_name == NULL)
-   elog(ERROR, "oid [%u] not found in table pg_class", object_oid);
-
-  int fetchCount=0;
-  Oid schema_name_oid = caql_getoid_plus(
-NULL,
-,
-NULL,
-cql("SELECT relnamespace FROM pg_class "
-  " WHERE oid = :1",
-  ObjectIdGetDatum(object_oid)));
-  if (schema_name_oid == InvalidOid)
- elog(ERROR, "oid [%u] not found in table pg_class", object_oid);
-
-  char* schema_name= caql_getcstring(
- NULL,
- cql("select nspname from pg_namespace "
-   " WHERE oid = :1",
-   ObjectIdGetDatum(schema_name_oid)));
-  if (schema_name == NULL)
- elog(ERROR, "oid [%u] not found in table pg_namespace", object_oid);
-
-  char* database_name = get_database_name(MyDatabaseId);
-  if (database_name == NULL)
-  elog(ERROR, "oid [%u] not found current database", object_oid);
-
-  appendStringInfo(, "%s", database_name);
-  appendStringInfoChar(, '.');
-  appendStringInfo(, "%s", schema_name);
-  appendStringInfoChar(, '.');
-  appendStringInfo(, "%s", seq_name);
-  pfree(seq_name);
-  pfree(schema_name);
-  pfree(database_name);
-
-  return tname.data;
-}
 char *getDatabaseNameFromOid(Oid object_oid)
 {
   Assert(OidIsValid(object_oid));
@@ -2671,9 +2625,8 @@ char *getNameFromOid(AclObjectKind objkind, Oid 
object_oid)
   switch (objkind)
   {
 case ACL_KIND_CLASS:
-  return getClassNameFromOid(object_oid);
 case ACL_KIND_SEQUENCE:
-  return getSequenceNameFromOid(object_oid);
+  return getClassNameFromOid(object_oid);
 case ACL_KIND_DATABASE:
   return getDatabaseNameFromOid(object_oid);
 case ACL_KIND_PROC:
@@ -2817,6 +2770,18 @@ bool fallBackToNativeChecks(AclObjectKind objkind, List* 
table_list, Oid roleid)
 }
 
 /*
+ * check whether rte is a sequence.
+ */
+bool is_sequence(Oid object_oid) {
+   char relkind = get_rel_relkind(object_oid);
+   if(relkind == 's' || relkind == 'S')
+   {
+   return true;
+   }
+   return false;
+}
+
+/*
  * return: List of RangerPrivilegeResults 
  * arg_list: List of RangerPrivilegeArgs
  */
@@ -3940,7 +3905,11 @@ pg_class_aclcheck(Oid table_oid, Oid roleid, AclMode 
mode)
 
   if(aclType == HAWQ_ACL_RANGER && !fallBackToNativeCheck(ACL_KIND_CLASS, 
table_oid, roleid, mode))
   {
-return pg_rangercheck(ACL_KIND_CLASS, table_oid, roleid, mode, 
ACLMASK_ANY);
+   AclObjectKind objkind = ACL_KIND_CLASS;
+   if (is_sequence(table_oid)) {
+   objkind = ACL_KIND_SEQUENCE;
+   }
+return pg_rangercheck(objkind, table_oid, roleid, mode, ACLMASK_ANY);
   }
   else
   {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8f4d0f52/src/include/utils/acl.h

incubator-hawq git commit: HAWQ-1328. Add deny and exclude policy template for hawq service in ranger.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master cc77b5de0 -> c7717d438


HAWQ-1328. Add deny and exclude policy template for hawq service in ranger.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c7717d43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c7717d43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c7717d43

Branch: refs/heads/master
Commit: c7717d43817f781c6ac9a8a7c256f509081fc4da
Parents: cc77b5d
Author: hubertzhang <hubertzh...@apache.org>
Authored: Tue Feb 14 14:20:13 2017 +0800
Committer: chunlingwang and hubertzhang 
<pair+chunlingwang+hubertzh...@apache.org>
Committed: Tue Feb 28 10:53:17 2017 +0800

--
 ranger-plugin/conf/ranger-servicedef-hawq.json | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c7717d43/ranger-plugin/conf/ranger-servicedef-hawq.json
--
diff --git a/ranger-plugin/conf/ranger-servicedef-hawq.json 
b/ranger-plugin/conf/ranger-servicedef-hawq.json
index 5e22ad1..38f88ea 100644
--- a/ranger-plugin/conf/ranger-servicedef-hawq.json
+++ b/ranger-plugin/conf/ranger-servicedef-hawq.json
@@ -4,6 +4,7 @@
   "label": "HAWQ",
   "description": "HAWQ",
   "guid": "1ebb27ee-549a-401d-85ab-818342ca54af",
+  "options": {"enableDenyAndExceptionsInPolicies": "true"},
   "resources":
   [
 {

incubator-hawq git commit: HAWQ-1356. Add waring when user does not have usage privilege of namespace.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 21d78d37a -> 52d5b553d


HAWQ-1356. Add waring when user does not have usage privilege of namespace.

Signed-off-by: Chuling Wang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/52d5b553
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/52d5b553
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/52d5b553

Branch: refs/heads/master
Commit: 52d5b553d6f3f8729e8c7eab227cbc3b4b3dfe0e
Parents: 21d78d3
Author: Hubert Zhang 
Authored: Fri Feb 24 10:31:56 2017 +0800
Committer: Chuling Wang 
Committed: Fri Feb 24 10:31:56 2017 +0800

--
 src/backend/catalog/namespace.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52d5b553/src/backend/catalog/namespace.c
--
diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c
index 3a90054..e4ee588 100644
--- a/src/backend/catalog/namespace.c
+++ b/src/backend/catalog/namespace.c
@@ -1939,7 +1939,7 @@ static void checkNamespaceInternal(List **oidlist, Oid 
namespaceId, Oid roleid)
*oidlist = lappend_oid(*oidlist, namespaceId);
}
else {
-   if (OidIsValid(namespaceId)) {
+   if (OidIsValid(namespaceId) && aclType == HAWQ_ACL_RANGER) {
elog(WARNING, "usage privilege of namespace %s is 
required.",
getNamespaceNameByOid(namespaceId));
}

incubator-hawq git commit: HAWQ-1358. Refactor gpfdist library in featuretest.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master b939206e1 -> b65d7561f


HAWQ-1358. Refactor gpfdist library in featuretest.

Signed-off-by: Chuling Wang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b65d7561
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b65d7561
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b65d7561

Branch: refs/heads/master
Commit: b65d7561fab6bbdfb48f10b4ca8f76980f8db0ef
Parents: b939206
Author: Hubert Zhang 
Authored: Thu Feb 23 16:48:27 2017 +0800
Committer: Chuling Wang 
Committed: Thu Feb 23 16:48:27 2017 +0800

--
 src/test/feature/lib/gpfdist.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b65d7561/src/test/feature/lib/gpfdist.cpp
--
diff --git a/src/test/feature/lib/gpfdist.cpp b/src/test/feature/lib/gpfdist.cpp
index 59b9df9..fbd4cf1 100644
--- a/src/test/feature/lib/gpfdist.cpp
+++ b/src/test/feature/lib/gpfdist.cpp
@@ -42,7 +42,7 @@ void GPfdist::init_gpfdist() {
 
util->execute(
"CREATE EXTERNAL WEB TABLE gpfdist_stop (x text) "
-   "execute E'(/bin/pkill gpfdist || 
killall gpfdist) > /dev/null 2>&1; echo stopping...' "
+   "execute E'(pkill gpfdist || killall 
gpfdist) > /dev/null 2>&1; echo stopping...' "
"on SEGMENT 0 "
"FORMAT 'text' (delimiter '|');");
util->execute("select * from gpfdist_stop;");

incubator-hawq git commit: HAWQ-1358. Refactor gpfdist library in featuretest.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 62392a2bf -> b939206e1


HAWQ-1358. Refactor gpfdist library in featuretest.

Signed-off-by: Hubert Zhang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b939206e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b939206e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b939206e

Branch: refs/heads/master
Commit: b939206e1c4c2de426c9f6847dff0ac2d51422c1
Parents: 62392a2
Author: Chuling Wang 
Authored: Thu Feb 23 16:45:21 2017 +0800
Committer: Hubert Zhang 
Committed: Thu Feb 23 16:45:21 2017 +0800

--
 .../feature/ExternalSource/test_errortbl.cpp| 45 ++---
 src/test/feature/lib/gpfdist.cpp| 71 
 src/test/feature/lib/gpfdist.h  | 50 ++
 3 files changed, 125 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b939206e/src/test/feature/ExternalSource/test_errortbl.cpp
--
diff --git a/src/test/feature/ExternalSource/test_errortbl.cpp 
b/src/test/feature/ExternalSource/test_errortbl.cpp
index 53a6dc0..a11a8df 100644
--- a/src/test/feature/ExternalSource/test_errortbl.cpp
+++ b/src/test/feature/ExternalSource/test_errortbl.cpp
@@ -20,6 +20,7 @@
 
 #include "lib/sql_util.h"
 #include "lib/string_util.h"
+#include "lib/gpfdist.h"
 
 using hawq::test::SQLUtility;
 
@@ -32,48 +33,10 @@ class TestErrorTable : public ::testing::Test {
 TEST_F(TestErrorTable, TestErrorTableAll) {
   
   SQLUtility util;
-  auto init_gpfdist = [&] () {
-auto sql = "CREATE EXTERNAL WEB TABLE gpfdist_status (x text) "
-"execute E'( python %s/bin/lib/gppinggpfdist.py localhost:7070 2>&1 || 
echo) ' "
-"on SEGMENT 0 "
-"FORMAT 'text' (delimiter '|');";
-auto GPHOME = getenv("GPHOME");
-util.execute(hawq::test::stringFormat(sql, GPHOME));
 
-sql = "CREATE EXTERNAL WEB TABLE gpfdist_start (x text) "
-"execute E'((%s/bin/gpfdist -p 7070 -d %s  /dev/null 2>&1 
&); sleep 2; echo \"starting\"...) ' "
-"on SEGMENT 0 "
-"FORMAT 'text' (delimiter '|');";
-std::string path = util.getTestRootPath() + "/ExternalSource/data";
-util.execute(hawq::test::stringFormat(sql, GPHOME, path.c_str()));
-  
-util.execute(
-"CREATE EXTERNAL WEB TABLE gpfdist_stop (x text) "
-"execute E'(/bin/pkill gpfdist || killall gpfdist) > /dev/null 2>&1; 
echo stopping...' "
-"on SEGMENT 0 "
-"FORMAT 'text' (delimiter '|');");
-util.execute("select * from gpfdist_stop;");
-util.execute("select * from gpfdist_status;");
-util.execute("select * from gpfdist_start;");
-util.execute("select * from gpfdist_status;");
-  };
-
-  auto finalize_gpfdist = [&] () {
-util.execute("drop external table EXT_NATION_WITH_EXIST_ERROR_TABLE;");
-util.execute("drop external table EXT_NATION1;");
-util.execute("drop table EXT_NATION_ERROR1 CASCADE;");
-util.execute("drop external table EXT_NATION2;");
-util.execute("drop table EXT_NATION_ERROR2 CASCADE;");
-util.execute("drop external table EXT_NATION3;");
-util.execute("drop table EXT_NATION_ERROR3 CASCADE;");
-util.execute("select * from gpfdist_stop;");
-util.execute("select * from gpfdist_status;");
-util.execute("drop external table gpfdist_status;");
-util.execute("drop external table gpfdist_start;");
-util.execute("drop external table gpfdist_stop;");
-  };
+  hawq::test::GPfdist gpdfist();
 
-  init_gpfdist();
+  gpdfist.init_gpfdist();
   
   // readable external table with error table
   util.execute(
@@ -139,5 +102,5 @@ TEST_F(TestErrorTable, TestErrorTableAll) {
   "LOG ERRORS INTO EXT_NATION_ERROR_WRITABLE SEGMENT REJECT LIMIT 5;",
   false);
 
-  finalize_gpfdist();
+  gpdfist.finalize_gpfdist();
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b939206e/src/test/feature/lib/gpfdist.cpp
--
diff --git a/src/test/feature/lib/gpfdist.cpp b/src/test/feature/lib/gpfdist.cpp
new file mode 100644
index 000..59b9df9
--- /dev/null
+++ b/src/test/feature/lib/gpfdist.cpp
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License 

incubator-hawq git commit: HAWQ-1357. Super user also need to check create privilege of public schema from Ranger.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master fada5f0b2 -> 62392a2bf


HAWQ-1357. Super user also need to check create privilege of public schema from 
Ranger.

Signed-off-by: Hubert Zhang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/62392a2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/62392a2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/62392a2b

Branch: refs/heads/master
Commit: 62392a2bf06ed639105474a046c7cc75f4cefa7a
Parents: fada5f0
Author: Chuling Wang 
Authored: Thu Feb 23 16:43:40 2017 +0800
Committer: Hubert Zhang 
Committed: Thu Feb 23 16:43:40 2017 +0800

--
 src/backend/catalog/aclchk.c | 53 ---
 src/include/utils/acl.h  |  2 +-
 2 files changed, 28 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/62392a2b/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index f6ac590..b968a2f 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -228,18 +228,19 @@ restrict_and_check_grant(bool is_grant, AclMode 
avail_goptions, bool all_privs,
 */
if (avail_goptions == ACL_NO_RIGHTS && Gp_role != GP_ROLE_EXECUTE)
{
- if (aclType == HAWQ_ACL_RANGER && !fallBackToNativeCheck(objkind, 
objectId, grantorId)) {
-   if (pg_rangercheck(objkind, objectId, grantorId,
-   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
-   ACLMASK_ANY) != ACLCHECK_OK)
- aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
- }
- else {
-   if (pg_aclmask(objkind, objectId, grantorId,
-   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
-   ACLMASK_ANY) == ACL_NO_RIGHTS)
- aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
- }
+   if (aclType == HAWQ_ACL_RANGER && 
!fallBackToNativeCheck(objkind, objectId,
+   grantorId, whole_mask | 
ACL_GRANT_OPTION_FOR(whole_mask))) {
+   if (pg_rangercheck(objkind, objectId, grantorId,
+   whole_mask | 
ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) != ACLCHECK_OK)
+   aclcheck_error(ACLCHECK_NO_PRIV, objkind, 
objname);
+   }
+   else {
+   if (pg_aclmask(objkind, objectId, grantorId,
+   whole_mask | 
ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) == ACL_NO_RIGHTS)
+   aclcheck_error(ACLCHECK_NO_PRIV, objkind, 
objname);
+   }
}
 
/*
@@ -306,7 +307,7 @@ bool checkACLNative(GrantObjectType type, Oid oid)
{
return false;
}
-   return fallBackToNativeCheck(kind, oid, GetUserId());
+   return fallBackToNativeCheck(kind, oid, GetUserId(), ACL_NO_RIGHTS);
 }
 
 /*
@@ -2751,7 +2752,7 @@ bool checkNamespaceFallback(Oid x)
   }
 }
 
-bool fallBackToNativeCheck(AclObjectKind objkind, Oid obj_oid, Oid roleid)
+bool fallBackToNativeCheck(AclObjectKind objkind, Oid obj_oid, Oid roleid, 
AclMode mode)
 {
   /* get the latest information_schema_namespcace_oid. Since caql access heap 
table
* directly without aclcheck, this function will not be called recursively
@@ -2776,7 +2777,7 @@ bool fallBackToNativeCheck(AclObjectKind objkind, Oid 
obj_oid, Oid roleid)
 {
   return true;
 }
-else if (obj_oid == PG_PUBLIC_NAMESPACE && superuser())
+else if (obj_oid == PG_PUBLIC_NAMESPACE && superuser() && mode == 
ACL_USAGE)
 {
   /* superuser's access to PUBLIC */
   return true;
@@ -2804,7 +2805,7 @@ bool fallBackToNativeChecks(AclObjectKind objkind, List* 
table_list, Oid roleid)
 foreach(l, table_list)
 {
   RangeTblEntry *rte=(RangeTblEntry *) lfirst(l);
-  bool ret = fallBackToNativeCheck(ACL_KIND_CLASS, rte->relid, roleid);
+  bool ret = fallBackToNativeCheck(ACL_KIND_CLASS, rte->relid, roleid, 
ACL_NO_RIGHTS);
   if(ret)
   {
 return true;
@@ -3937,7 +3938,7 @@ pg_class_aclcheck(Oid table_oid, Oid roleid, AclMode mode)
   if (Gp_role == GP_ROLE_EXECUTE)
 return ACLCHECK_OK;
 
-  if(aclType == HAWQ_ACL_RANGER && !fallBackToNativeCheck(ACL_KIND_CLASS, 
table_oid, roleid))
+  if(aclType == HAWQ_ACL_RANGER && !fallBackToNativeCheck(ACL_KIND_CLASS, 
table_oid, roleid, mode))
   {
 return pg_rangercheck(ACL_KIND_CLASS, table_oid, roleid, mode, 

incubator-hawq git commit: HAWQ-1356. Add waring when user does not have usage privilege of namespace.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master b37f18768 -> fada5f0b2


HAWQ-1356. Add waring when user does not have usage privilege of namespace.

Signed-off-by: Hubert Zhang 


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fada5f0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fada5f0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fada5f0b

Branch: refs/heads/master
Commit: fada5f0b28029af783dffddaa45a5cceb647584a
Parents: b37f187
Author: Chuling Wang 
Authored: Thu Feb 23 16:40:33 2017 +0800
Committer: Hubert Zhang 
Committed: Thu Feb 23 16:40:33 2017 +0800

--
 src/backend/catalog/namespace.c | 36 
 1 file changed, 28 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fada5f0b/src/backend/catalog/namespace.c
--
diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c
index e67570e..3a90054 100644
--- a/src/backend/catalog/namespace.c
+++ b/src/backend/catalog/namespace.c
@@ -167,6 +167,7 @@ char   *namespace_search_path = NULL;
 
 /* Local functions */
 static void recomputeNamespacePath(void);
+static void checkNamespaceInternal(List **oidlist, Oid namespaceId, Oid 
roleid);
 static void InitTempTableNamespace(void);
 static void RemoveTempRelations(Oid tempNamespaceId);
 static void RemoveTempRelationsCallback(int code, Datum arg);
@@ -1925,6 +1926,25 @@ FindDefaultConversionProc(int4 for_encoding, int4 
to_encoding)
return InvalidOid;
 }
 
+
+
+/*
+ *
+ */
+static void checkNamespaceInternal(List **oidlist, Oid namespaceId, Oid roleid)
+{
+   if (pg_namespace_aclcheck(namespaceId, roleid,
+   ACL_USAGE) == ACLCHECK_OK)
+   {
+   *oidlist = lappend_oid(*oidlist, namespaceId);
+   }
+   else {
+   if (OidIsValid(namespaceId)) {
+   elog(WARNING, "usage privilege of namespace %s is 
required.",
+   getNamespaceNameByOid(namespaceId));
+   }
+   }
+}
 /*
  * recomputeNamespacePath - recompute path derived variables if needed.
  */
@@ -2006,10 +2026,10 @@ recomputeNamespacePath(void)
namespaceId = LookupInternalNamespaceId(rname);
 
if (OidIsValid(namespaceId) &&
-   !list_member_oid(oidlist, namespaceId) 
&&
-   pg_namespace_aclcheck(namespaceId, 
roleid,
-   
  ACL_USAGE) == ACLCHECK_OK)
-   oidlist = lappend_oid(oidlist, 
namespaceId);
+   !list_member_oid(oidlist, 
namespaceId))
+   {
+   checkNamespaceInternal(, 
namespaceId, roleid);
+   }
}
}
else if (strcmp(curname, "pg_temp") == 0)
@@ -2034,10 +2054,10 @@ recomputeNamespacePath(void)
namespaceId = LookupInternalNamespaceId(curname);
 
if (OidIsValid(namespaceId) &&
-   !list_member_oid(oidlist, namespaceId) &&
-   pg_namespace_aclcheck(namespaceId, roleid,
- 
ACL_USAGE) == ACLCHECK_OK)
-   oidlist = lappend_oid(oidlist, namespaceId);
+   !list_member_oid(oidlist, namespaceId))
+   {
+   checkNamespaceInternal(, namespaceId, 
roleid);
+   }
}
}
 

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ran5 [deleted] 0edc300de

incubator-hawq git commit: HAWQ-1291. Fix the name of privilege when create temp table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master d2608dec4 -> eb2ea9074


HAWQ-1291. Fix the name of privilege when create temp table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/eb2ea907
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/eb2ea907
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/eb2ea907

Branch: refs/heads/master
Commit: eb2ea9074460d911eeed5205970bd4076188c3fc
Parents: d2608de
Author: hzhang2 
Authored: Tue Jan 24 14:57:13 2017 +0800
Committer: hzhang2 
Committed: Tue Jan 24 14:57:13 2017 +0800

--
 src/backend/catalog/aclchk.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eb2ea907/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index ed36330..33fa9ab 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2646,7 +2646,7 @@ char *getNameFromOid(AclObjectKind objkind, Oid 
object_oid)
 
 char actionName[12][12] = {"INSERT","SELECT","UPDATE", "DELETE",
 "TRUNCATE", "REFERENCES", "TRIGGER", "EXECUTE", "USAGE",
-"CREATE", "CREATE_TEMP", "CONNECT"};
+"CREATE", "TEMP", "CONNECT"};
 
 List *getActionName(AclMode mask)
 {

incubator-hawq git commit: HAWQ-1282. Shared Input Scan may result in endless loop.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 326fa4f91 -> efa1230a9


HAWQ-1282. Shared Input Scan may result in endless loop.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/efa1230a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/efa1230a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/efa1230a

Branch: refs/heads/master
Commit: efa1230a95133c954f843f4ced46fab6acdfd7d2
Parents: 326fa4f
Author: hubertzhang <hubertzh...@apache.org>
Authored: Thu Jan 19 13:58:48 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Jan 19 16:02:32 2017 +0800

--
 src/backend/executor/nodeShareInputScan.c | 17 -
 1 file changed, 16 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/efa1230a/src/backend/executor/nodeShareInputScan.c
--
diff --git a/src/backend/executor/nodeShareInputScan.c 
b/src/backend/executor/nodeShareInputScan.c
index cb303be..0f08848 100644
--- a/src/backend/executor/nodeShareInputScan.c
+++ b/src/backend/executor/nodeShareInputScan.c
@@ -640,8 +640,13 @@ read_retry:
else if(sz == 0 || errno == EINTR)
goto read_retry;
else
+   {
+   if(fd >= 0)
+   {
+   gp_retry_close(fd);
+   }
elog(ERROR, "could not read from fifo: %m");
-
+   }
Assert(!"Never be here");
return 0;
 }
@@ -658,7 +663,13 @@ write_retry:
else if(sz == 0 || errno == EINTR)
goto write_retry;
else
+   {
+   if(fd >= 0)
+   {
+   gp_retry_close(fd);
+   }
elog(ERROR, "could not write to fifo: %m");
+   }
 
Assert(!"Never be here");
return 0;
@@ -914,6 +925,10 @@ writer_wait_for_acks(ShareInput_Lk_Context *pctxt, int 
share_id, int xslice)
int save_errno = errno;
elog(LOG, "SISC WRITER (shareid=%d, slice=%d): notify 
still wait for an answer, errno %d",
share_id, currentSliceId, save_errno);
+   /*if error(except EINTR) happens in select, we just 
return to avoid endless loop*/
+   if(errno != EINTR){
+   return;
+   }
}
}
 }

incubator-hawq git commit: HAWQ-1276. hawq should error out directly when ranger plugin service is unavailable, not a warning.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 21f1e29a4 -> 326fa4f91


HAWQ-1276. hawq should error out directly when ranger plugin service is 
unavailable, not a warning.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/326fa4f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/326fa4f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/326fa4f9

Branch: refs/heads/master
Commit: 326fa4f918b2be14adb3d667aea6cd707a24fa16
Parents: 21f1e29
Author: stanlyxiang <stanly.sxi...@gmail.com>
Authored: Wed Jan 18 15:40:12 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Jan 19 11:54:40 2017 +0800

--
 src/backend/libpq/rangerrest.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/326fa4f9/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index dc5d193..c920575 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -425,7 +425,7 @@ static int call_ranger_rest(CURL_HANDLE curl_handle, const 
char* request)
/* check for errors */
if(res != CURLE_OK)
{
-   elog(WARNING, "ranger plugin service from http://%s:%d/%s is 
unavailable : %s.\n",
+   elog(ERROR, "ranger plugin service from http://%s:%d/%s is 
unavailable : %s.\n",
rps_addr_host, rps_addr_port, rps_addr_suffix, 
curl_easy_strerror(res));
}
else
@@ -467,7 +467,7 @@ int check_privilege_from_ranger(List *request_list, List 
*result_list)
int ret = parse_ranger_response(curl_context_ranger.response.buffer, 
result_list);
if (ret < 0)
{
-   elog(WARNING, "parse ranger response failed, ranger response 
content is %s",
+   elog(ERROR, "parse ranger response failed, ranger response 
content is %s",
curl_context_ranger.response.buffer == NULL? 
"empty.":curl_context_ranger.response.buffer);
}
if (curl_context_ranger.response.buffer != NULL)

incubator-hawq git commit: HAWQ-1279. Force to recompute namespace_path when enable_ranger

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 8261c13ef -> 21f1e29a4


HAWQ-1279. Force to recompute namespace_path when enable_ranger


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/21f1e29a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/21f1e29a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/21f1e29a

Branch: refs/heads/master
Commit: 21f1e29a43aee003e250e41375bf226fabbe2ce3
Parents: 8261c13
Author: interma <inte...@outlook.com>
Authored: Wed Jan 18 13:13:52 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Thu Jan 19 10:56:01 2017 +0800

--
 src/backend/catalog/namespace.c | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/21f1e29a/src/backend/catalog/namespace.c
--
diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c
index 4685bfa..7049d32 100644
--- a/src/backend/catalog/namespace.c
+++ b/src/backend/catalog/namespace.c
@@ -1933,7 +1933,12 @@ recomputeNamespacePath(void)
 * Do nothing if path is already valid.
 */
if (namespaceSearchPathValid && namespaceUser == roleid)
-   return;
+   {
+   if (!enable_ranger)
+   return;
+   else
+   elog(DEBUG3, "recompute search_path[%s] when 
enable_ranger", namespace_search_path);
+   }
 
/* Need a modifiable copy of namespace_search_path string */
rawname = pstrdup(namespace_search_path);

incubator-hawq git commit: HAWQ-1243. Add suffix name for ranger restful service.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master cf54c4180 -> ec7b4d9e9


HAWQ-1243. Add suffix name for ranger restful service.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ec7b4d9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ec7b4d9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ec7b4d9e

Branch: refs/heads/master
Commit: ec7b4d9e96834d86a1a7196c17e5068c
Parents: cf54c41
Author: hubertzhang <hubertzh...@apache.org>
Authored: Mon Jan 16 17:13:38 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Mon Jan 16 17:13:38 2017 +0800

--
 src/backend/utils/misc/guc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ec7b4d9e/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 21d705a..e87d514 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -8202,7 +8202,7 @@ static struct config_string ConfigureNamesString[] =
   NULL
 },
 _addr_suffix,
-"hawq", NULL, NULL
+"rps", NULL, NULL
   },
 
{

incubator-hawq git commit: HAWQ-1275. Check build-in catalogs, tables and functions in native aclcheck.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 0bc2c8c01 -> cf54c4180


HAWQ-1275. Check build-in catalogs, tables and functions in native aclcheck.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf54c418
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf54c418
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf54c418

Branch: refs/heads/master
Commit: cf54c41809627f5b9b38eb6322947ef12439b0af
Parents: 0bc2c8c
Author: hubertzhang <hubertzh...@apache.org>
Authored: Mon Jan 16 16:01:39 2017 +0800
Committer: hubertzhang <hubertzh...@apache.org>
Committed: Mon Jan 16 16:01:39 2017 +0800

--
 src/backend/catalog/aclchk.c| 43 ++--
 src/backend/utils/cache/lsyscache.c | 24 ++
 src/backend/utils/misc/guc.c| 13 +-
 src/include/catalog/pg_namespace.h  |  1 -
 src/include/utils/guc.h |  3 +++
 src/include/utils/lsyscache.h   |  1 +
 6 files changed, 76 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf54c418/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 73de11b..200d9cb 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2669,29 +2669,58 @@ List *getActionName(AclMode mask)
 
 bool fallBackToNativeCheck(AclObjectKind objkind, Oid obj_oid, Oid roleid)
 {
-  //for heap table, we fall back to native check.
-  if(objkind == ACL_KIND_CLASS)
+  /* get the latest information_schema_namespcace_oid. Since caql access heap 
table
+   * directly without aclcheck, this function will not be called recursively
+   */
+  if (information_schema_namespcace_oid == 0)
+  {
+ information_schema_namespcace_oid = 
(int)get_namespace_oid("information_schema");
+  }
+  /*for heap table, we fall back to native check.*/
+  if (objkind == ACL_KIND_CLASS)
   {
 char relstorage = get_rel_relstorage(obj_oid);
-if(relstorage == 'h')
+if (relstorage == 'h')
+{
+  return true;
+}
+  }
+  else if (objkind == ACL_KIND_NAMESPACE)
+  {
+   /*native check build-in schemas.*/
+if (obj_oid == PG_CATALOG_NAMESPACE || obj_oid == 
information_schema_namespcace_oid
+   || obj_oid == PG_AOSEGMENT_NAMESPACE || obj_oid == 
PG_TOAST_NAMESPACE
+   || obj_oid == PG_BITMAPINDEX_NAMESPACE)
 {
   return true;
 }
   }
+  else if (objkind == ACL_KIND_PROC)
+  {
+   /*native check functions under build-in schemas.*/
+Oid namespaceid = get_func_namespace(obj_oid);
+if (namespaceid == PG_CATALOG_NAMESPACE || namespaceid == 
information_schema_namespcace_oid
+   || namespaceid == PG_AOSEGMENT_NAMESPACE || namespaceid 
== PG_TOAST_NAMESPACE
+   || namespaceid == PG_BITMAPINDEX_NAMESPACE)
+{
+  return true;
+}
+  }
+
   return false;
 }
 
 bool fallBackToNativeChecks(AclObjectKind objkind, List* table_list, Oid 
roleid)
 {
-  //for heap table, we fall back to native check.
-  if(objkind == ACL_KIND_CLASS)
+  /*we only have range table here*/
+  if (objkind == ACL_KIND_CLASS)
   {
 ListCell   *l;
 foreach(l, table_list)
 {
   RangeTblEntry *rte=(RangeTblEntry *) lfirst(l);
-  char relstorage = get_rel_relstorage(rte->relid);
-  if(relstorage == 'h')
+  bool ret = fallBackToNativeCheck(ACL_KIND_CLASS, rte->relid, roleid);
+  if(ret)
   {
 return true;
   }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf54c418/src/backend/utils/cache/lsyscache.c
--
diff --git a/src/backend/utils/cache/lsyscache.c 
b/src/backend/utils/cache/lsyscache.c
index fa8fde5..3ccf847 100644
--- a/src/backend/utils/cache/lsyscache.c
+++ b/src/backend/utils/cache/lsyscache.c
@@ -3248,6 +3248,30 @@ get_namespace_name(Oid nspid)
return result;
 }
 
+/*
+ * get_namespace_oid
+ * Returns the oid of a namespace given its name
+ *
+ */
+Oid
+get_namespace_oid(const char* npname)
+{
+   Oid result;
+   int fetchCount;
+
+   result = caql_getoid_plus(
+   NULL,
+   ,
+   NULL,
+   cql("SELECT oid FROM pg_namespace "
+   " WHERE nspname = :1 ",
+   PointerGetDatum((char *) npname)));
+
+   if (!fetchCount)
+   return InvalidOid;
+
+   return result;
+}
 /*

incubator-hawq git commit: HAWQ-1256. non-superuser connect to db will do aclcheck through ranger and curl handle haven't been initialized.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 2a7c20f2b -> aa5792d85


HAWQ-1256. non-superuser connect to db will do aclcheck through ranger and curl 
handle haven't been initialized.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aa5792d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aa5792d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aa5792d8

Branch: refs/heads/master
Commit: aa5792d858f545815361c8247cdbb22eec9dd3f1
Parents: 2a7c20f
Author: stanlyxiang 
Authored: Wed Jan 11 18:10:11 2017 +0800
Committer: stanlyxiang 
Committed: Wed Jan 11 18:16:54 2017 +0800

--
 src/backend/libpq/rangerrest.c |  1 +
 src/backend/tcop/postgres.c| 38 ++---
 2 files changed, 19 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aa5792d8/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index fd8937a..5406251 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -384,6 +384,7 @@ int check_privilege_from_ranger(List *arg_list)
Assert(request != NULL);
 
/* call GET method to send request*/
+   Assert(curl_context_ranger.hasInited);
if (call_ranger_rest(_context_ranger, request) < 0)
{
return RANGERCHECK_NO_PRIV;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aa5792d8/src/backend/tcop/postgres.c
--
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index c8d7e33..e1bfb1d 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -4391,7 +4391,24 @@ PostgresMain(int argc, char *argv[], const char 
*username)
BuildFlatFiles(true);
}
 
-
+   /* for enable ranger*/
+   if (enable_ranger && !curl_context_ranger.hasInited)
+   {
+   memset(_context_ranger, 0, sizeof(curl_context_t));
+   curl_global_init(CURL_GLOBAL_ALL);
+   /* init the curl session */
+   curl_context_ranger.curl_handle = curl_easy_init();
+   if (curl_context_ranger.curl_handle == NULL) {
+   /* cleanup curl stuff */
+   /* no need to cleanup curl_handle since it's null. just 
cleanup curl global.*/
+   curl_global_cleanup();
+   }
+   curl_context_ranger.hasInited = true;
+   curl_context_ranger.response.buffer = 
palloc0(CURL_RES_BUFFER_SIZE);
+   curl_context_ranger.response.buffer_size = CURL_RES_BUFFER_SIZE;
+   elog(DEBUG3, "when enable ranger, init global struct for 
privileges check.");
+   on_proc_exit(curl_finalize, 0);
+   }
/*
 * Create a per-backend PGPROC struct in shared memory, except in the
 * EXEC_BACKEND case where this was done in SubPostmasterMain. We must 
do
@@ -4630,25 +4647,6 @@ PostgresMain(int argc, char *argv[], const char 
*username)
if (!ignore_till_sync)
send_ready_for_query = true;/* initially, or after error */
 
-   /* for enable ranger*/
-   if (AmIMaster() && enable_ranger && !curl_context_ranger.hasInited)
-   {
-   memset(_context_ranger, 0, sizeof(curl_context_t));
-   curl_global_init(CURL_GLOBAL_ALL);
-   /* init the curl session */
-   curl_context_ranger.curl_handle = curl_easy_init();
-   if (curl_context_ranger.curl_handle == NULL) {
-   /* cleanup curl stuff */
-   /* no need to cleanup curl_handle since it's null. just 
cleanup curl global.*/
-   curl_global_cleanup();
-   elog(ERROR, "initialize global curl context failed.");
-   }
-   curl_context_ranger.hasInited = true;
-   curl_context_ranger.response.buffer = 
palloc0(CURL_RES_BUFFER_SIZE);
-   curl_context_ranger.response.buffer_size = CURL_RES_BUFFER_SIZE;
-   elog(DEBUG3, "initialize global curl context for privileges 
check.");
-   on_proc_exit(curl_finalize, 0);
-   }
/*
 * Non-error queries loop here.
 */

incubator-hawq git commit: HAWQ-1246. Add generation of RequestID, ClientIP, queryContext(SQL Statement) and encapsulate these contents to JSON request to RPS.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 94239f5ed -> 60f093372


HAWQ-1246. Add generation of RequestID, ClientIP, queryContext(SQL Statement) 
and encapsulate these contents to JSON request to RPS.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/60f09337
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/60f09337
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/60f09337

Branch: refs/heads/master
Commit: 60f093372888fdead70ffea1d2b035c7d2bc343d
Parents: 94239f5
Author: stanlyxiang 
Authored: Thu Dec 22 10:28:17 2016 +0800
Committer: hzhang2 
Committed: Tue Jan 3 10:39:51 2017 +0800

--
 src/backend/catalog/aclchk.c   |  57 ---
 src/backend/libpq/rangerrest.c | 318 +++-
 src/include/utils/rangerrest.h |  11 +-
 3 files changed, 134 insertions(+), 252 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/60f09337/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index d3e4b64..d19a045 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2732,7 +2732,7 @@ List *pg_rangercheck_batch(List *arg_list)
 
   } // foreach
 
-  RangerACLResult ret = check_privilege_from_ranger_batch(requestargs);
+  RangerACLResult ret = check_privilege_from_ranger(requestargs);
 
   ListCell *result;
   int k = 0;
@@ -2752,7 +2752,7 @@ List *pg_rangercheck_batch(List *arg_list)
   (RangerRequestJsonArgs*)lfirst(tmp);
   pfree(requestarg->user);
   pfree(requestarg->object);
-  pfree(requestarg->actions);
+  list_free_deep(requestarg->actions);
 }
 
 list_free_deep(requestargs);
@@ -2770,27 +2770,38 @@ AclResult
 pg_rangercheck(AclObjectKind objkind, Oid object_oid, Oid roleid,
  AclMode mask, AclMaskHow how)
 {
-  char* objectname = getNameFromOid(objkind, object_oid);
-  char* rolename = getRoleName(roleid);
-  List* actions = getActionName(mask);
-  bool isAll = (how == ACLMASK_ALL) ? true: false;
-
-  elog(LOG, "rangeraclcheck 
kind:%d,objectname:%s,role:%s,mask:%u\n",objkind,objectname,rolename,mask);
-  int ret = check_privilege_from_ranger(rolename, objkind, objectname, 
actions, isAll);
-
-  if(objectname){
-pfree(objectname);
-objectname = NULL;
-  }
-  if(rolename){
-pfree(rolename);
-rolename = NULL;
-  }
-  if(actions){
-list_free_deep(actions);
-actions = NIL;
-  }
-  return ret;
+   char* objectname = getNameFromOid(objkind, object_oid);
+   char* rolename = getRoleName(roleid);
+   List* actions = getActionName(mask);
+   bool isAll = (how == ACLMASK_ALL) ? true: false;
+
+   elog(LOG, "rangeraclcheck 
kind:%d,objectname:%s,role:%s,mask:%u\n",objkind,objectname,rolename,mask);
+   List *requestargs = NIL;
+   RangerRequestJsonArgs *requestarg = (RangerRequestJsonArgs *) 
palloc(sizeof(RangerRequestJsonArgs));
+   requestarg->user = rolename;
+   requestarg->kind = objkind;
+   requestarg->object = objectname;
+   requestarg->actions = actions;
+   requestarg->isAll = isAll;
+   requestargs = lappend(requestargs, requestarg);
+   int ret = check_privilege_from_ranger(requestargs);
+
+   if (requestargs)
+   {
+   ListCell *cell = list_head(requestargs);
+   while (cell != NULL)
+   {
+   ListCell *tmp = cell;
+   cell = lnext(cell);
+   RangerRequestJsonArgs* requestarg = 
(RangerRequestJsonArgs*) lfirst(tmp);
+   pfree(requestarg->user);
+   pfree(requestarg->object);
+   list_free_deep(requestarg->actions);
+   }
+   list_free_deep(requestargs);
+   requestargs = NULL;
+   }
+   return ret;
 }
 
 /*

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/60f09337/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index 120f64f..56d30b5 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -49,10 +49,28 @@ char* AclObjectKindStr[] =
"none"   /* MUST BE LAST */
 };
 
+static int request_id = 1;
+
+static void getClientIP(char *remote_host)
+{
+   if( MyProcPort->remote_host == NULL || strlen(MyProcPort->remote_host) 
== 0 )
+   {
+   snprintf(remote_host, HOST_BUFFER_SIZE, "%s", "UNKNOWN");
+   return;
+   }
+   if (strcmp(MyProcPort->remote_host, 

incubator-hawq git commit: HAWQ-1243. Add suffix name for ranger restful service.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 3e3c41a72 -> eed9a0f25


HAWQ-1243. Add suffix name for ranger restful service.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/eed9a0f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/eed9a0f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/eed9a0f2

Branch: refs/heads/master
Commit: eed9a0f2535e7c7b39468252ee7ae184da009884
Parents: 3e3c41a
Author: hzhang2 
Authored: Wed Dec 28 13:36:45 2016 +0800
Committer: hzhang2 
Committed: Wed Dec 28 13:36:45 2016 +0800

--
 src/backend/libpq/rangerrest.c | 13 ++---
 src/backend/utils/misc/guc.c   | 12 +++-
 src/include/utils/guc.h|  1 +
 3 files changed, 14 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eed9a0f2/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index e50c3e1..59b33a8 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -426,25 +426,16 @@ int call_ranger_rest(CURL_HANDLE curl_handle, const char* 
request)
curl_easy_setopt(curl_handle->curl_handle, CURLOPT_TIMEOUT, 30L);
 
/* specify URL to get */
-   //curl_easy_setopt(curl_handle->curl_handle, CURLOPT_URL, 
"http://localhost:8089/checkprivilege;);
StringInfoData tname;
initStringInfo();
appendStringInfo(, "http://;);
appendStringInfo(, "%s", rps_addr_host);
appendStringInfo(, ":");
appendStringInfo(, "%d", rps_addr_port);
-   appendStringInfo(, "/rps");
+   appendStringInfo(, "/");
+   appendStringInfo(, "%s", rps_addr_suffix);
curl_easy_setopt(curl_handle->curl_handle, CURLOPT_URL, tname.data);
 
-   /* specify format */
-   // struct curl_slist *plist = curl_slist_append(NULL, 
"Content-Type:application/json;charset=UTF-8");
-   // curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, plist);
-
-
-   //curl_easy_setopt(curl_handle->curl_handle, 
CURLOPT_POSTFIELDSIZE_LARGE, 1000);
-   //curl_easy_setopt(curl_handle->curl_handle, CURLOPT_HTTPGET, 0);
-   //curl_easy_setopt(curl_handle->curl_handle, CURLOPT_CUSTOMREQUEST, 
"POST");
-
struct curl_slist *headers = NULL;
//curl_slist_append(headers, "Accept: application/json");
headers = curl_slist_append(headers, "Content-Type:application/json");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eed9a0f2/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 3d36a72..00b9bad 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -780,6 +780,7 @@ bool gp_plpgsql_clear_cache_always = false;
 bool gp_called_by_pgdump = false;
 
 char   *rps_addr_host;
+char   *rps_addr_suffix;
 int rps_addr_port;
 
 /*
@@ -6268,7 +6269,7 @@ static struct config_int ConfigureNamesInt[] =
   NULL
 },
 _addr_port,
-1, 1, 65535, NULL, NULL
+8080, 1, 65535, NULL, NULL
   },
 
{
@@ -8184,6 +8185,15 @@ static struct config_string ConfigureNamesString[] =
 "localhost", NULL, NULL
   },
 
+  {
+{"hawq_rps_address_suffix", PGC_POSTMASTER, PRESET_OPTIONS,
+  gettext_noop("ranger plugin server suffix of restful service address"),
+  NULL
+},
+_addr_suffix,
+"hawq", NULL, NULL
+  },
+
{
{"standby_address_host", PGC_POSTMASTER, PRESET_OPTIONS,
gettext_noop("standby server address hostname"),

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eed9a0f2/src/include/utils/guc.h
--
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 95e14a4..cb45a7c 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -454,6 +454,7 @@ extern bool optimizer_partition_selection_log;
  * rps host and port
  */
 extern char   *rps_addr_host;
+extern char   *rps_addr_suffix;
 extern int rps_addr_port;
 /*
  * During insertion in a table with parquet partitions,

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ranger [deleted] 64ae1a875

[2/2] incubator-hawq git commit: Fix some explanations

[hubertzhang]

Fix some explanations


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bbf897bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bbf897bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bbf897bc

Branch: refs/heads/ran5
Commit: bbf897bcf1632fd84844b92c5ba3a543b5db25c7
Parents: 5e42966
Author: hzhang2 
Authored: Wed Dec 14 11:50:51 2016 +0800
Committer: hzhang2 
Committed: Wed Dec 14 11:50:51 2016 +0800

--
 src/backend/parser/parse_relation.c | 2 +-
 src/backend/utils/misc/guc.c| 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bbf897bc/src/backend/parser/parse_relation.c
--
diff --git a/src/backend/parser/parse_relation.c 
b/src/backend/parser/parse_relation.c
index 62d815c..5ee7857 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -2766,7 +2766,7 @@ ExecCheckRTPermsWithRanger(List *rangeTable)
   aclresults = pg_rangercheck_batch(ranger_check_args);
   if (aclresults == NIL)
   {
-elog(ERROR, "ERROR\n");
+elog(ERROR, "ACL check failed\n");
 return;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bbf897bc/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 855e98c..3d36a72 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6264,7 +6264,7 @@ static struct config_int ConfigureNamesInt[] =
 
{
 {"hawq_rps_address_port", PGC_POSTMASTER, PRESET_OPTIONS,
-  gettext_noop("rps server address port number"),
+  gettext_noop("ranger plugin server address port number"),
   NULL
 },
 _addr_port,
@@ -8177,7 +8177,7 @@ static struct config_string ConfigureNamesString[] =
 
{
 {"hawq_rps_address_host", PGC_POSTMASTER, PRESET_OPTIONS,
-  gettext_noop("rps server address hostname"),
+  gettext_noop("ranger plugin server address hostname"),
   NULL
 },
 _addr_host,

[4/4] incubator-hawq git commit: HAWQ-1003. Implement bached ACL check through Ranger.

[hubertzhang]

HAWQ-1003. Implement bached ACL check through Ranger.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e408bcd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e408bcd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e408bcd6

Branch: refs/heads/ran5
Commit: e408bcd6236d4ee20994966072c5b3db81008395
Parents: a5a02b1
Author: hzhang2 
Authored: Mon Dec 12 14:47:46 2016 +0800
Committer: hzhang2 
Committed: Mon Dec 12 14:47:46 2016 +0800

--
 src/backend/catalog/aclchk.c | 56 ++-
 1 file changed, 55 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e408bcd6/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index a633d60..ac9ac30 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2684,7 +2684,61 @@ bool fallBackToNativeCheck(AclObjectKind objkind, Oid 
obj_oid, Oid roleid)
  */
 List *pg_rangercheck_batch(List *arg_list)
 {
-  return NIL;
+  List *aclresults = NIL;
+  List *requestargs = NIL;
+  ListCell *arg;
+  foreach(arg, arg_list) {
+RangerPrivilegeArgs *arg_ptr = (RangerPrivilegeArgs *) lfirst(arg);
+AclObjectKind objkind = arg_ptr->objkind;
+Oid object_oid = arg_ptr->object_oid;
+char *objectname = getNameFromOid(objkind, object_oid);
+char *rolename = getRoleName(arg_ptr->roleid);
+List* actions = getActionName(arg_ptr->mask);
+bool isAll = (arg_ptr->how == ACLMASK_ALL) ? true: false;
+RangerPrivilegeResults *aclresult = (RangerPrivilegeResults *) 
palloc(sizeof(RangerPrivilegeResults));
+aclresult->result = -1;
+aclresult->relOid = object_oid;
+aclresults = lappend(aclresults, aclresult);
+
+RangerRequestJsonArgs *requestarg = (RangerRequestJsonArgs *) 
palloc(sizeof(RangerRequestJsonArgs));
+requestarg->user = rolename;
+requestarg->kind = objkind;
+requestarg->object = objectname;
+requestarg->actions = actions;
+requestarg->how = isAll;
+requestargs = lappend(requestargs, requestarg);
+
+  } // foreach
+
+  RangerACLResult ret = check_privilege_from_ranger_batch(requestargs);
+
+  ListCell *result;
+  int k = 0;
+  foreach(result, aclresults) {
+RangerPrivilegeResults *result_ptr = (RangerPrivilegeResults *) 
lfirst(result);
+result_ptr->result = ret;
+++k;
+  }
+
+  if(requestargs) {
+ListCell   *cell = list_head(requestargs);
+while (cell != NULL)
+{
+  ListCell   *tmp = cell;
+  cell = lnext(cell);
+  RangerRequestJsonArgs* requestarg =
+  (RangerRequestJsonArgs*)lfirst(tmp);
+  pfree(requestarg->user);
+  pfree(requestarg->object);
+  pfree(requestarg->actions);
+}
+
+list_free_deep(requestargs);
+requestargs = NULL;
+  }
+
+  elog(LOG, "oids%d\n", arg_list->length);
+  return aclresults;
 }
 
 AclResult

[1/4] incubator-hawq git commit: HAWQ-1004. Implement calling Ranger REST Service using libcurl.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ran5 [created] e408bcd62


HAWQ-1004. Implement calling Ranger REST Service using libcurl.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0c5a8402
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0c5a8402
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0c5a8402

Branch: refs/heads/ran5
Commit: 0c5a8402354b89ac865ea82cf7c0e2741d744f2e
Parents: 6b2a18e
Author: hzhang2 
Authored: Mon Dec 12 14:29:29 2016 +0800
Committer: hzhang2 
Committed: Mon Dec 12 14:29:29 2016 +0800

--
 src/backend/libpq/Makefile |   2 +-
 src/backend/libpq/rangerrest.c | 549 
 src/include/utils/rangerrest.h |  39 +++
 3 files changed, 589 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0c5a8402/src/backend/libpq/Makefile
--
diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile
index 28a386f..16124c7 100644
--- a/src/backend/libpq/Makefile
+++ b/src/backend/libpq/Makefile
@@ -15,6 +15,6 @@ include $(top_builddir)/src/Makefile.global
 # be-fsstubs is here for historical reasons, probably belongs elsewhere
 
 OBJS = be-fsstubs.o be-secure.o auth.o crypt.o hba.o ip.o md5.o pqcomm.o \
-   pqformat.o pqsignal.o sha2.o pg_sha2.o
+   pqformat.o pqsignal.o sha2.o pg_sha2.o rangerrest.o
 
 include $(top_srcdir)/src/backend/common.mk

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0c5a8402/src/backend/libpq/rangerrest.c
--
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
new file mode 100644
index 000..81c2097
--- /dev/null
+++ b/src/backend/libpq/rangerrest.c
@@ -0,0 +1,549 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*-
+ *
+ * rangerrest.c
+ * routines to interact with Ranger REST API
+ *
+ *-
+ */
+#include 
+#include "utils/rangerrest.h"
+#include "utils/elog.h"
+#include "utils/palloc.h"
+#include "postgres.h"
+#include "utils/acl.h"
+#include "nodes/pg_list.h"
+
+/*
+ * Internal buffer for libcurl context
+ */
+typedef struct curl_context_t
+{
+CURL* curl_handle;
+
+char curl_error_buffer[CURL_ERROR_SIZE];
+
+int curl_still_running;
+
+struct
+{
+char* buffer;
+int size;
+} response;
+
+char* last_http_reponse;
+} curl_context_t;
+typedef curl_context_t* CURL_HANDLE;
+
+RangerACLResult parse_ranger_response(char* buffer)
+{
+Assert(buffer != NULL);
+if (strlen(buffer) == 0)
+return RANGERCHECK_UNKNOWN;
+
+elog(LOG, "RRread from Ranger Restful API: %s", buffer);
+
+struct json_object *response = json_tokener_parse(buffer);
+struct json_object *accessObj = json_object_object_get(response, "access");
+
+//json_object * jobj = json_object_object_get(jobj, key);
+int arraylen = json_object_array_length(accessObj);
+elog(LOG, "Array Length: %dn",arraylen);
+
+json_object * jvalue;
+for (int i=0; i< arraylen; i++){
+  jvalue = json_object_array_get_idx(accessObj, i);
+  //elog(LOG,"value[%d]: %sn",i, json_object_get_boolean(jvalue));
+}
+json_object * jallow = json_object_object_get(jvalue, "allowed");
+json_bool result = json_object_get_boolean(jallow);
+
+//char* szResult = json_object_get_boolean(result);
+//elog(LOG, "parse Ranger response, result:%s.", szResult);
+elog(LOG, "parse Ranger response, result:%d.", result);
+//if (strcmp(szResult, "true") == 0)
+if(result == 1)
+{
+return RANGERCHECK_OK;
+} else {
+return RANGERCHECK_NO_PRIV;
+}
+}
+
+/*
+ * A mapping from AclObjectKind to string
+ */
+char* 

[2/4] incubator-hawq git commit: HAWQ-1002. Implement configuration GUC related to Ranger.

[hubertzhang]

HAWQ-1002. Implement configuration GUC related to Ranger.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca9f6493
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca9f6493
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca9f6493

Branch: refs/heads/ran5
Commit: ca9f649338d0becbde15e24f43006aa8c5d20868
Parents: 0c5a840
Author: hzhang2 
Authored: Mon Dec 12 14:36:58 2016 +0800
Committer: hzhang2 
Committed: Mon Dec 12 14:36:58 2016 +0800

--
 src/backend/cdb/cdbvars.c|  2 ++
 src/backend/utils/misc/guc.c | 29 +
 src/include/cdb/cdbvars.h|  2 ++
 src/include/utils/guc.h  |  1 +
 4 files changed, 34 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca9f6493/src/backend/cdb/cdbvars.c
--
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index c2fca21..d7a2e59 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -292,6 +292,8 @@ bool enable_prefer_list_to_rm;
  * reserved to facilitate showing settings in hawq-site.xml. */
 char  *master_addr_host;
 intmaster_addr_port;
+char  *rps_addr_host;
+intrps_addr_port;
 char  *standby_addr_host;
 intseg_addr_port;
 char  *dfs_url;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca9f6493/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index e5f6548..f34a9b9 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -732,6 +732,7 @@ int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_limit;
+bool enable_ranger = false;
 double   optimizer_cost_threshold;
 double  optimizer_nestloop_factor;
 double  locality_upper_bound;
@@ -4326,6 +4327,16 @@ static struct config_bool ConfigureNamesBool[] =
},
 
{
+{"enable_ranger", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
+ gettext_noop("Enable Apache Ranger for HAWQ privilege management."),
+ NULL,
+ GUC_SUPERUSER_ONLY
+},
+_ranger,
+false, NULL, NULL
+  },
+
+   {
{"filesystem_support_truncate", PGC_USERSET, APPENDONLY_TABLES,
 gettext_noop("the file system support truncate feature."),
 NULL,
@@ -6250,6 +6261,15 @@ static struct config_int ConfigureNamesInt[] =
},
 
{
+{"hawq_rps_address_port", PGC_POSTMASTER, PRESET_OPTIONS,
+  gettext_noop("rps server address port number"),
+  NULL
+},
+_addr_port,
+1, 1, 65535, NULL, NULL
+  },
+
+   {
{"hawq_segment_address_port", PGC_POSTMASTER, PRESET_OPTIONS,
gettext_noop("segment address port number"),
NULL
@@ -8154,6 +8174,15 @@ static struct config_string ConfigureNamesString[] =
},
 
{
+{"hawq_rps_address_host", PGC_POSTMASTER, PRESET_OPTIONS,
+  gettext_noop("rps server address hostname"),
+  NULL
+},
+_addr_host,
+"localhost", NULL, NULL
+  },
+
+   {
{"standby_address_host", PGC_POSTMASTER, PRESET_OPTIONS,
gettext_noop("standby server address hostname"),
NULL

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca9f6493/src/include/cdb/cdbvars.h
--
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index 9f6c3b1..56b208f 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -1143,6 +1143,8 @@ extern bool debug_datalocality_time;
 /* New HAWQ 2.0 basic GUCs */
 extern char   *master_addr_host;
 extern int master_addr_port;
+extern char   *rps_addr_host;
+extern int rps_addr_port;
 extern char   *standby_addr_host;
 extern int seg_addr_port;
 extern char   *dfs_url;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca9f6493/src/include/utils/guc.h
--
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 8fdab3c..4041e41 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -275,6 +275,7 @@ extern bool gp_plpgsql_clear_cache_always;
 extern bool gp_disable_catalog_access_on_segment;
 
 extern bool gp_called_by_pgdump;
+extern bool enable_ranger;
 
 /* Debug DTM Action */
 typedef enum

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ran4 [deleted] 0c5a84023

[4/4] incubator-hawq git commit: HAWQ-1001. Translate resource oid to hierarchical name.

[hubertzhang]

HAWQ-1001. Translate resource oid to hierarchical name.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/afd1885b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/afd1885b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/afd1885b

Branch: refs/heads/ranger
Commit: afd1885bc88f6f510b7f5ba7055c8e141310f7cf
Parents: ee462d2
Author: hzhang2 
Authored: Wed Nov 9 16:03:00 2016 +0800
Committer: hzhang2 
Committed: Wed Nov 9 16:03:00 2016 +0800

--
 src/backend/catalog/aclchk.c| 462 ++-
 src/backend/parser/parse_relation.c |   2 +-
 src/include/utils/acl.h |  20 +-
 3 files changed, 469 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/afd1885b/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index dcb7c90..038dd69 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -25,6 +25,7 @@
 #include "catalog/catalog.h"
 #include "catalog/catquery.h"
 #include "catalog/dependency.h"
+
 #include "catalog/indexing.h"
 #include "catalog/gp_persistent.h"
 #include "catalog/pg_authid.h"
@@ -59,7 +60,7 @@
 #include "cdb/cdbdisp.h"
 #include "cdb/dispatcher.h"
 
-
+#define ACTION_LENGTH 12
 static void ExecGrant_Relation(InternalGrant *grantStmt);
 static void ExecGrant_Database(InternalGrant *grantStmt);
 static void ExecGrant_Fdw(InternalGrant *grantStmt);
@@ -226,7 +227,7 @@ restrict_and_check_grant(bool is_grant, AclMode 
avail_goptions, bool all_privs,
if (avail_goptions == ACL_NO_RIGHTS)
{
  if (enable_ranger) {
-   if (pg_rangercheck(objectId, grantorId,
+   if (pg_rangercheck(objkind, objectId, grantorId,
whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
ACLMASK_ANY) != ACLCHECK_OK)
  aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
@@ -2248,10 +2249,445 @@ has_rolcatupdate(Oid roleid)
return rolcatupdate;
 }
 
+char *getRoleName(Oid role_id)
+{
+  Assert(OidIsValid(role_id));
+  int fetchCount;
+  char* role_name = caql_getcstring_plus(
+NULL,
+,
+NULL,
+cql("SELECT rolname FROM pg_authid "
+  " WHERE oid = :1 ",
+  ObjectIdGetDatum(role_id)));
+
+ if (role_name == NULL)
+elog(ERROR, "oid [%u] not found in table pg_authid", role_id);
+  return role_name;
+}
+
+char *getClassNameFromOid(Oid object_oid)
+{
+  StringInfoData tname;
+  initStringInfo();
+
+  Assert(OidIsValid(object_oid));
+  char* rel_name = caql_getcstring(
+   NULL,
+   cql("SELECT relname FROM pg_class "
+ " WHERE oid = :1",
+ ObjectIdGetDatum(object_oid)));
+  if (rel_name == NULL)
+elog(ERROR, "oid [%u] not found in table pg_class", object_oid);
+
+  int fetchCount=0;
+  Oid schema_name_oid = caql_getoid_plus(
+ NULL,
+ ,
+ NULL,
+ cql("SELECT relnamespace FROM pg_class "
+   " WHERE oid = :1",
+   ObjectIdGetDatum(object_oid)));
+  if (schema_name_oid == InvalidOid)
+  elog(ERROR, "oid [%u] not found in table pg_class", object_oid);
+
+  char* schema_name= caql_getcstring(
+  NULL,
+  cql("select nspname from pg_namespace "
+" WHERE oid = :1",
+ObjectIdGetDatum(schema_name_oid)));
+  if (schema_name == NULL)
+  elog(ERROR, "oid [%u] not found in table pg_namespace", object_oid);
+
+  char* database_name = get_database_name(MyDatabaseId);
+  if (database_name == NULL)
+   elog(ERROR, "oid [%u] not found current database", object_oid);
+
+  appendStringInfo(, database_name);
+  appendStringInfo(, ".");
+  appendStringInfo(, schema_name);
+  appendStringInfo(, ".");
+  appendStringInfo(, rel_name);
+  pfree(rel_name);
+  pfree(schema_name);
+  pfree(database_name);
+
+  return tname.data;
+}
+
+char *getSequenceNameFromOid(Oid object_oid)
+{
+  StringInfoData tname;
+  initStringInfo();
+
+  Assert(OidIsValid(object_oid));
+  char* seq_name = caql_getcstring(
+  NULL,
+  cql("SELECT relname FROM pg_class "
+" WHERE oid = :1",
+ObjectIdGetDatum(object_oid)));
+  if (seq_name == NULL)
+   elog(ERROR, "oid [%u] not found in table pg_class", object_oid);
+
+  int fetchCount=0;
+  Oid schema_name_oid = caql_getoid_plus(
+NULL,
+,
+NULL,
+cql("SELECT relnamespace FROM pg_class "
+   

[1/4] incubator-hawq git commit: HAWQ-1001. Fix some explanations of GUC.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ranger [created] afd1885bc


HAWQ-1001. Fix some explanations of GUC.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ee462d2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ee462d2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ee462d2d

Branch: refs/heads/ranger
Commit: ee462d2d0cbd3c1df2f8e42b367fc899688ca367
Parents: 4a4da0c
Author: hzhang2 
Authored: Wed Sep 7 09:07:02 2016 +0800
Committer: hzhang2 
Committed: Mon Nov 7 16:35:28 2016 +0800

--
 src/backend/utils/misc/guc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ee462d2d/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index b44b482..afbafcb 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -349,7 +349,6 @@ bool
Debug_datumstream_write_use_small_initial_buffers = false;
 bool   gp_temporary_files_filespace_repair = false;
 bool   filesystem_support_truncate = true;
 bool   gp_allow_non_uniform_partitioning_ddl = true;
-bool  enable_ranger = false;
 
 intexplain_memory_verbosity = 0;
 char*  memory_profiler_run_id = "none";
@@ -733,6 +732,7 @@ int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_limit;
+bool enable_ranger = false;
 double   optimizer_cost_threshold;
 double  optimizer_nestloop_factor;
 double  locality_upper_bound;
@@ -4328,7 +4328,7 @@ static struct config_bool ConfigureNamesBool[] =
 
{
 {"enable_ranger", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
- gettext_noop("support to using ranger to manage hawq privilege."),
+ gettext_noop("Enable Apache Ranger for HAWQ privilege management."),
  NULL,
  GUC_SUPERUSER_ONLY
 },

[3/4] incubator-hawq git commit: HAWQ-1002. Implement a switch in hawq-site.xml to configure whether use Ranger or not for ACL.

[hubertzhang]

HAWQ-1002. Implement a switch in hawq-site.xml to configure whether use Ranger 
or not for ACL.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/85f57164
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/85f57164
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/85f57164

Branch: refs/heads/ranger
Commit: 85f571641e14d15edd810a001182644a60cb85ba
Parents: a1ab4ab
Author: hzhang2 
Authored: Thu Aug 18 17:08:58 2016 +0800
Committer: hzhang2 
Committed: Mon Nov 7 16:35:28 2016 +0800

--
 src/backend/utils/misc/guc.c | 11 +++
 src/include/utils/guc.h  |  1 +
 2 files changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/85f57164/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 03b6d88..b44b482 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -349,6 +349,7 @@ bool
Debug_datumstream_write_use_small_initial_buffers = false;
 bool   gp_temporary_files_filespace_repair = false;
 bool   filesystem_support_truncate = true;
 bool   gp_allow_non_uniform_partitioning_ddl = true;
+bool  enable_ranger = false;
 
 intexplain_memory_verbosity = 0;
 char*  memory_profiler_run_id = "none";
@@ -4326,6 +4327,16 @@ static struct config_bool ConfigureNamesBool[] =
},
 
{
+{"enable_ranger", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
+ gettext_noop("support to using ranger to manage hawq privilege."),
+ NULL,
+ GUC_SUPERUSER_ONLY
+},
+_ranger,
+false, NULL, NULL
+  },
+
+   {
{"filesystem_support_truncate", PGC_USERSET, APPENDONLY_TABLES,
 gettext_noop("the file system support truncate feature."),
 NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/85f57164/src/include/utils/guc.h
--
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 8fdab3c..4041e41 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -275,6 +275,7 @@ extern bool gp_plpgsql_clear_cache_always;
 extern bool gp_disable_catalog_access_on_segment;
 
 extern bool gp_called_by_pgdump;
+extern bool enable_ranger;
 
 /* Debug DTM Action */
 typedef enum

[2/4] incubator-hawq git commit: HAWQ-1001. Implement HAWQ user ACL check through Ranger.

[hubertzhang]

HAWQ-1001. Implement HAWQ user ACL check through Ranger.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4a4da0c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4a4da0c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4a4da0c1

Branch: refs/heads/ranger
Commit: 4a4da0c15bc7ba58c624c069e94403eacfbb89f9
Parents: 85f5716
Author: hzhang2 
Authored: Thu Aug 25 11:06:43 2016 +0800
Committer: hzhang2 
Committed: Mon Nov 7 16:35:28 2016 +0800

--
 src/backend/catalog/aclchk.c| 273 ++-
 src/backend/parser/parse_relation.c |  48 --
 src/include/utils/acl.h |  15 ++
 3 files changed, 276 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4a4da0c1/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 3fa616f..dcb7c90 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -225,10 +225,18 @@ restrict_and_check_grant(bool is_grant, AclMode 
avail_goptions, bool all_privs,
 */
if (avail_goptions == ACL_NO_RIGHTS)
{
-   if (pg_aclmask(objkind, objectId, grantorId,
-  whole_mask | 
ACL_GRANT_OPTION_FOR(whole_mask),
-  ACLMASK_ANY) == ACL_NO_RIGHTS)
-   aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ if (enable_ranger) {
+   if (pg_rangercheck(objectId, grantorId,
+   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) != ACLCHECK_OK)
+ aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ }
+ else {
+   if (pg_aclmask(objkind, objectId, grantorId,
+   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) == ACL_NO_RIGHTS)
+ aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ }
}
 
/*
@@ -2240,6 +2248,14 @@ has_rolcatupdate(Oid roleid)
return rolcatupdate;
 }
 
+AclResult
+pg_rangercheck(Oid table_oid, Oid roleid,
+ AclMode mask, AclMaskHow how)
+{
+  return ACLCHECK_OK;
+}
+
+
 /*
  * Relay for the various pg_*_mask routines depending on object kind
  */
@@ -3085,6 +3101,135 @@ pg_filesystem_aclmask(Oid fsysOid, Oid roleid,
return result;
 }
 
+
+/*
+ * Exported routine for checking a user's access privileges to a table
+ *
+ * Returns ACLCHECK_OK if the user has any of the privileges identified by
+ * 'mode'; otherwise returns a suitable error code (in practice, always
+ * ACLCHECK_NO_PRIV).
+ */
+AclResult
+pg_class_nativecheck(Oid table_oid, Oid roleid, AclMode mode)
+{
+  if (pg_class_aclmask(table_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a database
+ */
+AclResult
+pg_database_nativecheck(Oid db_oid, Oid roleid, AclMode mode)
+{
+  if (pg_database_aclmask(db_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a function
+ */
+AclResult
+pg_proc_nativecheck(Oid proc_oid, Oid roleid, AclMode mode)
+{
+  if (pg_proc_aclmask(proc_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a language
+ */
+AclResult
+pg_language_nativecheck(Oid lang_oid, Oid roleid, AclMode mode)
+{
+  if (pg_language_aclmask(lang_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a namespace
+ */
+AclResult
+pg_namespace_nativecheck(Oid nsp_oid, Oid roleid, AclMode mode)
+{
+  if (pg_namespace_aclmask(nsp_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a tablespace
+ */
+AclResult
+pg_tablespace_nativecheck(Oid spc_oid, Oid roleid, AclMode mode)
+{
+  if (pg_tablespace_aclmask(spc_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a foreign
+ * data wrapper
+ */
+AclResult
+pg_foreign_data_wrapper_nativecheck(Oid fdw_oid, Oid roleid, AclMode mode)
+{
+  if (pg_foreign_data_wrapper_aclmask(fdw_oid, roleid, mode, ACLMASK_ANY) != 0)
+return 

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/rangeracl [deleted] 65620be22

incubator-hawq git commit: HAWQ-1143. Libhdfs create semantic is not consistent with posix standard.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master b2c14cfc6 -> 68867d781


HAWQ-1143. Libhdfs create semantic is not consistent with posix standard.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/68867d78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/68867d78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/68867d78

Branch: refs/heads/master
Commit: 68867d781e138b0504145ef65c73b0d996fe5216
Parents: b2c14cf
Author: hzhang2 
Authored: Thu Nov 3 16:21:24 2016 +0800
Committer: hzhang2 
Committed: Thu Nov 3 16:21:24 2016 +0800

--
 depends/libhdfs3/src/client/Hdfs.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/68867d78/depends/libhdfs3/src/client/Hdfs.cpp
--
diff --git a/depends/libhdfs3/src/client/Hdfs.cpp 
b/depends/libhdfs3/src/client/Hdfs.cpp
index cafb289..9906a87 100644
--- a/depends/libhdfs3/src/client/Hdfs.cpp
+++ b/depends/libhdfs3/src/client/Hdfs.cpp
@@ -646,9 +646,9 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char * path, int 
flags, int bufferSize,
 if ((flags & O_CREAT) || (flags & O_APPEND) || (flags & O_WRONLY)) {
 int internalFlags = 0;
 
-if (flags & O_CREAT) {
+if ((flags & O_CREAT) && (flags & O_EXCL)) {
 internalFlags |= Hdfs::Create;
-} else if ((flags & O_APPEND) && (flags & O_WRONLY)) {
+} else if ((flags & O_CREAT) || ((flags & O_APPEND) && (flags & 
O_WRONLY))) {
 internalFlags |= Hdfs::Create;
 internalFlags |= Hdfs::Append;
 } else if (flags & O_WRONLY) {

incubator-hawq git commit: HAWQ-1135. MADlib: Raising exception leads to database connection termination.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1e5e996ec -> 56e21fce3


HAWQ-1135. MADlib: Raising exception leads to database connection termination.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/56e21fce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/56e21fce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/56e21fce

Branch: refs/heads/master
Commit: 56e21fce3d3e0c65d7988cb8b0fbdbc6f1975a60
Parents: 1e5e996
Author: hzhang2 
Authored: Wed Nov 2 10:24:04 2016 +0800
Committer: hzhang2 
Committed: Wed Nov 2 10:38:25 2016 +0800

--
 src/backend/access/transam/xact.c | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/56e21fce/src/backend/access/transam/xact.c
--
diff --git a/src/backend/access/transam/xact.c 
b/src/backend/access/transam/xact.c
index cfb5351..440b2ca 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -4822,7 +4822,11 @@ AbortSubTransaction(void)
AtSubAbort_Memory();
AtSubAbort_ResourceOwner();
AtAbort_AppendOnly(true);
-   AtSubAbort_ActiveQueryResource();
+   /* In sub transaction, ActiveQueryResource should not be set NULL
+* sub transaction inherits resource from parent.
+* parent transaction may still use ActiveQueryResource to reference 
the original resource.
+*/
+   // AtSubAbort_ActiveQueryResource();
 
/*
 * Release any LW locks we might be holding as quickly as possible.

incubator-hawq git commit: HAWQ-1113. Fix bug and add test for hawq register force mode.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master ced05d56f -> 6cefb5529


HAWQ-1113. Fix bug and add test for hawq register force mode.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6cefb552
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6cefb552
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6cefb552

Branch: refs/heads/master
Commit: 6cefb5529c536df99fb864105e31e083c37741ea
Parents: ced05d5
Author: hzhang2 
Authored: Fri Oct 28 08:22:48 2016 +0800
Committer: hzhang2 
Committed: Fri Oct 28 08:22:48 2016 +0800

--
 .../feature/ManagementTool/test_hawq_register.h |   2 +-
 .../test_hawq_register_usage2_case2.cpp |  14 -
 src/test/feature/ManagementTool/usage2case2/10  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/11  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/12  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/13  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/8   | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/9   | Bin 0 -> 543 bytes
 .../usage2case2/same_path_yaml_config_tpl.yml   |  57 +++
 tools/bin/hawqregister  |  35 +---
 10 files changed, 97 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register.h
--
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h 
b/src/test/feature/ManagementTool/test_hawq_register.h
index 4edf2aa..dce1c8c 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -94,7 +94,7 @@ class TestHawqRegister : public ::testing::Test {
 
 void runYamlCaseTableNotExists(std::string casename, std::string 
ymlname, int expectederror, int checknum);
 void runYamlCaseTableExists(std::string casename, std::string ymlname, 
int isexpectederror, int checknum);
-void runYamlCaseForceMode(std::string casename, std::string ymlname, 
int isexpectederror, int rows, int checknum);
+void runYamlCaseForceMode(std::string casename, std::string ymlname, 
int isexpectederror, int rows, int checknum, bool samepath);
 
 private:
 std::unique_ptr conn;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
--
diff --git 
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp 
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
index 492dbd3..ad28a0a 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
@@ -80,7 +80,7 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
 }
 }
 
-void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string 
ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200) {
+void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string 
ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200, bool 
samepathfile = false) {
 SQLUtility util;
 string test_root(util.getTestRootPath());
 string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", 
test_root.c_str(), ymlname.c_str()));
@@ -107,11 +107,17 @@ void TestHawqRegister::runYamlCaseForceMode(std::string 
casename, std::string ym
 hc.getNamenodeHost(hdfs_prefix);
 strs_src_dst["@PORT@"]= hdfs_prefix;
 frep.replace(t_yml_tpl, t_yml, strs_src_dst);
+if(samepathfile) {
+for(int i=8; i<=13; i++){
+  string 
file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", 
test_root.c_str()));
+  file.append(std::to_string(i));
+  EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put %s 
/hawq_default/16385/%s/%s/", file.c_str(), 
strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str(;
+}
+}
 //printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c 
%s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str());
 //sleep(60);
 EXPECT_EQ(isexpectederror, 
Command::getCommandStatus(hawq::test::stringFormat("hawq register --force -d %s 
-c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str(;
 util.query("select * from nt;", checknum);
-
 EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf 
%s", 

incubator-hawq git commit: HAWQ-1077. Table insert hangs due to stack overwrite which is caused by a bug in ao snappy code

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master fd0b25dd7 -> 98d48e78d


HAWQ-1077. Table insert hangs due to stack overwrite which is caused by a bug 
in ao snappy code


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/98d48e78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/98d48e78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/98d48e78

Branch: refs/heads/master
Commit: 98d48e78d5b4c57a2557412955a39edbcb2fe3bc
Parents: fd0b25d
Author: Paul Guo 
Authored: Mon Sep 26 16:42:14 2016 +0800
Committer: Paul Guo 
Committed: Tue Sep 27 18:46:06 2016 +0800

--
 src/backend/catalog/pg_compression.c | 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/98d48e78/src/backend/catalog/pg_compression.c
--
diff --git a/src/backend/catalog/pg_compression.c 
b/src/backend/catalog/pg_compression.c
index b073d00..245fd76 100644
--- a/src/backend/catalog/pg_compression.c
+++ b/src/backend/catalog/pg_compression.c
@@ -430,7 +430,7 @@ snappy_compress_internal(PG_FUNCTION_ARGS)
size_t  src_sz = PG_GETARG_INT32(1);
char*dst = PG_GETARG_POINTER(2);
size_t  dst_sz = PG_GETARG_INT32(3);
-   size_t  *dst_used = PG_GETARG_POINTER(4);
+   int32   *dst_used = PG_GETARG_POINTER(4);
size_t  compressed_length;
snappy_status   retval;
 
@@ -452,14 +452,14 @@ snappy_decompress_internal(PG_FUNCTION_ARGS)
const char  *src= PG_GETARG_POINTER(0);
size_t  src_sz = PG_GETARG_INT32(1);
char*dst= PG_GETARG_POINTER(2);
-   int32   dst_sz = PG_GETARG_INT32(3);
+   size_t  dst_sz = PG_GETARG_INT32(3);
int32   *dst_used = PG_GETARG_POINTER(4);
size_t  uncompressed_length;
snappy_status   retval;
 
Insist(src_sz > 0 && dst_sz > 0);
 
-   retval = snappy_uncompressed_length((char *) src, (size_t) src_sz,
+   retval = snappy_uncompressed_length(src, src_sz,

_length);
if (retval != SNAPPY_OK)
elog_snappy_error(retval, "snappy_uncompressed_length",
@@ -467,8 +467,7 @@ snappy_decompress_internal(PG_FUNCTION_ARGS)
 
Insist(dst_sz >= uncompressed_length);
 
-   retval = snappy_uncompress((char *) src, src_sz, (char *) dst,
-  
_length);
+   retval = snappy_uncompress(src, src_sz, dst, _length);
*dst_used = uncompressed_length;
 
if (retval != SNAPPY_OK)

[2/2] incubator-hawq git commit: HAWQ-991. Add note of hash distributed table in hawq register help.

[hubertzhang]

HAWQ-991. Add note of hash distributed table in hawq register help.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a37b3de0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a37b3de0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a37b3de0

Branch: refs/heads/master
Commit: a37b3de018f4229cd3075345cc80ba8cf4adcda0
Parents: a120125
Author: hzhang2 
Authored: Mon Sep 26 19:12:30 2016 +0800
Committer: hzhang2 
Committed: Tue Sep 27 07:11:54 2016 +0800

--
 tools/doc/hawqregister_help | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a37b3de0/tools/doc/hawqregister_help
--
diff --git a/tools/doc/hawqregister_help b/tools/doc/hawqregister_help
index b8577d7..ad6689d 100644
--- a/tools/doc/hawqregister_help
+++ b/tools/doc/hawqregister_help
@@ -52,6 +52,7 @@ Note. With --force option specified, if there are files under 
table directory wh
 Note. In usage2, if the table is hash distributed, hawq register just check 
the file number to be registered
   has to be multiple times of this table’s bucket number, and check 
whether the distribution key specified
   in .yml configuration file is same as that of table. It does not check 
whether files are actually distributed by the key.
+Note. To register a hash distributed table through yaml file , please make 
sure the order of the files in yaml keeps the hash distribution.
 
 To use "hawq register", HAWQ must have been started.
 Currently "hawq register" supports both AO and Parquet formats in this case.

[1/2] incubator-hawq git commit: HAWQ-991. Fix bug of yaml configuration file contains only files under table directory in --force mode.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master a12012581 -> 9c97bccfd


HAWQ-991. Fix bug of yaml configuration file contains only files under table 
directory in --force mode.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9c97bccf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9c97bccf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9c97bccf

Branch: refs/heads/master
Commit: 9c97bccfdd9a58ef16f13ecdb028b773c7699484
Parents: a37b3de
Author: hzhang2 
Authored: Mon Sep 26 19:12:52 2016 +0800
Committer: hzhang2 
Committed: Tue Sep 27 07:11:54 2016 +0800

--
 tools/bin/hawqregister | 44 ++--
 1 file changed, 30 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9c97bccf/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index c7c207d..6c01b77 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -564,7 +564,12 @@ class HawqRegister(object):
 self.failure_handler.rollback()
 sys.exit(1)
 
+if not self.yml:
+check_no_regex_filepath([self.filepath])
+self.files, self.sizes = self._get_files_in_hdfs(self.filepath)
+   
 self.do_not_move, self.files_update, self.sizes_update = False, [], []
+self.newfiles, self.newsizes = [f for f in self.files], [sz for sz in 
self.sizes]
 if self.mode == 'force':
 existed_files, _ = self._get_files_in_hdfs(self.tabledir)
 if len(self.files) == len(existed_files):
@@ -574,24 +579,21 @@ class HawqRegister(object):
 sys.exit(1)
 else:
 self.do_not_move, self.files_update, self.sizes_update = 
True, self.files, self.sizes
-self.files, self.sizes = [], []
 elif len(self.files) < len(existed_files):
 logger.error('In force mode, you should include existing table 
files in yaml configuration file. Otherwise you should drop the previous table 
before register --force.')
 self.failure_handler.rollback()
 sys.exit(1)
 else:
-files_old, sizes_old = [f for f in self.files], [sz for sz in 
self.sizes]
-for k, f in enumerate(files_old):
+for k, f in enumerate(self.files):
 if f in existed_files:
-self.files_update.append(files_old[k])
-self.sizes_update.append(sizes_old[k])
-self.files.remove(files_old[k])
-self.sizes.remove(sizes_old[k])
+self.files_update.append(self.files[k])
+self.sizes_update.append(self.sizes[k])
+self.newfiles.remove(self.files[k])
+self.newsizes.remove(self.sizes[k])
 if sorted(self.files_update) != sorted(existed_files):
 logger.error('In force mode, you should include existing 
table files in yaml configuration file. Otherwise you should drop the previous 
table before register --force.')
 self.failure_handler.rollback()
 sys.exit(1)
-
 elif self.mode == 'repair':
 self.do_not_move = True
 self.files_update, self.sizes_update = [fn for fn in self.files], 
[sz for sz in self.sizes]
@@ -603,9 +605,6 @@ class HawqRegister(object):
 
 self._check_files_and_table_in_same_hdfs_cluster(self.filepath, 
self.tabledir)
 
-if not self.yml:
-check_no_regex_filepath([self.filepath])
-self.files, self.sizes = self._get_files_in_hdfs(self.filepath)
 print 'New file(s) to be registered: ', self.files
 if self.files_update:
 print 'Catalog info need to be updated for these files: ', 
self.files_update
@@ -692,7 +691,7 @@ class HawqRegister(object):
 def _move_files_in_hdfs(self):
 '''Move file(s) in src path into the folder correspoding to the target 
table'''
 segno = self.firstsegno
-for f in self.files:
+for f in self.newfiles:
 srcfile = f
 dstfile = self.tabledir + str(segno)
 segno += 1
@@ -729,6 +728,23 @@ class HawqRegister(object):
 for k, eof in enumerate(eofs[1:]):
 query += ',(%d, %d, %d, %d, %d)' % (self.firstsegno + k + 
1, eof, -1, -1, -1)
 query += ';'
+elif mode == 'force':
+eofs = self.sizes
+query = "set 

incubator-hawq git commit: HAWQ-1035. Treat ddl of list and range partition table separately in hawq register.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 72553413c -> 17f555fb3


HAWQ-1035. Treat ddl of list and range partition table separately in hawq 
register.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17f555fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17f555fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17f555fb

Branch: refs/heads/master
Commit: 17f555fb3421e45f9c014475154fe061e061cd61
Parents: 7255341
Author: hzhang2 
Authored: Wed Sep 21 14:23:35 2016 +0800
Committer: hzhang2 
Committed: Wed Sep 21 15:46:24 2016 +0800

--
 tools/bin/hawqregister | 27 ++-
 1 file changed, 18 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17f555fb/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index d030854..fa23a1a 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -133,16 +133,23 @@ class GpRegisterAccessor(object):
 qry = """select count(*) from pg_class where relname = '%s';""" % 
tablename.split('.')[-1].lower()
 return self.exec_query(qry)[0]['count'] == 1
 
-def do_create_table(self, tablename, schema_info, fmt, distrbution_policy, 
file_locations, bucket_number, partitionby, partitions_constraint, 
partitions_name):
+def do_create_table(self, src_table_name, tablename, schema_info, fmt, 
distrbution_policy, file_locations, bucket_number, partitionby, 
partitions_constraint, partitions_name):
 if self.get_table_existed(tablename):
 return False
 schema = ','.join([k['name'] + ' ' + k['type'] for k in schema_info])
 partlist = ""
 for index in range(len(partitions_constraint)):
-if index > 0:
-partlist += ", "
-partlist = partlist + "partition " + partitions_name[index] + " " 
+ partitions_constraint[index]
-  
+  if index > 0:
+  partlist += ", "
+  partition_refine_name = partitions_name[index]
+  splitter =  src_table_name.split(".")[-1] + '_1_prt_'
+  partition_refine_name = partition_refine_name.split(splitter)[-1]
+  #in some case, constraint contains "partition XXX" but in other 
case, it doesn't contain. we need to treat them separately.
+  if partitions_constraint[index].strip().startswith("DEFAULT 
PARTITION") or partitions_constraint[index].strip().startswith("PARTITION") or 
(len(partition_refine_name) > 0 and partition_refine_name[0].isdigit()):
+  partlist = partlist + " " + partitions_constraint[index]
+  else:
+  partlist = partlist + "PARTITION " + partition_refine_name + " " 
+ partitions_constraint[index]
+  
 fmt = 'ROW' if fmt == 'AO' else fmt
 if fmt == 'ROW':
 if partitionby is None:
@@ -158,6 +165,7 @@ class GpRegisterAccessor(object):
 else:
 query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, pagesize=%s, 
rowgroupsize=%s, bucketnum=%s) %s %s (%s);'
  % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['PageSize'], file_locations['RowGroupSize'], bucket_number, 
distrbution_policy, partitionby, partlist))
+print query
 self.conn.query(query)
 return True
 
@@ -280,7 +288,7 @@ class HawqRegister(object):
 sys.exit(1)
 
 def create_table():
-return self.accessor.do_create_table(self.tablename, self.schema, 
self.file_format, self.distribution_policy, self.file_locations, 
self.bucket_number,
+return self.accessor.do_create_table(self.src_table_name, 
self.tablename, self.schema, self.file_format, self.distribution_policy, 
self.file_locations, self.bucket_number,
  self.partitionby, 
self.partitions_constraint, self.partitions_name)
 
 def get_seg_name():
@@ -318,12 +326,13 @@ class HawqRegister(object):
 logger.error('Bucket number of %s is not consistent with 
previous bucket number.' % self.tablename)
 sys.exit(1)
 
-def set_yml_dataa(file_format, files, sizes, schema, 
distribution_policy, file_locations,\
+def set_yml_dataa(file_format, files, sizes, tablename, schema, 
distribution_policy, file_locations,\
   bucket_number, partitionby, partitions_constraint, 
partitions_name, partitions_compression_level,\
   

incubator-hawq git commit: HAWQ-1035. Add ddl of partition table for hawq register.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 50c1aa9ea -> f676c58e4


HAWQ-1035. Add ddl of partition table for hawq register.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f676c58e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f676c58e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f676c58e

Branch: refs/heads/master
Commit: f676c58e4d9f007b13e0e21942134c936c6cb94d
Parents: 50c1aa9
Author: hzhang2 
Authored: Tue Sep 20 12:58:14 2016 +0800
Committer: hzhang2 
Committed: Tue Sep 20 16:34:39 2016 +0800

--
 tools/bin/hawqregister | 27 +--
 1 file changed, 21 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f676c58e/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index bdd6947..89e9f4b 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -133,17 +133,31 @@ class GpRegisterAccessor(object):
 qry = """select count(*) from pg_class where relname = '%s';""" % 
tablename.split('.')[-1].lower()
 return self.exec_query(qry)[0]['count'] == 1
 
-def do_create_table(self, tablename, schema_info, fmt, distrbution_policy, 
file_locations, bucket_number):
+def do_create_table(self, tablename, schema_info, fmt, distrbution_policy, 
file_locations, bucket_number, partitionby, partitions_constraint, 
partitions_name):
 if self.get_table_existed(tablename):
 return False
 schema = ','.join([k['name'] + ' ' + k['type'] for k in schema_info])
+partlist = ""
+for index in range(len(partitions_constraint)):
+  if index > 0:
+  partlist += ", "
+  partlist = partlist + "partition " + partitions_name[index] + " " + 
partitions_constraint[index]
+  
 fmt = 'ROW' if fmt == 'AO' else fmt
 if fmt == 'ROW':
-query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, checksum=%s, bucketnum=%s) 
%s;'
- % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['Checksum'], bucket_number, distrbution_policy))
+if partitionby is None:
+query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, checksum=%s, bucketnum=%s) 
%s;'
+ % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['Checksum'], bucket_number, distrbution_policy))
+else:
+query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, checksum=%s, bucketnum=%s) 
%s %s (%s);'
+ % (tablename, schema,fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['Checksum'], bucket_number, distrbution_policy, partitionby, 
partlist))
 else: # Parquet
-query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, pagesize=%s, 
rowgroupsize=%s, bucketnum=%s) %s;'
- % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['PageSize'], file_locations['RowGroupSize'], bucket_number, 
distrbution_policy))
+if partitionby is None:
+query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, pagesize=%s, 
rowgroupsize=%s, bucketnum=%s) %s;'
+ % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['PageSize'], file_locations['RowGroupSize'], bucket_number, 
distrbution_policy))
+else:
+query = ('create table %s(%s) with (appendonly=true, 
orientation=%s, compresstype=%s, compresslevel=%s, pagesize=%s, 
rowgroupsize=%s, bucketnum=%s) %s %s (%s);'
+ % (tablename, schema, fmt, 
file_locations['CompressionType'], file_locations['CompressionLevel'], 
file_locations['PageSize'], file_locations['RowGroupSize'], bucket_number, 
distrbution_policy, partitionby, partlist))
 self.conn.query(query)
 return True
 
@@ -266,7 +280,8 @@ class HawqRegister(object):
 sys.exit(1)
 
 def create_table():
-return self.accessor.do_create_table(self.tablename, self.schema, 
self.file_format, self.distribution_policy, self.file_locations, 
self.bucket_number)
+return 

[1/2] incubator-hawq git commit: HAWQ-1044. Add normal path testcase for hawq register when file exists, file does not exist and --force

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 410f35a7c -> 044411285


HAWQ-1044. Add normal path testcase for hawq register when file exists, file 
does not exist and --force


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f87c234e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f87c234e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f87c234e

Branch: refs/heads/master
Commit: f87c234e127315aa2a0ab00a0dfe9c3acdef530a
Parents: 410f35a
Author: Chunling Wang 
Authored: Mon Sep 19 11:15:38 2016 +0800
Committer: Chunling Wang 
Committed: Mon Sep 19 11:15:56 2016 +0800

--
 .../feature/ManagementTool/test_hawq_register.h | 54 +---
 .../test_hawq_register_usage2_case1.cpp | 51 +++
 .../test_hawq_register_usage2_case2.cpp | 65 
 .../ManagementTool/usage2case2/t_tpl_1.yml  | 31 ++
 .../ManagementTool/usage2case2/t_tpl_2.yml  | 33 ++
 .../ManagementTool/usage2case2/t_tpl_3.yml  | 21 +++
 .../ManagementTool/usage2case2/t_tpl_4.yml  | 23 +++
 .../ManagementTool/usage2case2/t_tpl_new_1.yml  | 43 +
 .../ManagementTool/usage2case2/t_tpl_new_2.yml  | 45 ++
 .../ManagementTool/usage2case2/t_tpl_new_3.yml  | 23 +++
 .../ManagementTool/usage2case2/t_tpl_new_4.yml  | 25 
 11 files changed, 405 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f87c234e/src/test/feature/ManagementTool/test_hawq_register.h
--
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h 
b/src/test/feature/ManagementTool/test_hawq_register.h
index 3cdc288..70a2998 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -2,19 +2,55 @@
 #define TEST_HAWQ_REGISTER_H
 
 #include 
+#include 
 #include "lib/hdfs_config.h"
 #include "gtest/gtest.h"
 
 class TestHawqRegister : public ::testing::Test {
- public:
-  TestHawqRegister() {}
-  ~TestHawqRegister() {}
-  std::string getHdfsLocation() {
-hawq::test::HdfsConfig hc;
-std::string namenodehost = "";
-EXPECT_EQ(true, hc.getNamenodeHost(namenodehost));
-return hawq::test::stringFormat("hdfs://%s", namenodehost.c_str());
-  }
+public:
+TestHawqRegister() {
+std::string user = HAWQ_USER;
+if(user.empty()) {
+struct passwd *pw;
+uid_t uid = geteuid();
+pw = getpwuid(uid);
+user.assign(pw->pw_name);
+}
+conn.reset(new hawq::test::PSQL(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, 
user, HAWQ_PASSWORD));
+}
+~TestHawqRegister() {}
+
+std::string getHdfsLocation() {
+hawq::test::HdfsConfig hc;
+std::string namenodehost = "";
+EXPECT_EQ(true, hc.getNamenodeHost(namenodehost));
+return hawq::test::stringFormat("hdfs://%s", namenodehost.c_str());
+}
+
+std::string getDatabaseOid() {
+const hawq::test::PSQLQueryResult  = conn->getQueryResult(
+hawq::test::stringFormat("SELECT oid from pg_database where 
datname = \'%s\';", HAWQ_DB));
+std::vector table = result.getRows();
+if (table.size() > 0) {
+return table[0][0];
+}
+
+return "";
+}
+
+std::string getTableOid(std::string relname) {
+const hawq::test::PSQLQueryResult  = conn->getQueryResult(
+hawq::test::stringFormat("SELECT oid from pg_class where 
relname = \'%s\';", relname.c_str()));
+std::vector table = result.getRows();
+if (table.size() > 0) {
+return table[0][0];
+}
+
+return "";
+}
+
+private:
+std::unique_ptr conn;
 };
 
 #endif

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f87c234e/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
--
diff --git 
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp 
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
new file mode 100644
index 000..fa0d00d
--- /dev/null
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -0,0 +1,51 @@
+#include 
+
+#include "lib/command.h"
+#include "lib/sql_util.h"
+#include "lib/string_util.h"
+#include "lib/hdfs_config.h"
+#include "test_hawq_register.h"
+
+#include "gtest/gtest.h"
+
+using std::string;
+using 

[2/2] incubator-hawq git commit: HAWQ-1044. Add normal path testcase for hawq register --repair

[hubertzhang]

HAWQ-1044. Add normal path testcase for hawq register --repair


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/04441128
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/04441128
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/04441128

Branch: refs/heads/master
Commit: 0444112857c1e23a744c555f4d7b3353187fa749
Parents: f87c234
Author: Chunling Wang 
Authored: Mon Sep 19 13:20:22 2016 +0800
Committer: Chunling Wang 
Committed: Mon Sep 19 13:20:22 2016 +0800

--
 .../test_hawq_register_usage2_case1.cpp |  3 +-
 .../test_hawq_register_usage2_case2.cpp |  7 +-
 .../test_hawq_register_usage2_case3.cpp | 68 
 .../ManagementTool/usage2case3/t_tpl_old_1.yml  | 31 +
 .../ManagementTool/usage2case3/t_tpl_old_2.yml  | 33 ++
 .../ManagementTool/usage2case3/t_tpl_old_3.yml  | 21 ++
 .../ManagementTool/usage2case3/t_tpl_old_4.yml  | 23 +++
 7 files changed, 183 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/04441128/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
--
diff --git 
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp 
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index fa0d00d..c967bb0 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -42,7 +42,8 @@ TEST_F(TestHawqRegister, TestUsage2Case1Expected) {
 EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o 
t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, 
std::to_string(suffix).c_str(), t.c_str(;
 EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c 
t_%s.yml testhawqregister_testusage2case1expected.%s", HAWQ_DB, 
std::to_string(suffix).c_str(), nt.c_str(;
 util.query(hawq::test::stringFormat("select * from %s;", 
nt.c_str()), 150);
-EXPECT_EQ(0, Command::getCommandStatus("rm -rf t.yml"));
+
+EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("rm -rf t_%s.yml", 
std::to_string(suffix).c_str(;
 util.execute(hawq::test::stringFormat("drop table %s;", 
t.c_str()));
 util.execute(hawq::test::stringFormat("drop table %s;", 
nt.c_str()));
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/04441128/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
--
diff --git 
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp 
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
index 43fd837..ab0e101 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
@@ -43,7 +43,6 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
 strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
 strs_src_dst["@TABLE_OID@"]= getTableOid(t);
 frep.replace(t_yml_tpl, t_yml, strs_src_dst);
-auto tmp = hawq::test::stringFormat("hawq register -d %s -c %s 
testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml.c_str(), 
nt.c_str());
 EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c %s 
testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml.c_str(), 
nt.c_str(;
 util.query(hawq::test::stringFormat("select * from %s;", 
nt.c_str()), 200);
 
@@ -56,9 +55,13 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
 strs_src_dst["@TABLE_OID_OLD@"]= getTableOid(nt);
 strs_src_dst["@TABLE_OID_NEW@"]= getTableOid(t);
 frep.replace(t_yml_tpl_new, t_yml_new, strs_src_dst);
-tmp = hawq::test::stringFormat("hawq register --force -d %s -c %s 
testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml_new.c_str(), 
nt.c_str());
 EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("hawq register --force -d %s 
-c %s testhawqregister_testusage2case2expected.%s", HAWQ_DB, t_yml_new.c_str(), 
nt.c_str(;
 util.query(hawq::test::stringFormat("select * from %s;", 
nt.c_str()), 150);
+
+EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", 
t_yml.c_str(;
+EXPECT_EQ(0, 
Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", 
t_yml_new.c_str(;
+

incubator-hawq git commit: HAWQ-1035. Add yml parser to support partition table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master ef2aef879 -> 77e245dde


HAWQ-1035. Add yml parser to support partition table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/77e245dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/77e245dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/77e245dd

Branch: refs/heads/master
Commit: 77e245ddeb3e619159eec90e4934eadc38ed259c
Parents: ef2aef8
Author: hzhang2 
Authored: Sun Sep 18 16:32:33 2016 +0800
Committer: hzhang2 
Committed: Sun Sep 18 16:32:33 2016 +0800

--
 tools/bin/hawqregister | 92 +++--
 1 file changed, 72 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/77e245dd/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index ffae437..0c75662 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -39,7 +39,6 @@ logger = get_default_logger()
 EXECNAME = os.path.split(__file__)[-1]
 setup_tool_logging(EXECNAME, getLocalHostname(), getUserName())
 
-
 def option_parser():
 '''option parser'''
 parser = OptParser(option_class=OptChecker,
@@ -110,23 +109,6 @@ def register_yaml_dict_check(D):
 logger.error('Wrong configuration yaml file format: "%s" 
attribute does not exist.\n See example in "hawq register --help".' % 
'AO_FileLocations.%s' % attr)
 sys.exit(1)
 
-
-def option_parser_yml(yml_file):
-import yaml
-with open(yml_file, 'r') as f:
-params = yaml.load(f)
-register_yaml_dict_check(params)
-if params['FileFormat'].lower() == 'parquet':
-if not len(params['Parquet_FileLocations']['Files']):
-return 'Parquet', [], [], params['Parquet_Schema'], 
params['Distribution_Policy'], params['Parquet_FileLocations'], 
params['Bucketnum']
-files, sizes = [params['DFS_URL'] + d['path'] for d in 
params['Parquet_FileLocations']['Files']], [d['size'] for d in 
params['Parquet_FileLocations']['Files']]
-return 'Parquet', files, sizes, params['Parquet_Schema'], 
params['Distribution_Policy'], params['Parquet_FileLocations'], 
params['Bucketnum']
-if not len(params['AO_FileLocations']['Files']):
-return 'AO', [], [], params['AO_Schema'], 
params['Distribution_Policy'], params['AO_FileLocations'], params['Bucketnum']
-files, sizes = [params['DFS_URL'] + d['path'] for d in 
params['AO_FileLocations']['Files']], [d['size'] for d in 
params['AO_FileLocations']['Files']]
-return 'AO', files, sizes, params['AO_Schema'], 
params['Distribution_Policy'], params['AO_FileLocations'], params['Bucketnum']
-
-
 class GpRegisterAccessor(object):
 def __init__(self, conn):
 self.conn = conn
@@ -297,9 +279,79 @@ class HawqRegister(object):
 if self.bucket_number != get_bucket_number():
 logger.error('Bucket number of %s is not consistent with 
previous bucket number.' % self.tablename)
 sys.exit(1)
-
+
+def set_yml_dataa(file_format, files, sizes, schema, 
distribution_policy, file_locations,\
+  bucket_number, partitionby, partitions_constraint, 
partitions_name, partitions_compression_level,\
+  partitions_compression_type, partitions_checksum, 
partitions_filepaths, partitions_filesizes):
+self.file_format = file_format
+self.files = files
+self.sizes = sizes
+self.schema = schema
+self.distribution_policy = distribution_policy
+self.file_locations = file_locations
+self.bucket_number = bucket_number
+self.partitionby = partitionby
+self.partitions_constraint = partitions_constraint
+self.partitions_name = partitions_name 
+self.partitions_compression_level = partitions_compression_level
+self.partitions_compression_type = partitions_compression_type
+self.partitions_checksum = partitions_checksum
+self.partitions_filepaths = partitions_filepaths 
+self.partitions_filesizes = partitions_filesizes
+
+def option_parser_yml(yml_file):
+import yaml
+with open(yml_file, 'r') as f:
+params = yaml.load(f)
+register_yaml_dict_check(params)
+partitions_filepaths = []
+partitions_filesizes = []
+partitions_constraint = []
+partitions_name = []
+partitions_checksum = []
+partitions_compression_level = []
+

incubator-hawq git commit: HAWQ-1033. Add --force option for hawq register, refactor related case using matrix style.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 4fbe8e2dd -> 0664852a8


HAWQ-1033. Add --force option for hawq register, refactor related case using 
matrix style.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0664852a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0664852a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0664852a

Branch: refs/heads/master
Commit: 0664852a8e6fd8fb2ce9fed8316d10f3857e67e2
Parents: 4fbe8e2
Author: xunzhang 
Authored: Tue Sep 13 10:55:07 2016 +0800
Committer: hzhang2 
Committed: Tue Sep 13 11:39:28 2016 +0800

--
 src/test/feature/Makefile   |   2 +-
 .../ManagementTool/data/parquet200/dat.paq  | Bin 0 -> 1229 bytes
 .../ManagementTool/data/parquet200sum/a.paq | Bin 0 -> 596 bytes
 .../ManagementTool/data/parquet200sum/b.paq | Bin 0 -> 596 bytes
 .../ManagementTool/test_hawq_register.cpp   |  70 +---
 .../feature/ManagementTool/test_hawq_register.h |  20 
 .../test_hawq_register_usage1.cpp   | 111 +++
 tools/bin/hawqregister  |  86 +++---
 8 files changed, 205 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0664852a/src/test/feature/Makefile
--
diff --git a/src/test/feature/Makefile b/src/test/feature/Makefile
index 7202ac3..d12b751 100644
--- a/src/test/feature/Makefile
+++ b/src/test/feature/Makefile
@@ -8,7 +8,7 @@ gmock_lib_path = 
$(abs_top_srcdir)/depends/thirdparty/googletest/build/googlemoc
 
 override CXX = g++
 override CXXFLAGS = -Wall -O0 -g -std=c++11
-override CPPFLAGS := -I/usr/include -I/usr/local/include 
-I/usr/include/libxml2 -I$(abs_top_srcdir)/src/test/feature/ 
-I$(abs_top_srcdir)/src/test/feature/lib/ 
-I$(abs_top_srcdir)/src/interfaces/libpq -I$(abs_top_srcdir)/src/interfaces 
-I$(abs_top_srcdir)/src/include  -I$(gtest_include) -I$(gmock_include)
+override CPPFLAGS := -I/usr/include -I/usr/local/include 
-I/usr/include/libxml2 -I$(abs_top_srcdir)/src/test/feature/ 
-I$(abs_top_srcdir)/src/test/feature/ManagementTool/ 
-I$(abs_top_srcdir)/src/test/feature/lib/ 
-I$(abs_top_srcdir)/src/interfaces/libpq -I$(abs_top_srcdir)/src/interfaces 
-I$(abs_top_srcdir)/src/include  -I$(gtest_include) -I$(gmock_include)
 override LIBS := $(LIBS) -lgtest -lpq -lxml2 -ltest
 override LDFLAGS += -L/usr/local/lib -L/usr/lib 
-L$(abs_top_srcdir)/src/test/feature/ -L$(abs_top_srcdir)/src/test/feature/lib/ 
-L$(abs_top_srcdir)/src/interfaces/libpq -L$(gtest_lib_path) -L$(gmock_lib_path)
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0664852a/src/test/feature/ManagementTool/data/parquet200/dat.paq
--
diff --git a/src/test/feature/ManagementTool/data/parquet200/dat.paq 
b/src/test/feature/ManagementTool/data/parquet200/dat.paq
new file mode 100644
index 000..0fefc6d
Binary files /dev/null and 
b/src/test/feature/ManagementTool/data/parquet200/dat.paq differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0664852a/src/test/feature/ManagementTool/data/parquet200sum/a.paq
--
diff --git a/src/test/feature/ManagementTool/data/parquet200sum/a.paq 
b/src/test/feature/ManagementTool/data/parquet200sum/a.paq
new file mode 100644
index 000..c42ade9
Binary files /dev/null and 
b/src/test/feature/ManagementTool/data/parquet200sum/a.paq differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0664852a/src/test/feature/ManagementTool/data/parquet200sum/b.paq
--
diff --git a/src/test/feature/ManagementTool/data/parquet200sum/b.paq 
b/src/test/feature/ManagementTool/data/parquet200sum/b.paq
new file mode 100644
index 000..6ccdc83
Binary files /dev/null and 
b/src/test/feature/ManagementTool/data/parquet200sum/b.paq differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0664852a/src/test/feature/ManagementTool/test_hawq_register.cpp
--
diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp 
b/src/test/feature/ManagementTool/test_hawq_register.cpp
index 385f959..791b522 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register.cpp
@@ -1,30 +1,17 @@
 #include 
 
+#include "gtest/gtest.h"
 #include "lib/command.h"
 #include "lib/sql_util.h"
 #include "lib/string_util.h"
 #include "lib/hdfs_config.h"
-
-#include "gtest/gtest.h"
+#include 

[3/3] incubator-hawq git commit: HAWQ-1001. Fix some explanations of GUC.

[hubertzhang]

HAWQ-1001. Fix some explanations of GUC.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/65620be2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/65620be2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/65620be2

Branch: refs/heads/rangeracl
Commit: 65620be22f9a15b58eb4047792c9890755075a89
Parents: 7f12454
Author: hzhang2 
Authored: Wed Sep 7 09:07:02 2016 +0800
Committer: hzhang2 
Committed: Wed Sep 7 09:07:02 2016 +0800

--
 src/backend/utils/misc/guc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65620be2/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 6d831ae..659b775 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -349,7 +349,6 @@ bool
Debug_datumstream_write_use_small_initial_buffers = false;
 bool   gp_temporary_files_filespace_repair = false;
 bool   filesystem_support_truncate = true;
 bool   gp_allow_non_uniform_partitioning_ddl = true;
-bool  enable_ranger = false;
 
 intexplain_memory_verbosity = 0;
 char*  memory_profiler_run_id = "none";
@@ -733,6 +732,7 @@ int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_limit;
+bool enable_ranger = false;
 double   optimizer_cost_threshold;
 double  optimizer_nestloop_factor;
 double  locality_upper_bound;
@@ -4325,7 +4325,7 @@ static struct config_bool ConfigureNamesBool[] =
 
{
 {"enable_ranger", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
- gettext_noop("support to using ranger to manage hawq privilege."),
+ gettext_noop("Enable Apache Ranger for HAWQ privilege management."),
  NULL,
  GUC_SUPERUSER_ONLY
 },

[2/3] incubator-hawq git commit: HAWQ-1001. Implement HAWQ user ACL check through Ranger.

[hubertzhang]

HAWQ-1001. Implement HAWQ user ACL check through Ranger.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7f124547
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7f124547
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7f124547

Branch: refs/heads/rangeracl
Commit: 7f1245477b80f3f1cf86947eb1401f57ac8f2938
Parents: 6e1f7f8
Author: hzhang2 
Authored: Thu Aug 25 11:06:43 2016 +0800
Committer: hzhang2 
Committed: Thu Aug 25 11:06:43 2016 +0800

--
 src/backend/catalog/aclchk.c| 273 ++-
 src/backend/parser/parse_relation.c |  48 --
 src/include/utils/acl.h |  15 ++
 3 files changed, 276 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7f124547/src/backend/catalog/aclchk.c
--
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 3fa616f..dcb7c90 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -225,10 +225,18 @@ restrict_and_check_grant(bool is_grant, AclMode 
avail_goptions, bool all_privs,
 */
if (avail_goptions == ACL_NO_RIGHTS)
{
-   if (pg_aclmask(objkind, objectId, grantorId,
-  whole_mask | 
ACL_GRANT_OPTION_FOR(whole_mask),
-  ACLMASK_ANY) == ACL_NO_RIGHTS)
-   aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ if (enable_ranger) {
+   if (pg_rangercheck(objectId, grantorId,
+   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) != ACLCHECK_OK)
+ aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ }
+ else {
+   if (pg_aclmask(objkind, objectId, grantorId,
+   whole_mask | ACL_GRANT_OPTION_FOR(whole_mask),
+   ACLMASK_ANY) == ACL_NO_RIGHTS)
+ aclcheck_error(ACLCHECK_NO_PRIV, objkind, objname);
+ }
}
 
/*
@@ -2240,6 +2248,14 @@ has_rolcatupdate(Oid roleid)
return rolcatupdate;
 }
 
+AclResult
+pg_rangercheck(Oid table_oid, Oid roleid,
+ AclMode mask, AclMaskHow how)
+{
+  return ACLCHECK_OK;
+}
+
+
 /*
  * Relay for the various pg_*_mask routines depending on object kind
  */
@@ -3085,6 +3101,135 @@ pg_filesystem_aclmask(Oid fsysOid, Oid roleid,
return result;
 }
 
+
+/*
+ * Exported routine for checking a user's access privileges to a table
+ *
+ * Returns ACLCHECK_OK if the user has any of the privileges identified by
+ * 'mode'; otherwise returns a suitable error code (in practice, always
+ * ACLCHECK_NO_PRIV).
+ */
+AclResult
+pg_class_nativecheck(Oid table_oid, Oid roleid, AclMode mode)
+{
+  if (pg_class_aclmask(table_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a database
+ */
+AclResult
+pg_database_nativecheck(Oid db_oid, Oid roleid, AclMode mode)
+{
+  if (pg_database_aclmask(db_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a function
+ */
+AclResult
+pg_proc_nativecheck(Oid proc_oid, Oid roleid, AclMode mode)
+{
+  if (pg_proc_aclmask(proc_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a language
+ */
+AclResult
+pg_language_nativecheck(Oid lang_oid, Oid roleid, AclMode mode)
+{
+  if (pg_language_aclmask(lang_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a namespace
+ */
+AclResult
+pg_namespace_nativecheck(Oid nsp_oid, Oid roleid, AclMode mode)
+{
+  if (pg_namespace_aclmask(nsp_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a tablespace
+ */
+AclResult
+pg_tablespace_nativecheck(Oid spc_oid, Oid roleid, AclMode mode)
+{
+  if (pg_tablespace_aclmask(spc_oid, roleid, mode, ACLMASK_ANY) != 0)
+return ACLCHECK_OK;
+  else
+return ACLCHECK_NO_PRIV;
+}
+
+/*
+ * Exported routine for checking a user's access privileges to a foreign
+ * data wrapper
+ */
+AclResult
+pg_foreign_data_wrapper_nativecheck(Oid fdw_oid, Oid roleid, AclMode mode)
+{
+  if (pg_foreign_data_wrapper_aclmask(fdw_oid, roleid, mode, ACLMASK_ANY) != 0)
+return 

incubator-hawq git commit: HAWQ-1032. Fix bug of set bucket number of child partition to parent partition only when policy type equals to POLICYTYPE_PARTITIONED.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 47892a1fd -> af3e3b722


HAWQ-1032. Fix bug of set bucket number of child partition to parent partition 
only when policy type equals to POLICYTYPE_PARTITIONED.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/af3e3b72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/af3e3b72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/af3e3b72

Branch: refs/heads/master
Commit: af3e3b722af4319f3a1fa3586a7a2860c5259db5
Parents: 47892a1
Author: hzhang2 
Authored: Fri Sep 2 16:43:30 2016 +0800
Committer: hzhang2 
Committed: Fri Sep 2 16:43:30 2016 +0800

--
 src/backend/cdb/cdbpartition.c | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/af3e3b72/src/backend/cdb/cdbpartition.c
--
diff --git a/src/backend/cdb/cdbpartition.c b/src/backend/cdb/cdbpartition.c
index eb157ed..3efeea2 100644
--- a/src/backend/cdb/cdbpartition.c
+++ b/src/backend/cdb/cdbpartition.c
@@ -6623,7 +6623,10 @@ atpxPartAddList(Relation rel,
/* propagate owner */
((CreateStmt *)q->utilityStmt)->ownerid = 
ownerid;
/* child partition should have the same bucket 
number with parent partition */
-   if (parPolicy && parPolicy->ptype == 
POLICYTYPE_PARTITIONED) {
+   if (parPolicy && ((CreateStmt 
*)q->utilityStmt)->policy
+   && ((CreateStmt 
*)q->utilityStmt)->policy->nattrs > 0
+   && ((CreateStmt 
*)q->utilityStmt)->policy->ptype ==
+   POLICYTYPE_PARTITIONED) {
  ((CreateStmt 
*)q->utilityStmt)->policy->bucketnum = parPolicy->bucketnum;
}
}

incubator-hawq git commit: HAWQ-1032. Set bucket number of child partition to parent partition only when policy type equals to POLICYTYPE_PARTITIONED.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1fce139d5 -> 47892a1fd


HAWQ-1032. Set bucket number of child partition to parent partition only when 
policy type equals to POLICYTYPE_PARTITIONED.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/47892a1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/47892a1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/47892a1f

Branch: refs/heads/master
Commit: 47892a1fd1a10363a6c83ca0b95d8ad2c5855434
Parents: 1fce139
Author: hzhang2 
Authored: Fri Sep 2 14:32:13 2016 +0800
Committer: hzhang2 
Committed: Fri Sep 2 14:36:53 2016 +0800

--
 src/backend/cdb/cdbpartition.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/47892a1f/src/backend/cdb/cdbpartition.c
--
diff --git a/src/backend/cdb/cdbpartition.c b/src/backend/cdb/cdbpartition.c
index 06e4805..eb157ed 100644
--- a/src/backend/cdb/cdbpartition.c
+++ b/src/backend/cdb/cdbpartition.c
@@ -6623,7 +6623,7 @@ atpxPartAddList(Relation rel,
/* propagate owner */
((CreateStmt *)q->utilityStmt)->ownerid = 
ownerid;
/* child partition should have the same bucket 
number with parent partition */
-   if (parPolicy) {
+   if (parPolicy && parPolicy->ptype == 
POLICYTYPE_PARTITIONED) {
  ((CreateStmt 
*)q->utilityStmt)->policy->bucketnum = parPolicy->bucketnum;
}
}

incubator-hawq git commit: HAWQ-1039. Add test case of bucket number may not be consistent with parent table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 193dad37e -> 25a4ab540


HAWQ-1039. Add test case of bucket number may not be consistent with parent 
table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/25a4ab54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/25a4ab54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/25a4ab54

Branch: refs/heads/master
Commit: 25a4ab5401d442c9b2d70d8c8a04ad5d2aba1080
Parents: 193dad3
Author: hzhang2 
Authored: Thu Sep 1 17:38:11 2016 +0800
Committer: hzhang2 
Committed: Thu Sep 1 17:38:11 2016 +0800

--
 .../ans/partition_negetive_and_basics.ans   | 96 +++-
 .../sql/partition_negetive_and_basics.sql   | 28 ++
 2 files changed, 103 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25a4ab54/src/test/feature/partition/ans/partition_negetive_and_basics.ans
--
diff --git a/src/test/feature/partition/ans/partition_negetive_and_basics.ans 
b/src/test/feature/partition/ans/partition_negetive_and_basics.ans
old mode 100755
new mode 100644
index 744e6a6..9d4a00f
--- a/src/test/feature/partition/ans/partition_negetive_and_basics.ans
+++ b/src/test/feature/partition/ans/partition_negetive_and_basics.ans
@@ -1,3 +1,7 @@
+-- start_ignore
+SET SEARCH_PATH=TestPartition_TestPartitionNegativeAndBasics;
+SET
+-- end_ignore
 set enable_partition_rules = false;
 SET
 set gp_enable_hash_partitioned_tables = true;
@@ -1085,7 +1089,7 @@ INSERT 0 1
 insert into ggg values (9, 9);
 INSERT 0 1
 insert into ggg values (10, 10);
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:566: ERROR:  no 
partition for partitioning key  (seg5 localhost:4 pid=172591)
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:566: ERROR:  no 
partition for partitioning key  (seg1 localhost:4 pid=35100)
 select * from ggg order by 1, 2;
  id | a 
 +---
@@ -1317,7 +1321,7 @@ 
psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:690: NOTICE:  CREATE
 ALTER TABLE
 -- works - anonymous partition MPP-3350
 alter table hhh add partition end ('2010-02-01');
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:693: NOTICE:  
CREATE TABLE will create partition "hhh_1_prt_r652516349" for table "hhh"
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:693: NOTICE:  
CREATE TABLE will create partition "hhh_1_prt_r1411808964" for table "hhh"
 ALTER TABLE
 -- MPP-3607 - ADD PARTITION with open intervals
 create table no_end1 (aa int, bb int) partition by range (bb)
@@ -1585,7 +1589,7 @@ insert into j select i from generate_series(1, 8) i;
 INSERT 0 8
 alter table j split partition for(1) at (2, 3) into (partition fa, partition
 fb);
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:870: NOTICE:  
exchanged partition "a" of relation "j" with relation "pg_temp_43517"
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:870: NOTICE:  
exchanged partition "a" of relation "j" with relation "pg_temp_116667"
 psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:870: NOTICE:  
dropped partition "a" for relation "j"
 psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:870: NOTICE:  
CREATE TABLE will create partition "j_1_prt_fa" for table "j"
 psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:870: NOTICE:  
CREATE TABLE will create partition "j_1_prt_fb" for table "j"
@@ -1605,10 +1609,10 @@ select * from j_1_prt_fb;
 (2 rows)
 
 alter table j split partition for(5) at (6);
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
exchanged partition "b" of relation "j" with relation "pg_temp_43517"
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
exchanged partition "b" of relation "j" with relation "pg_temp_116667"
 psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
dropped partition "b" for relation "j"
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
CREATE TABLE will create partition "j_1_prt_r630513650" for table "j"
-psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
CREATE TABLE will create partition "j_1_prt_r1373370768" for table "j"
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
CREATE TABLE will create partition "j_1_prt_r508808810" for table "j"
+psql:/tmp/TestPartition_TestPartitionNegativeAndBasics.sql:873: NOTICE:  
CREATE TABLE will create partition "j_1_prt_r468322643" for table "j"
 ALTER TABLE
 select * from j;
  i 
@@ -1658,7 +1662,7 @@ insert into k select i from generate_series(1, 30) i;
 INSERT 0 30
 alter table k split default 

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ranger [deleted] 8712bde26

[incubator-hawq] Git Push Summary

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ranger [deleted] 7f1245477

[1/2] incubator-hawq git commit: HAWQ-1002. Implement a switch in hawq-site.xml to configure whether use Ranger or not for ACL.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/ranger [created] 7f1245477


HAWQ-1002. Implement a switch in hawq-site.xml to configure whether use Ranger 
or not for ACL.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6e1f7f8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6e1f7f8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6e1f7f8b

Branch: refs/heads/ranger
Commit: 6e1f7f8b2a93e38e206ddfca4bec30820097a3f3
Parents: 8712bde
Author: hzhang2 
Authored: Thu Aug 18 17:08:58 2016 +0800
Committer: hzhang2 
Committed: Thu Aug 18 17:08:58 2016 +0800

--
 src/backend/utils/misc/guc.c | 11 +++
 src/include/utils/guc.h  |  1 +
 2 files changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e1f7f8b/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2cdcb62..6d831ae 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -349,6 +349,7 @@ bool
Debug_datumstream_write_use_small_initial_buffers = false;
 bool   gp_temporary_files_filespace_repair = false;
 bool   filesystem_support_truncate = true;
 bool   gp_allow_non_uniform_partitioning_ddl = true;
+bool  enable_ranger = false;
 
 intexplain_memory_verbosity = 0;
 char*  memory_profiler_run_id = "none";
@@ -4323,6 +4324,16 @@ static struct config_bool ConfigureNamesBool[] =
},
 
{
+{"enable_ranger", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
+ gettext_noop("support to using ranger to manage hawq privilege."),
+ NULL,
+ GUC_SUPERUSER_ONLY
+},
+_ranger,
+false, NULL, NULL
+  },
+
+   {
{"filesystem_support_truncate", PGC_USERSET, APPENDONLY_TABLES,
 gettext_noop("the file system support truncate feature."),
 NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e1f7f8b/src/include/utils/guc.h
--
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 41f7056..7cd7432 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -275,6 +275,7 @@ extern bool gp_plpgsql_clear_cache_always;
 extern bool gp_disable_catalog_access_on_segment;
 
 extern bool gp_called_by_pgdump;
+extern bool enable_ranger;
 
 /* Debug DTM Action */
 typedef enum

incubator-hawq git commit: HAWQ-999. Log_error when file count is not in proportion to bucket number of hash table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master a8da53f40 -> 8712bde26


HAWQ-999. Log_error when file count is not in proportion to bucket number of 
hash table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8712bde2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8712bde2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8712bde2

Branch: refs/heads/master
Commit: 8712bde26e516e1dbb69a145d5dba30c8804f359
Parents: a8da53f
Author: hzhang2 
Authored: Tue Aug 16 11:11:02 2016 +0800
Committer: hzhang2 
Committed: Tue Aug 16 11:11:02 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8712bde2/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index 3223a37..c475624 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -3797,7 +3797,7 @@ run_allocation_algorithm(SplitAllocResult *result, List 
*virtual_segments, Query
  FileCountBucketNumMismatch = fileCountInRelation %
targetPolicy->bucketnum == 0 ? false : true;
}
-   if (FileCountBucketNumMismatch && 
!allow_file_count_bucket_num_mismatch) {
+   if (isRelationHash && FileCountBucketNumMismatch && 
!allow_file_count_bucket_num_mismatch) {
  elog(ERROR, "file count %d in catalog is not in proportion to 
the bucket "
  "number %d of hash table with oid=%u, some data may be 
lost, if you "
  "still want to continue the query by considering the 
table as random, set GUC "

incubator-hawq git commit: HAWQ-999. Log_error when file count is not in proportion to bucket number of hash table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7920ad4ec -> a8da53f40


HAWQ-999. Log_error when file count is not in proportion to bucket number of 
hash table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a8da53f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a8da53f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a8da53f4

Branch: refs/heads/master
Commit: a8da53f40f07b25e74640f66ef1e0123ee74c5cf
Parents: 7920ad4
Author: hzhang2 
Authored: Tue Aug 16 10:54:48 2016 +0800
Committer: hzhang2 
Committed: Tue Aug 16 10:54:48 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c | 26 +++---
 src/backend/utils/misc/guc.c  | 12 
 src/include/utils/guc.h   |  1 +
 3 files changed, 36 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a8da53f4/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index aaa4c8d..3223a37 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -3791,12 +3791,27 @@ run_allocation_algorithm(SplitAllocResult *result, List 
*virtual_segments, Query
targetPolicy = GpPolicyFetch(CurrentMemoryContext, myrelid);
bool isRelationHash = is_relation_hash(targetPolicy);
 
+   int fileCountInRelation = list_length(rel_data->files);
+   bool FileCountBucketNumMismatch = false;
+   if (targetPolicy->bucketnum > 0) {
+ FileCountBucketNumMismatch = fileCountInRelation %
+   targetPolicy->bucketnum == 0 ? false : true;
+   }
+   if (FileCountBucketNumMismatch && 
!allow_file_count_bucket_num_mismatch) {
+ elog(ERROR, "file count %d in catalog is not in proportion to 
the bucket "
+ "number %d of hash table with oid=%u, some data may be 
lost, if you "
+ "still want to continue the query by considering the 
table as random, set GUC "
+ "allow_file_count_bucket_num_mismatch to on and try 
again.",
+ fileCountInRelation, targetPolicy->bucketnum, myrelid);
+   }
/* change the virtual segment order when keep hash.
 * order of idMap should also be changed.
+* if file count of the table is not equal to or multiple of
+* bucket number, we should process it as random table.
 */
if (isRelationHash && context->keep_hash
&& assignment_context.virtual_segment_num == 
targetPolicy->bucketnum
-   && !vSegOrderChanged) {
+   && !vSegOrderChanged && 
!FileCountBucketNumMismatch) {
change_hash_virtual_segments_order(resourcePtr, 
rel_data,
_context, );
for (int p = 0; p < idMap.target_segment_num; p++) {
@@ -3822,8 +3837,13 @@ run_allocation_algorithm(SplitAllocResult *result, List 
*virtual_segments, Query
uint64_t before_run_allocate_hash_or_random = 
gettime_microsec();
/*allocate hash relation*/
if (isRelationHash) {
-   if (context->keep_hash && 
assignment_context.virtual_segment_num
-   == targetPolicy->bucketnum) {
+ /*
+  * if file count of the table is not equal to or multiple of
+  * bucket number, we should process it as random table.
+  */
+   if (context->keep_hash
+   && assignment_context.virtual_segment_num== 
targetPolicy->bucketnum
+   && !FileCountBucketNumMismatch) {
ListCell* parlc;
bool parentIsHashExist=false;
bool parentIsHash =false;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a8da53f4/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 7300fee..2cdcb62 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -450,6 +450,7 @@ char   *Debug_dtm_action_protocol_str;
 
 /* Enable check for compatibility of encoding and locale in createdb */
 bool   gp_encoding_check_locale_compatibility;
+bool  

incubator-hawq git commit: HAWQ-961. Dispatch session user info, instead of current BOOTSTRAP_SUPERUSERID, from master to segments

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master ce75f5bf7 -> 17744320e


HAWQ-961. Dispatch session user info, instead of current BOOTSTRAP_SUPERUSERID, 
from master to segments


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17744320
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17744320
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17744320

Branch: refs/heads/master
Commit: 17744320ebfeef106a2ba5c428d07e412cd746cf
Parents: ce75f5b
Author: Paul Guo 
Authored: Thu Jul 28 11:38:16 2016 +0800
Committer: hzhang2 
Committed: Tue Aug 9 09:29:22 2016 +0800

--
 src/backend/cdb/executormgr.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17744320/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 7150346..4fecdea 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -438,8 +438,8 @@ executormgr_dispatch_and_run(struct DispatchData *data, 
QueryExecutor *executor)

parms->seqServerHost, parms->seqServerHostlen, parms->seqServerPort,

parms->primary_gang_id,

GetCurrentStatementStartTimestamp(),
-   
BOOTSTRAP_SUPERUSERID,
-   true,
+   
GetSessionUserId(), /* For external tools who want this info on segments. */
+   
IsAuthenticatedUserSuperUser(),

BOOTSTRAP_SUPERUSERID,
true,

BOOTSTRAP_SUPERUSERID,

incubator-hawq git commit: HAWQ-925. Set default locale, timezone & datastyle before running sql command/file

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 69f3c9026 -> 2f95286e0


HAWQ-925. Set default locale, timezone & datastyle before running sql 
command/file


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2f95286e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2f95286e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2f95286e

Branch: refs/heads/master
Commit: 2f95286e06006a362f3db8410491cf1ccf52bd78
Parents: 69f3c90
Author: Paul Guo 
Authored: Mon Jul 18 19:48:54 2016 +0800
Committer: Paul Guo 
Committed: Tue Jul 19 15:29:36 2016 +0800

--
 .../ans/alter-table-addcol-insert-alltypes.ans  |   6 +-
 src/test/feature/catalog/ans/date.ans   |   4 +-
 src/test/feature/catalog/sql/date.sql   |   2 +-
 .../ans/partition_negetive_and_basics.ans   | 166 +--
 src/test/feature/test_main.cpp  |  73 
 src/test/regress/known_good_schedule|  15 --
 6 files changed, 160 insertions(+), 106 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f95286e/src/test/feature/catalog/ans/alter-table-addcol-insert-alltypes.ans
--
diff --git 
a/src/test/feature/catalog/ans/alter-table-addcol-insert-alltypes.ans 
b/src/test/feature/catalog/ans/alter-table-addcol-insert-alltypes.ans
index 4aec294..3c50bdf 100644
--- a/src/test/feature/catalog/ans/alter-table-addcol-insert-alltypes.ans
+++ b/src/test/feature/catalog/ans/alter-table-addcol-insert-alltypes.ans
@@ -29,8 +29,8 @@ INSERT INTO tmp
'{1,2,3,4}');
 INSERT 0 1
 SELECT * FROM tmp;
- initial | a |  b   |  c   |  d  |  e  | f |   g   |   
h| i |  j   
|   k|   l   |  m  |n|   p   | 
q |   r   |  s  |  t
  |   u   |  v  |w |
 x | y | z 
--+---+--+--+-+-+---+---++---+--++---+-+-+---+---+---+-+-+---+-+--+---+---+---
- | 4 | name | text | 4.1 | 4.1 | 2 | ((4.1,4.1),(3.1,3.1)) | 
1995-05-01 00:30:30+00 | c | {"1995-05-01 00:30:30+00","1992-08-24 
14:43:07+00","1970-01-01 00:00:00+00"} | 314159 | (1,1) | 512 | 1 2 3 4 5 6 7 8 
| magnetic disk | (1.1,1.1) | [(4.1,4.1),(3.1,3.1)] | 
((0,2),(4.1,4.1),(3.1,3.1)) | (4.1,4.1),(3.1,3.1) | ["1970-01-01 00:00:00+00" 
"infinity"] | 1970-01-01 00:00:00 | 01:00:10 | {1,2,3,4} | {1,2,3,4} | {1,2,3,4}
+ initial | a |  b   |  c   |  d  |  e  | f |   g   |   
   h   | i |   j
|   k|   l   |  m  |n   
 |   p   | q |   r   |  s   
   |  t  |  u  |
v |w | x | y | z 
+-+---+--+--+-+-+---+---+--+---+++---+-+-+---+---+---+-+-+-+--+--+---+---+---
+ | 4 | name | text | 4.1 | 4.1 | 2 | ((4.1,4.1),(3.1,3.1)) | Mon May 
01 00:30:30 1995 UTC | c | {"Mon May 01 00:30:30 1995 UTC","Mon Aug 24 14:43:07 
1992 UTC","Thu Jan 01 00:00:00 1970 UTC"} | 314159 | (1,1) | 512 | 1 2 3 4 5 6 
7 8 | magnetic disk | (1.1,1.1) | [(4.1,4.1),(3.1,3.1)] | 
((0,2),(4.1,4.1),(3.1,3.1)) | (4.1,4.1),(3.1,3.1) | ["Thu Jan 01 00:00:00 1970 
UTC" "infinity"] | Thu Jan 01 00:00:00 1970 | 01:00:10 | {1,2,3,4} | {1,2,3,4} 
| {1,2,3,4}
 (1 row)
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f95286e/src/test/feature/catalog/ans/date.ans
--
diff --git a/src/test/feature/catalog/ans/date.ans 
b/src/test/feature/catalog/ans/date.ans
index 1fe3ad4..57bcd01 100755
--- a/src/test/feature/catalog/ans/date.ans
+++ 

incubator-hawq git commit: HAWQ-887. Query should be continued when bucket number of result hash relation is bigger than the on number of command external table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 745bdca4c -> c66cfbad1


HAWQ-887. Query should be continued when bucket number of result hash relation 
is bigger than the on number of command external table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c66cfbad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c66cfbad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c66cfbad

Branch: refs/heads/master
Commit: c66cfbad1361d7d8e249d3e312f95d9566d1bcb3
Parents: 745bdca
Author: hzhang2 
Authored: Mon Jul 4 14:54:51 2016 +0800
Committer: hzhang2 
Committed: Mon Jul 4 14:54:51 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c66cfbad/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index c56232c..aaa4c8d 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -4218,7 +4218,7 @@ calculate_planner_segment_num(Query *query, 
QueryResourceLife resourceLife,
int maxTargetSegmentNumber = 0;
/* we keep resultRelationHashSegNum in the highest 
priority*/
if (context.resultRelationHashSegNum != 0) {
-   if ((context.resultRelationHashSegNum != 
context.externTableForceSegNum
+   if ((context.resultRelationHashSegNum < 
context.externTableForceSegNum
&& 
context.externTableForceSegNum != 0)
|| 
(context.resultRelationHashSegNum < context.externTableLocationSegNum)) {
cleanup_allocation_algorithm();

incubator-hawq git commit: HAWQ-861. Wrong logerror position in insert into hashtable select * from gpfdist external table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 86c07d2a4 -> 10f778cda


HAWQ-861. Wrong logerror position in insert into hashtable select * from 
gpfdist external table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/10f778cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/10f778cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/10f778cd

Branch: refs/heads/master
Commit: 10f778cda19e58bd5afb2577903e3d99ac6d032d
Parents: 86c07d2
Author: hzhang2 
Authored: Tue Jun 28 10:49:00 2016 +0800
Committer: hzhang2 
Committed: Tue Jun 28 10:49:00 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c | 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/10f778cd/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index 1d14e80..adcc3b8 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -739,8 +739,6 @@ static void check_keep_hash_and_external_table(
{
context->keep_hash = true;
context->resultRelationHashSegNum = 
targetPolicy->bucketnum;
-   pfree(targetPolicy);
-   return;
}
pfree(targetPolicy);
}
@@ -752,7 +750,6 @@ static void check_keep_hash_and_external_table(
{
context->keep_hash = true;
context->resultRelationHashSegNum = intoPolicy->bucketnum;
-   return;
}
 
foreach(lc, context->rtc_context.range_tables)

incubator-hawq git commit: HAWQ-862. Make user defined function get_ao_distribution work.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master c31209db7 -> 87b5d8f9b


HAWQ-862. Make user defined function get_ao_distribution work.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/87b5d8f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/87b5d8f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/87b5d8f9

Branch: refs/heads/master
Commit: 87b5d8f9b2bfca2e865fa3004d8c5cf6a73a2e4a
Parents: c31209d
Author: hzhang2 
Authored: Fri Jun 24 09:10:33 2016 +0800
Committer: hzhang2 
Committed: Fri Jun 24 09:26:17 2016 +0800

--
 src/backend/access/appendonly/aosegfiles.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/87b5d8f9/src/backend/access/appendonly/aosegfiles.c
--
diff --git a/src/backend/access/appendonly/aosegfiles.c 
b/src/backend/access/appendonly/aosegfiles.c
index 6da6a21..993e489 100644
--- a/src/backend/access/appendonly/aosegfiles.c
+++ b/src/backend/access/appendonly/aosegfiles.c
@@ -1425,9 +1425,9 @@ get_ao_distribution_name(PG_FUNCTION_ARGS)
 * assemble our query string
 */
initStringInfo();
-   appendStringInfo(, "select gp_segment_id,sum(tupcount) "
-   "from 
gp_dist_random('pg_aoseg.%s') "
-   "group by (gp_segment_id)", 
aoseg_relname);
+   appendStringInfo(, "select segno,sum(tupcount) "
+   "from pg_aoseg.%s "
+   "group by (segno)", 
aoseg_relname);
 
PG_TRY();
{

incubator-hawq git commit: HAWQ-827. Refactor aggregate_with_groupingsets checkinstall cases using new test framework.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 0e62247f3 -> 583ef22e0


HAWQ-827. Refactor aggregate_with_groupingsets checkinstall cases using new 
test framework.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/583ef22e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/583ef22e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/583ef22e

Branch: refs/heads/master
Commit: 583ef22e0da0522fe13ad6b1fc1188fc2abca1c4
Parents: 0e62247
Author: hzhang2 
Authored: Fri Jun 17 15:38:55 2016 +0800
Committer: hzhang2 
Committed: Fri Jun 17 15:38:55 2016 +0800

--
 src/test/feature/query/test_aggregate.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/583ef22e/src/test/feature/query/test_aggregate.cpp
--
diff --git a/src/test/feature/query/test_aggregate.cpp 
b/src/test/feature/query/test_aggregate.cpp
index 76f2ede..4f07689 100644
--- a/src/test/feature/query/test_aggregate.cpp
+++ b/src/test/feature/query/test_aggregate.cpp
@@ -56,7 +56,7 @@ TEST_F(TestAggregate, TestAggregateWithGroupingsets) {
   dGen.genAggregateTable("t");
 
   util.query("SELECT a, b, sum(c) sum_c FROM (SELECT a, b, c FROM t F1 "
-  "LIMIT 3) F2 GROUP BY GROUPING SETS((a, b), (b)) ORDER BY a,"
+  "LIMIT 3) F2 GROUP BY GROUPING SETS((a, b), (b)) ORDER BY a, b,"
   " sum_c;", "1|aa|10|\n1|bb|20|\n2|cc|20|\n|aa|10|\n|bb|20|\n|cc|20|\n");
 }
 

incubator-hawq git commit: HAWQ-827. Refactor aggregate_with_groupingsets checkinstall cases using new test framework.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 54d90913a -> f2066eb4c


HAWQ-827. Refactor aggregate_with_groupingsets checkinstall cases using new 
test framework.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f2066eb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f2066eb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f2066eb4

Branch: refs/heads/master
Commit: f2066eb4ceed998442fad91673cc9f792ca9d466
Parents: 54d9091
Author: hzhang2 
Authored: Fri Jun 17 10:30:28 2016 +0800
Committer: hzhang2 
Committed: Fri Jun 17 10:30:41 2016 +0800

--
 src/test/feature/lib/data_gen.cpp | 16 
 src/test/feature/lib/data_gen.h   |  6 ++
 src/test/feature/query/test_aggregate.cpp | 11 +++
 3 files changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f2066eb4/src/test/feature/lib/data_gen.cpp
--
diff --git a/src/test/feature/lib/data_gen.cpp 
b/src/test/feature/lib/data_gen.cpp
index df19c09..b8fc444 100644
--- a/src/test/feature/lib/data_gen.cpp
+++ b/src/test/feature/lib/data_gen.cpp
@@ -20,6 +20,22 @@ void DataGenerator::genSimpleTable(string tableName, bool 
appendonly,
   sqlUtil->execute(insertSql);
 }
 
+void DataGenerator::genAggregateTable(string tableName, bool appendonly,
+   string orientation,
+   string compresstype,
+   int compresslevel) {
+  string desc =
+  genTableDesc(appendonly, orientation, compresstype, compresslevel);
+  string createSql =
+  "create table " + tableName + "(a int, b VARCHAR, c int) " + desc;
+  sqlUtil->execute(createSql);
+
+  string insertSql =
+  "insert into " + tableName + " values(1, 'aa', 10), (1, 'bb', 20), "
+  "(2, 'cc', 20), (3, 'cc', 20);";
+  sqlUtil->execute(insertSql);
+}
+
 void DataGenerator::genTableWithSeries(string tableName, bool appendonly,
string orientation,
string compresstype,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f2066eb4/src/test/feature/lib/data_gen.h
--
diff --git a/src/test/feature/lib/data_gen.h b/src/test/feature/lib/data_gen.h
index 5c2adae..8a0a352 100644
--- a/src/test/feature/lib/data_gen.h
+++ b/src/test/feature/lib/data_gen.h
@@ -28,6 +28,12 @@ class DataGenerator {
   std::string compresstype = "none",
   int compresslevel = -1);
 
+  void genAggregateTable(std::string tableName,
+ bool appendonly = true,
+ std::string orientation = "row",
+ std::string compresstype = "none",
+ int compresslevel = -1);
+
   void genTableWithFullTypes(std::string tableName,
  bool appendonly = true,
  std::string orientation = "row",

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f2066eb4/src/test/feature/query/test_aggregate.cpp
--
diff --git a/src/test/feature/query/test_aggregate.cpp 
b/src/test/feature/query/test_aggregate.cpp
index 337ef3c..76f2ede 100644
--- a/src/test/feature/query/test_aggregate.cpp
+++ b/src/test/feature/query/test_aggregate.cpp
@@ -49,4 +49,15 @@ TEST_F(TestAggregate, TestCreateAggregate) {
   util.query("select newavg(b) as bavg from t group by a order by 
bavg","3|\n15|\n62|\n");
 }
 
+TEST_F(TestAggregate, TestAggregateWithGroupingsets) {
+  hawq::test::SQLUtility util;
+  util.execute("drop table if exists t");
+  hawq::test::DataGenerator dGen();
+  dGen.genAggregateTable("t");
+
+  util.query("SELECT a, b, sum(c) sum_c FROM (SELECT a, b, c FROM t F1 "
+  "LIMIT 3) F2 GROUP BY GROUPING SETS((a, b), (b)) ORDER BY a,"
+  " sum_c;", "1|aa|10|\n1|bb|20|\n2|cc|20|\n|aa|10|\n|bb|20|\n|cc|20|\n");
+}
+
 

incubator-hawq git commit: HAWQ-819. Wrong timestamp file path for Makefile target pg_prepare

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 5f2b5b46a -> 2ae1c9569


HAWQ-819. Wrong timestamp file path for Makefile target pg_prepare


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2ae1c956
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2ae1c956
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2ae1c956

Branch: refs/heads/master
Commit: 2ae1c95690038e42309a3725644e089e8349494c
Parents: 5f2b5b4
Author: Paul Guo 
Authored: Wed Jun 15 15:06:48 2016 +0800
Committer: hzhang2 
Committed: Thu Jun 16 09:58:38 2016 +0800

--
 src/Makefile.global.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2ae1c956/src/Makefile.global.in
--
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 988c922..1afbc16 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -694,7 +694,7 @@ endif # enable_nls
 
 pg_prepare:
 ifneq "$(findstring yes, $(with_pgcrypto))" ""
-   if [ ! -e $(abs_top_builddir)/pg_prepare_timestamp ]; then \
+   if [ ! -e $(abs_top_builddir)/depends/thirdparty/pg_prepare_timestamp 
]; then \
cd $(abs_top_srcdir); \
git submodule update --init depends/thirdparty/postgres && \
touch 
$(abs_top_builddir)/depends/thirdparty/pg_prepare_timestamp; \

incubator-hawq git commit: HAWQ-818. Add order by for aggregate feature test.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 09c8b8589 -> 37dcd1ed6


HAWQ-818. Add order by for aggregate feature test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/37dcd1ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/37dcd1ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/37dcd1ed

Branch: refs/heads/master
Commit: 37dcd1ed6ae5db2d2a2cde0d906b6522edebf10e
Parents: 09c8b85
Author: hzhang2 
Authored: Wed Jun 15 14:50:03 2016 +0800
Committer: hzhang2 
Committed: Wed Jun 15 14:55:35 2016 +0800

--
 src/test/feature/query/test_aggregate.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/37dcd1ed/src/test/feature/query/test_aggregate.cpp
--
diff --git a/src/test/feature/query/test_aggregate.cpp 
b/src/test/feature/query/test_aggregate.cpp
index f8f93e9..337ef3c 100644
--- a/src/test/feature/query/test_aggregate.cpp
+++ b/src/test/feature/query/test_aggregate.cpp
@@ -45,8 +45,8 @@ TEST_F(TestAggregate, TestCreateAggregate) {
   util.execute("drop table if exists t");
   hawq::test::DataGenerator dGen();
   dGen.genSimpleTable("t");
-  util.query("select avg(b) from t group by a","3|\n62|\n15|\n");
-  util.query("select newavg(b) from t group by a","3|\n15|\n62|\n");
+  util.query("select avg(b) as bavg from t group by a order by 
bavg","3|\n15|\n62|\n");
+  util.query("select newavg(b) as bavg from t group by a order by 
bavg","3|\n15|\n62|\n");
 }
 
 

incubator-hawq git commit: HAWQ-801 Refactor create_aggregate checkinstall cases using new test framework.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master f5b0cadb9 -> ab5fc167e


HAWQ-801  Refactor create_aggregate checkinstall cases using new test framework.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ab5fc167
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ab5fc167
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ab5fc167

Branch: refs/heads/master
Commit: ab5fc167ef88156e85ace9df42cc3b58dcd3031c
Parents: f5b0cad
Author: hzhang2 
Authored: Wed Jun 15 10:02:37 2016 +0800
Committer: hzhang2 
Committed: Wed Jun 15 10:04:37 2016 +0800

--
 src/test/feature/query/test_aggregate.cpp | 52 ++
 1 file changed, 52 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ab5fc167/src/test/feature/query/test_aggregate.cpp
--
diff --git a/src/test/feature/query/test_aggregate.cpp 
b/src/test/feature/query/test_aggregate.cpp
new file mode 100644
index 000..f8f93e9
--- /dev/null
+++ b/src/test/feature/query/test_aggregate.cpp
@@ -0,0 +1,52 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "lib/command.h"
+#include "lib/data_gen.h"
+#include "lib/hawq_config.h"
+#include "lib/sql_util.h"
+
+#include "gtest/gtest.h"
+
+class TestAggregate : public ::testing::Test {
+ public:
+  TestAggregate() {}
+  ~TestAggregate() {}
+};
+
+TEST_F(TestAggregate, TestCreateAggregate) {
+  hawq::test::SQLUtility util;
+
+  // create a basic aggregate
+  util.execute("CREATE AGGREGATE newavg (sfunc = int8_avg_accum,"
+  " basetype = int8, stype = bytea, finalfunc = int8_avg,"
+  " initcond1 = '{0}');", true);
+
+  // create a full featured aggregate including schema.
+  util.execute("create schema testaggregateschema;");
+  util.execute("CREATE AGGREGATE testaggregateschema.newavg ("
+  "sfunc = int8_avg_accum, basetype = int8, stype = bytea,"
+  " finalfunc = int8_avg, initcond1 = '{0}');", true);
+  util.execute("drop schema testaggregateschema cascade;");
+
+  // multi argument aggregate
+  util.execute("create function sum3(int8,int8,int8) returns int8 as "
+  "'select \\$1 + \\$2 + \\$3' language sql strict immutable;",true);
+  util.execute("create ordered aggregate sum2(int8,int8) ( sfunc = sum3, "
+  "stype = int8, initcond = '0');", true);
+
+  // test basic aggregate
+  util.execute("drop table if exists t");
+  hawq::test::DataGenerator dGen();
+  dGen.genSimpleTable("t");
+  util.query("select avg(b) from t group by a","3|\n62|\n15|\n");
+  util.query("select newavg(b) from t group by a","3|\n15|\n62|\n");
+}
+
+

incubator-hawq git commit: HAWQ-741. Set system login user as default when PGUSER is not set.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 9078784a5 -> 025eecbad


HAWQ-741. Set system login user as default when PGUSER is not set.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/025eecba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/025eecba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/025eecba

Branch: refs/heads/master
Commit: 025eecbad197ee8cbc4bf329c2a72f32472886ed
Parents: 9078784
Author: hzhang2 
Authored: Wed May 18 22:13:45 2016 +0800
Committer: hzhang2 
Committed: Thu May 19 10:01:15 2016 +0800

--
 src/test/feature/lib/sql-util.cpp | 11 ++-
 src/test/feature/lib/sql-util.h   |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/025eecba/src/test/feature/lib/sql-util.cpp
--
diff --git a/src/test/feature/lib/sql-util.cpp 
b/src/test/feature/lib/sql-util.cpp
index 08e0d09..61b3d87 100644
--- a/src/test/feature/lib/sql-util.cpp
+++ b/src/test/feature/lib/sql-util.cpp
@@ -1,5 +1,7 @@
 #include "sql-util.h"
 
+#include 
+#include 
 #include 
 
 #include 
@@ -77,8 +79,15 @@ void SQLUtility::execSQLFile(const std::string ,
 }
 
 std::unique_ptr SQLUtility::getConnection() {
+  std::string user = HAWQ_USER;
+  if(user == ""){
+struct passwd *pw;
+uid_t uid = geteuid();
+pw = getpwuid(uid);
+user.assign(pw->pw_name);;
+  }
   std::unique_ptr psql(
-  new PSQL(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD));
+  new PSQL(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, user, HAWQ_PASSWORD));
   return std::move(psql);
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/025eecba/src/test/feature/lib/sql-util.h
--
diff --git a/src/test/feature/lib/sql-util.h b/src/test/feature/lib/sql-util.h
index 5c5a59d..8070d3f 100644
--- a/src/test/feature/lib/sql-util.h
+++ b/src/test/feature/lib/sql-util.h
@@ -9,7 +9,7 @@
 #define HAWQ_DB (getenv("PGDATABASE") ? getenv("PGDATABASE") : "postgres")
 #define HAWQ_HOST (getenv("PGHOST") ? getenv("PGHOST") : "localhost")
 #define HAWQ_PORT (getenv("PGPORT") ? getenv("PGPORT") : "5432")
-#define HAWQ_USER (getenv("PGUSER") ? getenv("PGUSER") : "gpadmin")
+#define HAWQ_USER (getenv("PGUSER") ? getenv("PGUSER") : "")
 #define HAWQ_PASSWORD (getenv("PGPASSWORD") ? getenv("PGPASSWORD") : "")
 
 struct FilePath {

incubator-hawq git commit: HAWQ-727. Fix featuretest makefile error.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 5a3f69976 -> 02411abad


HAWQ-727. Fix featuretest makefile error.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/02411aba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/02411aba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/02411aba

Branch: refs/heads/master
Commit: 02411abad13329f216ff48bc6e6cde246fa36744
Parents: 5a3f699
Author: hzhang2 
Authored: Wed May 11 11:03:04 2016 +0800
Committer: hzhang2 
Committed: Wed May 11 11:06:01 2016 +0800

--
 GNUmakefile.in   | 2 +-
 configure.in | 2 +-
 src/Makefile | 4 ++--
 src/test/Makefile| 4 +++-
 src/test/feature/lib/hawq-config.cpp | 1 +
 src/test/feature/lib/string-util.cpp | 2 +-
 src/test/feature/lib/xml-parser.cpp  | 1 +
 7 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/GNUmakefile.in
--
diff --git a/GNUmakefile.in b/GNUmakefile.in
index 4da5951..d38273c 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -86,8 +86,8 @@ distclean maintainer-clean:
-$(MAKE) -C depends/libyarn $@
-$(MAKE) -C config $@
-$(MAKE) -C tools $@
+   -$(MAKE) -C src feature_test clean
-$(MAKE) -C src $@
-   $(MAKE) -C src feature_test clean
-rm -f config.cache config.log config.status GNUmakefile
 # Garbage from autoconf:
@rm -rf autom4te.cache/

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/configure.in
--
diff --git a/configure.in b/configure.in
index d2459b8..391e17e 100644
--- a/configure.in
+++ b/configure.in
@@ -2058,7 +2058,7 @@ fi
 AC_SUBST(vpath_build)
 
 
-AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk depends/libhdfs3/Makefile.global 
depends/libyarn/Makefile.global src/Makefile.global 
src/pl/pljava/src/java/Makefile.global])
+AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk src/Makefile.global 
depends/libhdfs3/Makefile.global depends/libyarn/Makefile.global 
src/Makefile.global src/pl/pljava/src/java/Makefile.global])
 
 AC_CONFIG_LINKS([
   src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/src/Makefile
--
diff --git a/src/Makefile b/src/Makefile
index 866a9bf..2b08a89 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -63,7 +63,7 @@ clean:
$(MAKE) -C tutorial NO_PGXS=1 $@
$(MAKE) -C test/thread $@
$(MAKE) -C test/regress $@
-   $(MAKE) -C test/feature/example $@
+   $(MAKE) -C test/feature $@
$(MAKE) -C tools/fsync $@
 
 distclean maintainer-clean:
@@ -80,7 +80,7 @@ distclean maintainer-clean:
$(MAKE) -C tutorial NO_PGXS=1 $@
$(MAKE) -C test/thread $@
$(MAKE) -C test/regress $@
-   $(MAKE) -C test/feature/example clean
+   $(MAKE) -C test/feature clean
$(MAKE) -C tools/fsync $@
rm -f Makefile.port Makefile.global
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/src/test/Makefile
--
diff --git a/src/test/Makefile b/src/test/Makefile
index ba4e729..9bc9b5c 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -14,10 +14,12 @@
 #.DEFAULT:
 #  $(MAKE) -C regress $@
 
-.PHONY: feature_test clean
+.PHONY: feature_test clean distclean
 
 feature_test:
$(MAKE) -C feature all
 
 clean:
$(MAKE) -C feature clean
+
+distclean: clean

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/src/test/feature/lib/hawq-config.cpp
--
diff --git a/src/test/feature/lib/hawq-config.cpp 
b/src/test/feature/lib/hawq-config.cpp
index d973328..263e54a 100644
--- a/src/test/feature/lib/hawq-config.cpp
+++ b/src/test/feature/lib/hawq-config.cpp
@@ -5,6 +5,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "command.h"
 #include "psql.h"

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/02411aba/src/test/feature/lib/string-util.cpp
--
diff --git a/src/test/feature/lib/string-util.cpp 
b/src/test/feature/lib/string-util.cpp
index 4474c80..e4bb7f5 100644
--- a/src/test/feature/lib/string-util.cpp
+++ b/src/test/feature/lib/string-util.cpp
@@ -42,7 +42,7 @@ std::string StringUtil::lower(const std::string ) {
 result.append(, sizeof(char));
   }
 
-  

incubator-hawq git commit: HAWQ-689. Create table with (oids) in PBE should check whether arg of oids is NULL.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 95197ad7e -> 76658bc1a


HAWQ-689. Create table with (oids) in PBE should check whether arg of oids is 
NULL.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/76658bc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/76658bc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/76658bc1

Branch: refs/heads/master
Commit: 76658bc1ae4dea666ac6fe86e7fd3ae54474fe61
Parents: 95197ad
Author: hzhang2 
Authored: Wed Apr 20 13:44:07 2016 +0800
Committer: hzhang2 
Committed: Wed Apr 20 13:44:07 2016 +0800

--
 src/backend/executor/execMain.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/76658bc1/src/backend/executor/execMain.c
--
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 5e8c6b1..3a43cec 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -4333,7 +4333,7 @@ CreateIntoRel(QueryDesc *queryDesc)
 
if (!IsA(e, DefElem))
continue;
-   if (!IsA(e->arg, String))
+   if (!e->arg || !IsA(e->arg, String))
continue;
if (pg_strcasecmp(e->defname, "appendonly"))
continue;

[1/5] incubator-hawq git commit: Revert "HAWQ-673. Unify out put of explain analyze."

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 8ba02a818 -> e9c5dc3d7


Revert "HAWQ-673. Unify out put of explain analyze."

This reverts commit 8ba02a8188d26b52615889bc44624db2cdf964b6.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f786bb87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f786bb87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f786bb87

Branch: refs/heads/master
Commit: f786bb876f30dcff9f439e1d8f324b3ca9e51c86
Parents: 8ba02a8
Author: hzhang2 
Authored: Thu Apr 14 15:50:53 2016 +0800
Committer: hzhang2 
Committed: Thu Apr 14 15:50:53 2016 +0800

--
 src/backend/cdb/cdbexplain.c | 1 +
 src/include/cdb/cdbexplain.h | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f786bb87/src/backend/cdb/cdbexplain.c
--
diff --git a/src/backend/cdb/cdbexplain.c b/src/backend/cdb/cdbexplain.c
index d6583b8..f20a7e5 100644
--- a/src/backend/cdb/cdbexplain.c
+++ b/src/backend/cdb/cdbexplain.c
@@ -306,6 +306,7 @@ cdbexplain_localExecStats(struct PlanState 
*planstate,
 
 /* Set up a temporary StatHdr for both collecting and depositing stats. */
 gethostname(ctx.send.hdr.hostname,SEGMENT_IDENTITY_NAME_LENGTH-1);
+
//strncpy(ctx.send.hdr.hostname,gethostname(),SEGMENT_IDENTITY_NAME_LENGTH-1);
 ctx.msgptrs[0] = 
 ctx.send.hdr.segindex = GetQEIndex();
 ctx.send.hdr.nInst = 1;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f786bb87/src/include/cdb/cdbexplain.h
--
diff --git a/src/include/cdb/cdbexplain.h b/src/include/cdb/cdbexplain.h
index 761b48d..719eb38 100644
--- a/src/include/cdb/cdbexplain.h
+++ b/src/include/cdb/cdbexplain.h
@@ -74,7 +74,7 @@ cdbexplain_agg_init1(CdbExplain_Agg *agg, double v, int id)
 static inline bool
 cdbexplain_agg_upd(CdbExplain_Agg *agg, double v, int id,char* hostname)
 {
-if(v == 0){
+/*if(v == 0){
   if (agg->vcnt == 0)
 {
 agg->vmax = v;
@@ -84,7 +84,7 @@ cdbexplain_agg_upd(CdbExplain_Agg *agg, double v, int 
id,char* hostname)
 return true;
 }
 }
-else if (v > 0)
+else */if (v > 0)
 {
 agg->vsum += v;
 agg->vcnt++;

[4/5] incubator-hawq git commit: Revert "fff"

[hubertzhang]

Revert "fff"

This reverts commit d4e27a41443d4bd4bb1dfb058cfec05459fac22b.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/63ef5499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/63ef5499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/63ef5499

Branch: refs/heads/master
Commit: 63ef5499e09abe944da3848f8a7c52fc98317bb5
Parents: 5a4583e
Author: hzhang2 
Authored: Thu Apr 14 15:51:53 2016 +0800
Committer: hzhang2 
Committed: Thu Apr 14 15:51:53 2016 +0800

--
 src/backend/cdb/cdbexplain.c | 34 +-
 1 file changed, 29 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/63ef5499/src/backend/cdb/cdbexplain.c
--
diff --git a/src/backend/cdb/cdbexplain.c b/src/backend/cdb/cdbexplain.c
index 84dafc1..bea98ca 100644
--- a/src/backend/cdb/cdbexplain.c
+++ b/src/backend/cdb/cdbexplain.c
@@ -1424,7 +1424,7 @@ cdbexplain_formatSegNoParenthesis(char *outbuf, int 
bufsize, int segindex, int n
Assert(outbuf != NULL &&  "CDBEXPLAIN: char buffer is null");
Assert(bufsize > 0 &&  "CDBEXPLAIN: size of char buffer is zero");
 
-if ( nInst > 0 && segindex >= 0){
+if ( nInst > 1 && segindex >= 0){
/*check if truncation occurs */
 #ifdef USE_ASSERT_CHECKING
int nchars_written =
@@ -1564,7 +1564,7 @@ cdbexplain_showExecStats(struct PlanState  
*planstate,
 case T_BitmapIndexScanState:
 s_row = "";
 s_rows = "";
-if (ns->ntuples.vcnt > 0){
+if (ns->ntuples.vcnt > 1){
 appendStringInfo(str,
  "Bitmaps out:  Avg %.1f x %d workers."
  "  Max/Last(%s/%s) %.0f/%.0f rows",
@@ -1574,9 +1574,15 @@ cdbexplain_showExecStats(struct PlanState  
*planstate,

ns->ntuples.vmax,ns->ntuples.vlast);
 containMaxRowAndLast = true;
 }
+else
+appendStringInfo(str,
+ "Bitmaps out:  %s%.0f%s",
+noRowRequested,
+ ns->ntuples.vmax,
+
segbufWithParenthese);
 break;
 case T_HashState:
-if (ns->ntuples.vcnt > 0){
+if (ns->ntuples.vcnt > 1){
 appendStringInfo(str,
  "Rows in:  Avg %.1f rows x %d workers."
  "  Max/Last(%s/%s) %.0f/%.0f rows",
@@ -1586,9 +1592,15 @@ cdbexplain_showExecStats(struct PlanState  
*planstate,

ns->ntuples.vmax,ns->ntuples.vlast);
 containMaxRowAndLast = true;
 }
+else
+appendStringInfo(str,
+ "Rows in:  %s%.0f rows%s",
+noRowRequested,
+ ns->ntuples.vmax,
+   
segbufWithParenthese);
 break;
 case T_MotionState:
-if (ns->ntuples.vcnt > 0){
+if (ns->ntuples.vcnt > 1){
 appendStringInfo(str,
  "Rows out:  Avg %.1f rows x %d workers"
  " at destination.  Max/Last(%s/%s) %.0f/%.0f 
rows",
@@ -1598,9 +1610,15 @@ cdbexplain_showExecStats(struct PlanState  
*planstate,

ns->ntuples.vmax,ns->ntuples.vlast);
 containMaxRowAndLast = true;
 }
+else
+appendStringInfo(str,
+ "Rows out:  %s%.0f rows at destination%s",
+noRowRequested,
+ ns->ntuples.vmax,
+   
segbufWithParenthese);
 break;
 default:
-if (ns->ntuples.vcnt > 0){
+if (ns->ntuples.vcnt > 1){
 appendStringInfo(str,
  "Rows out:  Avg %.1f rows x %d workers."
  "  Max/Last(%s/%s) %.0f/%.0f rows",
@@ -1610,6 +1628,12 @@ cdbexplain_showExecStats(struct PlanState  
*planstate,
  

[2/5] incubator-hawq git commit: Revert "HAWQ-673. Unify out put of explain analyze."

[hubertzhang]

Revert "HAWQ-673. Unify out put of explain analyze."

This reverts commit cbf85995d93b04c9f2d41a83c0afac2639f21297.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b802bf67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b802bf67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b802bf67

Branch: refs/heads/master
Commit: b802bf6789badbc70b1e9cbe79242ae78c07d861
Parents: f786bb8
Author: hzhang2 
Authored: Thu Apr 14 15:51:19 2016 +0800
Committer: hzhang2 
Committed: Thu Apr 14 15:51:19 2016 +0800

--
 src/backend/cdb/cdbexplain.c |  6 ++
 src/include/cdb/cdbexplain.h | 14 ++
 2 files changed, 4 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b802bf67/src/backend/cdb/cdbexplain.c
--
diff --git a/src/backend/cdb/cdbexplain.c b/src/backend/cdb/cdbexplain.c
index f20a7e5..6ddb095 100644
--- a/src/backend/cdb/cdbexplain.c
+++ b/src/backend/cdb/cdbexplain.c
@@ -305,8 +305,6 @@ cdbexplain_localExecStats(struct PlanState 
*planstate,
 ctx.send.notebuf = >extratextbuf;
 
 /* Set up a temporary StatHdr for both collecting and depositing stats. */
-gethostname(ctx.send.hdr.hostname,SEGMENT_IDENTITY_NAME_LENGTH-1);
-
//strncpy(ctx.send.hdr.hostname,gethostname(),SEGMENT_IDENTITY_NAME_LENGTH-1);
 ctx.msgptrs[0] = 
 ctx.send.hdr.segindex = GetQEIndex();
 ctx.send.hdr.nInst = 1;
@@ -923,7 +921,7 @@ cdbexplain_depStatAcc_upd(CdbExplain_DepStatAcc*acc,
 acc->nsimax = nsi;
 }
 
-if (acc->max_total <= nsi->total)
+if (acc->max_total < nsi->total)
 {
acc->rshLast = rsh;
acc->rsiLast = rsi;
@@ -1426,7 +1424,7 @@ cdbexplain_formatSegNoParenthesis(char *outbuf, int 
bufsize, int segindex, int n
Assert(outbuf != NULL &&  "CDBEXPLAIN: char buffer is null");
Assert(bufsize > 0 &&  "CDBEXPLAIN: size of char buffer is zero");
 
-if ( nInst >= 0 && segindex >= -1){
+if ( nInst >= 0 && segindex >= 0){
/*check if truncation occurs */
 #ifdef USE_ASSERT_CHECKING
int nchars_written =

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b802bf67/src/include/cdb/cdbexplain.h
--
diff --git a/src/include/cdb/cdbexplain.h b/src/include/cdb/cdbexplain.h
index 719eb38..8dde233 100644
--- a/src/include/cdb/cdbexplain.h
+++ b/src/include/cdb/cdbexplain.h
@@ -74,22 +74,12 @@ cdbexplain_agg_init1(CdbExplain_Agg *agg, double v, int id)
 static inline bool
 cdbexplain_agg_upd(CdbExplain_Agg *agg, double v, int id,char* hostname)
 {
-/*if(v == 0){
-  if (agg->vcnt == 0)
-{
-agg->vmax = v;
-agg->imax = id;
-if(hostname!=NULL)
-strncpy(agg->hostnamemax, 
hostname,SEGMENT_IDENTITY_NAME_LENGTH-1);
-return true;
-}
-}
-else */if (v > 0)
+if (v > 0)
 {
 agg->vsum += v;
 agg->vcnt++;
 
-if (v >= agg->vmax ||
+if (v > agg->vmax ||
 agg->vcnt == 0)
 {
 agg->vmax = v;

incubator-hawq git commit: HAWQ-662. Fix memory leak when mppExecutorCleanup throws error.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 3ab82d76a -> 3db63cf3c


HAWQ-662. Fix memory leak when mppExecutorCleanup throws error.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3db63cf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3db63cf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3db63cf3

Branch: refs/heads/master
Commit: 3db63cf3c46822f2522d5bcf180353e1bd6edd81
Parents: 3ab82d7
Author: hzhang2 
Authored: Tue Apr 12 17:49:32 2016 +0800
Committer: hzhang2 
Committed: Tue Apr 12 17:49:32 2016 +0800

--
 src/backend/executor/execMain.c | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3db63cf3/src/backend/executor/execMain.c
--
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 58d8d9b..bb2e8b9 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -1360,6 +1360,12 @@ ExecutorEnd(QueryDesc *queryDesc)
}
PG_CATCH();
{
+ /* Cleanup the global resource reference for spi/function resource 
inheritate. */
+if (Gp_role == GP_ROLE_DISPATCH) {
+  AutoFreeResource(queryDesc->resource);
+  queryDesc->resource = NULL;
+}
+
/*
 * we got an error. do all the necessary cleanup.
 */
@@ -1376,11 +1382,6 @@ ExecutorEnd(QueryDesc *queryDesc)
 */
FreeExecutorState(estate);
 
-   /* Cleanup the global resource reference for spi/function 
resource inheritate. */
-   if (Gp_role == GP_ROLE_DISPATCH) {
- AutoFreeResource(queryDesc->resource);
- queryDesc->resource = NULL;
-   }
PG_RE_THROW();
}
PG_END_TRY();

incubator-hawq git commit: HAWQ-652. Slicemap in dispatch is mismatched when run multiple query in one session.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master cad586c95 -> 69e8dddbc


HAWQ-652. Slicemap in dispatch is mismatched when run multiple query in one 
session.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/69e8dddb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/69e8dddb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/69e8dddb

Branch: refs/heads/master
Commit: 69e8dddbc3d03b40759cc582525a351feba19a27
Parents: cad586c
Author: hzhang2 
Authored: Mon Apr 11 19:32:35 2016 +0800
Committer: hzhang2 
Committed: Mon Apr 11 19:32:35 2016 +0800

--
 src/backend/cdb/cdbdispatchresult.c | 18 ++--
 src/backend/cdb/dispatcher.c| 28 ++-
 src/backend/cdb/dispatcher_mgt.c| 47 +---
 src/include/cdb/dispatcher_mgt.h| 16 ++-
 4 files changed, 46 insertions(+), 63 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/69e8dddb/src/backend/cdb/cdbdispatchresult.c
--
diff --git a/src/backend/cdb/cdbdispatchresult.c 
b/src/backend/cdb/cdbdispatchresult.c
index d97dda5..c70c43f 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -150,10 +150,9 @@ cdbdisp_makeResult(struct CdbDispatchResults   
*meleeResults,
 }
 else
 {
-if (si->resultBegin > meleeIndex)
-si->resultBegin = meleeIndex;
-if (si->resultEnd <= meleeIndex)
-si->resultEnd = meleeIndex + 1;
+Assert(si->resultBegin <= meleeIndex);
+Assert(si->resultEnd <= meleeIndex);
+si->resultEnd = meleeIndex + 1;
 }
 }
 
@@ -685,10 +684,11 @@ cdbdisp_makeDispatchResults(int resultCapacity,
 int sliceCapacity,
 boolcancelOnError)
 {
-CdbDispatchResults *results = palloc0(sizeof(*results));
-int nbytes = resultCapacity * sizeof(results->resultArray[0]);
+CdbDispatchResults *results = (CdbDispatchResults*) 
palloc0(sizeof(CdbDispatchResults));
+int nbytes = resultCapacity * sizeof(CdbDispatchResult);
+
+results->resultArray = (CdbDispatchResult*) palloc0(nbytes);
 
-results->resultArray = palloc0(nbytes);
 results->resultCapacity = resultCapacity;
 results->resultCount = 0;
 results->iFirstError = -1;
@@ -699,8 +699,8 @@ cdbdisp_makeDispatchResults(int resultCapacity,
 results->sliceCapacity = sliceCapacity;
 if (sliceCapacity > 0)
 {
-nbytes = sliceCapacity * sizeof(results->sliceMap[0]);
-results->sliceMap = palloc0(nbytes);
+nbytes = sliceCapacity * sizeof(CdbDispatchResults_SliceInfo);
+results->sliceMap = (CdbDispatchResults_SliceInfo*) palloc0(nbytes);
 }
 
 return results;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/69e8dddb/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 67530fd..634c52a 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -928,19 +928,22 @@ 
dispatcher_split_logical_tasks_for_statemet_or_node(DispatchData *data)
 static bool
 dispatcher_bind_executor(DispatchData *data)
 {
-   int i,
-   j;
+   int i, j;
struct QueryExecutor*executor;
QueryExecutorIterator   iterator;
List*concurrent_connect_executors = NIL;
ListCell*lc;
-   boolret;
+   boolret = true;
+   boolbindExecutorError = false;
 
dispmgt_init_query_executor_iterator(data->query_executor_team, 
);
for (i = 0; i < data->job.used_slices_num; i++)
{
DispatchSlice   *slice = >job.slices[i];
 
+   if (bindExecutorError)
+ break;
+
for (j = 0; j < slice->tasks_num; j++)
{
DispatchTask*task = >tasks[j];
@@ -963,6 +966,11 @@ dispatcher_bind_executor(DispatchData *data)
concurrent_connect_executors = 
lappend(concurrent_connect_executors,
  info);
data->num_of_new_connected_executors++;
+   if (!executormgr_bind_executor_task(data, executor, 
info->desc, task, slice))
+   {
+ bindExecutorError = true;
+ break;
+   }
   

incubator-hawq git commit: Revert "HAWQ-652. Slicemap in dispatch is mismatched when run multiple query in one session."

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 3953930d9 -> 421b97350


Revert "HAWQ-652. Slicemap in dispatch is mismatched when run multiple query in 
one session."

This reverts commit 4538dcd82731bd11c5b28ac52ad906b86bcbfb54.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/421b9735
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/421b9735
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/421b9735

Branch: refs/heads/master
Commit: 421b97350ee64b70e7a8da6018e81ef5b471033f
Parents: 3953930
Author: hzhang2 
Authored: Mon Apr 11 11:38:48 2016 +0800
Committer: hzhang2 
Committed: Mon Apr 11 11:38:48 2016 +0800

--
 src/backend/cdb/cdbdispatchresult.c | 18 ++---
 src/backend/cdb/dispatcher.c|  2 --
 src/backend/cdb/dispatcher_mgt.c| 44 +++-
 src/include/cdb/dispatcher_mgt.h| 16 +---
 4 files changed, 53 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/421b9735/src/backend/cdb/cdbdispatchresult.c
--
diff --git a/src/backend/cdb/cdbdispatchresult.c 
b/src/backend/cdb/cdbdispatchresult.c
index c70c43f..d97dda5 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -150,9 +150,10 @@ cdbdisp_makeResult(struct CdbDispatchResults   
*meleeResults,
 }
 else
 {
-Assert(si->resultBegin <= meleeIndex);
-Assert(si->resultEnd <= meleeIndex);
-si->resultEnd = meleeIndex + 1;
+if (si->resultBegin > meleeIndex)
+si->resultBegin = meleeIndex;
+if (si->resultEnd <= meleeIndex)
+si->resultEnd = meleeIndex + 1;
 }
 }
 
@@ -684,11 +685,10 @@ cdbdisp_makeDispatchResults(int resultCapacity,
 int sliceCapacity,
 boolcancelOnError)
 {
-CdbDispatchResults *results = (CdbDispatchResults*) 
palloc0(sizeof(CdbDispatchResults));
-int nbytes = resultCapacity * sizeof(CdbDispatchResult);
-
-results->resultArray = (CdbDispatchResult*) palloc0(nbytes);
+CdbDispatchResults *results = palloc0(sizeof(*results));
+int nbytes = resultCapacity * sizeof(results->resultArray[0]);
 
+results->resultArray = palloc0(nbytes);
 results->resultCapacity = resultCapacity;
 results->resultCount = 0;
 results->iFirstError = -1;
@@ -699,8 +699,8 @@ cdbdisp_makeDispatchResults(int resultCapacity,
 results->sliceCapacity = sliceCapacity;
 if (sliceCapacity > 0)
 {
-nbytes = sliceCapacity * sizeof(CdbDispatchResults_SliceInfo);
-results->sliceMap = (CdbDispatchResults_SliceInfo*) palloc0(nbytes);
+nbytes = sliceCapacity * sizeof(results->sliceMap[0]);
+results->sliceMap = palloc0(nbytes);
 }
 
 return results;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/421b9735/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 349c077..67530fd 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -963,8 +963,6 @@ dispatcher_bind_executor(DispatchData *data)
concurrent_connect_executors = 
lappend(concurrent_connect_executors,
  info);
data->num_of_new_connected_executors++;
-   if (!executormgr_bind_executor_task(data, executor, 
info->desc, task, slice))
- return false;
  }
  continue;
}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/421b9735/src/backend/cdb/dispatcher_mgt.c
--
diff --git a/src/backend/cdb/dispatcher_mgt.c b/src/backend/cdb/dispatcher_mgt.c
index 1f7a155..7cf2573 100644
--- a/src/backend/cdb/dispatcher_mgt.c
+++ b/src/backend/cdb/dispatcher_mgt.c
@@ -62,11 +62,28 @@ typedef struct QueryExecutorGroupIterator
int group_id;
 } QueryExecutorGroupIterator;
 
+/*
+ * ConcurrentConnectExecutorInfo
+ * Used to create the executor concurrently.
+ */
+typedef struct ConcurrentConnectExecutorInfo {
+   boolis_writer;
+   boolis_superuser;
+   struct QueryExecutor*executor;
+   struct DispatchData *data;
+   struct DispatchSlice*slice;
+   

incubator-hawq git commit: HAWQ-629. Insert into table select generate_series free resource too early.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1fbdf8b9a -> 0aa695293


HAWQ-629. Insert into table select generate_series free resource too early.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0aa69529
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0aa69529
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0aa69529

Branch: refs/heads/master
Commit: 0aa6952930e6f7d155624947dbb612e1516b0a00
Parents: 1fbdf8b
Author: hzhang2 
Authored: Wed Apr 6 10:59:15 2016 +0800
Committer: hzhang2 
Committed: Wed Apr 6 11:00:07 2016 +0800

--
 src/backend/executor/execMain.c | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0aa69529/src/backend/executor/execMain.c
--
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 6fa5cd2..cc030be 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -1306,12 +1306,6 @@ ExecutorEnd(QueryDesc *queryDesc)
/* sanity checks */
Assert(queryDesc != NULL);
 
-   /* Cleanup the global resource reference for spi/function resource 
inheritate. */
-   if ( Gp_role == GP_ROLE_DISPATCH ) {
-   AutoFreeResource(queryDesc->resource);
-   queryDesc->resource = NULL;
-   }
-
estate = queryDesc->estate;
 
Assert(estate != NULL);
@@ -1386,6 +1380,12 @@ ExecutorEnd(QueryDesc *queryDesc)
}
PG_END_TRY();
 
+   /* Cleanup the global resource reference for spi/function resource 
inheritate. */
+   if ( Gp_role == GP_ROLE_DISPATCH ) {
+   AutoFreeResource(queryDesc->resource);
+   queryDesc->resource = NULL;
+   }
+
/*
 * If normal termination, let each operator clean itself up.
 * Otherwise don't risk it... an error might have left some

incubator-hawq git commit: HAWQ-616. Mininum resource needed by copy random table from a file should be one.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7fef48058 -> 071f276bc


HAWQ-616. Mininum resource needed by copy random table from a file should be 
one.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/071f276b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/071f276b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/071f276b

Branch: refs/heads/master
Commit: 071f276bc3c3e14b818ecf584a5697a2ebf24870
Parents: 7fef480
Author: hzhang2 
Authored: Fri Apr 1 10:26:28 2016 +0800
Committer: hzhang2 
Committed: Fri Apr 1 10:26:28 2016 +0800

--
 src/backend/commands/copy.c | 11 +++
 1 file changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/071f276b/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 2ed87e6..6dc41c1 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1557,7 +1557,8 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
Oid relid = RelationGetRelid(cstate->rel);
List*all_relids = NIL;
GpPolicy *target_policy = NULL;
-   int target_segment_num = 0;
+   int min_target_segment_num = 0;
+   int max_target_segment_num = 0;
QueryResource *savedResource = NULL;
 
target_policy = GpPolicyFetch(CurrentMemoryContext, 
relid);
@@ -1567,14 +1568,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 * For random table, we use a fixed GUC value to 
request vsegs.
 */
if(target_policy->nattrs > 0){
-   target_segment_num = target_policy->bucketnum;
+   min_target_segment_num = 
target_policy->bucketnum;
+   max_target_segment_num = 
target_policy->bucketnum;
}
else{
-   target_segment_num = 
hawq_rm_nvseg_for_copy_from_perquery;
+   min_target_segment_num = 1;
+   max_target_segment_num = 
hawq_rm_nvseg_for_copy_from_perquery;
}
pfree(target_policy);
 
-   cstate->resource = AllocateResource(QRL_ONCE, 1, 1, 
target_segment_num, target_segment_num,NULL,0);
+   cstate->resource = AllocateResource(QRL_ONCE, 1, 1, 
max_target_segment_num, min_target_segment_num,NULL,0);
savedResource = GetActiveQueryResource();
SetActiveQueryResource(cstate->resource);
all_relids = lappend_oid(all_relids, relid);

incubator-hawq git commit: HAWQ-573. Refix the bug as last fix has imported more warning messages.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master b65815a3c -> 323f265aa


HAWQ-573. Refix the bug as last fix has imported more warning messages.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/323f265a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/323f265a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/323f265a

Branch: refs/heads/master
Commit: 323f265aa28cfb49d29311cc1626e7bd2048e4bf
Parents: b65815a
Author: doli 
Authored: Thu Mar 31 13:59:29 2016 +0800
Committer: hzhang2 
Committed: Thu Mar 31 17:36:12 2016 +0800

--
 src/backend/tcop/pquery.c | 81 +++---
 1 file changed, 77 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/323f265a/src/backend/tcop/pquery.c
--
diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c
index 23b59a1..04420c3 100644
--- a/src/backend/tcop/pquery.c
+++ b/src/backend/tcop/pquery.c
@@ -85,6 +85,7 @@ static List *ActiveRelsType =NULL;
 typedef struct QueryResourceItem {
   bool alive;
   int resource_id;
+  bool allocateSucceed;
 } QueryResourceItem;
 
 static List *GlobalQueryResources = NIL;
@@ -111,7 +112,8 @@ static int64 DoPortalRunFetch(Portal portal,
 static void DoPortalRewind(Portal portal);
 
 static void AddToGlobalQueryResources(int resourceId, QueryResourceLife life);
-
+static void RemoveFromGlobalQueryResources(int resourceId, QueryResourceLife 
life);
+static void SetResourcesAllocatedSucceed(int resourceId, QueryResourceLife 
life);
 static int compare_segment(const void *e1, const void *e2);
 /*
  * CreateQueryDesc
@@ -805,7 +807,10 @@ AllocateResource(QueryResourceLife   life,
errorbuf,

sizeof(errorbuf));
if (ret != FUNC_RETURN_OK) {
+   RemoveFromGlobalQueryResources(resourceId, life);
ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), 
errmsg("%s",errorbuf)));
+   } else {
+   SetResourcesAllocatedSucceed(resourceId, life);
}
 
elog(DEBUG3, "Acquired resource from resource manager.");
@@ -948,10 +953,73 @@ AddToGlobalQueryResources(int resourceId, 
QueryResourceLife life)
   newItem = palloc(sizeof(QueryResourceItem));
   newItem->alive = true;
   newItem->resource_id = resourceId;
+  /*
+   * Only if the allocate resource RPC returns OK,
+   * the allocateSucceed can be set true by using 
SetResourcesAllocatedSucceed()
+   */
+  newItem->allocateSucceed = false;
   GlobalQueryResources = lappend(GlobalQueryResources, newItem);
   MemoryContextSwitchTo(old);
 }
 
+static void
+RemoveFromGlobalQueryResources(int resourceId, QueryResourceLife life)
+{
+  ListCell *lc;
+  QueryResourceItem *newItem;
+  MemoryContext old;
+
+  if (life == QRL_NONE)
+  {
+return;
+  }
+
+  foreach(lc, GlobalQueryResources)
+  {
+QueryResourceItem *qri = lfirst(lc);
+if(qri->resource_id == resourceId)
+{
+  /*
+   * found, delete it.
+   */
+  break;
+}
+  }
+
+  /*
+   * remove from the global query resource.
+   */
+  old = MemoryContextSwitchTo(TopMemoryContext);
+  GlobalQueryResources = list_delete_ptr(GlobalQueryResources,lfirst(lc));
+  MemoryContextSwitchTo(old);
+}
+
+static void
+SetResourcesAllocatedSucceed(int resourceId, QueryResourceLife life)
+{
+   ListCell *lc;
+   QueryResourceItem *newItem;
+   MemoryContext old;
+
+   if (life == QRL_NONE)
+   {
+ return;
+   }
+
+   foreach(lc, GlobalQueryResources)
+   {
+ QueryResourceItem *qri = lfirst(lc);
+ if(qri->resource_id == resourceId)
+ {
+/*
+* found, set it succeed.
+*/
+qri->allocateSucceed = true;
+return;
+ }
+   }
+}
+
 void
 FreeResource(QueryResource *resource)
 {
@@ -1068,17 +1136,22 @@ CleanupGlobalQueryResources(void)
if (qri->alive)
{
ret = returnResource(qri->resource_id, errorbuf, 
sizeof(errorbuf));
-   if (ret != FUNC_RETURN_OK)
+   /*
+* If qri->allocateSucceed == false, that means it 
interrupts during acquireResourceFromRM().
+* And we don't know if it has been allocated succeed.
+* It also means it may return resource failed as if 
the resource hasn't been allocated yet.
+* So don't report warning message in this situation.
+*/
+   

incubator-hawq git commit: HAWQ-580. Set analyze vseg number for partition and no-partition table separately.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 12b029ff1 -> 3db0b6fea


HAWQ-580. Set analyze vseg number for partition and no-partition table 
separately.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3db0b6fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3db0b6fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3db0b6fe

Branch: refs/heads/master
Commit: 3db0b6feaa1f0ee3821bf39a3a94a44eeffafb6a
Parents: 12b029f
Author: hzhang2 
Authored: Mon Mar 28 10:11:00 2016 +0800
Committer: hzhang2 
Committed: Mon Mar 28 10:28:15 2016 +0800

--
 src/backend/postmaster/identity.c |  6 --
 src/backend/utils/misc/guc.c  | 16 +---
 src/include/utils/guc.h   |  3 ++-
 3 files changed, 19 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3db0b6fe/src/backend/postmaster/identity.c
--
diff --git a/src/backend/postmaster/identity.c 
b/src/backend/postmaster/identity.c
index a17958b..0418a3a 100644
--- a/src/backend/postmaster/identity.c
+++ b/src/backend/postmaster/identity.c
@@ -456,12 +456,14 @@ int   GetAnalyzeVSegNumLimit(bool 
isPartitionTableExist)
 {
int perSegLimit = isPartitionTableExist ? 
hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit
: 
hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
+   int perQueryLimit = isPartitionTableExist ? 
hawq_rm_nvseg_for_analyze_part_perquery_limit
+   : 
hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
int nvseg = perSegLimit * slaveHostNumber;
-   while(nvseg > hawq_rm_nvseg_for_analyze_perquery_limit){
+   while(nvseg > perQueryLimit){
nvseg -= slaveHostNumber;
}
if(nvseg <= 0){
-   nvseg = hawq_rm_nvseg_for_analyze_perquery_limit;
+   nvseg = perQueryLimit;
}
return nvseg;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3db0b6fe/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9477dbc..851dfcb 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -737,7 +737,8 @@ int default_hash_table_bucket_number;
 int hawq_rm_nvseg_for_copy_from_perquery;
 int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
-int hawq_rm_nvseg_for_analyze_perquery_limit;
+int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
+int hawq_rm_nvseg_for_analyze_part_perquery_limit;
 double   optimizer_cost_threshold;
 double  optimizer_nestloop_factor;
 double  locality_upper_bound;
@@ -4548,11 +4549,20 @@ static struct config_int ConfigureNamesInt[] =
},
 
{
-   {"hawq_rm_nvseg_for_analyze_perquery_limit", PGC_USERSET, 
QUERY_TUNING_OTHER,
+   {"hawq_rm_nvseg_for_analyze_nopart_perquery_limit", 
PGC_USERSET, QUERY_TUNING_OTHER,
gettext_noop("Sets default virtual segment number per 
query limit for analyze statement"),
NULL
},
-   _rm_nvseg_for_analyze_perquery_limit,
+   _rm_nvseg_for_analyze_nopart_perquery_limit,
+   512, 1, 65535, NULL, NULL
+   },
+
+   {
+   {"hawq_rm_nvseg_for_analyze_part_perquery_limit", PGC_USERSET, 
QUERY_TUNING_OTHER,
+   gettext_noop("Sets default virtual segment number per 
query limit for analyze statement"),
+   NULL
+   },
+   _rm_nvseg_for_analyze_part_perquery_limit,
256, 1, 65535, NULL, NULL
},
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3db0b6fe/src/include/utils/guc.h
--
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index edab452..c70c491 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -312,7 +312,8 @@ extern int default_hash_table_bucket_number;
 extern int hawq_rm_nvseg_for_copy_from_perquery;
 extern int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 extern int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
-extern int hawq_rm_nvseg_for_analyze_perquery_limit;
+extern int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
+extern int hawq_rm_nvseg_for_analyze_part_perquery_limit;
 
 extern char *ConfigFileName;
 extern char *HbaFileName;

incubator-hawq git commit: HAWQ-562. Refactor bucket number of external table.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master eb229d974 -> 12b029ff1


HAWQ-562. Refactor bucket number of external table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/12b029ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/12b029ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/12b029ff

Branch: refs/heads/master
Commit: 12b029ff1b4bf92599470e5cc3efaa243476ee12
Parents: eb229d9
Author: hzhang2 
Authored: Mon Mar 28 10:17:17 2016 +0800
Committer: hzhang2 
Committed: Mon Mar 28 10:17:17 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c |  7 ++-
 src/backend/commands/tablecmds.c  | 23 ++-
 src/backend/postmaster/identity.c |  2 +-
 3 files changed, 21 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12b029ff/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index 4c67429..cbe1222 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -768,7 +768,8 @@ static void check_keep_hash_and_external_table(
 * whose default value is set to default_segment_num
 */
ExtTableEntry* extEnrty = GetExtTableEntry(rel->rd_id);
-   if(extEnrty->isweb){
+   if(extEnrty->command){
+   // command external table case
if (context->externTableOnClauseSegNum == 0) {
context->externTableOnClauseSegNum = 
targetPolicy->bucketnum;
} else {
@@ -782,6 +783,7 @@ static void check_keep_hash_and_external_table(
}
}
else{
+   // gpfdist location case.
if (context->externTableLocationSegNum < 
targetPolicy->bucketnum) {
context->externTableLocationSegNum = 
targetPolicy->bucketnum;
context->minimum_segment_num =  
targetPolicy->bucketnum;
@@ -4256,6 +4258,9 @@ calculate_planner_segment_num(Query *query, 
QueryResourceLife resourceLife,
}
} else {
maxTargetSegmentNumber = context.randomSegNum;
+   if(context.externTableLocationSegNum > 0 && 
maxTargetSegmentNumber < GetQueryVsegNum()){
+   maxTargetSegmentNumber = 
GetQueryVsegNum();
+   }
minTargetSegmentNumber = 
context.minimum_segment_num;
}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/12b029ff/src/backend/commands/tablecmds.c
--
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index d858d5c..4595f9a 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -387,7 +387,7 @@ static void update_ri_trigger_args(Oid relid,
   const char *newname,
   bool fk_scan,
   bool update_relname);
-static Datum transformLocationUris(List *locs, List* fmtopts, bool isweb, bool 
iswritable);
+static Datum transformLocationUris(List *locs, List* fmtopts, bool isweb, bool 
iswritable, bool* isCustom);
 static Datum transformExecOnClause(List*on_clause, int 
*preferred_segment_num, bool iswritable);
 static char transformFormatType(char *formatname);
 static Datum transformFormatOpts(char formattype, List *formatOpts, int 
numcols, bool iswritable);
@@ -959,19 +959,21 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt)
createStmt->tablespacename = NULL;
createStmt->policy = createExtStmt->policy; /* policy was set in 
transform */

-   
+   bool isCustom = false;
switch(exttypeDesc->exttabletype)
{
case EXTTBL_TYPE_LOCATION:
 
/* Parse and validate URI strings (LOCATION clause) */
locationUris = 
transformLocationUris(exttypeDesc->location_list,
-   
 createExtStmt->formatOpts,
-   
 

incubator-hawq git commit: HAWQ-580. Set analyze vseg number for partition and no-partition table separately.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 60039f499 -> 1c4388a4f


HAWQ-580. Set analyze vseg number for partition and no-partition table 
separately.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1c4388a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1c4388a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1c4388a4

Branch: refs/heads/master
Commit: 1c4388a4f8b51ce070a18e1789709732689408f3
Parents: 60039f4
Author: hzhang2 
Authored: Thu Mar 24 10:12:36 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 25 10:08:38 2016 +0800

--
 src/backend/commands/analyze.c| 14 +-
 src/backend/commands/copy.c   |  2 +-
 src/backend/postmaster/identity.c |  6 --
 src/backend/utils/misc/guc.c  | 16 +---
 src/include/utils/guc.h   |  3 ++-
 5 files changed, 29 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c4388a4/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index ac12837..ae869da 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -821,19 +821,23 @@ static int calculate_virtual_segment_number(List* 
candidateOids) {
int64 totalDataSize = 0;
bool isHashRelationExist = false;
int maxHashBucketNumber = 0;
-
+   bool isPartitionTableExist = false;
foreach (le1, candidateOids)
{
Oid candidateOid  = InvalidOid;
candidateOid = lfirst_oid(le1);
 
-   //Relation rel = (Relation)lfirst(le1);
+   PartStatus ps = rel_part_status(candidateOid);
+   if(ps != PART_STATUS_NONE){
+   isPartitionTableExist = true;
+   }
+
Relation rel = relation_open(candidateOid, 
ShareUpdateExclusiveLock);
if (candidateOid > 0 ) {
GpPolicy *targetPolicy = 
GpPolicyFetch(CurrentMemoryContext,
candidateOid);
if(targetPolicy == NULL){
-   return GetAnalyzeVSegNumLimit();
+   return 
GetAnalyzeVSegNumLimit(isPartitionTableExist);
}
if (targetPolicy->nattrs > 0) {
isHashRelationExist = true;
@@ -860,8 +864,8 @@ static int calculate_virtual_segment_number(List* 
candidateOids) {
}
Assert(vsegNumber > 0);
/*vsegNumber should be less than GetUtilPartitionNum*/
-   if(vsegNumber > GetAnalyzeVSegNumLimit()){
-   vsegNumber = GetAnalyzeVSegNumLimit();
+   if(vsegNumber > GetAnalyzeVSegNumLimit(isPartitionTableExist)){
+   vsegNumber = GetAnalyzeVSegNumLimit(isPartitionTableExist);
}
 
return vsegNumber;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c4388a4/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e37fb83..2ed87e6 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1786,7 +1786,7 @@ static int calculate_virtual_segment_number(List* 
candidateOids) {
GpPolicy *targetPolicy = 
GpPolicyFetch(CurrentMemoryContext,
candidateOid);
if(targetPolicy == NULL){
-   return GetAnalyzeVSegNumLimit();
+   return GetQueryVsegNum();
}
if (targetPolicy->nattrs > 0) {
isHashRelationExist = true;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c4388a4/src/backend/postmaster/identity.c
--
diff --git a/src/backend/postmaster/identity.c 
b/src/backend/postmaster/identity.c
index 1bb558d..02a2038 100644
--- a/src/backend/postmaster/identity.c
+++ b/src/backend/postmaster/identity.c
@@ -452,9 +452,11 @@ GetQEGangNum(void)
return SegmentId.pid.init ? SegmentId.pid.gang_member_num : 0;
 }
 
-intGetAnalyzeVSegNumLimit(void)
+intGetAnalyzeVSegNumLimit(bool isPartitionTableExist)
 {
-   int nvseg = hawq_rm_nvseg_for_analyze_perquery_perseg_limit * 
slaveHostNumber;
+   int perSegLimit = isPartitionTableExist ? 
hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit
+   : 

incubator-hawq git commit: HAWQ-566. Fix analyze and truncate hung.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7daee4002 -> 76dc63191


HAWQ-566. Fix analyze and truncate hung.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/76dc6319
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/76dc6319
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/76dc6319

Branch: refs/heads/master
Commit: 76dc63191fb21b1321c00f00954afc1cb98c6f45
Parents: 7daee40
Author: hzhang2 
Authored: Mon Mar 21 16:38:15 2016 +0800
Committer: hzhang2 
Committed: Mon Mar 21 16:38:15 2016 +0800

--
 src/backend/commands/analyze.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/76dc6319/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 32b3416..ac12837 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -828,7 +828,7 @@ static int calculate_virtual_segment_number(List* 
candidateOids) {
candidateOid = lfirst_oid(le1);
 
//Relation rel = (Relation)lfirst(le1);
-   Relation rel = relation_open(candidateOid, AccessShareLock);
+   Relation rel = relation_open(candidateOid, 
ShareUpdateExclusiveLock);
if (candidateOid > 0 ) {
GpPolicy *targetPolicy = 
GpPolicyFetch(CurrentMemoryContext,
candidateOid);
@@ -848,7 +848,7 @@ static int calculate_virtual_segment_number(List* 
candidateOids) {
totalDataSize += calculate_relation_size(rel);
}
}
-   relation_close(rel, AccessShareLock);
+   relation_close(rel, NoLock);
}
 
if (isHashRelationExist) {

[2/2] incubator-hawq git commit: HAWQ-562. Refactor bucket number of external table.

[hubertzhang]

HAWQ-562. Refactor bucket number of external table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7daee400
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7daee400
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7daee400

Branch: refs/heads/master
Commit: 7daee4002780797587c3dcca265c646fd0c92e1e
Parents: 695606f
Author: hzhang2 
Authored: Mon Mar 21 10:26:11 2016 +0800
Committer: hzhang2 
Committed: Mon Mar 21 16:29:42 2016 +0800

--
 src/backend/cdb/cdbcopy.c |  2 +-
 src/backend/cdb/cdbdatalocality.c | 97 --
 src/backend/cdb/cdbfilesplit.c| 67 +++
 src/backend/commands/tablecmds.c  | 11 ++--
 src/backend/parser/analyze.c  |  2 +-
 src/include/access/filesplit.h|  2 +-
 src/include/cdb/cdbdatalocality.h |  1 -
 7 files changed, 88 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbcopy.c
--
diff --git a/src/backend/cdb/cdbcopy.c b/src/backend/cdb/cdbcopy.c
index 17ffa96..ee2ff80 100644
--- a/src/backend/cdb/cdbcopy.c
+++ b/src/backend/cdb/cdbcopy.c
@@ -222,7 +222,7 @@ cdbCopyStart(CdbCopy *c, char *copyCmd, Oid relid, Oid 
relerror, List *err_aoseg
List *scantable_splits = NIL;
prepareDispatchedCatalogRelation(q->contextdisp, relid, FALSE, 
NULL);
scantable_splits = AssignAOSegFileSplitToSegment(relid, NIL,
-   
true, c->partition_num,
+   
c->partition_num,

scantable_splits);
((CopyStmt *)q->utilityStmt)->scantable_splits = 
scantable_splits;
}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index 3f8fcb5..4c67429 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -34,6 +34,7 @@
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
 #include "catalog/catquery.h"
+#include "catalog/pg_exttable.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
@@ -296,11 +297,13 @@ typedef struct split_to_segment_mapping_context {
int64 split_size;
MemoryContext old_memorycontext;
MemoryContext datalocality_memorycontext;
-   int externTableSegNum;  //expected virtual segment number when external 
table exists
+   int externTableOnClauseSegNum;  //expected virtual segment number when 
external table exists
+   int externTableLocationSegNum;  //expected virtual segment number when 
external table exists
int tableFuncSegNum;  //expected virtual segment number when table 
function exists
int hashSegNum;  // expected virtual segment number when there is hash 
table in from clause
int randomSegNum; // expected virtual segment number when there is 
random table in from clause
int resultRelationHashSegNum; // expected virtual segment number when 
hash table as a result relation
+   int minimum_segment_num; //default is 1.
int64 randomRelSize; //all the random relation size
int64 hashRelSize; //all the hash relation size
 
@@ -529,13 +532,15 @@ static void 
init_datalocality_context(split_to_segment_mapping_context *context)
context->rtc_context.range_tables = NIL;
context->rtc_context.full_range_tables = NIL;
 
-   context->externTableSegNum = 0;
+   context->externTableOnClauseSegNum = 0;
+   context->externTableLocationSegNum = 0;
context->tableFuncSegNum = 0;
context->hashSegNum = 0;
context->resultRelationHashSegNum = 0;
context->randomSegNum = 0;
context->randomRelSize = 0;
context->hashRelSize = 0;
+   context->minimum_segment_num = 1;
/*
 * initialize the data distribution
 * static context.
@@ -762,17 +767,24 @@ static void check_keep_hash_and_external_table(
/* targetPolicy->bucketnum is bucket number of external 
table,
 * whose default value is set to default_segment_num
 */
-   if (context->externTableSegNum == 0) {
- 

[2/2] incubator-hawq git commit: Revert "HAWQ-529. Allocate resource for udf in resource negotiator."

[hubertzhang]

Revert "HAWQ-529. Allocate resource for udf in resource negotiator."

This reverts commit 8e38e844f0d2a1c45f7d5cd25a8f5b639c8b6fb7.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fd202277
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fd202277
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fd202277

Branch: refs/heads/master
Commit: fd2022772d1f39e5f205347a4c0cfd0a0f04021f
Parents: bc065a5
Author: hzhang2 
Authored: Thu Mar 17 09:19:04 2016 +0800
Committer: hzhang2 
Committed: Thu Mar 17 09:19:04 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c| 194 ++
 src/backend/executor/spi.c   |  72 ++-
 src/backend/optimizer/plan/planner.c |  57 +
 src/backend/optimizer/util/clauses.c |  11 +-
 src/backend/optimizer/util/walkers.c |   1 -
 src/include/cdb/cdbdatalocality.h|  19 +--
 src/include/executor/spi.h   |   7 --
 src/include/optimizer/planner.h  |   4 -
 8 files changed, 76 insertions(+), 289 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index d15d6b0..c1f829b 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,9 +33,6 @@
 #include "access/filesplit.h"
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
-#include "catalog/catquery.h"
-#include "catalog/pg_inherits.h"
-#include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
 #include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
@@ -44,7 +41,6 @@
 #include "utils/tqual.h"
 #include "utils/memutils.h"
 #include "executor/execdesc.h"
-#include "executor/spi.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "optimizer/walkers.h"
@@ -58,7 +54,11 @@
 #include "catalog/pg_proc.h"
 #include "postgres.h"
 #include "resourcemanager/utils/hashtable.h"
+#include "catalog/pg_inherits.h"
+//#include "utils/misc/guc.c"
 
+#define PRONAME 1
+#define PROISAGG 5
 /* We need to build a mapping from host name to host index */
 
 extern booloptimizer; /* Enable the optimizer */
@@ -372,7 +372,7 @@ static Block_Host_Index * update_data_dist_stat(
 static HostDataVolumeInfo *search_host_in_stat_context(
split_to_segment_mapping_context *context, char *hostname);
 
-static bool IsBuildInFunction(Oid funcOid);
+static bool IsAggFunction(char* funcName);
 
 static bool allocate_hash_relation(Relation_Data* rel_data,
Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -630,36 +630,39 @@ static void collect_range_tables(Query *query, List* 
full_range_table,
 /*
  *
  */
-static bool IsBuildInFunction(Oid foid) {
+static bool IsAggFunction(char* funcName) {
+   if (funcName == NULL) {
+   return false;
+   }
+   Relation pg_proc_rel;
+   TupleDesc pg_proc_dsc;
+   HeapTuple tuple;
+   SysScanDesc pg_proc_scan;
 
-   cqContext  *pcqCtx;
-   HeapTuple readtup = NULL;
-   HeapTuple   procedureTuple;
-   Form_pg_proc procedureStruct;
+   pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
+   pg_proc_dsc = RelationGetDescr(pg_proc_rel);
+   ScanKeyData skey;
 
-   /*
-* get the procedure tuple corresponding to the given function Oid
-*/
-   pcqCtx = caql_beginscan(
-   NULL,
-   cql("SELECT * FROM pg_proc "
-   " WHERE oid = :1 ",
-   ObjectIdGetDatum(foid)));
-
-   procedureTuple = caql_getnext(pcqCtx);
-
-   if (!HeapTupleIsValid(procedureTuple))
-   elog(ERROR, "cache lookup failed for function %u", foid);
-   procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
-   caql_endscan(pcqCtx);
-   /* we treat proc namespace = 11 to build in function.*/
-   if (procedureStruct->pronamespace == 11) {
-   return true;
-   } else {
-   return false;
+   ScanKeyInit(, PRONAME, BTEqualStrategyNumber,
+   F_NAMEEQ, CStringGetDatum(funcName));
+
+   pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
+   ActiveSnapshot, 1, );
+   while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+
+   bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, 
pg_proc_dsc, NULL));
+   systable_endscan(pg_proc_scan);
+   heap_close(pg_proc_rel, AccessShareLock);
+   if (isAgg) {
+   return 

incubator-hawq git commit: HAWQ-532. Optimise vseg number for copy to statement.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master b3a352186 -> 36571bce2


HAWQ-532. Optimise vseg number for copy to statement.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/36571bce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/36571bce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/36571bce

Branch: refs/heads/master
Commit: 36571bce2c96fce04ceb94f9ab78ace4cc90c66d
Parents: b3a3521
Author: hzhang2 
Authored: Fri Mar 18 12:36:39 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 12:36:39 2016 +0800

--
 src/backend/commands/copy.c | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/36571bce/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 2f4cf39..e37fb83 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1888,11 +1888,12 @@ DoCopyTo(CopyState cstate)
 */
PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 
/*level*/ ,

0 /*parent*/, false /* inctemplate */, 
CurrentMemoryContext, true /*includesubparts*/);
-   Assert(pn);
List*lFullRelOids = NIL;
-   lFullRelOids = all_leaf_partition_relids(pn);
-   lFullRelOids = lappend_oid(lFullRelOids, 
cstate->rel->rd_id); /* root partition */
-   lFullRelOids = list_concat(lFullRelOids, 
all_interior_partition_relids(pn)); /* interior partitions */
+   if(pn){
+   lFullRelOids = all_leaf_partition_relids(pn);
+   lFullRelOids = list_concat(lFullRelOids, 
all_interior_partition_relids(pn)); /* interior partitions */
+   }
+   lFullRelOids = lappend_oid(lFullRelOids, 
cstate->rel->rd_id);
 
target_segment_num = 
calculate_virtual_segment_number(lFullRelOids);
elog(LOG, "virtual segment number of copy to is: %d\n", 
target_segment_num);

[1/5] incubator-hawq git commit: HAWQ-554. Create web external table shouldnot omit ON clause.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 2c033d9f0 -> 17af44cb1


HAWQ-554. Create web external table shouldnot omit ON clause.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17af44cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17af44cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17af44cb

Branch: refs/heads/master
Commit: 17af44cb139ddbe6b0e903d041d1177b73d2e556
Parents: 2513ad1
Author: hzhang2 
Authored: Fri Mar 18 10:33:46 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 10:42:44 2016 +0800

--
 src/backend/parser/gram.y| 9 ++---
 src/test/regress/output/exttab1.source   | 6 --
 src/test/regress/output/exttab1_optimizer.source | 6 --
 3 files changed, 14 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/backend/parser/gram.y
--
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 4d6f76f..dc0e13b 100755
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -4610,9 +4610,12 @@ CreateExternalStmt:  CREATE OptWritable EXTERNAL 
OptWeb OptTemp TABLE qualified_n

/* if no ON 
clause specified, default to "ON ALL" */

if(extdesc->on_clause == NIL)
-   {   

-   
extdesc->on_clause = lappend(extdesc->on_clause, 
-   
 makeDefElem("all", (Node 
*)makeInteger(TRUE)));
+   {
+   
ereport(ERROR,
+   
(errcode(ERRCODE_SYNTAX_ERROR),
+   
errmsg("ON clause must be specified in external table"),
+   
errhint("Use ON number instread"),
+   
errOmitLocation(true)));
}
}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1.source
--
diff --git a/src/test/regress/output/exttab1.source 
b/src/test/regress/output/exttab1.source
index bd9dec8..ec8ed3b 100755
--- a/src/test/regress/output/exttab1.source
+++ b/src/test/regress/output/exttab1.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
 -- test for exec child process stderr showing in error message
 --
 create external web table ext_stderr1(a text) execute 'nosuchcommand' format 
'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' 
format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 --
 -- bad csv (quote must be a single char)
 --

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1_optimizer.source
--
diff --git a/src/test/regress/output/exttab1_optimizer.source 
b/src/test/regress/output/exttab1_optimizer.source
index 9519769..3f03143 100755
--- a/src/test/regress/output/exttab1_optimizer.source
+++ b/src/test/regress/output/exttab1_optimizer.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
 -- test for exec child process stderr showing in error message
 --
 create external web table ext_stderr1(a text) execute 'nosuchcommand' format 
'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' 
format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON 

[2/5] incubator-hawq git commit: HAWQ-532. Optimise vseg number for copy to statement.

[hubertzhang]

HAWQ-532. Optimise vseg number for copy to statement.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c29d6476
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c29d6476
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c29d6476

Branch: refs/heads/master
Commit: c29d647658e5e4219a370b4869f68cdcd577ab65
Parents: eaaf945
Author: hzhang2 
Authored: Fri Mar 18 10:30:20 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 10:42:44 2016 +0800

--
 src/backend/cdb/cdbmutate.c  |  2 +-
 src/backend/commands/copy.c  | 96 ---
 src/backend/commands/prepare.c   |  2 +-
 src/backend/optimizer/util/relnode.c |  2 +-
 src/backend/parser/analyze.c |  8 +--
 src/backend/postmaster/identity.c|  2 +-
 src/backend/utils/misc/guc.c | 12 ++--
 src/include/postmaster/identity.h|  2 +-
 8 files changed, 103 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/cdb/cdbmutate.c
--
diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c
index 6959c64..9a998da 100644
--- a/src/backend/cdb/cdbmutate.c
+++ b/src/backend/cdb/cdbmutate.c
@@ -312,7 +312,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query)

targetPolicy = 
palloc0(sizeof(GpPolicy)- sizeof(targetPolicy->attrs) + maxattrs * 
sizeof(targetPolicy->attrs[0]));
targetPolicy->nattrs = 0;
-   targetPolicy->bucketnum = 
GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, 
GetRandomDistPartitionNum());
+   targetPolicy->bucketnum = 
GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, 
GetDefaultPartitionNum());
targetPolicy->ptype = 
POLICYTYPE_PARTITIONED;

/* Find out what the flow is 
partitioned on */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index bcd5384..2f4cf39 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,6 +91,11 @@
 #include "postmaster/autovacuum.h"
 #include "cdb/dispatcher.h"
 
+/*
+ * in dbsize.c
+ */
+extern int64 calculate_relation_size(Relation rel);
+
 /* DestReceiver for COPY (SELECT) TO */
 typedef struct
 {
@@ -137,6 +142,7 @@ static void copy_in_error_callback(void *arg);
 static void CopyInitPartitioningState(EState *estate);
 static void CopyInitDataParser(CopyState cstate);
 static bool CopyCheckIsLastLine(CopyState cstate);
+static int calculate_virtual_segment_number(List* candidateRelations);
 
 /* ==
  * The follwing macros aid in major refactoring of data processing code (in
@@ -1556,7 +1562,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 
target_policy = GpPolicyFetch(CurrentMemoryContext, 
relid);
Assert(target_policy);
-   target_segment_num = target_policy->bucketnum;
+   /*
+* For hash table we use table bucket number to request 
vsegs
+* For random table, we use a fixed GUC value to 
request vsegs.
+*/
+   if(target_policy->nattrs > 0){
+   target_segment_num = target_policy->bucketnum;
+   }
+   else{
+   target_segment_num = 
hawq_rm_nvseg_for_copy_from_perquery;
+   }
pfree(target_policy);
 
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, 
target_segment_num, target_segment_num,NULL,0);
@@ -1749,6 +1764,63 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 }
 
 /*
+ * calculate virtual segment number for copy statement.
+ * if there is hash distributed relations exist, use the max bucket number.
+ * if all relation are random, use the data size to determine vseg number.
+ */
+static int calculate_virtual_segment_number(List* candidateOids) {
+   ListCell* le1;
+   int vsegNumber = 1;
+   int64 totalDataSize = 0;
+   bool isHashRelationExist = false;
+   int maxHashBucketNumber = 0;
+
+   foreach (le1, candidateOids)
+   {
+

[4/5] incubator-hawq git commit: HAWQ-553. Analyze ROOTPATITION using too few vsegs.

[hubertzhang]

HAWQ-553. Analyze ROOTPATITION using too few vsegs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2513ad10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2513ad10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2513ad10

Branch: refs/heads/master
Commit: 2513ad108e0212b071da4301f19420c04c4db9d3
Parents: c29d647
Author: hzhang2 
Authored: Fri Mar 18 10:31:58 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 10:42:44 2016 +0800

--
 src/backend/commands/analyze.c | 54 -
 1 file changed, 35 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2513ad10/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index dd5d5c0..32b3416 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -143,7 +143,7 @@ static void analyzeComputeAttributeStatistics(Oid 
relationOid,
float4 sampleTableRelTuples, 
bool mergeStats,
AttributeStatistics *stats);
-static List* analyzableRelations(bool rootonly);
+static List* analyzableRelations(bool rootonly, List** fullRelOids);
 static bool analyzePermitted(Oid relationOid);
 static List *analyzableAttributes(Relation candidateRelation);
 static int calculate_virtual_segment_number(List* candidateRelations);
@@ -300,6 +300,7 @@ void analyzeStatement(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
 void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 {
List*lRelOids = NIL;
+   List*lFullRelOids = NIL;
MemoryContext   callerContext = NULL;
MemoryContext   analyzeStatementContext = NULL;
MemoryContext   analyzeRelationContext = NULL;
@@ -379,7 +380,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
/**
 * ANALYZE entire DB.
 */
-   lRelOids = analyzableRelations(stmt->rootonly);
+   lRelOids = analyzableRelations(stmt->rootonly, );
if (stmt->rootonly && NIL == lRelOids)
{
ereport(WARNING,
@@ -392,6 +393,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
 * ANALYZE called by autovacuum.
 */
lRelOids = relids;
+   lFullRelOids = relids;
}
else
{
@@ -419,11 +421,14 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
{
lRelOids = all_leaf_partition_relids(pn); /* 
all leaves */
}
+   lFullRelOids = all_leaf_partition_relids(pn);
lRelOids = lappend_oid(lRelOids, relationOid); /* root 
partition */
+   lFullRelOids = lappend_oid(lFullRelOids, relationOid); 
/* root partition */
if (optimizer_analyze_midlevel_partition)
{
lRelOids = list_concat(lRelOids, 
all_interior_partition_relids(pn)); /* interior partitions */
}
+   lFullRelOids = list_concat(lFullRelOids, 
all_interior_partition_relids(pn)); /* interior partitions */
}
else if (ps == PART_STATUS_INTERIOR) /* analyze an interior 
partition directly */
{
@@ -440,6 +445,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
else
{
lRelOids = list_make1_oid(relationOid);
+   lFullRelOids = list_make1_oid(relationOid);
}
}
 
@@ -472,6 +478,17 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
}
 
/**
+*  we use preferred_seg_num as default and
+*  compute target_seg_num based on data size and distributed type
+*  if there is no preferred_seg_num.
+*/
+   int target_seg_num = preferred_seg_num;
+   if (target_seg_num <= 0) {
+   target_seg_num = calculate_virtual_segment_number(lFullRelOids);
+   }
+   elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
+
+   /**
 * We open relations with appropreciate locks
 */
List *candidateRelations = NIL;
@@ -499,17 +516,6 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 

incubator-hawq git commit: Revert "HAWQ-532. Optimise vseg number for copy to statement."

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 36571bce2 -> 86d9b0c03


Revert "HAWQ-532. Optimise vseg number for copy to statement."


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/86d9b0c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/86d9b0c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/86d9b0c0

Branch: refs/heads/master
Commit: 86d9b0c03de349398ddb0337f7ac5dac3b58d2b9
Parents: 36571bc
Author: hzhang2 
Authored: Fri Mar 18 14:22:29 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 14:22:29 2016 +0800

--
 src/backend/commands/copy.c | 97 
 1 file changed, 8 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/86d9b0c0/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e37fb83..bcd5384 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,11 +91,6 @@
 #include "postmaster/autovacuum.h"
 #include "cdb/dispatcher.h"
 
-/*
- * in dbsize.c
- */
-extern int64 calculate_relation_size(Relation rel);
-
 /* DestReceiver for COPY (SELECT) TO */
 typedef struct
 {
@@ -142,7 +137,6 @@ static void copy_in_error_callback(void *arg);
 static void CopyInitPartitioningState(EState *estate);
 static void CopyInitDataParser(CopyState cstate);
 static bool CopyCheckIsLastLine(CopyState cstate);
-static int calculate_virtual_segment_number(List* candidateRelations);
 
 /* ==
  * The follwing macros aid in major refactoring of data processing code (in
@@ -1562,16 +1556,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 
target_policy = GpPolicyFetch(CurrentMemoryContext, 
relid);
Assert(target_policy);
-   /*
-* For hash table we use table bucket number to request 
vsegs
-* For random table, we use a fixed GUC value to 
request vsegs.
-*/
-   if(target_policy->nattrs > 0){
-   target_segment_num = target_policy->bucketnum;
-   }
-   else{
-   target_segment_num = 
hawq_rm_nvseg_for_copy_from_perquery;
-   }
+   target_segment_num = target_policy->bucketnum;
pfree(target_policy);
 
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, 
target_segment_num, target_segment_num,NULL,0);
@@ -1764,63 +1749,6 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 }
 
 /*
- * calculate virtual segment number for copy statement.
- * if there is hash distributed relations exist, use the max bucket number.
- * if all relation are random, use the data size to determine vseg number.
- */
-static int calculate_virtual_segment_number(List* candidateOids) {
-   ListCell* le1;
-   int vsegNumber = 1;
-   int64 totalDataSize = 0;
-   bool isHashRelationExist = false;
-   int maxHashBucketNumber = 0;
-
-   foreach (le1, candidateOids)
-   {
-   Oid candidateOid  = InvalidOid;
-   candidateOid = lfirst_oid(le1);
-
-   //Relation rel = (Relation)lfirst(le1);
-   Relation rel = relation_open(candidateOid, AccessShareLock);
-   if (candidateOid > 0 ) {
-   GpPolicy *targetPolicy = 
GpPolicyFetch(CurrentMemoryContext,
-   candidateOid);
-   if(targetPolicy == NULL){
-   return GetAnalyzeVSegNumLimit();
-   }
-   if (targetPolicy->nattrs > 0) {
-   isHashRelationExist = true;
-   if(maxHashBucketNumber < 
targetPolicy->bucketnum){
-   maxHashBucketNumber = 
targetPolicy->bucketnum;
-   }
-   }
-   /*
-* if no hash relation, we calculate the data size of 
all the relations.
-*/
-   if (!isHashRelationExist) {
-   totalDataSize += calculate_relation_size(rel);
-   }
-   }
-   relation_close(rel, AccessShareLock);
-   }
-
-   if (isHashRelationExist) {
-   vsegNumber = maxHashBucketNumber;
-   } else {

[5/5] incubator-hawq git commit: HAWQ-531. Change GUC default value.

[hubertzhang]

HAWQ-531. Change GUC default value.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/eaaf9451
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/eaaf9451
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/eaaf9451

Branch: refs/heads/master
Commit: eaaf94514396c8f60716a78a76852017f7614808
Parents: 4d997b5
Author: hzhang2 
Authored: Fri Mar 18 10:16:51 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 10:42:44 2016 +0800

--
 src/backend/utils/misc/guc.c | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eaaf9451/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 3d13eb4..a6a8bee 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4516,7 +4516,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
_hash_table_bucket_number,
-   6, 1, INT_MAX, NULL, NULL
+   6, 1, 1, NULL, NULL
},
 
{
@@ -4525,7 +4525,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
_rm_nvseg_for_copy_from_perquery,
-   6, 1, INT_MAX, NULL, NULL
+   6, 1, 1, NULL, NULL
},
 
{
@@ -4534,7 +4534,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
_rm_nvseg_for_analyze_perquery_perseg_limit,
-   4, 1, INT_MAX, NULL, NULL
+   4, 1, 1, NULL, NULL
},
 
{
@@ -4543,7 +4543,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
_rm_nvseg_for_analyze_perquery_limit,
-   256, 1, INT_MAX, NULL, NULL
+   256, 1, 1, NULL, NULL
},
 
{
@@ -6388,7 +6388,7 @@ static struct config_int ConfigureNamesInt[] =
 NULL
 },
 _nvseg_perquery_limit,
-1000, 1, 65535, NULL, NULL
+512, 1, 1, NULL, NULL
 },
 
 {
@@ -6398,7 +6398,7 @@ static struct config_int ConfigureNamesInt[] =
 NULL
 },
 _nvseg_perquery_perseg_limit,
-6, 1, 65535, NULL, NULL
+6, 1, 1, NULL, NULL
 },
 
 {

[3/5] incubator-hawq git commit: HAWQ-529. Allocate resource for udf in resource negotiator.

[hubertzhang]

HAWQ-529. Allocate resource for udf in resource negotiator.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d997b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d997b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d997b57

Branch: refs/heads/master
Commit: 4d997b5714bb09ed9020a56acfce13b1d6fd637c
Parents: 2c033d9
Author: hzhang2 
Authored: Fri Mar 18 10:14:42 2016 +0800
Committer: hzhang2 
Committed: Fri Mar 18 10:42:44 2016 +0800

--
 src/backend/cdb/cdbdatalocality.c| 213 --
 src/backend/executor/spi.c   |  71 --
 src/backend/optimizer/plan/planner.c |  59 -
 src/backend/optimizer/util/clauses.c |  11 +-
 src/backend/optimizer/util/walkers.c |   1 +
 src/include/cdb/cdbdatalocality.h|  19 ++-
 src/include/executor/spi.h   |   7 +
 src/include/optimizer/planner.h  |   4 +
 8 files changed, 294 insertions(+), 91 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/cdb/cdbdatalocality.c
--
diff --git a/src/backend/cdb/cdbdatalocality.c 
b/src/backend/cdb/cdbdatalocality.c
index eec87b4..3f8fcb5 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,6 +33,9 @@
 #include "access/filesplit.h"
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
+#include "catalog/catquery.h"
+#include "catalog/pg_inherits.h"
+#include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
 #include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
@@ -41,6 +44,7 @@
 #include "utils/tqual.h"
 #include "utils/memutils.h"
 #include "executor/execdesc.h"
+#include "executor/spi.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "optimizer/walkers.h"
@@ -54,11 +58,7 @@
 #include "catalog/pg_proc.h"
 #include "postgres.h"
 #include "resourcemanager/utils/hashtable.h"
-#include "catalog/pg_inherits.h"
-//#include "utils/misc/guc.c"
 
-#define PRONAME 1
-#define PROISAGG 5
 /* We need to build a mapping from host name to host index */
 
 extern booloptimizer; /* Enable the optimizer */
@@ -376,7 +376,7 @@ static Block_Host_Index * update_data_dist_stat(
 static HostDataVolumeInfo *search_host_in_stat_context(
split_to_segment_mapping_context *context, char *hostname);
 
-static bool IsAggFunction(char* funcName);
+static bool IsBuildInFunction(Oid funcOid);
 
 static bool allocate_hash_relation(Relation_Data* rel_data,
Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -638,39 +638,35 @@ static void collect_range_tables(Query *query, List* 
full_range_table,
 /*
  *
  */
-static bool IsAggFunction(char* funcName) {
-   if (funcName == NULL) {
-   return false;
-   }
-   Relation pg_proc_rel;
-   TupleDesc pg_proc_dsc;
-   HeapTuple tuple;
-   SysScanDesc pg_proc_scan;
-
-   pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
-   pg_proc_dsc = RelationGetDescr(pg_proc_rel);
-   ScanKeyData skey;
+static bool IsBuildInFunction(Oid foid) {
 
-   ScanKeyInit(, PRONAME, BTEqualStrategyNumber,
-   F_NAMEEQ, CStringGetDatum(funcName));
-
-   pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
-   ActiveSnapshot, 1, );
-   while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+   cqContext  *pcqCtx;
+   HeapTuple   procedureTuple;
+   Form_pg_proc procedureStruct;
 
-   bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, 
pg_proc_dsc, NULL));
-   systable_endscan(pg_proc_scan);
-   heap_close(pg_proc_rel, AccessShareLock);
-   if (isAgg) {
-   return true;
-   } else {
-   return false;
-   }
+   /*
+* get the procedure tuple corresponding to the given function Oid
+*/
+   pcqCtx = caql_beginscan(
+   NULL,
+   cql("SELECT * FROM pg_proc "
+   " WHERE oid = :1 ",
+   ObjectIdGetDatum(foid)));
+
+   procedureTuple = caql_getnext(pcqCtx);
+
+   if (!HeapTupleIsValid(procedureTuple))
+   elog(ERROR, "cache lookup failed for function %u", foid);
+   procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
+   caql_endscan(pcqCtx);
+   /* we treat proc namespace = 11 to build in function.*/
+   if (procedureStruct->pronamespace == 11) {
+   return true;
+   } else {
+   return false;
}
-   

[1/2] incubator-hawq git commit: Revert "HAWQ-533. Cursor failed, if don't allocate resource in prepare."

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master d259efffb -> fd2022772


Revert "HAWQ-533. Cursor failed, if don't allocate resource in prepare."

This reverts commit 96345af194bfd3ca05227037479c9835a5e495b0.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bc065a51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bc065a51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bc065a51

Branch: refs/heads/master
Commit: bc065a51fe1e04c15c398a6d15c5d01424efb52f
Parents: d259eff
Author: hzhang2 
Authored: Thu Mar 17 09:18:50 2016 +0800
Committer: hzhang2 
Committed: Thu Mar 17 09:18:50 2016 +0800

--
 src/backend/executor/spi.c | 24 +---
 1 file changed, 5 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bc065a51/src/backend/executor/spi.c
--
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 9465de8..8e7645c 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1146,20 +1146,6 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
qtlist = copyObject(qtlist);
ptlist = copyObject(ptlist);
 
-   PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
-
-   if ( (Gp_role == GP_ROLE_DISPATCH) &&
-(stmt->resource_parameters != NULL) )
-   {
-   /*
-* Now, we want to allocate resource.
-*/
-   stmt->resource = 
AllocateResource(stmt->resource_parameters->life, 
stmt->resource_parameters->slice_size,
-   stmt->resource_parameters->iobytes, 
stmt->resource_parameters->max_target_segment_num,
-   
stmt->resource_parameters->min_target_segment_num, 
stmt->resource_parameters->vol_info,
-   stmt->resource_parameters->vol_info_size);
-   }
-
/* If the plan has parameters, set them up */
if (spiplan->nargs > 0)
{
@@ -1860,12 +1846,12 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, 
const char *Nulls,
 * We only allocate resource for multiple 
executions of queries, NOT for utility commands.
 * SELECT/INSERT are supported at present.
 */
-   if((queryTree->commandType == CMD_SELECT) ||
-   (queryTree->commandType == 
CMD_INSERT))
+   if( (queryTree->commandType == CMD_SELECT) ||
+   (queryTree->commandType == CMD_INSERT) )
{
-   if ((Gp_role == GP_ROLE_DISPATCH) &&
-   (stmt->resource == 
NULL) &&
-   
(stmt->resource_parameters != NULL))
+   if ( (Gp_role == GP_ROLE_DISPATCH) &&
+(stmt->resource == NULL) &&
+(stmt->resource_parameters != 
NULL) )
{
SplitAllocResult *allocResult = 
NULL;
 

[2/2] incubator-hawq git commit: Revert "HAWQ-531. Change GUC default value."

[hubertzhang]

Revert "HAWQ-531. Change GUC default value."

This reverts commit 4d1a85405b87145fb931a16be9e22bd40f16eca8.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/94b7a822
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/94b7a822
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/94b7a822

Branch: refs/heads/master
Commit: 94b7a8225d23729d01cfc481375146d06078b601
Parents: 1897ba1
Author: hubertzhang <hzh...@pivotal.io>
Authored: Thu Mar 17 15:38:46 2016 +0800
Committer: hubertzhang <hzh...@pivotal.io>
Committed: Thu Mar 17 15:38:46 2016 +0800

--
 src/backend/utils/misc/guc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/94b7a822/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ba8e747..fa58f86 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6379,7 +6379,7 @@ static struct config_int ConfigureNamesInt[] =
 NULL
 },
 _nvseg_perquery_limit,
-512, 1, 65535, NULL, NULL
+1000, 1, 65535, NULL, NULL
 },
 
 {
@@ -6535,7 +6535,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
_rejectrequest_nseg_limit,
-   4, 0, 65535, NULL, NULL
+   2, 0, 65535, NULL, NULL
},
 
{

incubator-hawq git commit: HAWQ-505. Fix InputFormatError caused by GUC change.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master e29e13345 -> 71911c1ce


HAWQ-505. Fix InputFormatError caused by GUC change.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/71911c1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/71911c1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/71911c1c

Branch: refs/heads/master
Commit: 71911c1ce05c9cea1ae694632c319d13da05b03f
Parents: e29e133
Author: hubertzhang <hzh...@pivotal.io>
Authored: Thu Mar 10 09:49:40 2016 +0800
Committer: hubertzhang <hzh...@pivotal.io>
Committed: Thu Mar 10 09:50:07 2016 +0800

--
 .../test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/71911c1c/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java
--
diff --git 
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java
 
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java
index fdaed2e..130c53a 100644
--- 
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java
+++ 
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHClusterTester.java
@@ -64,8 +64,8 @@ public class TPCHClusterTester extends TPCHTester {
 
// 1. load TPCH data
Map<String, String> rs = 
HAWQJdbcUtils.executeSafeQueryForSingleRow(
-   conn, "SHOW default_segment_num;");
-   int segnum = 
Integer.parseInt(rs.get("default_segment_num"));
+   conn, "SHOW 
default_hash_table_bucket_number;");
+   int segnum = 
Integer.parseInt(rs.get("default_hash_table_bucket_number"));

MRFormatTestUtils.runShellCommand(tpchSpec.getLoadCmd(segnum));
 
// 2. generate answer

incubator-hawq git commit: HAWQ-487. Ensure virtual segment of analyze not greate than GetUtilPartitionNum.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7715e6c30 -> 7924d56fb


HAWQ-487. Ensure virtual segment of analyze not greate than GetUtilPartitionNum.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7924d56f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7924d56f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7924d56f

Branch: refs/heads/master
Commit: 7924d56fb9818543d1145defe075ee6c0d337f08
Parents: 7715e6c
Author: hubertzhang <hzh...@pivotal.io>
Authored: Fri Mar 4 17:15:39 2016 +0800
Committer: hubertzhang <hzh...@pivotal.io>
Committed: Fri Mar 4 17:15:39 2016 +0800

--
 src/backend/commands/analyze.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7924d56f/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 91451d8..afa4cc0 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -807,7 +807,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int 
preferred_seg_num)
 static int calculate_virtual_segment_number(List* candidateRelations) {
ListCell* le1;
int vsegNumber = 1;
-   int64_t totalDataSize = 0;
+   int64 totalDataSize = 0;
bool isHashRelationExist = false;
int maxHashBucketNumber = 0;
 
@@ -843,7 +843,7 @@ static int calculate_virtual_segment_number(List* 
candidateRelations) {
vsegNumber = totalDataSize + 1;
}
/*vsegNumber should be less than GetUtilPartitionNum*/
-   if(vsegNumber > GetUtilPartitionNum()){
+   if(vsegNumber > GetUtilPartitionNum() || vsegNumber <= 0){
vsegNumber = GetUtilPartitionNum();
}
 

incubator-hawq git commit: HAWQ-487. Ensure virtual segment of analyze not greate than GetUtilPartitionNum.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master e88a257ad -> 2d36f1340


HAWQ-487. Ensure virtual segment of analyze not greate than GetUtilPartitionNum.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2d36f134
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2d36f134
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2d36f134

Branch: refs/heads/master
Commit: 2d36f1340a85758821bbef645b69756adab93713
Parents: e88a257
Author: hubertzhang <hzh...@pivotal.io>
Authored: Thu Mar 3 15:05:04 2016 +0800
Committer: hubertzhang <hzh...@pivotal.io>
Committed: Thu Mar 3 15:05:04 2016 +0800

--
 src/backend/commands/analyze.c | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2d36f134/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 779aea5..91451d8 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -842,6 +842,10 @@ static int calculate_virtual_segment_number(List* 
candidateRelations) {
totalDataSize >>= 27;
vsegNumber = totalDataSize + 1;
}
+   /*vsegNumber should be less than GetUtilPartitionNum*/
+   if(vsegNumber > GetUtilPartitionNum()){
+   vsegNumber = GetUtilPartitionNum();
+   }
 
return vsegNumber;
 }

incubator-hawq git commit: HAWQ-484. Fix bug in gpcopy and pgdump bug.

[hubertzhang]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 419d89e7b -> e88a257ad


HAWQ-484. Fix bug in gpcopy and pgdump bug.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e88a257a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e88a257a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e88a257a

Branch: refs/heads/master
Commit: e88a257adf45e6ed730210e4e360d0674a65b06a
Parents: 419d89e
Author: hubertzhang <hzh...@pivotal.io>
Authored: Thu Mar 3 11:29:49 2016 +0800
Committer: hubertzhang <hzh...@pivotal.io>
Committed: Thu Mar 3 11:29:49 2016 +0800

--
 src/backend/commands/analyze.c | 3 +++
 src/backend/commands/copy.c| 5 ++---
 2 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e88a257a/src/backend/commands/analyze.c
--
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index b4ad0c9..779aea5 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -817,6 +817,9 @@ static int calculate_virtual_segment_number(List* 
candidateRelations) {
if (rel ) {
GpPolicy *targetPolicy = 
GpPolicyFetch(CurrentMemoryContext,
rel->rd_id);
+   if(targetPolicy == NULL){
+   return GetUtilPartitionNum();
+   }
if (targetPolicy->nattrs > 0) {
isHashRelationExist = true;
if(maxHashBucketNumber < 
targetPolicy->bucketnum){

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e88a257a/src/backend/commands/copy.c
--
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e78c1ca..bcd5384 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2273,8 +2273,7 @@ CopyTo(CopyState cstate)
MemTupleBinding *mt_bind = 
create_memtuple_binding(tupDesc);
 
aoscandesc = appendonly_beginscan(rel, 
ActiveSnapshot, 0, NULL);
-   aoscandesc->splits = 
GetFileSplitsOfSegment(cstate->splits,
-   
rel->rd_id, GetQEIndex());
+   aoscandesc->splits = 
GetFileSplitsOfSegment(cstate->splits,rel->rd_id, GetQEIndex());
 
 
while ((tuple = appendonly_getnext(aoscandesc, 
ForwardScanDirection, slot)) != NULL)
@@ -2314,7 +2313,7 @@ CopyTo(CopyState cstate)
proj[i] = true;
 
scan = parquet_beginscan(rel, ActiveSnapshot, 
0, proj);
-   scan->splits = cstate->splits;
+   scan->splits = 
GetFileSplitsOfSegment(cstate->splits, rel->rd_id, GetQEIndex());
for(;;)
{
parquet_getnext(scan, 
ForwardScanDirection, slot);