This is an automated email from the ASF dual-hosted git repository. djwang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 6fa21e7cd6196d13eb5ee8250b4e13c01f575454 Author: Himanshu Pandey <[email protected]> AuthorDate: Thu Sep 22 10:42:40 2022 -0700 updated ProjectionInfo related code for GP7 and code refactor --- external-table/src/pxfheaders.c | 245 ++++++++++++++++++++++++++++++---------- 1 file changed, 185 insertions(+), 60 deletions(-) diff --git a/external-table/src/pxfheaders.c b/external-table/src/pxfheaders.c index d07a9595..a9bd3661 100644 --- a/external-table/src/pxfheaders.c +++ b/external-table/src/pxfheaders.c @@ -23,6 +23,7 @@ #if PG_VERSION_NUM >= 120000 #include "access/external.h" #include "extension/gp_exttable_fdw/extaccess.h" +#include "executor/execExpr.h" #else #include "access/fileam.h" #include "catalog/pg_exttable.h" @@ -39,7 +40,12 @@ static char *get_format_name(ExtTableEntry *exttbl); static char *getFormatterString(ExtTableEntry *exttbl); static bool isFormatterPxfDelimited(ExtTableEntry *exttbl); static bool isFormatterPxfWritable(ExtTableEntry *exttbl); -static void add_projection_desc_httpheaders(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel); +#if PG_VERSION_NUM >= 120000 +static void add_projection_desc_httpheader_pg12(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel); +#else +static void add_projection_desc_httpheader_pg94(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel); +#endif + static bool add_attnums_from_targetList(Node *node, List *attnums); static void add_projection_index_header(CHURL_HEADERS pVoid, StringInfoData data, int attno, char number[32]); static List *appendCopyEncodingOptionToList(List *copyFmtOpts, int encoding); @@ -155,7 +161,11 @@ build_http_headers(PxfInputData *input) if (qualsAreSupported && (qualsAttributes != NIL || list_length(input->quals) == 0)) { - add_projection_desc_httpheaders(headers, proj_info, qualsAttributes, rel); + #if PG_VERSION_NUM >= 120000 + add_projection_desc_httpheader_pg12(headers, proj_info, qualsAttributes, rel); + #else + add_projection_desc_httpheader_pg94(headers, proj_info, qualsAttributes, rel); + #endif } else { @@ -272,7 +282,7 @@ add_tuple_desc_httpheader(CHURL_HEADERS headers, Relation rel) for (i = 0, attrIx = 0; i < tuple->natts; ++i) { #if PG_VERSION_NUM >= 120000 - Form_pg_attribute attribute = &tuple->attrs[i]; + FormData_pg_attribute *attribute = &tuple->attrs[i]; #else FormData_pg_attribute *attribute = tuple->attrs[i]; #endif @@ -494,8 +504,10 @@ int getNumSimpleVars(ProjectionInfo *projInfo) * indices will be 0, 2. This is because we use 0-based indexing and because * col2 was dropped, the indices for col3 and col4 get shifted by -1. */ + +#if PG_VERSION_NUM < 120000 static void -add_projection_desc_httpheaders(CHURL_HEADERS headers, +add_projection_desc_httpheader_pg94(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel) @@ -504,22 +516,11 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, int dropped_count; int number; int numTargetList; -#if PG_VERSION_NUM < 90400 || PG_VERSION_NUM == 120000 - int numSimpleVars = 0; +#if PG_VERSION_NUM < 90400 + int numSimpleVars; #endif char long_number[sizeof(int32) * 8]; - /* FIXME: to get it to compile assign it to NULL */ - // pi_varNumbers is not available anymore in the postgters code - // https://doxygen.postgresql.org/structProjectionInfo.html - //int *varNumbers = projInfo->pi_varNumbers; -#if PG_VERSION_NUM < 120000 // This will work for 6.x and lower versions of GPDB - int *varNumbers = projInfo->pi_varNumbers; - List *targetList = projInfo->pi_targetlist; -#else - int *varNumbers = NULL; //TODO Need to find a proper value here - List *targetList = (List *)projInfo->pi_state.resultslot; -#endif - + int *varNumbers = projInfo->pi_varNumbers; Bitmapset *attrs_used; StringInfoData formatter; List *targetList; // targetList from the query plan @@ -528,18 +529,14 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, // STEP 0: initialize variables initStringInfo(&formatter); numTargetList = 0; + #if PG_VERSION_NUM >= 90400 /* * Non-simpleVars are added to the targetlist * we use expression_tree_walker to access attrno information * we do it through a helper function add_attnums_from_targetList */ - /* FIXME: commenting this out for compilation success */ - // pi_targetlist is not available anymore in the postgters code - // https://doxygen.postgresql.org/structProjectionInfo.html - //if (projInfo->pi_targetlist) - - if (targetList) + if (projInfo->pi_targetlist) { #else numSimpleVars = 0; @@ -553,17 +550,10 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, List *l = lappend_int(NIL, 0); ListCell *lc1; - /* FIXME: commenting this out to make it compile */ - - foreach(lc1, targetList) + foreach(lc1, projInfo->pi_targetlist) { - #if PG_VERSION_NUM >= 120000 - ExprState *gstate = (ExprState *) lfirst(lc1); - add_attnums_from_targetList((Node *) gstate->expr, l); - #else GenericExprState *gstate = (GenericExprState *) lfirst(lc1); add_attnums_from_targetList((Node *) gstate->arg->expr, l); - #endif } foreach(lc1, l) @@ -579,29 +569,19 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, list_free(l); } -#if PG_VERSION_NUM < 90400 || PG_VERSION_NUM >= 120000 +#if PG_VERSION_NUM < 90400 else { - #if PG_VERSION_NUM >= 120000 - numSimpleVars = list_length(targetList); - #else - numSimpleVars = list_length(projInfo->pi_targetlist); - #endif + numSimpleVars = list_length(projInfo->pi_targetlist); } #endif number = numTargetList + - - - #if PG_VERSION_NUM >= 90400 && PG_VERSION_NUM < 120000 - // FIXME: Commenting this out for compilation success - // pi_numSimpleVars is not available anymore in the postgters 12 code - // https://doxygen.postgresql.org/structProjectionInfo.html - projInfo->pi_numSimpleVars + - #else - numSimpleVars + - #endif - +#if PG_VERSION_NUM >= 90400 + projInfo->pi_numSimpleVars + +#else + numSimpleVars + +#endif list_length(qualsAttributes); if (number == 0) return; @@ -612,13 +592,11 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, pg_ltoa(number, long_number); churl_headers_append(headers, "X-GP-ATTRS-PROJ", long_number); - -#if PG_VERSION_NUM >= 90400 && PG_VERSION_NUM < 120000 - /* FIXME: commenting out to get compile to work */ - //for (i = 0; i < projInfo->pi_numSimpleVars; i++) - for (i = 0; i < projInfo->pi_numSimpleVars; i++) +#if PG_VERSION_NUM >= 90400 + for (i = 0; i < projInfo->pi_numSimpleVars; i++) #else for (i = 0; varNumbers && i < numSimpleVars; i++) +#endif { attrs_used = bms_add_member(attrs_used, @@ -642,13 +620,8 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, for (i = 1; i <= tupdesc->natts; i++) { - #if PG_VERSION_NUM >= 120000 - Form_pg_attribute attr = &tupdesc->attrs[i-1]; - #else - FormData_pg_attribute *attr = tupdesc->attrs[i-1]; - #endif /* Ignore dropped attributes. */ - if (attr->attisdropped) + if (tupdesc->attrs[i - 1]->attisdropped) { /* keep a counter of the number of dropped attributes */ droppedCount++; @@ -675,6 +648,158 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers, pfree(formatter.data); bms_free(attrs_used); } +#endif + +/* + * Report projection description to the remote component, the indices of + * dropped columns do not get reported, as if they never existed, and + * column indices that follow dropped columns will be shifted by the number + * of dropped columns that precede it. For example, + * + * --------------------------------------------- + * | col1 | col2 (dropped) | col3 | col4 | + * --------------------------------------------- + * + * Let's assume that col1 and col4 are projected, the reported projected + * indices will be 0, 2. This is because we use 0-based indexing and because + * col2 was dropped, the indices for col3 and col4 get shifted by -1. + */ + +#if PG_VERSION_NUM >= 120000 +static void +add_projection_desc_httpheader_pg12(CHURL_HEADERS headers, + ProjectionInfo *projInfo, + List *qualsAttributes, + Relation rel) +{ + int i; + int dropped_count; + int number; + int numTargetList; + char long_number[sizeof(int32) * 8]; + //int *varNumbers = projInfo->pi_varNumbers; + Bitmapset *attrs_used; + StringInfoData formatter; + TupleDesc tupdesc; + ExprEvalStep step; + initStringInfo(&formatter); + numTargetList = 0; + + List *targetList = (List *) projInfo->pi_state.expr; + int numSimpleVars = 0; + +#if PG_VERSION_NUM >= 90400 + + int numNonSimpleVars = 0; + + for (int i = 0; i < projInfo->pi_state.steps_len; i++) + { + step = projInfo->pi_state.steps[i]; + if (step.opcode == EEOP_ASSIGN_INNER_VAR || + step.opcode == EEOP_ASSIGN_OUTER_VAR || + step.opcode == EEOP_ASSIGN_SCAN_VAR) + numSimpleVars++; + else if (step.opcode == EEOP_ASSIGN_TMP_MAKE_RO || + step.opcode == EEOP_ASSIGN_TMP) + numNonSimpleVars++; + } + int * varNumbers = &numNonSimpleVars; + /* + * Non-simpleVars are added to the targetlist + * we use expression_tree_walker to access attrno information + * we do it through a helper function add_attnums_from_targetList + */ + if (targetList) + { +#else + + if (numNonSimpleVars > 0) + { + /* + * When there are not just simple Vars we need to + * walk the tree to get attnums + */ +#endif + List *l = lappend_int(NIL, 0); + ListCell *lc1; + + foreach(lc1, targetList) + { + ExprState *gstate = (ExprState *) lfirst(lc1); + add_attnums_from_targetList( (Node *) gstate, l); + } + + foreach(lc1, l) + { + int attno = lfirst_int(lc1); + if (attno > InvalidAttrNumber) + { + add_projection_index_header(headers, + formatter, attno - 1, long_number); + numTargetList++; + } + } + + list_free(l); + } + + number = numTargetList + numSimpleVars + list_length(qualsAttributes); + if (number == 0) + return; + + attrs_used = NULL; + + /* Convert the number of projection columns to a string */ + pg_ltoa(number, long_number); + churl_headers_append(headers, "X-GP-ATTRS-PROJ", long_number); + + for (i = 0; varNumbers && i < numSimpleVars ; i++) + { + attrs_used = + bms_add_member(attrs_used, + varNumbers[i] - FirstLowInvalidHeapAttributeNumber); + } + + ListCell *attribute = NULL; + + /* + * AttrNumbers coming from quals + */ + foreach(attribute, qualsAttributes) + { + AttrNumber attrNumber = (AttrNumber) lfirst_int(attribute); + attrs_used = + bms_add_member(attrs_used, + attrNumber + 1 - FirstLowInvalidHeapAttributeNumber); + } + + tupdesc = RelationGetDescr(rel); + dropped_count = 0; + + for (i = 1; i <= tupdesc->natts; i++) + { + /* Ignore dropped attributes. */ + if (tupdesc->attrs[i - 1].attisdropped) + { + /* keep a counter of the number of dropped attributes */ + dropped_count++; + continue; + } + + if (bms_is_member(i - FirstLowInvalidHeapAttributeNumber, attrs_used)) + { + /* Shift the column index by the running dropped_count */ + add_projection_index_header(headers, formatter, + i - 1 - dropped_count, long_number); + } + } + + list_free(qualsAttributes); + pfree(formatter.data); + bms_free(attrs_used); +} +#endif + /* * Adds the projection index header for the given attno --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
