Repository: incubator-hawq Updated Branches: refs/heads/master 9f33d8dd0 -> 1c189fc12
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c189fc1/src/include/cdb/cdbdatalocality.h ---------------------------------------------------------------------- diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h index c3753ce..0afc45c 100644 --- a/src/include/cdb/cdbdatalocality.h +++ b/src/include/cdb/cdbdatalocality.h @@ -32,6 +32,7 @@ #include "catalog/gp_policy.h" #include "nodes/parsenodes.h" #include "executor/execdesc.h" +#include "catalog/pg_exttable.h" /* * structure containing information about data residence @@ -71,12 +72,18 @@ typedef struct VirtualSegmentNode char *hostname; } VirtualSegmentNode; +typedef struct blocklocation_file{ + BlockLocation *locations; + int block_num; + char *file_uri; +}blocklocation_file; + /* * calculate_planner_segment_num: based on the parse tree, * we calculate the appropriate planner segment_num. */ -SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife, - List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum); +SplitAllocResult * calculate_planner_segment_num(PlannedStmt *plannedstmt, Query *query, + QueryResourceLife resourceLife, int fixedVsegNum); /* * udf_collector_walker: the routine to file udfs. http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c189fc1/src/include/commands/tablecmds.h ---------------------------------------------------------------------- diff --git a/src/include/commands/tablecmds.h b/src/include/commands/tablecmds.h index 8404a7f..e23df2f 100644 --- a/src/include/commands/tablecmds.h +++ b/src/include/commands/tablecmds.h @@ -52,7 +52,7 @@ typedef struct AttrMapContext{ extern const char *synthetic_sql; -extern Oid DefineRelation(CreateStmt *stmt, char relkind, char relstorage); +extern Oid DefineRelation(CreateStmt *stmt, char relkind, char relstorage, const char *formattername); extern void DefineExternalRelation(CreateExternalStmt *stmt); @@ -89,6 +89,8 @@ extern void CheckTableNotInUse(Relation rel, const char *stmt); extern void ExecuteTruncate(TruncateStmt *stmt); + + extern void renameatt(Oid myrelid, const char *oldattname, const char *newattname, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c189fc1/src/include/nodes/parsenodes.h ---------------------------------------------------------------------- diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index a9ca4a0..173a35c 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1354,14 +1354,15 @@ typedef struct GrantRoleStmt * Node that represents the single row error handling (SREH) clause. * used in COPY and External Tables. */ -typedef struct SingleRowErrorDesc -{ - NodeTag type; - RangeVar *errtable; /* error table for data format errors */ - int rejectlimit; /* per segment error reject limit */ - bool is_keep; /* true if KEEP indicated (COPY only) */ - bool is_limit_in_rows; /* true for ROWS false for PERCENT */ - bool reusing_existing_errtable; /* var used later in trasform... */ +typedef struct SingleRowErrorDesc { + NodeTag type; + RangeVar *errtable; /* error table for data format errors */ + int rejectlimit; /* per segment error reject limit */ + bool is_keep; /* true if KEEP indicated (COPY only) */ + bool is_limit_in_rows; /* true for ROWS false for PERCENT */ + bool reusing_existing_errtable; /* var used later in trasform... */ + bool is_hdfs_protocol_text; /* hdfs protocol text format table */ + char *hdfsLoc; /* error table location for hdfs protocol text only */ } SingleRowErrorDesc; /* ---------------------- @@ -1403,37 +1404,40 @@ typedef struct CopyStmt * implementation). * ---------------------- */ - -typedef struct CreateStmt -{ - NodeTag type; - RangeVar *relation; /* relation to create */ - List *tableElts; /* column definitions (list of ColumnDef) */ - List *inhRelations; /* relations to inherit from (list of - * inhRelation) */ - List *constraints; /* constraints (list of Constraint nodes) */ - List *options; /* options from WITH clause */ - OnCommitAction oncommit; /* what do we do at COMMIT? */ - char *tablespacename; /* table space to use, or NULL */ - List *distributedBy; /* what columns we distribute the data by */ - Node *partitionBy; /* what columns we partition the data by */ - TableOidInfo oidInfo; - char relKind; /* CDB: force relkind to this */ - char relStorage; - struct GpPolicy *policy; - Node *postCreate; /* CDB: parse and process after the CREATE */ - List *deferredStmts; /* CDB: Statements, e.g., partial indexes, that can't be - * analyzed until after CREATE (until the target table - * is created and visible). */ - bool is_part_child; /* CDB: child table in a partition? Marked during analysis for - * interior or leaf parts of the new table. Not marked for a - * a partition root or ordinary table. - */ - bool is_add_part; /* CDB: is create adding a part to a partition? */ - bool is_split_part; /* CDB: is create spliting a part? */ - Oid ownerid; /* OID of the role to own this. if InvalidOid, GetUserId() */ - bool buildAoBlkdir; /* whether to build the block directory for an AO table */ - List *attr_encodings; /* attribute storage directives */ +typedef struct CreateStmtBase { + char relKind; // CDB: force relkind to this + RangeVar *relation; // relation to create + List *tableElts; // column definitions (list of ColumnDef) + List *inhRelations; // relations to inherit from (list of inhRelation) + List *constraints; // constraints (list of Constraint nodes) + List *options; // options from WITH clause + OnCommitAction oncommit; // what do we do at COMMIT? + char *tablespacename; // table space to use, or NULL + List *distributedBy; // what columns we distribute the data by + Node *postCreate; // CDB: parse and process after the CREATE + // CDB: child table in a partition? Marked during analysis for + // interior or leaf parts of the new table. Not marked for a + // a partition root or ordinary table. + bool is_part_child; + bool is_add_part; // CDB: is create adding a part to a partition? + Node *partitionBy; // what columns we partition the data by +} CreateStmtBase; + + +typedef struct CreateStmt { + NodeTag type; + CreateStmtBase base; + TableOidInfo oidInfo; + char relStorage; + struct GpPolicy *policy; + List *deferredStmts; /* CDB: Statements, e.g., partial indexes, that can't be + * analyzed until after CREATE + * (until the target table + * is created and visible). */ + bool is_split_part; /* CDB: is create spliting a part? */ + Oid ownerid; /* OID of the role to own this. if InvalidOid, GetUserId() */ + bool buildAoBlkdir; /* whether to build the block directory for an AO table */ + List *attr_encodings; /* attribute storage directives */ } CreateStmt; typedef enum SharedStorageOp @@ -1459,8 +1463,9 @@ typedef struct SharedStorageOpStmt */ typedef enum ExtTableType { - EXTTBL_TYPE_LOCATION, /* table defined with LOCATION clause */ - EXTTBL_TYPE_EXECUTE /* table defined with EXECUTE clause */ + EXTTBL_TYPE_LOCATION, /* table defined with LOCATION clause */ + EXTTBL_TYPE_EXECUTE, /* table defined with EXECUTE clause */ + EXTTBL_TYPE_UNKNOWN /* table defined with unknown store */ } ExtTableType; typedef struct ExtTableTypeDesc @@ -1472,22 +1477,21 @@ typedef struct ExtTableTypeDesc char *command_string; } ExtTableTypeDesc; -typedef struct CreateExternalStmt -{ - NodeTag type; - RangeVar *relation; /* external relation to create */ - List *tableElts; /* column definitions (list of ColumnDef) */ - Node *exttypedesc; /* LOCATION or EXECUTE information */ - char *format; /* data format name */ - List *formatOpts; /* List of DefElem nodes for data format */ - bool isweb; - bool iswritable; - Node *sreh; /* Single row error handling info */ - List *encoding; /* List (size 1 max) of DefElem nodes for - data encoding */ - List *distributedBy; /* what columns we distribute the data by */ - struct GpPolicy *policy; /* used for writable tables */ - +typedef struct CreateExternalStmt { + NodeTag type; + CreateStmtBase base; + Node *exttypedesc; /* LOCATION or EXECUTE information */ + char *format; /* data format name */ + bool isweb; + bool iswritable; + bool isexternal; + Node *sreh; /* Single row error handling info */ + List *encoding; /* List (size 1 max) of DefElem nodes for + data encoding */ + struct GpPolicy *policy; /* used for writable tables */ + bool forceCreateDir; /* true to create external dir */ + char *parentPath; // keep the parent relative path for partition + struct IndexStmt *pkey; /* PRIMARY KEY index, if any */ } CreateExternalStmt; /* ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c189fc1/src/include/parser/analyze.h ---------------------------------------------------------------------- diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h index ccb54f7..1c7df3f 100644 --- a/src/include/parser/analyze.h +++ b/src/include/parser/analyze.h @@ -34,27 +34,33 @@ extern void applyLockingClause(Query *qry, Index rtindex, Datum partition_arg_get_val(Node *node, bool *isnull); /* State shared by transformCreateStmt and its subroutines */ -typedef struct -{ - const char *stmtType; /* "CREATE TABLE" or "ALTER TABLE" */ - RangeVar *relation; /* relation to create */ - List *inhRelations; /* relations to inherit from */ - bool hasoids; /* does relation have an OID column? */ - bool isalter; /* true if altering existing table */ - bool isaddpart; /* true if create in service of adding a part */ - List *columns; /* ColumnDef items */ - List *ckconstraints; /* CHECK constraints */ - List *fkconstraints; /* FOREIGN KEY constraints */ - List *ixconstraints; /* index-creating constraints */ - List *inh_indexes; /* cloned indexes from INCLUDING INDEXES */ - List *blist; /* "before list" of things to do before - * creating the table */ - List *alist; /* "after list" of things to do after creating - * the table */ - List *dlist; /* "deferred list" of utility statements to - * transfer to the list CreateStmt->deferredStmts - * for later parse_analyze and dispatch */ - IndexStmt *pkey; /* PRIMARY KEY index, if any */ +typedef struct { + bool isExternalTable; + const char *stmtType; /* "CREATE TABLE" or "ALTER TABLE" */ + RangeVar *relation; /* relation to create */ + List *inhRelations; /* relations to inherit from */ + bool hasoids; /* does relation have an OID column? */ + bool isalter; /* true if altering existing table */ + bool isaddpart; /* true if create in service of adding a part */ + List *columns; /* ColumnDef items */ + List *ckconstraints; /* CHECK constraints */ + List *fkconstraints; /* FOREIGN KEY constraints */ + List *ixconstraints; /* index-creating constraints */ + List *inh_indexes; /* cloned indexes from INCLUDING INDEXES */ + List *blist; /* "before list" of things to do before + * creating the table */ + List *alist; /* "after list" of things to do after creating + * the table */ + List *dlist; /* "deferred list" of utility statements to + * transfer to the list CreateStmt->deferredStmts + * for later parse_analyze and dispatch */ + IndexStmt *pkey; /* PRIMARY KEY index, if any */ + + // for external table only + Node *exttypedesc; + char *format; + bool iswritable; + char *parentPath; } CreateStmtContext; Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt, @@ -83,4 +89,6 @@ extern List * form_default_storage_directive(List *enc); extern struct GpPolicy *createRandomDistribution(int maxattrs); +extern void recognizeExternalRelationFormatterOptions( + CreateExternalStmt *createExtStmt); #endif /* ANALYZE_H */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1c189fc1/src/include/utils/uri.h ---------------------------------------------------------------------- diff --git a/src/include/utils/uri.h b/src/include/utils/uri.h index a37d9e7..9c5aedd 100644 --- a/src/include/utils/uri.h +++ b/src/include/utils/uri.h @@ -32,7 +32,8 @@ typedef enum UriProtocol URI_HTTP, URI_GPFDIST, URI_CUSTOM, - URI_GPFDISTS + URI_GPFDISTS, + URI_HDFS } UriProtocol; #define PROTOCOL_FILE "file://" @@ -41,6 +42,7 @@ typedef enum UriProtocol #define PROTOCOL_GPFDIST "gpfdist://" #define PROTOCOL_GPFDISTS "gpfdists://" #define PROTOCOL_PXF "pxf://" +#define PROTOCOL_HDFS "hdfs://" /* * sometimes we don't want to parse the whole URI but just take a peek at @@ -52,6 +54,7 @@ typedef enum UriProtocol #define IS_GPFDISTS_URI(uri_str) (pg_strncasecmp(uri_str, PROTOCOL_GPFDISTS, strlen(PROTOCOL_GPFDISTS)) == 0) #define IS_FTP_URI(uri_str) (pg_strncasecmp(uri_str, PROTOCOL_FTP, strlen(PROTOCOL_FTP)) == 0) #define IS_PXF_URI(uri_str) (pg_strncasecmp(uri_str, PROTOCOL_PXF, strlen(PROTOCOL_PXF)) == 0) +#define IS_HDFS_URI(uri_str) (pg_strncasecmp(uri_str, PROTOCOL_HDFS, strlen(PROTOCOL_HDFS)) == 0) typedef struct Uri {
