HAWQ-808. Refactor feature test for external_oid with new framework.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0e6fe7a4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0e6fe7a4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0e6fe7a4 Branch: refs/heads/2.0.0.0-incubating Commit: 0e6fe7a4f1431ed5d01f0fdc8b48e648869c177e Parents: e308af6 Author: xunzhang <xunzhang...@gmail.com> Authored: Tue Jul 12 14:14:08 2016 +0800 Committer: rlei <r...@pivotal.io> Committed: Tue Jul 19 10:49:42 2016 +0800 ---------------------------------------------------------------------- .../ExternalSource/ans/external_oid.ans.source | 209 ++++++ .../ExternalSource/data/multi_table.json | 1 + .../ExternalSource/data/single_table.json | 1 + src/test/feature/ExternalSource/lib/Makefile | 44 ++ src/test/feature/ExternalSource/lib/function.c | 727 +++++++++++++++++++ .../ExternalSource/sql/external_oid.sql.source | 108 +++ .../ExternalSource/test_external_oid.cpp | 35 + src/test/feature/Makefile | 2 + src/test/regress/input/external_oid.source | 118 --- src/test/regress/known_good_schedule | 1 - src/test/regress/output/external_oid.source | 211 ------ 11 files changed, 1127 insertions(+), 330 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/ans/external_oid.ans.source ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/ans/external_oid.ans.source b/src/test/feature/ExternalSource/ans/external_oid.ans.source new file mode 100644 index 0000000..3f3f3ae --- /dev/null +++ b/src/test/feature/ExternalSource/ans/external_oid.ans.source @@ -0,0 +1,209 @@ +-- -------------------------------------- +-- test first external Oid initialization +-- -------------------------------------- +-- start_matchsubs +-- +-- # create a match/subs expression to handle ip addresses that change +-- +-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/ +-- s/oid \d+/oid SOME_OID/ +-- +-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/ +-- s/oid=\d+/oid=OID/ +-- +-- end_matchsubs +-- Create function that returns the first external Oid boundary +CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'min_external_oid' + LANGUAGE C; +CREATE FUNCTION +-- Create function that returns the current external Oid +CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid' + LANGUAGE C; +CREATE FUNCTION +-- Create function that sets the current external Oid +CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid' + LANGUAGE C; +CREATE FUNCTION +-- Create function to insert and scan in-memory data to pg_class +CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'load_json_data' + LANGUAGE C; +CREATE FUNCTION +-- Create function that inserts tuple with given Oid +CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class' + LANGUAGE C; +CREATE FUNCTION +-- Create function that inserts tuple with given Oid +CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class' + LANGUAGE C; +CREATE FUNCTION +-- -------------------------------------- +-- Test hcat table external oid initialization +-- -------------------------------------- +-- Boundary should be at FirstExternalObjectId +--SELECT min_external_oid(); +-- NextExternalObjectId is uninitialized +SELECT get_next_external_oid(); + get_next_external_oid +----------------------- + 0 +(1 row) + +SELECT load_json_data('@abs_datadir@/single_table.json'); + load_json_data +------------------ + default.mytable +(1 row) + +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 0 +(1 row) + +BEGIN TRANSACTION; +BEGIN +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 0 +(1 row) + +-- load default.mytable -> +3 oids +-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype +SELECT load_json_data('@abs_datadir@/single_table.json'); + load_json_data +------------------ + default.mytable +(1 row) + +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 3 +(1 row) + +-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids +-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2) +SELECT load_json_data('@abs_datadir@/multi_table.json'); + load_json_data +-------------------------- + db1.ht1 db2.ht1 db2.ht2 +(1 row) + +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 11 +(1 row) + +END TRANSACTION; +COMMIT +-- New transaction will reset external Oid start point +-- Yields the same result as previous transaction +BEGIN TRANSACTION; +BEGIN +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 0 +(1 row) + +SELECT load_json_data('@abs_datadir@/single_table.json'); + load_json_data +------------------ + default.mytable +(1 row) + +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 3 +(1 row) + +SELECT load_json_data('@abs_datadir@/multi_table.json'); + load_json_data +-------------------------- + db1.ht1 db2.ht1 db2.ht2 +(1 row) + +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 11 +(1 row) + +END TRANSACTION; +COMMIT +-- -------------------------------------- +-- Test external oid rollover +-- -------------------------------------- +BEGIN TRANSACTION; +BEGIN +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + ?column? +---------- + 0 +(1 row) + +SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0; + ?column? +---------- + t +(1 row) + +SELECT load_json_data('@abs_datadir@/multi_table.json'); + load_json_data +-------------------------- + db1.ht1 db2.ht1 db2.ht2 +(1 row) + +-- Used up external Oids result in Oid overflow +SELECT get_next_external_oid(); + get_next_external_oid +----------------------- + 0 +(1 row) + +-- Rollover disallowed! +SELECT load_json_data('@abs_datadir@/single_table.json'); +psql:/tmp/TestExternalOid_TestExternalOidAll.sql:92: ERROR: number of external objects from HCatalog exceeded 10M during transaction +HINT: Separate HCatalog queries into different transactions to process. +END TRANSACTION; +ROLLBACK +-- -------------------------------------- +-- Test external Oid boundary +-- -------------------------------------- +-- Create a tuple with Oid larger than FirstExternalObjectId +-- Will fail during next session when try to query HCatalog +-- Because external Oid boundary is violated +SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl'); + caql_insert_into_heap_pg_class +-------------------------------------------------------------------------- + inserted tuple to heap table pg_class (oid 4284481555, relname table_xl) +(1 row) + +-- cleanup +SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20); + caql_delete_from_heap_pg_class +------------------------------------------------------- + deleted tuple oid=4284481555 from heap table pg_class +(1 row) + +DROP FUNCTION caql_delete_from_heap_pg_class(relid oid); +DROP FUNCTION +DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text); +DROP FUNCTION +DROP FUNCTION load_json_data(filename text); +DROP FUNCTION +DROP FUNCTION get_next_external_oid(); +DROP FUNCTION +DROP FUNCTION set_next_external_oid(ext_oid oid); +DROP FUNCTION +DROP FUNCTION min_external_oid(); +DROP FUNCTION http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/data/multi_table.json ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/data/multi_table.json b/src/test/feature/ExternalSource/data/multi_table.json new file mode 100644 index 0000000..82e70b0 --- /dev/null +++ b/src/test/feature/ExternalSource/data/multi_table.json @@ -0,0 +1 @@ +{"PXFMetadata":[{"item":{"path":"db1","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht2"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]}]} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/data/single_table.json ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/data/single_table.json b/src/test/feature/ExternalSource/data/single_table.json new file mode 100644 index 0000000..b571e5d --- /dev/null +++ b/src/test/feature/ExternalSource/data/single_table.json @@ -0,0 +1 @@ +{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}]}]} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/lib/Makefile ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/lib/Makefile b/src/test/feature/ExternalSource/lib/Makefile new file mode 100644 index 0000000..46f6c20 --- /dev/null +++ b/src/test/feature/ExternalSource/lib/Makefile @@ -0,0 +1,44 @@ +top_builddir = ../../../../.. +include $(top_builddir)/src/Makefile.global + +OS = $(shell uname) + +CXX = gcc +CXXFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC + +CC = gcc +CFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC + +CPPFLAGS = -I$(abs_top_srcdir)/src/include +CPPFLAGS += -I$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/include +CPPFLAGS += -I$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/include + +LDFLAGS = -L$(libdir) +LDFLAGS += -L$(abs_top_srcdir)/src/port +LDFLAGS += -L$(abs_top_builddir)/src/port +LDFLAGS += -L$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/lib +LDFLAGS += -L$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/lib + +POSTGRES = $(abs_top_srcdir)/src/backend/postgres + +PROG = function.c +OBJS = function.o +TARGET = function.so + +RM = rm -rf + +all: $(TARGET) + +$(OBJS): $(PROG) + $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $(OBJS) $(PROG) + +$(TARGET): $(OBJS) +ifeq ($(OS),Darwin) + $(CXX) $(CXXFLAGS) -bundle $(OBJS) -bundle_loader $(POSTGRES) $(LDFLAGS) -o $@ +else + $(CXX) $(CXXFLAGS) -shared $(OBJS) $(LDFLAGS) -Wl,-rpath,'$(abs_top_builddir)/lib' -o $@ +endif + +clean: + $(RM) *.o + $(RM) *.so http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/lib/function.c ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/lib/function.c b/src/test/feature/ExternalSource/lib/function.c new file mode 100644 index 0000000..4275edc --- /dev/null +++ b/src/test/feature/ExternalSource/lib/function.c @@ -0,0 +1,727 @@ +/* + * json_utils.c + * + * Created on: Mar 5, 2015 + * Author: antova + */ + +#include "catalog/external/externalmd.h" +#include "postgres.h" +#include "access/hd_work_mgr.h" +#include "funcapi.h" +#include "catalog/catquery.h" +#include "catalog/gp_policy.h" +#include "catalog/namespace.h" +#include "catalog/pg_database.h" +#include "catalog/pg_exttable.h" +#include "catalog/pg_namespace.h" +#include "cdb/cdbinmemheapam.h" +#include "nodes/makefuncs.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "access/transam.h" + +/* + * number of output columns for the UDFs for scanning in memory catalog tables + */ +#define NUM_COLS 3 + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +static +char *read_file(const char *filename); + +static +Oid GetRelationOid(char *tableName); + +/* utility functions for loading json files */ +extern Datum load_json_data(PG_FUNCTION_ARGS); +extern Datum caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS); +extern Datum caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS); +extern Datum caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS); +extern Datum caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS); +extern Datum caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS); +extern Datum caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS); +extern Datum caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS); +extern Datum get_next_external_oid(PG_FUNCTION_ARGS); +extern Datum set_next_external_oid(PG_FUNCTION_ARGS); +extern Datum min_external_oid(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(load_json_data); +PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_namespace); +PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_type); +PG_FUNCTION_INFO_V1(caql_scan_in_memory_gp_distribution_policy); +PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_exttable); +PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_attribute); +PG_FUNCTION_INFO_V1(caql_insert_into_heap_pg_class); +PG_FUNCTION_INFO_V1(caql_delete_from_heap_pg_class); +PG_FUNCTION_INFO_V1(get_next_external_oid); +PG_FUNCTION_INFO_V1(set_next_external_oid); +PG_FUNCTION_INFO_V1(min_external_oid); + +Datum +load_json_data(PG_FUNCTION_ARGS) +{ + text *filename = PG_GETARG_TEXT_P(0); + char *filenameStr = text_to_cstring(filename); + + char *pcbuf = read_file(filenameStr); + StringInfoData buf; + initStringInfo(&buf); + appendStringInfo(&buf, "%s", pcbuf); + + List *items = ParsePxfEntries(&buf, HiveProfileName, HcatalogDbOid); + pfree(buf.data); + + StringInfoData tblNames; + initStringInfo(&tblNames); + ListCell *lc = NULL; + foreach (lc, items) + { + PxfItem *item = (PxfItem *) lfirst(lc); + appendStringInfo(&tblNames, "%s.%s ", item->path, item->name); + } + + PG_RETURN_TEXT_P(cstring_to_text(tblNames.data)); +} + +char * +read_file(const char *filename) +{ + FILE *pf = fopen(filename, "r"); + if (NULL == pf) + { + elog(ERROR, "Could not open file %s for reading", filename); + } + + StringInfoData strinfo; + initStringInfo(&strinfo); + + const int len = 1024; + char buf[len]; + + while (NULL != fgets(buf, len, pf)) + { + appendStringInfo(&strinfo, "%s", buf); + } + + fclose(pf); + return strinfo.data; +} + +Datum +caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS) +{ + text *namespaceNameText = PG_GETARG_TEXT_P(0); + char *namespaceNameStr = text_to_cstring(namespaceNameText); + + FuncCallContext *funcctx; + Datum result; + MemoryContext oldcontext; + TupleDesc tupdesc; + HeapTuple restuple; + HeapTuple readtup = NULL; + cqContext *pcqCtx; + StringInfoData buf; + initStringInfo(&buf); + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + /* switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + tupdesc = CreateTemplateTupleDesc(NUM_COLS, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "nspname", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "nspdboid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "oid", + OIDOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* create tuples for pg_namespace table */ + pcqCtx = caql_beginscan( + NULL, + cql("SELECT * FROM pg_namespace " + " WHERE nspname = :1 and nspdboid = :2", + CStringGetDatum((char *) namespaceNameStr), ObjectIdGetDatum(HcatalogDbOid))); + + funcctx->user_fctx = pcqCtx; + + /* return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + pcqCtx = (cqContext *)funcctx->user_fctx; + + if (NULL != (readtup = caql_getnext(pcqCtx))) + { + Datum values[NUM_COLS]; + bool nulls[NUM_COLS]; + + Relation relation = RelationIdGetRelation(NamespaceRelationId); + TupleDesc tupleDesc = RelationGetDescr(relation); + + char *nspname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspname)); + text *t = cstring_to_text(nspname); + + values[0] = PointerGetDatum(t); + nulls[0] = false; + + values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspdboid); + nulls[1] = false; + + values[2] = ObjectIdGetDatum(HeapTupleGetOid(readtup)); + nulls[2] = false; + + elog(DEBUG1, "Namespace oid: %d", HeapTupleGetOid(readtup)); + + /* build tuple */ + restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + /* make the tuple into a datum */ + result = HeapTupleGetDatum(restuple); + + RelationClose(relation); + SRF_RETURN_NEXT(funcctx, result); + } + else + { + caql_endscan(pcqCtx); + SRF_RETURN_DONE(funcctx); + } +} + +Datum +caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS) +{ + text *relNameText = PG_GETARG_TEXT_P(0); + char *relNameStr = text_to_cstring(relNameText); + + FuncCallContext *funcctx; + Datum result; + MemoryContext oldcontext; + TupleDesc tupdesc; + HeapTuple restuple; + HeapTuple readtup = NULL; + cqContext *pcqCtx; + StringInfoData buf; + initStringInfo(&buf); + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + /* switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + tupdesc = CreateTemplateTupleDesc(NUM_COLS, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "typoid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "typname", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "typnamespace", + OIDOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* create tuples for pg_type table */ + pcqCtx = caql_beginscan( + NULL, + cql("SELECT * FROM pg_type " + " WHERE typname = :1", + CStringGetDatum((char *) relNameStr))); + + funcctx->user_fctx = pcqCtx; + + /* return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + pcqCtx = (cqContext *)funcctx->user_fctx; + + if (NULL != (readtup = caql_getnext(pcqCtx))) + { + Datum values[NUM_COLS]; + bool nulls[NUM_COLS]; + + Relation relation = RelationIdGetRelation(TypeRelationId); + TupleDesc tupleDesc = RelationGetDescr(relation); + + values[0] = ObjectIdGetDatum(HeapTupleGetOid(readtup)); + nulls[0] = false; + + elog(DEBUG1, "Type oid: %d", HeapTupleGetOid(readtup)); + + char *typname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_type_typname)); + text *t = cstring_to_text(typname); + + values[1] = PointerGetDatum(t); + nulls[1] = false; + + values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_type_typnamespace); + nulls[2] = false; + + /* build tuple */ + restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + /* make the tuple into a datum */ + result = HeapTupleGetDatum(restuple); + + RelationClose(relation); + SRF_RETURN_NEXT(funcctx, result); + } + else + { + caql_endscan(pcqCtx); + SRF_RETURN_DONE(funcctx); + } +} + +Datum +caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS) +{ + text *tableName = PG_GETARG_TEXT_P(0); + char *tableNameStr = text_to_cstring(tableName); + Oid reloid = InvalidOid; + + FuncCallContext *funcctx; + Datum result; + MemoryContext oldcontext; + TupleDesc tupdesc; + HeapTuple restuple; + HeapTuple pgclasstup = NULL; + HeapTuple readtup = NULL; + cqContext *pcqCtx; + StringInfoData buf; + initStringInfo(&buf); + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + /* switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + tupdesc = CreateTemplateTupleDesc(NUM_COLS, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "tableoid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tablename", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "distpolicy", + TEXTOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* iterate on pg_class and query + * gp_distribution_policy by given oid */ + pcqCtx = caql_beginscan( + NULL, + cql("SELECT oid FROM pg_class " + " WHERE relname = :1", + CStringGetDatum(tableNameStr))); + + funcctx->user_fctx = pcqCtx; + + /* return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + pcqCtx = (cqContext *)funcctx->user_fctx; + + if (NULL != (pgclasstup = caql_getnext(pcqCtx))) + { + Datum values[NUM_COLS]; + bool nulls[NUM_COLS]; + + /* create tuples for pg_exttable table */ + cqContext* pcqCtx1 = caql_beginscan( + NULL, + cql("SELECT * FROM gp_distribution_policy " + " WHERE localoid = :1", + ObjectIdGetDatum(HeapTupleGetOid(pgclasstup)))); + + Relation relation = RelationIdGetRelation(GpPolicyRelationId); + TupleDesc tupleDesc = RelationGetDescr(relation); + + readtup = caql_getnext(pcqCtx1); + + values[0] = ObjectIdGetDatum(reloid); + nulls[0] = false; + + text *t1 = cstring_to_text(tableNameStr); + + values[1] = PointerGetDatum(t1); + nulls[1] = false; + + text *t2 = NULL; + bool isnull; + Datum policy = heap_getattr(readtup, Anum_gp_policy_attrnums, tupleDesc, &isnull); + if (isnull) + { + t2 = cstring_to_text("null"); + } + else + { + /* not tested! */ + Datum* elems = NULL; + int nelems; + deconstruct_array(DatumGetArrayTypeP(policy), + INT2OID, -1, false, 'i', + &elems, NULL, &nelems); + Assert(nelems > 0); + StringInfoData elems_str; + initStringInfo(&elems_str); + for (int i = 0; i < nelems; i++) + { + appendStringInfo(&elems_str, "%d", DatumGetInt16(elems[0])); + if (i != nelems) + { + appendStringInfo(&elems_str, ", "); + } + } + t2 = cstring_to_text(elems_str.data); + pfree(elems_str.data); + } + + values[2] = PointerGetDatum(t2); + nulls[2] = false; + + /* build tuple */ + restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + /* make the tuple into a datum */ + result = HeapTupleGetDatum(restuple); + + RelationClose(relation); + /* there should be only one value per oid */ + Assert(NULL == caql_getnext(pcqCtx1)) ; + caql_endscan(pcqCtx1); + + SRF_RETURN_NEXT(funcctx, result); + } + else + { + caql_endscan(pcqCtx); + SRF_RETURN_DONE(funcctx); + } +} + +Datum +caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS) +{ + text *tableName = PG_GETARG_TEXT_P(0); + char *tableNameStr = text_to_cstring(tableName); + Oid reloid = InvalidOid; + + FuncCallContext *funcctx; + Datum result; + MemoryContext oldcontext; + TupleDesc tupdesc; + HeapTuple restuple; + HeapTuple pgclasstup = NULL; + HeapTuple readtup = NULL; + cqContext *pcqCtx; + StringInfoData buf; + initStringInfo(&buf); + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + /* switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + tupdesc = CreateTemplateTupleDesc(NUM_COLS, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "exttableoid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "exttablename", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "exttablelocation", + TEXTOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* iterate on pg_class and query + * pg_exttable by given oid */ + pcqCtx = caql_beginscan( + NULL, + cql("SELECT oid FROM pg_class " + " WHERE relname = :1", + CStringGetDatum(tableNameStr))); + + funcctx->user_fctx = pcqCtx; + + /* return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + pcqCtx = (cqContext *)funcctx->user_fctx; + + if (NULL != (pgclasstup = caql_getnext(pcqCtx))) + { + Datum values[NUM_COLS]; + bool nulls[NUM_COLS]; + + /* create tuples for pg_exttable table */ + cqContext* pcqCtx1 = caql_beginscan( + NULL, + cql("SELECT * FROM pg_exttable " + " WHERE reloid = :1", + ObjectIdGetDatum(HeapTupleGetOid(pgclasstup)))); + + Relation relation = RelationIdGetRelation(ExtTableRelationId); + TupleDesc tupleDesc = RelationGetDescr(relation); + + readtup = caql_getnext(pcqCtx1); + + values[0] = ObjectIdGetDatum(reloid); + nulls[0] = false; + + text *t1 = cstring_to_text(tableNameStr); + + values[1] = PointerGetDatum(t1); + nulls[1] = false; + + Datum locations = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_location); + Datum* elems = NULL; + int nelems; + deconstruct_array(DatumGetArrayTypeP(locations), + TEXTOID, -1, false, 'i', + &elems, NULL, &nelems); + Assert(nelems > 0); + char *loc_str = DatumGetCString(DirectFunctionCall1(textout, elems[0])); + text *t2 = cstring_to_text(loc_str); + + values[2] = PointerGetDatum(t2); + nulls[2] = false; + + /* build tuple */ + restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + /* make the tuple into a datum */ + result = HeapTupleGetDatum(restuple); + + RelationClose(relation); + /* there should be only one value per oid */ + Assert(NULL == caql_getnext(pcqCtx1)) ; + caql_endscan(pcqCtx1); + + SRF_RETURN_NEXT(funcctx, result); + } + else + { + caql_endscan(pcqCtx); + SRF_RETURN_DONE(funcctx); + } +} + +Datum +caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS) +{ + text *tableNameText = PG_GETARG_TEXT_P(0); + char *tableNameStr = text_to_cstring(tableNameText); + + FuncCallContext *funcctx; + Datum result; + MemoryContext oldcontext; + TupleDesc tupdesc; + HeapTuple restuple; + HeapTuple readtup = NULL; + cqContext *pcqCtx; + StringInfoData buf; + initStringInfo(&buf); + + Oid relid = GetRelationOid(tableNameStr); + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + /* switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + tupdesc = CreateTemplateTupleDesc(NUM_COLS, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "attname", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "atttype", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "atttypmod", + INT4OID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* create tuples for pg_attribute table */ + pcqCtx = caql_beginscan( + NULL, + cql("SELECT * FROM pg_attribute " + " WHERE attrelid = :1", + ObjectIdGetDatum(relid))); + funcctx->user_fctx = pcqCtx; + + /* return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + pcqCtx = (cqContext *)funcctx->user_fctx; + + if (NULL != (readtup = caql_getnext(pcqCtx))) + { + Datum values[NUM_COLS]; + bool nulls[NUM_COLS]; + + Relation relation = RelationIdGetRelation(AttributeRelationId); + TupleDesc tupleDesc = RelationGetDescr(relation); + + char *attname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_attname)); + text *t = cstring_to_text(attname); + + values[0] = PointerGetDatum(t); + nulls[0] = false; + + values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypid); + nulls[1] = false; + + values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypmod); + nulls[2] = false; + + /* build tuple */ + restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + /* make the tuple into a datum */ + result = HeapTupleGetDatum(restuple); + + RelationClose(relation); + SRF_RETURN_NEXT(funcctx, result); + } + else + { + caql_endscan(pcqCtx); + SRF_RETURN_DONE(funcctx); + } +} + +Oid GetRelationOid(char *tableName) +{ + cqContext *pcqCtx = caql_beginscan( + NULL, + cql("SELECT * FROM pg_class " + " WHERE relname = :1", + CStringGetDatum((char *) tableName))); + + HeapTuple ht = caql_getnext(pcqCtx); + Oid result = InvalidOid; + if (NULL != ht) + { + result = HeapTupleGetOid(ht); + } + + elog(DEBUG1, "Found relname %s, oid: %d", tableName, result); + caql_endscan(pcqCtx); + return result; +} + +/* + * This function must be used in conjunction with + * caql_delete_from_heap_pg_class + * and exclusively for testing external Oid setting + */ +Datum +caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS) + { + Oid relid = PG_GETARG_OID(0); + char *tblname = text_to_cstring(PG_GETARG_TEXT_P(1)); + + Datum values[Natts_pg_class]; + bool nulls[Natts_pg_class]; + + for (int i = 0; i < Natts_pg_class; i++) + { + nulls[i] = true; + values[i] = (Datum) 0; + } + + NameData name; + namestrcpy(&name, tblname); + + values[Anum_pg_class_relname - 1] = NameGetDatum(&name); + values[Anum_pg_class_relnamespace - 1] = ObjectIdGetDatum((Oid) NSPDBOID_CURRENT); + nulls[Anum_pg_class_relname - 1] = false; + nulls[Anum_pg_class_relnamespace - 1] = false; + + cqContext *pcqCtx = caql_beginscan( + NULL, + cql("INSERT INTO pg_class", NULL)); + + HeapTuple tup = caql_form_tuple(pcqCtx, values, nulls); + HeapTupleSetOid(tup, relid); + + caql_insert(pcqCtx, tup); + caql_endscan(pcqCtx); + + StringInfoData buf; + initStringInfo(&buf); + + appendStringInfo(&buf, "inserted tuple to heap table pg_class (oid %u, relname %s)", relid, tblname); + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + +/* + * This function must be used in conjunction with + * caql_insert_into_heap_pg_class + * and exclusively for testing external Oid setting + */ +Datum +caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + cqContext *pcqCtx = caql_beginscan( + NULL, + cql("SELECT * FROM pg_class " + " WHERE oid = :1 " + " FOR UPDATE ", + ObjectIdGetDatum(relid))); + + HeapTuple tuple = caql_getnext(pcqCtx); + + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("pg_class table relid=%u does not exist", relid))); + + /* Delete the pg_class table entry from the catalog (pg_class) */ + caql_delete_current(pcqCtx); + + /* Finish up scan and close pg_class catalog */ + caql_endscan(pcqCtx); + + StringInfoData buf; + initStringInfo(&buf); + + appendStringInfo(&buf, "deleted tuple oid=%u from heap table pg_class", relid); + + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + +Datum +get_next_external_oid(PG_FUNCTION_ARGS) +{ + PG_RETURN_OID(GetCurrentExternalObjectId()); +} + +Datum +set_next_external_oid(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + SetCurrentExternalObjectId(oid); + + PG_RETURN_OID(GetCurrentExternalObjectId()); +} + +Datum +min_external_oid(PG_FUNCTION_ARGS) +{ + PG_RETURN_OID(FirstExternalObjectId); +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/sql/external_oid.sql.source ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/sql/external_oid.sql.source b/src/test/feature/ExternalSource/sql/external_oid.sql.source new file mode 100644 index 0000000..bded23e --- /dev/null +++ b/src/test/feature/ExternalSource/sql/external_oid.sql.source @@ -0,0 +1,108 @@ +-- -------------------------------------- +-- test first external Oid initialization +-- -------------------------------------- + +-- start_matchsubs +-- +-- # create a match/subs expression to handle ip addresses that change +-- +-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/ +-- s/oid \d+/oid SOME_OID/ +-- +-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/ +-- s/oid=\d+/oid=OID/ +-- +-- end_matchsubs + +-- Create function that returns the first external Oid boundary +CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'min_external_oid' + LANGUAGE C; + +-- Create function that returns the current external Oid +CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid' + LANGUAGE C; + +-- Create function that sets the current external Oid +CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid + AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid' + LANGUAGE C; + +-- Create function to insert and scan in-memory data to pg_class +CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'load_json_data' + LANGUAGE C; + +-- Create function that inserts tuple with given Oid +CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class' + LANGUAGE C; + +-- Create function that inserts tuple with given Oid +CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text + AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class' + LANGUAGE C; + +-- -------------------------------------- +-- Test hcat table external oid initialization +-- -------------------------------------- +-- Boundary should be at FirstExternalObjectId +--SELECT min_external_oid(); +-- NextExternalObjectId is uninitialized +SELECT get_next_external_oid(); +SELECT load_json_data('@abs_datadir@/single_table.json'); +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; + +BEGIN TRANSACTION; +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +-- load default.mytable -> +3 oids +-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype +SELECT load_json_data('@abs_datadir@/single_table.json'); +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids +-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2) +SELECT load_json_data('@abs_datadir@/multi_table.json'); +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +END TRANSACTION; + +-- New transaction will reset external Oid start point +-- Yields the same result as previous transaction +BEGIN TRANSACTION; +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +SELECT load_json_data('@abs_datadir@/single_table.json'); +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +SELECT load_json_data('@abs_datadir@/multi_table.json'); +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +END TRANSACTION; + +-- -------------------------------------- +-- Test external oid rollover +-- -------------------------------------- +BEGIN TRANSACTION; +SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; +SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0; +SELECT load_json_data('@abs_datadir@/multi_table.json'); +-- Used up external Oids result in Oid overflow +SELECT get_next_external_oid(); +-- Rollover disallowed! +SELECT load_json_data('@abs_datadir@/single_table.json'); +END TRANSACTION; + +-- -------------------------------------- +-- Test external Oid boundary +-- -------------------------------------- +-- Create a tuple with Oid larger than FirstExternalObjectId +-- Will fail during next session when try to query HCatalog +-- Because external Oid boundary is violated +SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl'); + +-- cleanup +SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20); + +DROP FUNCTION caql_delete_from_heap_pg_class(relid oid); +DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text); +DROP FUNCTION load_json_data(filename text); +DROP FUNCTION get_next_external_oid(); +DROP FUNCTION set_next_external_oid(ext_oid oid); +DROP FUNCTION min_external_oid(); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/ExternalSource/test_external_oid.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ExternalSource/test_external_oid.cpp b/src/test/feature/ExternalSource/test_external_oid.cpp new file mode 100644 index 0000000..3326514 --- /dev/null +++ b/src/test/feature/ExternalSource/test_external_oid.cpp @@ -0,0 +1,35 @@ +#include <string> +#include <iostream> +#include "gtest/gtest.h" + +#include "lib/sql_util.h" +#include "lib/file_replace.h" + +using hawq::test::SQLUtility; +using hawq::test::FileReplace; + +class TestExternalOid : public ::testing::Test { + public: + TestExternalOid() {} + ~TestExternalOid() {} +}; + +TEST_F(TestExternalOid, TestExternalOidAll) { + SQLUtility util; + FileReplace frep; + auto test_root = util.getTestRootPath(); + std::cout << test_root << std::endl; + + std::unordered_map<std::string, std::string> D; + D["@SHARE_LIBRARY_PATH@"] = test_root + "/ExternalSource/lib/function.so"; + D["@abs_datadir@"] = test_root + "/ExternalSource/data"; + frep.replace(test_root + "/ExternalSource/sql/external_oid.sql.source", + test_root + "/ExternalSource/sql/external_oid.sql", + D); + frep.replace(test_root + "/ExternalSource/ans/external_oid.ans.source", + test_root + "/ExternalSource/ans/external_oid.ans", + D); + + util.execSQLFile("ExternalSource/sql/external_oid.sql", + "ExternalSource/ans/external_oid.ans"); +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/feature/Makefile ---------------------------------------------------------------------- diff --git a/src/test/feature/Makefile b/src/test/feature/Makefile index adc6acc..f393a76 100644 --- a/src/test/feature/Makefile +++ b/src/test/feature/Makefile @@ -27,9 +27,11 @@ all: $(OBJS) sharelib sharelib: cd UDF/lib || exit 1; $(MAKE) || exit 2 + cd ExternalSource/lib || exit 1; $(MAKE) || exit 2 sharelibclean: cd UDF/lib || exit 1; $(RM) *.o *.so || exit 2 + cd ExternalSource/lib || exit 1; $(MAKE) || exit 2 doc: doxygen doxygen_template http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/regress/input/external_oid.source ---------------------------------------------------------------------- diff --git a/src/test/regress/input/external_oid.source b/src/test/regress/input/external_oid.source deleted file mode 100644 index bd4724a..0000000 --- a/src/test/regress/input/external_oid.source +++ /dev/null @@ -1,118 +0,0 @@ --- -------------------------------------- --- test first external Oid initialization --- -------------------------------------- - --- start_matchsubs --- --- # create a match/subs expression to handle ip addresses that change --- --- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/ --- s/oid \d+/oid SOME_OID/ --- --- m/.*deleted tuple oid=\d+ from heap table pg_class.*/ --- s/oid=\d+/oid=OID/ --- --- end_matchsubs - --- Create function that returns the first external Oid boundary -CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid' - LANGUAGE C; - --- Create function that returns the current external Oid -CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid' - LANGUAGE C; - --- Create function that sets the current external Oid -CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid' - LANGUAGE C; - --- Create function to insert and scan in-memory data to pg_class -CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data' - LANGUAGE C; - --- Create function that inserts tuple with given Oid -CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class' - LANGUAGE C; - --- Create function that inserts tuple with given Oid -CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class' - LANGUAGE C; - --- -------------------------------------- --- Test hcat table external oid initialization --- -------------------------------------- --- Boundary should be at FirstExternalObjectId ---SELECT min_external_oid(); --- NextExternalObjectId is uninitialized -SELECT get_next_external_oid(); -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; --- load default.mytable -> +3 oids --- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; --- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids --- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2) -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; -END TRANSACTION; - --- New transaction will reset external Oid start point --- Yields the same result as previous transaction -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; -END TRANSACTION; - --- -------------------------------------- --- Test external oid rollover --- -------------------------------------- -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; -SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0; -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); --- Used up external Oids result in Oid overflow -SELECT get_next_external_oid(); --- Rollover disallowed! -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -END TRANSACTION; - --- -------------------------------------- --- Test external Oid boundary --- -------------------------------------- --- Create a tuple with Oid larger than FirstExternalObjectId --- Will fail during next session when try to query HCatalog --- Because external Oid boundary is violated -SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl'); - --- Fresh session -\c --- NextExternalObjectId is uninitialized -SELECT get_next_external_oid(); --- Should fail -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); --- NextExternalObjectId is still uninitialized -SELECT get_next_external_oid(); - --- cleanup -SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20); - -DROP FUNCTION caql_delete_from_heap_pg_class(relid oid); -DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text); -DROP FUNCTION load_json_data(filename text); -DROP FUNCTION get_next_external_oid(); -DROP FUNCTION set_next_external_oid(ext_oid oid); -DROP FUNCTION min_external_oid(); - http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/regress/known_good_schedule ---------------------------------------------------------------------- diff --git a/src/test/regress/known_good_schedule b/src/test/regress/known_good_schedule index 0cd48f3..1f95e49 100755 --- a/src/test/regress/known_good_schedule +++ b/src/test/regress/known_good_schedule @@ -202,5 +202,4 @@ ignore: co_disabled test: caqlinmem test: hcatalog_lookup test: json_load -test: external_oid test: validator_function http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0e6fe7a4/src/test/regress/output/external_oid.source ---------------------------------------------------------------------- diff --git a/src/test/regress/output/external_oid.source b/src/test/regress/output/external_oid.source deleted file mode 100644 index d0a3730..0000000 --- a/src/test/regress/output/external_oid.source +++ /dev/null @@ -1,211 +0,0 @@ --- -------------------------------------- --- test first external Oid initialization --- -------------------------------------- --- start_matchsubs --- --- # create a match/subs expression to handle ip addresses that change --- --- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/ --- s/oid \d+/oid SOME_OID/ --- --- m/.*deleted tuple oid=\d+ from heap table pg_class.*/ --- s/oid=\d+/oid=OID/ --- --- end_matchsubs --- Create function that returns the first external Oid boundary -CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid' - LANGUAGE C; --- Create function that returns the current external Oid -CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid' - LANGUAGE C; --- Create function that sets the current external Oid -CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid - AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid' - LANGUAGE C; --- Create function to insert and scan in-memory data to pg_class -CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data' - LANGUAGE C; --- Create function that inserts tuple with given Oid -CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class' - LANGUAGE C; --- Create function that inserts tuple with given Oid -CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text - AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class' - LANGUAGE C; --- -------------------------------------- --- Test hcat table external oid initialization --- -------------------------------------- --- Boundary should be at FirstExternalObjectId ---SELECT min_external_oid(); --- NextExternalObjectId is uninitialized -SELECT get_next_external_oid(); - get_next_external_oid ------------------------ - 0 -(1 row) - -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); - load_json_data ------------------- - default.mytable -(1 row) - -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 0 -(1 row) - -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 0 -(1 row) - --- load default.mytable -> +3 oids --- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); - load_json_data ------------------- - default.mytable -(1 row) - -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 3 -(1 row) - --- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids --- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2) -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); - load_json_data --------------------------- - db1.ht1 db2.ht1 db2.ht2 -(1 row) - -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 11 -(1 row) - -END TRANSACTION; --- New transaction will reset external Oid start point --- Yields the same result as previous transaction -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 0 -(1 row) - -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); - load_json_data ------------------- - default.mytable -(1 row) - -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 3 -(1 row) - -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); - load_json_data --------------------------- - db1.ht1 db2.ht1 db2.ht2 -(1 row) - -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 11 -(1 row) - -END TRANSACTION; --- -------------------------------------- --- Test external oid rollover --- -------------------------------------- -BEGIN TRANSACTION; -SELECT get_next_external_oid()::bigint - min_external_oid()::bigint; - ?column? ----------- - 0 -(1 row) - -SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0; - ?column? ----------- - t -(1 row) - -SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json'); - load_json_data --------------------------- - db1.ht1 db2.ht1 db2.ht2 -(1 row) - --- Used up external Oids result in Oid overflow -SELECT get_next_external_oid(); - get_next_external_oid ------------------------ - 0 -(1 row) - --- Rollover disallowed! -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -ERROR: number of external objects from HCatalog exceeded 10M during transaction -HINT: Separate HCatalog queries into different transactions to process. -END TRANSACTION; --- -------------------------------------- --- Test external Oid boundary --- -------------------------------------- --- Create a tuple with Oid larger than FirstExternalObjectId --- Will fail during next session when try to query HCatalog --- Because external Oid boundary is violated -SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl'); - caql_insert_into_heap_pg_class --------------------------------------------------------------------------- - inserted tuple to heap table pg_class (oid 4293918750, relname table_xl) -(1 row) - --- Fresh session -\c --- NextExternalObjectId is uninitialized -SELECT get_next_external_oid(); - get_next_external_oid ------------------------ - 0 -(1 row) - --- Should fail -SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json'); -ERROR: database does not have enough available Oids to support HCatalog queries -HINT: Database VACUUM may recycle unused Oids. --- NextExternalObjectId is still uninitialized -SELECT get_next_external_oid(); - get_next_external_oid ------------------------ - 0 -(1 row) - --- cleanup -SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20); - caql_delete_from_heap_pg_class -------------------------------------------------------- - deleted tuple oid=4293918750 from heap table pg_class -(1 row) - -DROP FUNCTION caql_delete_from_heap_pg_class(relid oid); -DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text); -DROP FUNCTION load_json_data(filename text); -DROP FUNCTION get_next_external_oid(); -DROP FUNCTION set_next_external_oid(ext_oid oid); -DROP FUNCTION min_external_oid();
