This is an automated email from the ASF dual-hosted git repository.
jgemignani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-age.git
The following commit(s) were added to refs/heads/master by this push:
new b558c70 VLE memory usage bug fixes
b558c70 is described below
commit b558c7023997dd8f7342a98f55353b7647f6edfd
Author: John Gemignani <[email protected]>
AuthorDate: Wed Mar 23 18:01:33 2022 -0700
VLE memory usage bug fixes
Added some bug fixes to the VLE memory management system.
Moved the age_vertex_stats function to global_graph.
---
age--1.0.0.sql | 7 +
src/backend/utils/adt/age_global_graph.c | 263 +++++++++++++++++++++++++++++++
src/backend/utils/adt/age_vle.c | 176 +++++++--------------
3 files changed, 327 insertions(+), 119 deletions(-)
diff --git a/age--1.0.0.sql b/age--1.0.0.sql
index 10b3dea..2a6e717 100644
--- a/age--1.0.0.sql
+++ b/age--1.0.0.sql
@@ -3924,6 +3924,13 @@ STABLE
PARALLEL SAFE
AS 'MODULE_PATHNAME';
+CREATE FUNCTION ag_catalog.age_delete_global_graphs(agtype)
+RETURNS boolean
+LANGUAGE c
+STABLE
+PARALLEL SAFE
+AS 'MODULE_PATHNAME';
+
--
-- End
--
diff --git a/src/backend/utils/adt/age_global_graph.c
b/src/backend/utils/adt/age_global_graph.c
index ca6a9ec..9615ba8 100644
--- a/src/backend/utils/adt/age_global_graph.c
+++ b/src/backend/utils/adt/age_global_graph.c
@@ -87,6 +87,8 @@ static GRAPH_global_context *global_graph_contexts = NULL;
/* declarations */
/* GRAPH global context functions */
static void free_specific_GRAPH_global_context(GRAPH_global_context *ggctx);
+static bool delete_specific_GRAPH_global_contexts(char *graph_name);
+static bool delete_GRAPH_global_contexts(void);
static void create_GRAPH_global_hashtables(GRAPH_global_context *ggctx);
static void load_GRAPH_global_hashtables(GRAPH_global_context *ggctx);
static void load_vertex_hashtable(GRAPH_global_context *ggctx);
@@ -146,6 +148,7 @@ static void
create_GRAPH_global_hashtables(GRAPH_global_context *ggctx)
ggctx->vertex_hashtable = hash_create(vhn, VERTEX_HTAB_INITIAL_SIZE,
&vertex_ctl,
HASH_ELEM | HASH_FUNCTION);
+ pfree(vhn);
/* initialize the edge hashtable */
MemSet(&edge_ctl, 0, sizeof(edge_ctl));
@@ -154,6 +157,7 @@ static void
create_GRAPH_global_hashtables(GRAPH_global_context *ggctx)
edge_ctl.hash = tag_hash;
ggctx->edge_hashtable = hash_create(ehn, EDGE_HTAB_INITIAL_SIZE, &edge_ctl,
HASH_ELEM | HASH_FUNCTION);
+ pfree(ehn);
}
/* helper function to get a List of all label names for the specified graph */
@@ -573,6 +577,10 @@ static void
free_specific_GRAPH_global_context(GRAPH_global_context *ggctx)
/* free the graph name */
pfree(ggctx->graph_name);
+ ggctx->graph_name = NULL;
+
+ ggctx->graph_oid = InvalidOid;
+ ggctx->next = NULL;
/* free the vertex edge lists, starting with the head */
curr_vertex = peek_stack_head(ggctx->vertices);
@@ -601,19 +609,28 @@ static void
free_specific_GRAPH_global_context(GRAPH_global_context *ggctx)
free_ListGraphId(value->edges_out);
free_ListGraphId(value->edges_self);
+ value->edges_in = NULL;
+ value->edges_out = NULL;
+ value->edges_self = NULL;
+
/* move to the next vertex */
curr_vertex = next_vertex;
}
/* free the vertices list */
free_ListGraphId(ggctx->vertices);
+ ggctx->vertices = NULL;
/* free the hashtables */
hash_destroy(ggctx->vertex_hashtable);
hash_destroy(ggctx->edge_hashtable);
+ ggctx->vertex_hashtable = NULL;
+ ggctx->edge_hashtable = NULL;
+
/* free the context */
pfree(ggctx);
+ ggctx = NULL;
}
/*
@@ -733,6 +750,95 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char
*graph_name,
}
/*
+ * Helper function to delete all of the global graph contexts used by the
+ * process. When done the global global_graph_contexts will be NULL.
+ */
+static bool delete_GRAPH_global_contexts(void)
+{
+ GRAPH_global_context *curr_ggctx = NULL;
+ bool retval = false;
+
+ /* get the first context, if any */
+ curr_ggctx = global_graph_contexts;
+
+ /* free all GRAPH global contexts */
+ while (curr_ggctx != NULL)
+ {
+ GRAPH_global_context *next_ggctx = curr_ggctx->next;
+
+ /* free the current graph context */
+ free_specific_GRAPH_global_context(curr_ggctx);
+
+ /* advance to the next context */
+ curr_ggctx = next_ggctx;
+
+ retval = true;
+ }
+
+ /* clear the global variable */
+ global_graph_contexts = NULL;
+
+ return retval;
+}
+
+/*
+ * Helper function to delete a specific global graph context used by the
+ * process.
+ */
+static bool delete_specific_GRAPH_global_contexts(char *graph_name)
+{
+ GRAPH_global_context *prev_ggctx = NULL;
+ GRAPH_global_context *curr_ggctx = NULL;
+ Oid graph_oid = InvalidOid;
+
+ if (graph_name == NULL)
+ {
+ return false;
+ }
+
+ /* get the graph oid */
+ graph_oid = get_graph_oid(graph_name);
+
+ /* get the first context, if any */
+ curr_ggctx = global_graph_contexts;
+
+ /* find the specified GRAPH global context */
+ while (curr_ggctx != NULL)
+ {
+ GRAPH_global_context *next_ggctx = curr_ggctx->next;
+
+ if (curr_ggctx->graph_oid == graph_oid)
+ {
+ /*
+ * If prev_ggctx is NULL then we are freeing the top of the
+ * contexts. So, we need to point the global variable to the
+ * new (next) top context, if there is one.
+ */
+ if (prev_ggctx == NULL)
+ {
+ global_graph_contexts = next_ggctx;
+ }
+ else
+ {
+ prev_ggctx->next = curr_ggctx->next;
+ }
+
+ /* free the current graph context */
+ free_specific_GRAPH_global_context(curr_ggctx);
+
+ /* we found and freed it, return true */
+ return true;
+ }
+
+ /* advance to the next one */
+ curr_ggctx = next_ggctx;
+ }
+
+ /* we didn't find it, return false */
+ return false;
+}
+
+/*
* Helper function to retrieve a vertex_entry from the graph's vertex hash
* table. If there isn't one, it returns a NULL. The latter is necessary for
* checking if the vsid and veid entries exist.
@@ -853,3 +959,160 @@ graphid get_edge_entry_end_vertex_id(edge_entry *ee)
{
return ee->end_vertex_id;
}
+
+/* PostgreSQL SQL facing functions */
+
+/* PG wrapper function for age_delete_global_graphs */
+PG_FUNCTION_INFO_V1(age_delete_global_graphs);
+
+Datum age_delete_global_graphs(PG_FUNCTION_ARGS)
+{
+ agtype_value *agtv_temp = NULL;
+ bool success = false;
+
+ /* get the graph name if supplied */
+ if (!PG_ARGISNULL(0))
+ {
+ agtv_temp = get_agtype_value("delete_global_graphs",
+ AG_GET_ARG_AGTYPE_P(0),
+ AGTV_STRING, false);
+ }
+
+ if (agtv_temp == NULL || agtv_temp->type == AGTV_NULL)
+ {
+ success = delete_GRAPH_global_contexts();
+ }
+ else if (agtv_temp->type == AGTV_STRING)
+ {
+ char *graph_name = NULL;
+
+ graph_name = agtv_temp->val.string.val;
+ success = delete_specific_GRAPH_global_contexts(graph_name);
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("delete_global_graphs: invalid graph name type")));
+ }
+
+ PG_RETURN_BOOL(success);
+}
+
+/* PG wrapper function for age_vertex_degree */
+PG_FUNCTION_INFO_V1(age_vertex_stats);
+
+Datum age_vertex_stats(PG_FUNCTION_ARGS)
+{
+ GRAPH_global_context *ggctx = NULL;
+ vertex_entry *ve = NULL;
+ ListGraphId *edges = NULL;
+ agtype_value *agtv_vertex = NULL;
+ agtype_value *agtv_temp = NULL;
+ agtype_value agtv_integer;
+ agtype_in_state result;
+ char *graph_name = NULL;
+ Oid graph_oid = InvalidOid;
+ graphid vid = 0;
+ int64 self_loops = 0;
+ int64 degree = 0;
+
+ /* the graph name is required, but this generally isn't user supplied */
+ if (PG_ARGISNULL(0))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("vertex_stats: graph name cannot be NULL")));
+ }
+
+ /* get the graph name */
+ agtv_temp = get_agtype_value("vertex_stats", AG_GET_ARG_AGTYPE_P(0),
+ AGTV_STRING, true);
+
+ /* we need the vertex */
+ if (PG_ARGISNULL(1))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("vertex_stats: vertex cannot be NULL")));
+ }
+
+ /* get the vertex */
+ agtv_vertex = get_agtype_value("vertex_stats", AG_GET_ARG_AGTYPE_P(1),
+ AGTV_VERTEX, true);
+
+ graph_name = pnstrdup(agtv_temp->val.string.val,
+ agtv_temp->val.string.len);
+
+ /* get the graph oid */
+ graph_oid = get_graph_oid(graph_name);
+
+ /*
+ * Create or retrieve the GRAPH global context for this graph. This
function
+ * will also purge off invalidated contexts.
+ */
+ ggctx = manage_GRAPH_global_contexts(graph_name, graph_oid);
+
+ /* free the graph name */
+ pfree(graph_name);
+
+ /* get the id */
+ agtv_temp = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_vertex, "id");
+ vid = agtv_temp->val.int_value;
+
+ /* get the vertex entry */
+ ve = get_vertex_entry(ggctx, vid);
+
+ /* zero the state */
+ memset(&result, 0, sizeof(agtype_in_state));
+
+ /* start the object */
+ result.res = push_agtype_value(&result.parse_state, WAGT_BEGIN_OBJECT,
+ NULL);
+ /* store the id */
+ result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
+ string_to_agtype_value("id"));
+ result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
+
+ /* store the label */
+ agtv_temp = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_vertex, "label");
+ result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
+ string_to_agtype_value("label"));
+ result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
+
+ /* set up an integer for returning values */
+ agtv_temp = &agtv_integer;
+ agtv_temp->type = AGTV_INTEGER;
+ agtv_temp->val.int_value = 0;
+
+ /* get and store the self_loops */
+ edges = get_vertex_entry_edges_self(ve);
+ self_loops = (edges != NULL) ? get_list_size(edges) : 0;
+ agtv_temp->val.int_value = self_loops;
+ result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
+ string_to_agtype_value("self_loops"));
+ result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
+
+ /* get and store the in_degree */
+ edges = get_vertex_entry_edges_in(ve);
+ degree = (edges != NULL) ? get_list_size(edges) : 0;
+ agtv_temp->val.int_value = degree + self_loops;
+ result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
+ string_to_agtype_value("in_degree"));
+ result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
+
+ /* get and store the out_degree */
+ edges = get_vertex_entry_edges_out(ve);
+ degree = (edges != NULL) ? get_list_size(edges) : 0;
+ agtv_temp->val.int_value = degree + self_loops;
+ result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
+ string_to_agtype_value("out_degree"));
+ result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
+
+ /* close the object */
+ result.res = push_agtype_value(&result.parse_state, WAGT_END_OBJECT, NULL);
+
+ result.res->type = AGTV_OBJECT;
+
+ PG_RETURN_POINTER(agtype_value_to_agtype(result.res));
+}
diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c
index 4ec40f9..1b5b3f3 100644
--- a/src/backend/utils/adt/age_vle.c
+++ b/src/backend/utils/adt/age_vle.c
@@ -158,7 +158,7 @@ static VLE_local_context
*get_cached_VLE_local_context(int64 vle_grammar_node_id
while (vlelctx != NULL)
{
/* purge any contexts past the maximum cache size */
- if (cache_size > MAXIMUM_NUMBER_OF_CACHED_LOCAL_CONTEXTS)
+ if (cache_size >= MAXIMUM_NUMBER_OF_CACHED_LOCAL_CONTEXTS)
{
/* set the next pointer to the context that follows */
next = vlelctx->next;
@@ -185,8 +185,29 @@ static VLE_local_context
*get_cached_VLE_local_context(int64 vle_grammar_node_id
/* and isn't dirty */
if (vlelctx->is_dirty == false)
{
- /* if the context isn't the head of the list, promote it */
- if (vlelctx != prev)
+ GRAPH_global_context *ggctx = NULL;
+
+ /*
+ * Get the GRAPH global context associated with this local VLE
+ * context. We need to verify it still exists and that the
+ * pointer is valid.
+ */
+ ggctx = find_GRAPH_global_context(vlelctx->graph_oid);
+
+ /*
+ * If ggctx == NULL, vlelctx is bad and vlelctx needs to be
+ * removed.
+ * If ggctx == vlelctx->ggctx, then vlelctx is good.
+ * If ggctx != vlelctx->ggctx, then vlelctx needs to be
updated.
+ * In the end, vlelctx->ggctx will be set to ggctx.
+ */
+ vlelctx->ggctx = ggctx;
+
+ /*
+ * If the context is good and isn't at the head of the cache,
+ * promote it to the head.
+ */
+ if (ggctx != NULL && vlelctx != prev)
{
/* adjust the links to cut out the node */
prev->next = vlelctx->next;
@@ -195,24 +216,29 @@ static VLE_local_context
*get_cached_VLE_local_context(int64 vle_grammar_node_id
/* point the head to this context */
global_vle_local_contexts = vlelctx;
}
- return vlelctx;
+
+ /* if we have a good one, return it. */
+ if (ggctx != NULL)
+ {
+ return vlelctx;
+ }
}
+
/* otherwise, clean and remove it, and return NULL */
+
+ /* set the top if necessary and unlink it */
+ if (prev == NULL)
+ {
+ global_vle_local_contexts = vlelctx->next;
+ }
else
{
- /* set the top if necessary and unlink it */
- if (prev == NULL)
- {
- global_vle_local_contexts = vlelctx->next;
- }
- else
- {
- prev->next = vlelctx->next;
- }
- /* now free it and return NULL */
- free_VLE_local_context(vlelctx);
- return NULL;
+ prev->next = vlelctx->next;
}
+
+ /* now free it and return NULL */
+ free_VLE_local_context(vlelctx);
+ return NULL;
}
/* save the previous context */
prev = vlelctx;
@@ -274,6 +300,7 @@ static void
create_VLE_local_state_hashtable(VLE_local_context *vlelctx)
EDGE_STATE_HTAB_INITIAL_SIZE,
&edge_state_ctl,
HASH_ELEM | HASH_FUNCTION);
+ pfree(eshn);
}
/*
@@ -375,10 +402,20 @@ static void free_VLE_local_context(VLE_local_context
*vlelctx)
hash_destroy(vlelctx->edge_state_hashtable);
vlelctx->edge_state_hashtable = NULL;
- /* we need to free our stacks */
- free_graphid_stack(vlelctx->dfs_vertex_stack);
- free_graphid_stack(vlelctx->dfs_edge_stack);
- free_graphid_stack(vlelctx->dfs_path_stack);
+ /*
+ * We need to free the contents of our stacks if the context is not dirty.
+ * These stacks are created in a more volatile memory context. If the
+ * process was interupted, they will be garbage collected by PG. The only
+ * time we will ever clean them here is if the cache isn't being used.
+ */
+ if (vlelctx->is_dirty == false)
+ {
+ free_graphid_stack(vlelctx->dfs_vertex_stack);
+ free_graphid_stack(vlelctx->dfs_edge_stack);
+ free_graphid_stack(vlelctx->dfs_path_stack);
+ }
+
+ /* free the containers */
pfree(vlelctx->dfs_vertex_stack);
pfree(vlelctx->dfs_edge_stack);
pfree(vlelctx->dfs_path_stack);
@@ -2429,102 +2466,3 @@ Datum _ag_enforce_edge_uniqueness(PG_FUNCTION_ARGS)
hash_destroy(exists_hash);
PG_RETURN_BOOL(true);
}
-
-/* PG wrapper function for age_vertex_degree */
-PG_FUNCTION_INFO_V1(age_vertex_stats);
-
-Datum age_vertex_stats(PG_FUNCTION_ARGS)
-{
- GRAPH_global_context *ggctx = NULL;
- vertex_entry *ve = NULL;
- ListGraphId *edges = NULL;
- agtype_value *agtv_vertex = NULL;
- agtype_value *agtv_temp = NULL;
- agtype_value agtv_integer;
- agtype_in_state result;
- char *graph_name = NULL;
- Oid graph_oid = InvalidOid;
- graphid vid = 0;
- int64 self_loops = 0;
- int64 degree = 0;
-
- /* get the graph name (required) */
- agtv_temp = get_agtype_value("vertex_stats", AG_GET_ARG_AGTYPE_P(0),
- AGTV_STRING, true);
-
- /* get the vertex (required) */
- agtv_vertex = get_agtype_value("vertex_stats", AG_GET_ARG_AGTYPE_P(1),
- AGTV_VERTEX, true);
-
- graph_name = pnstrdup(agtv_temp->val.string.val,
- agtv_temp->val.string.len);
-
- /* get the graph oid */
- graph_oid = get_graph_oid(graph_name);
-
- /*
- * Create or retrieve the GRAPH global context for this graph. This
function
- * will also purge off invalidated contexts.
- */
- ggctx = manage_GRAPH_global_contexts(graph_name, graph_oid);
-
- /* get the id */
- agtv_temp = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_vertex, "id");
- vid = agtv_temp->val.int_value;
-
- /* get the vertex entry */
- ve = get_vertex_entry(ggctx, vid);
-
- /* zero the state */
- memset(&result, 0, sizeof(agtype_in_state));
-
- /* start the object */
- result.res = push_agtype_value(&result.parse_state, WAGT_BEGIN_OBJECT,
- NULL);
- /* store the id */
- result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
- string_to_agtype_value("id"));
- result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
-
- /* store the label */
- agtv_temp = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_vertex, "label");
- result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
- string_to_agtype_value("label"));
- result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
-
- /* set up an integer for returning values */
- agtv_temp = &agtv_integer;
- agtv_temp->type = AGTV_INTEGER;
- agtv_temp->val.int_value = 0;
-
- /* get and store the self_loops */
- edges = get_vertex_entry_edges_self(ve);
- self_loops = (edges != NULL) ? get_list_size(edges) : 0;
- agtv_temp->val.int_value = self_loops;
- result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
- string_to_agtype_value("self_loops"));
- result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
-
- /* get and store the in_degree */
- edges = get_vertex_entry_edges_in(ve);
- degree = (edges != NULL) ? get_list_size(edges) : 0;
- agtv_temp->val.int_value = degree + self_loops;
- result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
- string_to_agtype_value("in_degree"));
- result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
-
- /* get and store the out_degree */
- edges = get_vertex_entry_edges_out(ve);
- degree = (edges != NULL) ? get_list_size(edges) : 0;
- agtv_temp->val.int_value = degree + self_loops;
- result.res = push_agtype_value(&result.parse_state, WAGT_KEY,
- string_to_agtype_value("out_degree"));
- result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp);
-
- /* close the object */
- result.res = push_agtype_value(&result.parse_state, WAGT_END_OBJECT, NULL);
-
- result.res->type = AGTV_OBJECT;
-
- PG_RETURN_POINTER(agtype_value_to_agtype(result.res));
-}
