Author: patrick
Date: 2008-11-14 23:17:58 EST (Fri, 14 Nov 2008)
New Revision: 20003
URL: https://svn.open-mpi.org/trac/ompi/changeset/20003
Log:
Define a "fake" mpool to provide a memory release callback for the
memory hooks (munmap) and initialize the mallopt component, and
nothing else.
Use this mpool in the MX common initialization, supporting both BTL
and MTL. Automatically set the MX_RCACHE environment variable to
enable registration cache in MX.
Tested with success for munmap() and large free().
Added:
trunk/ompi/mca/mpool/fake/
trunk/ompi/mca/mpool/fake/Makefile.am
trunk/ompi/mca/mpool/fake/configure.params
trunk/ompi/mca/mpool/fake/mpool_fake.h
trunk/ompi/mca/mpool/fake/mpool_fake_component.c
trunk/ompi/mca/mpool/fake/mpool_fake_module.c
Text files modified:
trunk/ompi/mca/common/mx/common_mx.c | 56 ++++++++++++++++++++
+++++++++++++++++++
1 files changed, 55 insertions(+), 1 deletions(-)
Modified: trunk/ompi/mca/common/mx/common_mx.c
=
=
=
=
=
=
=
=
=
=
====================================================================
--- trunk/ompi/mca/common/mx/common_mx.c (original)
+++ trunk/ompi/mca/common/mx/common_mx.c 2008-11-14 23:17:58 EST
(Fri, 14 Nov 2008)
@@ -9,6 +9,8 @@
* University of Stuttgart. All rights
reserved.
* Copyright (c) 2004-2006 The Regents of the University of
California.
* All rights reserved.
+ * Copyright (c) 2008 Myricom. All rights reserved.
+ *
* $COPYRIGHT$
*
* Additional copyrights may follow
@@ -21,11 +23,29 @@
#include "ompi/constants.h"
#include "common_mx.h"
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+#include "opal/memoryhooks/memory.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/mpool/mpool.h"
+#include "ompi/mca/mpool/base/base.h"
+#include "ompi/mca/mpool/fake/mpool_fake.h"
+
+
+int mx__regcache_clean(void *ptr, size_t size);
+
static int ompi_common_mx_initialize_ref_cnt = 0;
+static mca_mpool_base_module_t *ompi_common_mx_fake_mpool = 0;
+
int
ompi_common_mx_initialize(void)
{
mx_return_t mx_return;
+ struct mca_mpool_base_resources_t mpool_resources;
+ int index, value;
+
ompi_common_mx_initialize_ref_cnt++;
if(ompi_common_mx_initialize_ref_cnt == 1) {
@@ -35,7 +55,37 @@
* library does not exit the application.
*/
mx_set_error_handler(MX_ERRORS_RETURN);
-
+
+ /* If we have a memory manager available, and
+ mpi_leave_pinned == -1, then set mpi_leave_pinned to 1.
+
+ We have a memory manager if:
+ - we have both FREE and MUNMAP support
+ - we have MUNMAP support and the linux mallopt */
+ value = opal_mem_hooks_support_level();
+ if (((value & (OPAL_MEMORY_FREE_SUPPORT |
OPAL_MEMORY_MUNMAP_SUPPORT))
+ == (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
+ || ((value & OPAL_MEMORY_MUNMAP_SUPPORT) &&
+ OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT)) {
+ index = mca_base_param_find("mpi", NULL, "leave_pinned");
+ if (index >= 0)
+ if ((mca_base_param_lookup_int(index, &value) ==
OPAL_SUCCESS)
+ && (value == -1)) {
+
+ ompi_mpi_leave_pinned = 1;
+ setenv("MX_RCACHE", "2", 1);
+ mpool_resources.regcache_clean = mx__regcache_clean;
+ ompi_common_mx_fake_mpool =
+ mca_mpool_base_module_create("fake", NULL, &mpool_resources);
+ if (!ompi_common_mx_fake_mpool) {
+ ompi_mpi_leave_pinned = 0;
+ setenv("MX_RCACHE", "0", 1);
+ opal_output(0, "Error creating fake mpool (error %s)\n",
+ strerror(errno));
+ }
+ }
+ }
+
/* initialize the mx library */
mx_return = mx_init();
@@ -57,6 +107,10 @@
mx_return_t mx_return;
ompi_common_mx_initialize_ref_cnt--;
if( 0 == ompi_common_mx_initialize_ref_cnt ) {
+
+ if (ompi_common_mx_fake_mpool)
+ mca_mpool_base_module_destroy(ompi_common_mx_fake_mpool);
+
mx_return = mx_finalize();
if(mx_return != MX_SUCCESS){
opal_output(0, "Error in mx_finalize (error %s)\n",
mx_strerror(mx_return));
Added: trunk/ompi/mca/mpool/fake/Makefile.am
=
=
=
=
=
=
=
=
=
=
====================================================================
--- (empty file)
+++ trunk/ompi/mca/mpool/fake/Makefile.am 2008-11-14 23:17:58 EST
(Fri, 14 Nov 2008)
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and
Indiana
+# University Research and Technology
+# Corporation. All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The
University
+# of Tennessee Research Foundation. All
rights
+# reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center
Stuttgart,
+# University of Stuttgart. All rights
reserved.
+# Copyright (c) 2004-2005 The Regents of the University of
California.
+# All rights reserved.
+# Copyright (c) 2008 Myricom. All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(mpool_fake_CPPFLAGS)
+
+sources = \
+ mpool_fake_module.c \
+ mpool_fake_component.c
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/mpool/fake
+ompi_HEADERS = mpool_fake.h
+else
+ompidir = $(includedir)
+ompi_HEADERS =
+endif
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mpool_fake_DSO
+component_noinst =
+component_install = mca_mpool_fake.la
+else
+component_noinst = libmca_mpool_fake.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mpool_fake_la_SOURCES = $(sources)
+mca_mpool_fake_la_LDFLAGS = -module -avoid-version
+mca_mpool_fake_la_LIBADD = $(mpool_fake_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mpool_fake_la_SOURCES = $(sources)
+libmca_mpool_fake_la_LDFLAGS = -module -avoid-version
+libmca_mpool_fake_la_LIBADD = $(mpool_fake_LIBS)
Added: trunk/ompi/mca/mpool/fake/configure.params
=
=
=
=
=
=
=
=
=
=
====================================================================
--- (empty file)
+++ trunk/ompi/mca/mpool/fake/configure.params 2008-11-14 23:17:58
EST (Fri, 14 Nov 2008)
@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and
Indiana
+# University Research and Technology
+# Corporation. All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The
University
+# of Tennessee Research Foundation. All
rights
+# reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center
Stuttgart,
+# University of Stuttgart. All rights
reserved.
+# Copyright (c) 2004-2005 The Regents of the University of
California.
+# All rights reserved.
+# Copyright (c) 2008 Myricom. All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"
Added: trunk/ompi/mca/mpool/fake/mpool_fake.h
=
=
=
=
=
=
=
=
=
=
====================================================================
--- (empty file)
+++ trunk/ompi/mca/mpool/fake/mpool_fake.h 2008-11-14 23:17:58 EST
(Fri, 14 Nov 2008)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and
Indiana
+ * University Research and Technology
+ * Corporation. All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The
University
+ * of Tennessee Research Foundation. All
rights
+ * reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center
Stuttgart,
+ * University of Stuttgart. All rights
reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of
California.
+ * All rights reserved.
+ * Copyright (c) 2008 Myricom. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+
+#ifndef MCA_MPOOL_FAKE_H
+#define MCA_MPOOL_FAKE_H
+
+#include "ompi/mca/mpool/mpool.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+typedef struct mca_mpool_fake_component_t {
+ mca_mpool_base_component_t super;
+} mca_mpool_fake_component_t;
+
+OMPI_DECLSPEC extern mca_mpool_fake_component_t
mca_mpool_fake_component;
+
+typedef struct mca_mpool_base_resources_t {
+ int (*regcache_clean)(void *ptr, size_t size);
+} mca_mpool_base_resources_t;
+
+typedef struct mca_mpool_fake_module_t {
+ mca_mpool_base_module_t super;
+ mca_mpool_base_resources_t resources;
+} mca_mpool_fake_module_t;
+
+/*
+ * Initializes the mpool module.
+ */
+void mca_mpool_fake_module_init(mca_mpool_fake_module_t *mpool);
+
+/**
+ * unregister all registration covering the block of memory
+ */
+int mca_mpool_fake_release_memory(mca_mpool_base_module_t* mpool,
void *base,
+ size_t size);
+
+/**
+ * finalize mpool
+ */
+void mca_mpool_fake_finalize(struct mca_mpool_base_module_t
*mpool);
+
+/**
+ * Fault Tolerance Event Notification Function
+ * @param state Checkpoint Stae
+ * @return OMPI_SUCCESS or failure status
+ */
+int mca_mpool_fake_ft_event(int state);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
Added: trunk/ompi/mca/mpool/fake/mpool_fake_component.c
=
=
=
=
=
=
=
=
=
=
====================================================================
--- (empty file)
+++ trunk/ompi/mca/mpool/fake/mpool_fake_component.c 2008-11-14
23:17:58 EST (Fri, 14 Nov 2008)
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and
Indiana
+ * University Research and Technology
+ * Corporation. All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The
University
+ * of Tennessee Research Foundation. All
rights
+ * reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center
Stuttgart,
+ * University of Stuttgart. All rights
reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of
California.
+ * All rights reserved.
+ * Copyright (c) 2008 Myricom. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#define OMPI_DISABLE_ENABLE_MEM_DEBUG 1
+#include "ompi_config.h"
+#include "orte/util/show_help.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "mpool_fake.h"
+#include "orte/util/proc_info.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+
+/*
+ * Local functions
+ */
+static int mca_mpool_fake_open(void);
+static mca_mpool_base_module_t* mca_mpool_fake_init(
+ struct mca_mpool_base_resources_t* resources);
+
+mca_mpool_fake_component_t mca_mpool_fake_component = {
+ {
+ /* First, the mca_base_component_t struct containing meta
+ information about the component itself */
+
+ {
+ MCA_MPOOL_BASE_VERSION_2_0_0,
+
+ "fake", /* MCA component name */
+ OMPI_MAJOR_VERSION, /* MCA component major version */
+ OMPI_MINOR_VERSION, /* MCA component minor version */
+ OMPI_RELEASE_VERSION, /* MCA component release version
*/
+ mca_mpool_fake_open, /* component open */
+ NULL
+ },
+ {
+ /* The component is checkpoint ready */
+ MCA_BASE_METADATA_PARAM_CHECKPOINT
+ },
+
+ mca_mpool_fake_init
+ }
+};
+
+/**
+ * component open/close/init function
+ */
+static int mca_mpool_fake_open(void)
+{
+ return OMPI_SUCCESS;
+}
+
+static mca_mpool_base_module_t* mca_mpool_fake_init(
+ struct mca_mpool_base_resources_t* resources)
+{
+ mca_mpool_fake_module_t* mpool_module;
+
+ mpool_module = (mca_mpool_fake_module_t*)
+ malloc(sizeof(mca_mpool_fake_module_t));
+
+ mpool_module->resources = *resources;
+
+ mca_mpool_fake_module_init(mpool_module);
+
+ return &mpool_module->super;
+}
Added: trunk/ompi/mca/mpool/fake/mpool_fake_module.c
=
=
=
=
=
=
=
=
=
=
====================================================================
--- (empty file)
+++ trunk/ompi/mca/mpool/fake/mpool_fake_module.c 2008-11-14
23:17:58 EST (Fri, 14 Nov 2008)
@@ -0,0 +1,72 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and
Indiana
+ * University Research and Technology
+ * Corporation. All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The
University
+ * of Tennessee Research Foundation. All
rights
+ * reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center
Stuttgart,
+ * University of Stuttgart. All rights
reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of
California.
+ * All rights reserved.
+ * Copyright (c) 2008 Myricom. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#define OMPI_DISABLE_ENABLE_MEM_DEBUG 1
+#include "ompi_config.h"
+#include "opal/include/opal/align.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/name_fns.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/util/show_help.h"
+#include "mpool_fake.h"
+#include <errno.h>
+#include <string.h>
+#include "ompi/mca/mpool/base/base.h"
+#include "ompi/runtime/params.h"
+
+/*
+ * Initializes the mpool module.
+ */
+void mca_mpool_fake_module_init(mca_mpool_fake_module_t* mpool)
+{
+ mpool->super.mpool_component = &mca_mpool_fake_component.super;
+ mpool->super.mpool_base = NULL; /* no base .. */
+ mpool->super.mpool_alloc = NULL;
+ mpool->super.mpool_realloc = NULL;
+ mpool->super.mpool_free = NULL;
+ mpool->super.mpool_register = NULL;
+ mpool->super.mpool_find = NULL;
+ mpool->super.mpool_deregister = NULL;
+ mpool->super.mpool_release_memory =
mca_mpool_fake_release_memory;
+ mpool->super.mpool_finalize = mca_mpool_fake_finalize;
+ mpool->super.mpool_ft_event = mca_mpool_fake_ft_event;
+}
+
+
+int mca_mpool_fake_release_memory(struct mca_mpool_base_module_t
*mpool,
+ void *base, size_t size)
+{
+ mca_mpool_fake_module_t *mpool_module =
(mca_mpool_fake_module_t*)mpool;
+
+ mpool_module->resources.regcache_clean(base, size);
+
+ return OMPI_SUCCESS;
+}
+
+void mca_mpool_fake_finalize(struct mca_mpool_base_module_t *mpool)
+{
+ ;
+}
+
+int mca_mpool_fake_ft_event(int state)
+{
+ return OMPI_SUCCESS;
+}
_______________________________________________
svn-full mailing list
svn-f...@open-mpi.org
http://www.open-mpi.org/mailman/listinfo.cgi/svn-full
