What: Allocate free list payload even if a payload size is specified even if no 
mpool is specified.

When: Thursday, Feb 23, 2012

Why: The current behavior is to ignore the payload size if no mpool is 
specified. I see no reason why a payload buffer should't be allocated in the no 
mpool case. Thoughts?

Patch is attached.

-Nathan Hjelm
HPC-3, LANL
diff --git a/ompi/class/ompi_free_list.c b/ompi/class/ompi_free_list.c
index d468a70..e3c0988 100644
--- a/ompi/class/ompi_free_list.c
+++ b/ompi/class/ompi_free_list.c
@@ -1,4 +1,4 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -13,6 +13,8 @@
  * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
  * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -70,23 +72,19 @@ static void ompi_free_list_destruct(ompi_free_list_t* fl)
     }
 #endif
 
-    if( NULL != fl->fl_mpool ) {
-        while(NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
-            fl_mem = (ompi_free_list_memory_t*)item;
+    while(NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
+        fl_mem = (ompi_free_list_memory_t*)item;
 
+        if( NULL != fl->fl_mpool ) {
             fl->fl_mpool->mpool_free(fl->fl_mpool, fl_mem->ptr,
                                      fl_mem->registration);
-
-            /* destruct the item (we constructed it), then free the memory 
chunk */
-            OBJ_DESTRUCT(item);
-            free(item);
-        }
-    } else {
-        while(NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
-            /* destruct the item (we constructed it), then free the memory 
chunk */
-            OBJ_DESTRUCT(item);
-            free(item);
+        } else if (fl_mem->ptr) {
+            free (fl_mem->ptr);
         }
+
+        /* destruct the item (we constructed it), then free the memory chunk */
+        OBJ_DESTRUCT(item);
+        free(item);
     }
 
     OBJ_DESTRUCT(&fl->fl_allocations);
@@ -171,7 +169,7 @@ int ompi_free_list_init_ex_new(
 }
 int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
 {
-    unsigned char *ptr, *mpool_alloc_ptr = NULL;
+    unsigned char *ptr, *mpool_alloc_ptr = NULL, *payload_ptr;
     ompi_free_list_memory_t *alloc_ptr;
     size_t i, alloc_size, head_size, elem_size = 0;
     mca_mpool_base_registration_t *reg = NULL;
@@ -201,7 +199,7 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t 
num_elements)
         elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size, 
                 flist->fl_payload_buffer_alignment, size_t);
         if(elem_size != 0) {
-            mpool_alloc_ptr = (unsigned char *) 
flist->fl_mpool->mpool_alloc(flist->fl_mpool,
+            payload_ptr = mpool_alloc_ptr = (unsigned char *) 
flist->fl_mpool->mpool_alloc(flist->fl_mpool,
                    num_elements * elem_size, 
flist->fl_payload_buffer_alignment,
                    MCA_MPOOL_FLAGS_CACHE_BYPASS | 
MCA_MPOOL_FLAGS_CUDA_REGISTER_MEM, &reg);
             if(NULL == mpool_alloc_ptr) {
@@ -209,6 +207,26 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t 
num_elements)
                 return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
             }
         }
+    } else if (0 != flist->fl_payload_buffer_size) {
+        elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size, 
+                               flist->fl_payload_buffer_alignment, size_t);
+        if(elem_size != 0) {
+#ifdef HAVE_POSIX_MEMALIGN
+            posix_memalign ((void **) &mpool_alloc_ptr, 
flist->fl_payload_buffer_alignment,
+                            num_elements * elem_size);
+            payload_ptr = mpool_alloc_ptr;
+#else
+            mpool_alloc_ptr = malloc (num_elements * elem_size +
+                                      flist->fl_payload_buffer_alignment);
+            payload_ptr = (void*)OPAL_ALIGN((uintptr_t)mpool_alloc_ptr, 
+                                            flist->fl_payload_buffer_alignment,
+                                            uintptr_t);
+#endif
+            if(NULL == mpool_alloc_ptr) {
+                free(alloc_ptr);
+                return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+            }
+        }
     }
 
     /* make the alloc_ptr a list item, save the chunk in the allocations list,
@@ -225,7 +243,7 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t 
num_elements)
     for(i=0; i<num_elements; i++) {
         ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr;
         item->registration = reg;
-        item->ptr = mpool_alloc_ptr;
+        item->ptr = payload_ptr;
 
         OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class);
         
@@ -236,7 +254,7 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t 
num_elements)
 
         opal_atomic_lifo_push(&(flist->super), &(item->super));
         ptr += head_size;
-        mpool_alloc_ptr += elem_size;
+        payload_ptr += elem_size;
         
     }
     flist->fl_num_allocated += num_elements;
diff --git a/ompi/mca/btl/ugni/btl_ugni_frag.c 
b/ompi/mca/btl/ugni/btl_ugni_frag.c
index de7857b..16f3612 100644
--- a/ompi/mca/btl/ugni/btl_ugni_frag.c
+++ b/ompi/mca/btl/ugni/btl_ugni_frag.c
@@ -17,14 +17,11 @@ static inline void mca_btl_ugni_smsg_frag_constructor 
(mca_btl_ugni_base_frag_t
 {
     /* send memory does not need to be registered so we do not need a mpool */
     memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof 
(frag->base));
-    frag->buffer = frag->segments[0].seg_addr.pval = calloc (1, 
mca_btl_ugni_component.smsg_max_data);
+    frag->segments[0].seg_addr.pval = frag->base.super.ptr;
 }
 
 static inline void mca_btl_ugni_frag_destructor (mca_btl_ugni_base_frag_t 
*frag)
 {
-    if (NULL != frag->buffer) {
-        free (frag->buffer);
-    }
 }
 
 static inline void mca_btl_ugni_rdma_frag_constructor 
(mca_btl_ugni_base_frag_t *frag)
diff --git a/ompi/mca/btl/ugni/btl_ugni_frag.h 
b/ompi/mca/btl/ugni/btl_ugni_frag.h
index b115963..d01ff17 100644
--- a/ompi/mca/btl/ugni/btl_ugni_frag.h
+++ b/ompi/mca/btl/ugni/btl_ugni_frag.h
@@ -48,7 +48,6 @@ typedef struct mca_btl_ugni_base_frag_t {
     mca_btl_base_endpoint_t *endpoint;
     mca_btl_ugni_reg_t *registration;
     ompi_free_list_t *my_list;
-    void *buffer;
     uint32_t msg_id;
 } mca_btl_ugni_base_frag_t;
 

Reply via email to