Index: ompi/mca/coll/portals4/coll_portals4_barrier.c =================================================================== --- ompi/mca/coll/portals4/coll_portals4_barrier.c (revision 32401) +++ ompi/mca/coll/portals4/coll_portals4_barrier.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -40,7 +43,7 @@ ompi_coll_portals4_get_md(0, &md_h, &base); - count = opal_atomic_add_size_t(&portals4_module->barrier_count, 1); + count = 1 + opal_atomic_add_size_t(&portals4_module->barrier_count, 1); ret = PtlCTAlloc(mca_coll_portals4_component.ni_h, &ct_h); @@ -196,7 +199,7 @@ *ompi_req = &request->super; request->type = OMPI_COLL_PORTALS4_TYPE_BARRIER; - count = opal_atomic_add_size_t(&portals4_module->barrier_count, 1); + count = 1 + opal_atomic_add_size_t(&portals4_module->barrier_count, 1); ret = PtlCTAlloc(mca_coll_portals4_component.ni_h, &request->ct_h); Index: ompi/mca/coll/sm/coll_sm.h =================================================================== --- ompi/mca/coll/sm/coll_sm.h (revision 32401) +++ ompi/mca/coll/sm/coll_sm.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -356,7 +356,7 @@ * Macro to release an in-use flag from this process */ #define FLAG_RELEASE(flag) \ - opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1) + (void)opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1) /** * Macro to copy a single segment in from a user buffer to a shared Index: ompi/mca/coll/sm/coll_sm_barrier.c =================================================================== --- ompi/mca/coll/sm/coll_sm_barrier.c (revision 32401) +++ ompi/mca/coll/sm/coll_sm_barrier.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -101,7 +101,7 @@ if (0 != rank) { /* Get parent *in* buffer */ parent = &data->mcb_barrier_control_parent[buffer_set]; - opal_atomic_add(parent, 1); + (void)opal_atomic_add(parent, 1); SPIN_CONDITION(0 != *me_out, exit_label2); *me_out = 0; Index: ompi/mca/coll/sm/coll_sm_module.c =================================================================== --- ompi/mca/coll/sm/coll_sm_module.c (revision 32401) +++ ompi/mca/coll/sm/coll_sm_module.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -460,7 +460,7 @@ OBJ_RETAIN(sm_module->previous_reduce_module); /* Indicate that we have successfully attached and setup */ - opal_atomic_add(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1); + (void)opal_atomic_add(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1); /* Wait for everyone in this communicator to attach and setup */ opal_output_verbose(10, ompi_coll_base_framework.framework_output, Index: ompi/mca/mtl/portals4/mtl_portals4_flowctl.c =================================================================== --- ompi/mca/mtl/portals4/mtl_portals4_flowctl.c (revision 32401) +++ ompi/mca/mtl/portals4/mtl_portals4_flowctl.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2012 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -321,7 +324,7 @@ int64_t epoch_counter; ompi_mtl_portals4.flowctl.flowctl_active = true; - epoch_counter = opal_atomic_add_64(&ompi_mtl_portals4.flowctl.epoch_counter, 1); + epoch_counter = 1 + opal_atomic_add_64(&ompi_mtl_portals4.flowctl.epoch_counter, 1); opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "Entering flowctl_start_recover %d", Index: ompi/mca/osc/portals4/osc_portals4_comm.c =================================================================== --- ompi/mca/osc/portals4/osc_portals4_comm.c (revision 32401) +++ ompi/mca/osc/portals4/osc_portals4_comm.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -217,7 +220,7 @@ "MPI_Rput: transfer of non-contiguous memory is not currently supported.\n"); return OMPI_ERR_NOT_SUPPORTED; } else { - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected = 1; ret = ompi_datatype_type_size(origin_dt, &length); if (OMPI_SUCCESS != ret) { @@ -287,7 +290,7 @@ "MPI_Rget: transfer of non-contiguous memory is not currently supported.\n"); return OMPI_ERR_NOT_SUPPORTED; } else { - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected = 1; ret = ompi_datatype_type_size(origin_dt, &length); if (OMPI_SUCCESS != ret) { @@ -374,7 +377,7 @@ do { size_t msg_length = MIN(module->atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected++; if (MPI_REPLACE == op) { @@ -491,7 +494,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected++; ret = PtlSwap(result_md_h, @@ -528,7 +531,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected++; ret = PtlGet(md_h, @@ -567,7 +570,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); request->ops_expected++; ret = PtlFetchAtomic(result_md_h, @@ -630,7 +633,7 @@ "MPI_Put: transfer of non-contiguous memory is not currently supported.\n"); return OMPI_ERR_NOT_SUPPORTED; } else { - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = ompi_datatype_type_size(origin_dt, &length); if (OMPI_SUCCESS != ret) { return ret; @@ -690,7 +693,7 @@ "MPI_Get: transfer of non-contiguous memory is not currently supported.\n"); return OMPI_ERR_NOT_SUPPORTED; } else { - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = ompi_datatype_type_size(origin_dt, &length); if (OMPI_SUCCESS != ret) { return ret; @@ -766,7 +769,7 @@ do { size_t msg_length = MIN(module->atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); if (MPI_REPLACE == op) { ret = PtlPut(md_h, @@ -873,7 +876,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = PtlSwap(result_md_h, result_md_offset + sent, @@ -908,7 +911,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = PtlGet(md_h, md_offset + sent, @@ -946,7 +949,7 @@ do { size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = PtlFetchAtomic(result_md_h, result_md_offset + sent, @@ -1016,7 +1019,7 @@ ompi_osc_portals4_get_md(origin_addr, module->md_h, &origin_md_h, &origin_md_base); origin_md_offset = ((char*) origin_addr - (char*) origin_md_base); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ret = PtlSwap(result_md_h, result_md_offset, @@ -1076,7 +1079,7 @@ assert(length < module->fetch_atomic_max); - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); if (MPI_REPLACE == op) { ptl_handle_md_t result_md_h, origin_md_h; Index: ompi/mca/osc/portals4/osc_portals4_component.c =================================================================== --- ompi/mca/osc/portals4/osc_portals4_component.c (revision 32401) +++ ompi/mca/osc/portals4/osc_portals4_component.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -219,8 +222,8 @@ /* can't disable send events, but they don't count in ops */ if (ev.type == PTL_EVENT_SEND) continue; req = (ompi_osc_portals4_request_t*) ev.user_ptr; - opal_atomic_add_size_t(&req->super.req_status._ucount, ev.mlength); - ops = opal_atomic_add_32(&req->ops_committed, 1); + (void)opal_atomic_add_size_t(&req->super.req_status._ucount, ev.mlength); + ops = 1 + opal_atomic_add_32(&req->ops_committed, 1); if (ops == req->ops_expected) { OPAL_THREAD_LOCK(&ompi_request_lock); ompi_request_complete(&req->super, true); Index: ompi/mca/osc/portals4/osc_portals4_passive_target.c =================================================================== --- ompi/mca/osc/portals4/osc_portals4_passive_target.c (revision 32401) +++ ompi/mca/osc/portals4/osc_portals4_passive_target.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -44,7 +47,7 @@ ptl_handle_md_t result_md_h, write_md_h; void *result_base, *write_base; - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ompi_osc_portals4_get_md(result_val, module->md_h, &result_md_h, &result_base); ompi_osc_portals4_get_md(&write_val, module->md_h, &write_md_h, &write_base); @@ -82,7 +85,7 @@ ptl_handle_md_t md_h; void *base; - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ompi_osc_portals4_get_md(&write_val, module->md_h, &md_h, &base); @@ -116,7 +119,7 @@ ptl_handle_md_t result_md_h, write_md_h; void *result_base, *write_base; - opal_atomic_add_64(&module->opcount, 1); + (void)opal_atomic_add_64(&module->opcount, 1); ompi_osc_portals4_get_md(result_val, module->md_h, &result_md_h, &result_base); ompi_osc_portals4_get_md(&write_val, module->md_h, &write_md_h, &write_base); Index: ompi/mca/osc/sm/osc_sm_active_target.c =================================================================== --- ompi/mca/osc/sm/osc_sm_active_target.c (revision 32401) +++ ompi/mca/osc/sm/osc_sm_active_target.c (working copy) @@ -3,6 +3,9 @@ * Copyright (c) 2012 Sandia National Laboratories. All rights reserved. * Copyright (c) 2014 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -100,7 +103,7 @@ for (int j = 0 ; j < csize ; ++j) { if (ompi_group_peer_lookup(module->start_group, i) == ompi_comm_peer_lookup(module->comm, j)) { - opal_atomic_add_32(&module->node_states[j].complete_count, 1); + (void)opal_atomic_add_32(&module->node_states[j].complete_count, 1); } } } @@ -136,7 +139,7 @@ for (int j = 0 ; j < csize ; ++j) { if (ompi_group_peer_lookup(module->post_group, i) == ompi_comm_peer_lookup(module->comm, j)) { - opal_atomic_add_32(&module->node_states[j].post_count, 1); + (void)opal_atomic_add_32(&module->node_states[j].post_count, 1); } } } Index: ompi/mca/osc/sm/osc_sm_passive_target.c =================================================================== --- ompi/mca/osc/sm/osc_sm_passive_target.c (revision 32401) +++ ompi/mca/osc/sm/osc_sm_passive_target.c (working copy) @@ -3,6 +3,9 @@ * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. * Copyright (c) 2014 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -25,10 +28,7 @@ size_t offset, uint32_t delta) { - /* opal_atomic_add_32 is an add then fetch so delta needs to be subtracted out to get the - * old value */ - return opal_atomic_add_32((int32_t*) ((char*) &module->node_states[target].lock + offset), - delta) - delta; + return opal_atomic_add_32((int32_t*) ((char*) &module->node_states[target].lock + offset), delta); } Index: ompi/mpi/tool/init_thread.c =================================================================== --- ompi/mpi/tool/init_thread.c (revision 32401) +++ ompi/mpi/tool/init_thread.c (working copy) @@ -3,6 +3,9 @@ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,7 +38,7 @@ static volatile int32_t first_init = 1; int rc = MPI_SUCCESS; - if (opal_atomic_cmpset (&first_init, 1, 0) == 1) { + if (1 == opal_atomic_cmpset (&first_init, 1, 0)) { OBJ_CONSTRUCT(&mpit_big_lock, opal_mutex_t); initted = 1; } Index: ompi/runtime/ompi_mpi_finalize.c =================================================================== --- ompi/runtime/ompi_mpi_finalize.c (revision 32401) +++ ompi/runtime/ompi_mpi_finalize.c (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2011 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -103,7 +103,7 @@ ompi_comm_free() (or run into other nasty lions, tigers, or bears) */ - if (! opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1)) { + if (opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1)) { /* Note that if we're already finalized, we cannot raise an MPI exception. The best that we can do is write something to stderr. */ Index: opal/asm/asm.c =================================================================== --- opal/asm/asm.c (revision 32401) +++ opal/asm/asm.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -59,7 +59,8 @@ opal_atomic_lock(FIND_LOCK(addr)); - ret = (*addr += delta); + ret = *addr; + *addr += delta; opal_atomic_unlock(FIND_LOCK(addr)); @@ -74,7 +75,8 @@ opal_atomic_lock(FIND_LOCK(addr)); - ret = (*addr -= delta); + ret = *addr; + *addr -= delta; opal_atomic_unlock(FIND_LOCK(addr)); Index: opal/class/opal_list.c =================================================================== --- opal/class/opal_list.c (revision 32401) +++ opal/class/opal_list.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, @@ -144,13 +144,13 @@ /* Spot check: ensure this item is only on the list that we just insertted it into */ - opal_atomic_add( &(item->opal_list_item_refcount), 1 ); + (void)opal_atomic_add( &(item->opal_list_item_refcount), 1 ); assert(1 == item->opal_list_item_refcount); item->opal_list_item_belong_to = list; #endif } - - list->opal_list_length++; + + list->opal_list_length++; return true; } Index: opal/class/opal_object.h =================================================================== --- opal/class/opal_object.h (revision 32401) +++ opal/class/opal_object.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2007 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart, @@ -490,7 +490,7 @@ static inline int opal_obj_update(opal_object_t *object, int inc) __opal_attribute_always_inline__; static inline int opal_obj_update(opal_object_t *object, int inc) { - return opal_atomic_add_32(&(object->obj_reference_count), inc); + return inc + opal_atomic_add_32(&(object->obj_reference_count), inc); } END_C_DECLS Index: opal/include/opal/sys/alpha/atomic.h =================================================================== --- opal/include/opal/sys/alpha/atomic.h (revision 32401) +++ opal/include/opal/sys/alpha/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -84,8 +84,8 @@ *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { int32_t ret; @@ -107,11 +107,11 @@ } -static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr, - int32_t oldval, - int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32(volatile int32_t *addr, + int32_t oldval, + int32_t newval) { - int rc; + int32_t rc; rc = opal_atomic_cmpset_32(addr, oldval, newval); opal_atomic_rmb(); @@ -120,9 +120,9 @@ } -static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, - int32_t oldval, - int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32(volatile int32_t *addr, + int32_t oldval, + int32_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_32(addr, oldval, newval); @@ -129,10 +129,10 @@ } -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int32_t ret; + int64_t ret; __asm__ __volatile__ ( "1: ldq_l %0, %1 \n\t" @@ -152,11 +152,11 @@ } -static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr, - int64_t oldval, - int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, + int64_t oldval, + int64_t newval) { - int rc; + int64_t rc; rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); @@ -165,9 +165,9 @@ } -static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, - int64_t oldval, - int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, + int64_t oldval, + int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); @@ -174,7 +174,142 @@ } +#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1 +static inline int32_t opal_atomic_add_32(volatile int32_t *addr, + int32_t inc) +{ + int32_t temp, result; + + __asm__ __volatile__( + "1: ldl_l %0,%1 \n" + " addl %0,%3,%2 \n" + " addl %0,%3,%0 \n" + " stl_c %0,%1 \n" + " beq %0,2f \n" + ".subsection 2 \n" + "2: br 1b \n" + ".previous" + :"=&r" (temp), "=m" (*addr), "=&r" (result) + :"Ir" (inc), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return result-inc; +} + +#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1 +static inline int64_t opal_atomic_add_64(volatile int64_t *addr, + int64_t inc) +{ + int64_t temp, result; + + __asm__ __volatile__( + "1: ldq_l %0,%1\n" + " addq %0,%3,%2\n" + " addq %0,%3,%0\n" + " stq_c %0,%1\n" + " beq %0,2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + :"=&r" (temp), "=m" (*addr), "=&r" (result) + :"Ir" (inc), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return result-inc; +} + +#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1 +static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, + int32_t dec) +{ + int32_t temp, result; + + __asm__ __volatile__( + "1: ldl_l %0,%1 \n" + " subl %0,%3,%2 \n" + " subl %0,%3,%0 \n" + " stl_c %0,%1 \n" + " beq %0,2f \n" + ".subsection 2 \n" + "2: br 1b \n" + ".previous" + :"=&r" (temp), "=m" (*addr), "=&r" (result) + :"Ir" (dec), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return result+dec; +} + +#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1 +static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, + int64_t dec) +{ + int64_t temp, result; + + __asm__ __volatile__( + "1: ldq_l %0,%1 \n" + " subq %0,%3,%2 \n" + " subq %0,%3,%0 \n" + " stq_c %0,%1 \n" + " beq %0,2f \n" + ".subsection 2 \n" + "2: br 1b \n" + ".previous" + :"=&r" (temp), "=m" (*addr), "=&r" (result) + :"Ir" (dec), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return result+dec; +} + +#define OPAL_HAVE_ATOMIC_SWAP_32 1 +static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, + int32_t newval) +{ + int32_t dummy; + + __asm__ __volatile__( + "1: ldl_l %0,%4 \n" + " bis $31,%3,%1 \n" + " stl_c %1,%2 \n" + " beq %1,2f \n" + ".subsection 2 \n" + "2: br 1b \n" + ".previous" + : "=&r" (newval), "=&r" (dummy), "=m" (*addr) + : "rI" (newval), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return newval; +} + +#define OPAL_HAVE_ATOMIC_SWAP_64 1 +static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, + int64_t newval) +{ + int32_t dummy; + + __asm__ __volatile__( + "1: ldq_l %0,%4 \n" + " bis $31,%3,%1 \n" + " stq_c %1,%2 \n" + " beq %1,2f \n" + ".subsection 2 \n" + "2: br 1b \n" + ".previous" + : "=&r" (newval), "=&r" (dummy), "=m" (*addr) + : "rI" (newval), "m" (*addr) + : "memory"); + opal_atomic_mb(); + + return newval; +} + #endif /* OPAL_GCC_INLINE_ASSEMBLY */ - #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ Index: opal/include/opal/sys/amd64/atomic.h =================================================================== --- opal/include/opal/sys/amd64/atomic.h (revision 32401) +++ opal/include/opal/sys/amd64/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -78,18 +78,17 @@ *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { - unsigned char ret; + int32_t prev; __asm__ __volatile__ ( - SMPLOCK "cmpxchgl %3,%2 \n\t" - "sete %0 \n\t" - : "=qm" (ret), "+a" (oldval), "+m" (*addr) - : "q"(newval) + SMPLOCK "cmpxchgl %1,%2 \n\t" + : "=a" (prev) + : "q"(newval), "m" (*addr), "0" (oldval) : "memory", "cc"); - return (int)ret; + return prev; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -99,19 +98,18 @@ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { - unsigned char ret; + int64_t prev; __asm__ __volatile__ ( - SMPLOCK "cmpxchgq %3,%2 \n\t" - "sete %0 \n\t" - : "=qm" (ret), "+a" (oldval), "+m" (*((volatile long*)addr)) - : "q"(newval) + SMPLOCK "cmpxchgq %1,%2 \n\t" + : "=a" (prev) + : "r"(newval), "m" (*addr), "0" (oldval) : "memory", "cc" ); - return (int)ret; + return prev; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -126,15 +124,13 @@ #define OPAL_HAVE_ATOMIC_SWAP_64 1 static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, - int32_t newval) + int32_t newval) { - int32_t oldval; - - __asm__ __volatile__("xchg %1, %0" : - "=r" (oldval), "=m" (*addr) : - "0" (newval), "m" (*addr) : - "memory"); - return oldval; + __asm__ __volatile__("xchg %1, %0" /* SMPLOCK is implicit */: + "+r" (newval), "+m" (*addr) : + : + "memory"); + return newval; /* beware this has been swaped in xchgl */ } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -144,13 +140,11 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, int64_t newval) { - int64_t oldval; - - __asm__ __volatile__("xchgq %1, %0" : - "=r" (oldval) : - "m" (*addr), "0" (newval) : - "memory"); - return oldval; + __asm__ __volatile__("xchgq %1, %0" /* SMPLOCK is implicit */: + "+r" (newval), "+m" (*addr) : + : + "memory"); + return newval; /* beware this has been swaped in xchgl */ } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -169,18 +163,17 @@ * @i: integer value to add * @v: pointer of type int * - * Atomically adds @i to @v. + * Atomically adds @value to @*v. */ -static inline int32_t opal_atomic_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_add_32(volatile int32_t* v, int32_t value) { - int ret = i; __asm__ __volatile__( SMPLOCK "xaddl %1,%0" - :"=m" (*v), "+r" (ret) - :"m" (*v) + :"+m" (*v), "+r" (value) + : :"memory", "cc" ); - return (ret+i); + return value; } #define OPAL_HAVE_ATOMIC_ADD_64 1 @@ -190,18 +183,17 @@ * @i: integer value to add * @v: pointer of type int * - * Atomically adds @i to @v. + * Atomically adds @value to @*v. */ -static inline int64_t opal_atomic_add_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_add_64(volatile int64_t* v, int64_t value) { - int64_t ret = i; __asm__ __volatile__( SMPLOCK "xaddq %1,%0" - :"=m" (*v), "+r" (ret) - :"m" (*v) + :"+m" (*v), "+r" (value) + : :"memory", "cc" ); - return (ret+i); + return value; } #define OPAL_HAVE_ATOMIC_SUB_32 1 @@ -211,7 +203,7 @@ * @i: integer value to subtract * @v: pointer of type int * - * Atomically subtracts @i from @v. + * Atomically subtracts @i from @*v. */ static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int i) { @@ -218,11 +210,11 @@ int ret = -i; __asm__ __volatile__( SMPLOCK "xaddl %1,%0" - :"=m" (*v), "+r" (ret) - :"m" (*v) + :"+m" (*v), "+r" (ret) + : :"memory", "cc" ); - return (ret-i); + return ret; } #define OPAL_HAVE_ATOMIC_SUB_64 1 @@ -232,7 +224,7 @@ * @i: integer value to subtract * @v: pointer of type int * - * Atomically subtracts @i from @v. + * Atomically subtracts @i from @*v. */ static inline int64_t opal_atomic_sub_64(volatile int64_t* v, int64_t i) { @@ -239,11 +231,11 @@ int64_t ret = -i; __asm__ __volatile__( SMPLOCK "xaddq %1,%0" - :"=m" (*v), "+r" (ret) - :"m" (*v) + :"+m" (*v), "+r" (ret) + : :"memory", "cc" ); - return (ret-i); + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ Index: opal/include/opal/sys/arm/atomic.h =================================================================== --- opal/include/opal/sys/arm/atomic.h (revision 32401) +++ opal/include/opal/sys/arm/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -113,8 +113,8 @@ #define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1 -static inline int opal_atomic_cmpset_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { int32_t ret, tmp; @@ -131,7 +131,7 @@ : "r" (addr), "r" (oldval), "r" (newval) : "cc", "memory"); - return (ret == oldval); + return ret; } /* these two functions aren't inlined in the non-gcc case because then @@ -139,10 +139,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { - int rc; + int32_t rc; rc = opal_atomic_cmpset_32(addr, oldval, newval); opal_atomic_rmb(); @@ -151,8 +151,8 @@ } -static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_32(addr, oldval, newval); @@ -161,8 +161,8 @@ #if (OPAL_ASM_SUPPORT_64BIT == 1) #define OPAL_HAVE_ATOMIC_CMPSET_64 1 -static inline int opal_atomic_cmpset_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { int64_t ret; int tmp; @@ -183,7 +183,7 @@ : "r" (addr), "r" (oldval), "r" (newval) : "cc", "memory"); - return (ret == oldval); + return ret; } /* these two functions aren't inlined in the non-gcc case because then @@ -191,10 +191,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int rc; + int64_t rc; rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); @@ -203,8 +203,8 @@ } -static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); @@ -212,46 +212,84 @@ #endif +#if OPAL_GCC_INLINE_ASSEMBLY +#define OPAL_HAVE_ATOMIC_SWAP_32 1 + +static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, + int32_t newval) +{ + int32_t res, tmp; + + __asm__ __volatile__( + "1: ldxr %w0, %2\n" + " stlxr %w1, %w3, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (res), "=&r" (tmp), "+Q" (*addr) + : "r" (newval) + : "memory"); + return res; +} + +#if (OPAL_ASM_SUPPORT_64BIT == 1) +#define OPAL_HAVE_ATOMIC_SWAP_64 1 + +static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, + int64_t newval) +{ + int64_t res, tmp; + + __asm__ __volatile__( + "1: ldrexd %0, %H0, [%3]\n" + " strexd %1, %4, %H4, [%3]\n" + " teq %1, #0\n" + " bne 1b" + : "=&r" (res), "=&r" (tmp), "+Qo" (*addr) + : "r" (addr), "r" (newval) + : "cc"); + return res; +} + +#endif /* OPAL_GCC_INLINE_ASSEMBLY */ +#endif /* (OPAL_ASM_SUPPORT_64BIT == 1) */ + #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc) +static inline int32_t opal_atomic_add_32(volatile int32_t* v, int32_t inc) { - int32_t t; - int tmp; + int32_t t, prev, tmp; __asm__ __volatile__( - "1: ldrex %0, [%2] \n" - " add %0, %0, %3 \n" - " strex %1, %0, [%2] \n" - " cmp %1, #0 \n" + "1: ldrex %0, [%4] \n" + " add %1, %0, %5 \n" + " strex %2, %1, [%4] \n" + " teq %2, #0 \n" " bne 1b \n" - : "=&r" (t), "=&r" (tmp) - : "r" (v), "r" (inc) + : "=&r" (prev), "=&r" (tmp), "=&r" (t), "+Qo" (*v) + : "r" (v), "Ir" (inc) : "cc", "memory"); - return t; + return prev; } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec) +static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int32_t dec) { - int32_t t; - int tmp; + int32_t t, prev, tmp; __asm__ __volatile__( - "1: ldrex %0, [%2] \n" - " sub %0, %0, %3 \n" - " strex %1, %0, [%2] \n" - " cmp %1, #0 \n" + "1: ldrex %0, [%4] \n" + " sub %1, %0, %5 \n" + " strex %2, %1, [%4] \n" + " teq %2, #0 \n" " bne 1b \n" - : "=&r" (t), "=&r" (tmp) - : "r" (v), "r" (dec) + : "=&r" (prev), "=&r" (tmp), "=&r" (t), "+Qo" (*v) + : "r" (v), "Ir" (dec) : "cc", "memory"); - return t; + return prev; } #else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */ @@ -280,4 +318,47 @@ #endif +#if (OPAL_ASM_SUPPORT_64BIT == 1) + +#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1 +static inline int64_t opal_atomic_add_64(volatile int64_t* v, int64_t inc) +{ + int64_t result; + int tmp; + + __asm__ __volatile__( + "1: ldrexd %0, %H0, [%3]\n" + " adds %Q0, %Q0, %Q4\n" + " adc %R0, %R0, %R4\n" + " strexd %1, %0, %H0, [%3]\n" + " teq %1, #0\n" + " bne 1b" + : "=&r" (result), "=&r" (tmp), "+Qo" (*v) + : "r" (v), "r" (i) + : "cc"); + + return result; +} + +#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1 +static inline int64_t opal_atomic_sub_64(volatile int64_t* v, int64_t dec) +{ + int64_t result; + int tmp; + + __asm__ __volatile__( + "1: ldrexd %0, %H0, [%3]\n" + " subs %Q0, %Q0, %Q4\n" + " sbc %R0, %R0, %R4\n" + " strexd %1, %0, %H0, [%3]\n" + " teq %1, #0\n" + " bne 1b" + : "=&r" (result), "=&r" (tmp), "+Qo" (*v) + : "r" (v), "r" (i) + : "cc"); + + return result; +} +#endif /* (OPAL_ASM_SUPPORT_64BIT == 1) */ + #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ Index: opal/include/opal/sys/atomic.h =================================================================== --- opal/include/opal/sys/atomic.h (revision 32401) +++ opal/include/opal/sys/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -103,6 +103,14 @@ }; typedef struct opal_atomic_lock_t opal_atomic_lock_t; +/** + * Enumeration of lock states + */ +enum { + OPAL_ATOMIC_UNLOCKED = 0, + OPAL_ATOMIC_LOCKED = 1 +}; + /********************************************************************** * * Set or unset these macros in the architecture-specific atomic.h @@ -257,15 +265,7 @@ #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) -/** - * Enumeration of lock states - */ -enum { - OPAL_ATOMIC_UNLOCKED = 0, - OPAL_ATOMIC_LOCKED = 1 -}; - /** * Initialize a lock to value * @@ -334,20 +334,20 @@ #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 static inline #endif -int opal_atomic_cmpset_32(volatile int32_t *addr, int32_t oldval, - int32_t newval); +int32_t opal_atomic_cmpset_32(volatile int32_t *addr, int32_t oldval, + int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 static inline #endif -int opal_atomic_cmpset_acq_32(volatile int32_t *addr, int32_t oldval, - int32_t newval); +int32_t opal_atomic_cmpset_acq_32(volatile int32_t *addr, int32_t oldval, + int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 static inline #endif -int opal_atomic_cmpset_rel_32(volatile int32_t *addr, int32_t oldval, - int32_t newval); +int32_t opal_atomic_cmpset_rel_32(volatile int32_t *addr, int32_t oldval, + int32_t newval); #endif @@ -359,20 +359,20 @@ #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 static inline #endif -int opal_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval, - int64_t newval); +int64_t opal_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval, + int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 static inline #endif -int opal_atomic_cmpset_acq_64(volatile int64_t *addr, int64_t oldval, - int64_t newval); +int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, int64_t oldval, + int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 static inline #endif -int opal_atomic_cmpset_rel_64(volatile int64_t *addr, int64_t oldval, - int64_t newval); +int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, int64_t oldval, + int64_t newval); #endif @@ -440,7 +440,7 @@ /* provide a size_t add/subtract. When in debug mode, make it an * inline function so that we don't have any casts in the - * interface and can catch type errors. When not in debug mode, + * interface and can catch type errors. When not in debug mode, * just make it a macro, so that there's no performance penalty */ #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG @@ -481,24 +481,24 @@ #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) /* these are always done with inline functions, so always mark as static inline */ -static inline int opal_atomic_cmpset_xx(volatile void* addr, int64_t oldval, - int64_t newval, size_t length); -static inline int opal_atomic_cmpset_acq_xx(volatile void* addr, - int64_t oldval, int64_t newval, - size_t length); -static inline int opal_atomic_cmpset_rel_xx(volatile void* addr, - int64_t oldval, int64_t newval, - size_t length); +static inline int64_t opal_atomic_cmpset_xx(volatile void* addr, int64_t oldval, + int64_t newval, size_t length); +static inline int64_t opal_atomic_cmpset_acq_xx(volatile void* addr, + int64_t oldval, int64_t newval, + size_t length); +static inline int64_t opal_atomic_cmpset_rel_xx(volatile void* addr, + int64_t oldval, int64_t newval, + size_t length); -static inline int opal_atomic_cmpset_ptr(volatile void* addr, - void* oldval, - void* newval); -static inline int opal_atomic_cmpset_acq_ptr(volatile void* addr, - void* oldval, - void* newval); -static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr, - void* oldval, - void* newval); +static inline void* opal_atomic_cmpset_ptr(volatile void* addr, + void* oldval, + void* newval); +static inline void* opal_atomic_cmpset_acq_ptr(volatile void* addr, + void* oldval, + void* newval); +static inline void* opal_atomic_cmpset_rel_ptr(volatile void* addr, + void* oldval, + void* newval); /** * Atomic compare and set of pointer with relaxed semantics. This @@ -556,16 +556,16 @@ #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) -static inline void opal_atomic_add_xx(volatile void* addr, - int32_t value, size_t length); -static inline void opal_atomic_sub_xx(volatile void* addr, - int32_t value, size_t length); +static inline int64_t opal_atomic_add_xx(volatile void* addr, + int32_t value, size_t length); +static inline int64_t opal_atomic_sub_xx(volatile void* addr, + int32_t value, size_t length); #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 -static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta ); -static inline int32_t opal_atomic_sub_ptr( volatile void* addr, void* delta ); +static inline void* opal_atomic_add_ptr( volatile void* addr, void* delta ); +static inline void* opal_atomic_sub_ptr( volatile void* addr, void* delta ); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 -static inline int64_t opal_atomic_add_ptr( volatile void* addr, void* delta ); -static inline int64_t opal_atomic_sub_ptr( volatile void* addr, void* delta ); +static inline void* opal_atomic_add_ptr( volatile void* addr, void* delta ); +static inline void* opal_atomic_sub_ptr( volatile void* addr, void* delta ); #else #error Atomic arithmetic on pointers not supported #endif Index: opal/include/opal/sys/atomic_impl.h =================================================================== --- opal/include/opal/sys/atomic_impl.h (revision 32401) +++ opal/include/opal/sys/atomic_impl.h (working copy) @@ -22,14 +22,15 @@ #ifdef HAVE_STDLIB_H #include #endif +#include /********************************************************************** * - * Atomic math operations + * Atomic SWAP operations * * All the architectures provide a compare_and_set atomic operations. If - * they dont provide atomic additions and/or substractions then we can - * define these operations using the atomic compare_and_set. + * they dont provide atomic swap then we can define these operations + * using the atomic compare_and_set. * * Some architectures do not provide support for the 64 bits * atomic operations. Until we find a better solution let's just @@ -46,12 +47,28 @@ int32_t old; do { old = *addr; - } while (0 == opal_atomic_cmpset_32(addr, old, newval)); + } while (old != opal_atomic_cmpset_32(addr, old, newval)); return old; } #endif /* OPAL_HAVE_ATOMIC_SWAP_32 */ +/********************************************************************** + * + * Atomic math operations + * + * All the architectures provide a compare_and_set atomic operations. If + * they dont provide atomic additions and/or substractions then we can + * define these operations using the atomic compare_and_set. + * + * Some architectures do not provide support for the 64 bits + * atomic operations. Until we find a better solution let's just + * undefine all those functions if there is no 64 bit cmpset + * + * The current implementation correspond to a fetch_and_ meaning that + * the old value before the operation is returned in all cases. + * + *********************************************************************/ #if !defined(OPAL_HAVE_ATOMIC_ADD_32) #define OPAL_HAVE_ATOMIC_ADD_32 1 static inline int32_t @@ -58,11 +75,11 @@ opal_atomic_add_32(volatile int32_t *addr, int delta) { int32_t oldval; - + do { oldval = *addr; - } while (0 == opal_atomic_cmpset_32(addr, oldval, oldval + delta)); - return (oldval + delta); + } while (oldval != opal_atomic_cmpset_32(addr, oldval, oldval + delta)); + return oldval; } #endif /* OPAL_HAVE_ATOMIC_ADD_32 */ @@ -73,11 +90,11 @@ opal_atomic_sub_32(volatile int32_t *addr, int delta) { int32_t oldval; - + do { oldval = *addr; - } while (0 == opal_atomic_cmpset_32(addr, oldval, oldval - delta)); - return (oldval - delta); + } while (oldval != opal_atomic_cmpset_32(addr, oldval, oldval - delta)); + return oldval; } #endif /* OPAL_HAVE_ATOMIC_SUB_32 */ @@ -94,7 +111,7 @@ int64_t old; do { old = *addr; - } while (0 == opal_atomic_cmpset_64(addr, old, newval)); + } while (old != opal_atomic_cmpset_64(addr, old, newval)); return old; } #endif /* OPAL_HAVE_ATOMIC_SWAP_32 */ @@ -105,11 +122,11 @@ opal_atomic_add_64(volatile int64_t *addr, int64_t delta) { int64_t oldval; - + do { oldval = *addr; - } while (0 == opal_atomic_cmpset_64(addr, oldval, oldval + delta)); - return (oldval + delta); + } while (oldval != opal_atomic_cmpset_64(addr, oldval, oldval + delta)); + return oldval; } #endif /* OPAL_HAVE_ATOMIC_ADD_64 */ @@ -123,8 +140,8 @@ do { oldval = *addr; - } while (0 == opal_atomic_cmpset_64(addr, oldval, oldval - delta)); - return (oldval - delta); + } while (oldval != opal_atomic_cmpset_64(addr, oldval, oldval - delta)); + return oldval; } #endif /* OPAL_HAVE_ATOMIC_SUB_64 */ @@ -143,7 +160,7 @@ #if (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) -static inline int +static inline int64_t opal_atomic_cmpset_xx(volatile void* addr, int64_t oldval, int64_t newval, size_t length) { @@ -166,7 +183,7 @@ } -static inline int +static inline int64_t opal_atomic_cmpset_acq_xx(volatile void* addr, int64_t oldval, int64_t newval, size_t length) { @@ -189,7 +206,7 @@ } -static inline int +static inline int64_t opal_atomic_cmpset_rel_xx(volatile void* addr, int64_t oldval, int64_t newval, size_t length) { @@ -212,17 +229,17 @@ } -static inline int +static inline void* opal_atomic_cmpset_ptr(volatile void* addr, void* oldval, void* newval) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 - return opal_atomic_cmpset_32((int32_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_32((int32_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 - return opal_atomic_cmpset_64((int64_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_64((int64_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #else abort(); #endif @@ -229,17 +246,17 @@ } -static inline int +static inline void* opal_atomic_cmpset_acq_ptr(volatile void* addr, void* oldval, void* newval) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 - return opal_atomic_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 - return opal_atomic_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #else abort(); #endif @@ -246,16 +263,16 @@ } -static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr, - void* oldval, - void* newval) +static inline void* opal_atomic_cmpset_rel_ptr(volatile void* addr, + void* oldval, + void* newval) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 - return opal_atomic_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 - return opal_atomic_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval, - (unsigned long) newval); + return (void*)opal_atomic_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval, + (unsigned long) newval); #else abort(); #endif @@ -276,20 +293,18 @@ #if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 -static inline void +static inline int64_t opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_ADD_32 case 4: - opal_atomic_add_32( (volatile int32_t*)addr, (int32_t)value ); - break; + return opal_atomic_add_32( (volatile int32_t*)addr, (int32_t)value ); #endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #if OPAL_HAVE_ATOMIC_ADD_64 case 8: - opal_atomic_add_64( (volatile int64_t*)addr, (int64_t)value ); - break; + return opal_atomic_add_64( (volatile int64_t*)addr, (int64_t)value ); #endif /* OPAL_HAVE_ATOMIC_ADD_64 */ default: /* This should never happen, so deliberately abort (hopefully @@ -299,20 +314,18 @@ } -static inline void +static inline int64_t opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_SUB_32 case 4: - opal_atomic_sub_32( (volatile int32_t*)addr, (int32_t)value ); - break; + return opal_atomic_sub_32( (volatile int32_t*)addr, (int32_t)value ); #endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #if OPAL_HAVE_ATOMIC_SUB_64 case 8: - opal_atomic_sub_64( (volatile int64_t*)addr, (int64_t)value ); - break; + return opal_atomic_sub_64( (volatile int64_t*)addr, (int64_t)value ); #endif /* OPAL_HAVE_ATOMIC_SUB_64 */ default: /* This should never happen, so deliberately abort (hopefully @@ -322,44 +335,44 @@ } #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 -static inline int32_t opal_atomic_add_ptr( volatile void* addr, - void* delta ) +static inline void* opal_atomic_add_ptr( volatile void* addr, + void* delta ) { - return opal_atomic_add_32((int32_t*) addr, (unsigned long) delta); + return (void*)opal_atomic_add_32((int32_t*) addr, (unsigned long) delta); } #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 -static inline int64_t opal_atomic_add_ptr( volatile void* addr, - void* delta ) +static inline void* opal_atomic_add_ptr( volatile void* addr, + void* delta ) { - return opal_atomic_add_64((int64_t*) addr, (unsigned long) delta); + return (void*)opal_atomic_add_64((int64_t*) addr, (unsigned long) delta); } #else -static inline int32_t opal_atomic_add_ptr( volatile void* addr, +static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta ) { abort(); - return 0; + return (void*)0; } #endif #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 -static inline int32_t opal_atomic_sub_ptr( volatile void* addr, - void* delta ) +static inline void* opal_atomic_sub_ptr( volatile void* addr, + void* delta ) { - return opal_atomic_sub_32((int32_t*) addr, (unsigned long) delta); + return (void*)opal_atomic_sub_32((int32_t*) addr, (unsigned long) delta); } #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 -static inline int64_t opal_atomic_sub_ptr( volatile void* addr, - void* delta ) +static inline void* opal_atomic_sub_ptr( volatile void* addr, + void* delta ) { - return opal_atomic_sub_64((int64_t*) addr, (unsigned long) delta); + return (void*)opal_atomic_sub_64((int64_t*) addr, (unsigned long) delta); } #else -static inline int32_t opal_atomic_sub_ptr( volatile void* addr, - void* delta ) +static inline void* opal_atomic_sub_ptr( volatile void* addr, + void* delta ) { abort(); - return 0; + return (void*)0; } #endif @@ -372,13 +385,14 @@ *********************************************************************/ #ifdef OPAL_NEED_INLINE_ATOMIC_SPINLOCKS -/* - * Lock initialization function. It set the lock to UNLOCKED. +/* + * Lock initialization function. It set the lock to @value. */ static inline void opal_atomic_init( opal_atomic_lock_t* lock, int32_t value ) { - lock->u.lock = value; + assert( (OPAL_ATOMIC_UNLOCKED == value) || (OPAL_ATOMIC_LOCKED == value) ); + lock->u.lock = value; } @@ -387,7 +401,7 @@ { int ret = opal_atomic_cmpset_acq_32( &(lock->u.lock), OPAL_ATOMIC_UNLOCKED, OPAL_ATOMIC_LOCKED); - return (ret == 0) ? 1 : 0; + return (ret != OPAL_ATOMIC_UNLOCKED) ? 1 : 0; } @@ -394,8 +408,9 @@ static inline void opal_atomic_lock(opal_atomic_lock_t *lock) { - while( !opal_atomic_cmpset_acq_32( &(lock->u.lock), - OPAL_ATOMIC_UNLOCKED, OPAL_ATOMIC_LOCKED) ) { + while( OPAL_ATOMIC_UNLOCKED != + opal_atomic_cmpset_acq_32( &(lock->u.lock), + OPAL_ATOMIC_UNLOCKED, OPAL_ATOMIC_LOCKED) ) { while (lock->u.lock == OPAL_ATOMIC_LOCKED) { /* spin */ ; } @@ -407,7 +422,7 @@ opal_atomic_unlock(opal_atomic_lock_t *lock) { opal_atomic_wmb(); - lock->u.lock=OPAL_ATOMIC_UNLOCKED; + lock->u.lock = OPAL_ATOMIC_UNLOCKED; } #endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */ Index: opal/include/opal/sys/ia32/atomic.h =================================================================== --- opal/include/opal/sys/ia32/atomic.h (revision 32401) +++ opal/include/opal/sys/ia32/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -85,19 +85,18 @@ *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_32(volatile int32_t *addr, - int32_t oldval, - int32_t newval) +static inline int32_t opal_atomic_cmpset_32(volatile int32_t *addr, + int32_t oldval, + int32_t newval) { - unsigned char ret; + int32_t ret; __asm__ __volatile__ ( SMPLOCK "cmpxchgl %3,%2 \n\t" - "sete %0 \n\t" - : "=qm" (ret), "+a" (oldval), "+m" (*addr) + : "=a" (ret), "+a" (oldval), "+m" (*addr) : "q"(newval) : "memory", "cc"); - - return (int)ret; + + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -162,16 +161,15 @@ * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_add_32(volatile int32_t* v, int32_t i) { - int ret = i; __asm__ __volatile__( SMPLOCK "xaddl %1,%0" - :"=m" (*v), "+r" (ret) + :"=m" (*v), "+r" (i) :"m" (*v) :"memory", "cc" ); - return (ret+i); + return i; } @@ -182,9 +180,9 @@ * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int32_t i) { - int ret = -i; + int32_t ret = -i; __asm__ __volatile__( SMPLOCK "xaddl %1,%0" :"=m" (*v), "+r" (ret) @@ -191,7 +189,7 @@ :"m" (*v) :"memory", "cc" ); - return (ret-i); + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ Index: opal/include/opal/sys/ia64/atomic.h =================================================================== --- opal/include/opal/sys/ia64/atomic.h (revision 32401) +++ opal/include/opal/sys/ia64/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -78,8 +78,8 @@ ia64_intri_res; \ }) -static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { int64_t ret; @@ -87,12 +87,12 @@ __asm__ __volatile__ ("cmpxchg4.acq %0=[%1],%2,ar.ccv": "=r"(ret) : "r"(addr), "r"(newval) : "memory"); - return ((int32_t)ret == oldval); + return (int32_t)ret; } -static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { int64_t ret; @@ -100,7 +100,7 @@ __asm__ __volatile__ ("cmpxchg4.rel %0=[%1],%2,ar.ccv": "=r"(ret) : "r"(addr), "r"(newval) : "memory"); - return ((int32_t)ret == oldval); + return (int32_t)ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ @@ -110,8 +110,8 @@ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { int64_t ret; @@ -119,12 +119,12 @@ __asm__ __volatile__ ("cmpxchg8.acq %0=[%1],%2,ar.ccv": "=r"(ret) : "r"(addr), "r"(newval) : "memory"); - return (ret == oldval); + return ret; } -static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { int64_t ret; @@ -132,7 +132,7 @@ __asm__ __volatile__ ("cmpxchg8.rel %0=[%1],%2,ar.ccv": "=r"(ret) : "r"(addr), "r"(newval) : "memory"); - return (ret == oldval); + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ Index: opal/include/opal/sys/mips/atomic.h =================================================================== --- opal/include/opal/sys/mips/atomic.h (revision 32401) +++ opal/include/opal/sys/mips/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -94,8 +94,8 @@ *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { int32_t ret; @@ -114,13 +114,13 @@ #endif /* note: ret will be 0 if failed, 1 if succeeded */ "beqz $1, 1b \n" /* if 0 jump back to 1b */ - "nop \n" /* fill delay slots */ + "nop \n" /* fill delay slots */ "2: \n" ".set reorder \n" : "=&r"(ret), "=m"(*addr) : "m"(*addr), "r"(oldval), "r"(newval) : "cc", "memory"); - return (ret == oldval); + return ret; } @@ -129,10 +129,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { - int rc; + int32_t rc; rc = opal_atomic_cmpset_32(addr, oldval, newval); opal_atomic_rmb(); @@ -141,8 +141,8 @@ } -static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_32(addr, oldval, newval); @@ -149,8 +149,8 @@ } #ifdef OPAL_HAVE_ATOMIC_CMPSET_64 -static inline int opal_atomic_cmpset_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { int64_t ret; @@ -163,7 +163,7 @@ "scd $1, %2 \n\t" /* store tmp in *addr */ /* note: ret will be 0 if failed, 1 if succeeded */ "beqz $1, 1b \n\t" /* if 0 jump back to 1b */ - "nop \n\t" /* fill delay slot */ + "nop \n\t" /* fill delay slot */ "2: \n\t" ".set reorder \n" : "=&r" (ret), "=m" (*addr) @@ -170,7 +170,7 @@ : "m" (*addr), "r" (oldval), "r" (newval) : "cc", "memory"); - return (ret == oldval); + return ret; } @@ -179,10 +179,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int rc; + int64_t rc; rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); @@ -191,8 +191,8 @@ } -static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); Index: opal/include/opal/sys/osx/atomic.h =================================================================== --- opal/include/opal/sys/osx/atomic.h (revision 32401) +++ opal/include/opal/sys/osx/atomic.h (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -27,7 +27,7 @@ #if OPAL_WANT_SMP_LOCKS -#define MB() OSMemoryBarrier +#define MB() OSMemoryBarrier() #else #define MB() #endif @@ -76,13 +76,13 @@ * Atomic math operations * *********************************************************************/ -static inline int opal_atomic_cmpset_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicCompareAndSwap32Barrier(oldval, newval, addr); + return (OSAtomicCompareAndSwap32Barrier(oldval, newval, addr)) ? oldval : *addr; #else - return OSAtomicCompareAndSwap32(oldval, newval, addr); + return (OSAtomicCompareAndSwap32(oldval, newval, addr)) ? oldval : *addr; #endif } @@ -90,13 +90,13 @@ #define opal_atomic_cmpset_rel_32 opal_atomic_cmpset_32 -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicCompareAndSwap64Barrier(oldval, newval, addr); + return (OSAtomicCompareAndSwap64Barrier(oldval, newval, addr)) ? oldval : *addr; #else - return OSAtomicCompareAndSwap64(oldval, newval, addr); + return (OSAtomicCompareAndSwap64(oldval, newval, addr)) ? oldval : *addr; #endif } @@ -110,12 +110,12 @@ * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_add_32(volatile int32_t* v, int32_t i) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicAdd32Barrier (i, v); + return OSAtomicAdd32Barrier (i, v) - i; #else - return OSAtomicAdd32 (i, v); + return OSAtomicAdd32 (i, v) - i; #endif } @@ -129,9 +129,9 @@ static inline int64_t opal_atomic_add_64(volatile int64_t* v, int64_t i) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicAdd64Barrier (i, v); + return OSAtomicAdd64Barrier (i, v) - i; #else - return OSAtomicAdd64 (i, v); + return OSAtomicAdd64 (i, v) - i; #endif } @@ -142,12 +142,12 @@ * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int32_t i) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicAdd32Barrier (-i, v); + return OSAtomicAdd32Barrier (-i, v) + i; #else - return OSAtomicAdd32 (-i, v); + return OSAtomicAdd32 (-i, v) + i; #endif } @@ -161,9 +161,9 @@ static inline int64_t opal_atomic_sub_64(volatile int64_t* v, int64_t i) { #if OPAL_WANT_SMP_LOCKS - return OSAtomicAdd64Barrier (-i, v); + return OSAtomicAdd64Barrier (-i, v) + i; #else - return OSAtomicAdd64 (-i, v); + return OSAtomicAdd64 (-i, v) + i; #endif } @@ -170,7 +170,7 @@ static inline void opal_atomic_init(opal_atomic_lock_t* lock, int32_t value) { lock->u.lock = OS_SPINLOCK_INIT; - if (value) { + if (OPAL_ATOMIC_LOCKED == value) { OSSpinLockLock (&lock->u.lock); } } Index: opal/include/opal/sys/powerpc/atomic.h =================================================================== --- opal/include/opal/sys/powerpc/atomic.h (revision 32401) +++ opal/include/opal/sys/powerpc/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -126,8 +126,8 @@ #define OPAL_ASM_ADDR(a) (a) #endif -static inline int opal_atomic_cmpset_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { int32_t ret; @@ -138,11 +138,11 @@ " stwcx. %4, 0, %2 \n\t" " bne- 1b \n\t" "2:" - : "=&r" (ret), "=m" (*addr) - : "r" OPAL_ASM_ADDR(addr), "r" (oldval), "r" (newval), "m" (*addr) + : "=&r" (ret), "+m" (*addr) + : "r" OPAL_ASM_ADDR(addr), "r" (oldval), "r" (newval) : "cc", "memory"); - return (ret == oldval); + return ret; } /* these two functions aren't inlined in the non-gcc case because then @@ -150,10 +150,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { - int rc; + int32_t rc; rc = opal_atomic_cmpset_32(addr, oldval, newval); opal_atomic_rmb(); @@ -162,8 +162,8 @@ } -static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32(volatile int32_t *addr, + int32_t oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_32(addr, oldval, newval); @@ -175,8 +175,8 @@ #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { int64_t ret; @@ -187,11 +187,11 @@ " stdcx. %4, 0, %2 \n\t" " bne- 1b \n\t" "2:" - : "=&r" (ret), "=m" (*addr) - : "r" (addr), "r" (oldval), "r" (newval), "m" (*addr) + : "=&r" (ret), "+m" (*addr) + : "r" (addr), "r" (oldval), "r" (newval) : "cc", "memory"); - - return (ret == oldval); + + return ret; } /* these two functions aren't inlined in the non-gcc case because then @@ -199,10 +199,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int rc; + int64_t rc; rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); @@ -211,8 +211,8 @@ } -static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); @@ -229,10 +229,10 @@ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int ret; + int64_t ret; /* * We force oldval and newval into memory because PPC doesn't @@ -261,7 +261,7 @@ : "r"OPAL_ASM_ADDR(addr), "m"(oldval), "m"(newval) : "r4", "r5", "r9", "cc", "memory"); - + return ret; } @@ -270,10 +270,10 @@ atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline int opal_atomic_cmpset_acq_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int rc; + int64_t rc; rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); @@ -282,8 +282,8 @@ } -static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64(volatile int64_t *addr, + int64_t oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); @@ -296,37 +296,37 @@ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int32_t opal_atomic_add_32(volatile int32_t* v, int inc) +static inline int32_t opal_atomic_add_32(volatile int32_t* v, int32_t inc) { - int32_t t; + int32_t prev, t; __asm__ __volatile__( - "1: lwarx %0, 0, %3 \n\t" - " add %0, %2, %0 \n\t" - " stwcx. %0, 0, %3 \n\t" + "1: lwarx %0, 0, %4 \n\t" + " add %1, %3, %0 \n\t" + " stwcx. %1, 0, %4 \n\t" " bne- 1b \n\t" - : "=&r" (t), "=m" (*v) - : "r" (inc), "r" OPAL_ASM_ADDR(v), "m" (*v) + : "=&r" (prev), "=&r" (t), "+m" (*v) + : "r" (inc), "r" OPAL_ASM_ADDR(v) : "cc"); - return t; + return prev; } -static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec) +static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int32_t dec) { - int32_t t; + int32_t prev, t; __asm__ __volatile__( - "1: lwarx %0,0,%3 \n\t" - " subf %0,%2,%0 \n\t" - " stwcx. %0,0,%3 \n\t" + "1: lwarx %0,0,%4 \n\t" + " subf %1,%3,%0 \n\t" + " stwcx. %1,0,%4 \n\t" " bne- 1b \n\t" - : "=&r" (t), "=m" (*v) - : "r" (dec), "r" OPAL_ASM_ADDR(v), "m" (*v) + : "=&r" (prev), "=&r" (t), "+m" (*v) + : "r" (dec), "r" OPAL_ASM_ADDR(v) : "cc"); - return t; + return prev; } Index: opal/include/opal/sys/sparcv9/atomic.h =================================================================== --- opal/include/opal/sys/sparcv9/atomic.h (revision 32401) +++ opal/include/opal/sys/sparcv9/atomic.h (working copy) @@ -2,10 +2,10 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. @@ -44,6 +44,7 @@ #define OPAL_HAVE_ATOMIC_CMPSET_64 1 +#define OPAL_HAVE_ATOMIC_SWAP_64 0 /* updated later */ /********************************************************************** * @@ -79,8 +80,8 @@ *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline int opal_atomic_cmpset_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -95,12 +96,12 @@ __asm__ __volatile__("casa [%1] " ASI_P ", %2, %0" : "+r" (ret) : "r" (addr), "r" (oldval)); - return (ret == oldval); + return ret; } -static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_acq_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { int rc; @@ -111,8 +112,8 @@ } -static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) +static inline int32_t opal_atomic_cmpset_rel_32( volatile int32_t *addr, + int32_t oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_32(addr, oldval, newval); @@ -121,8 +122,8 @@ #if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -136,13 +137,13 @@ __asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0" : "+r" (ret) : "r" (addr), "r" (oldval)); - return (ret == oldval); + return ret; } #else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -152,7 +153,7 @@ * reg(rd) = *(reg(rs1)) * */ - long long ret = newval; + int64_t ret = newval; __asm__ __volatile__( "ldx %0, %%g1 \n\t" /* g1 = ret */ @@ -164,25 +165,24 @@ : "%g1", "%g2" ); - return (ret == oldval); + return ret; } #endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_acq_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { - int rc; - + int64_t rc; + rc = opal_atomic_cmpset_64(addr, oldval, newval); opal_atomic_rmb(); - + return rc; } - -static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) +static inline int64_t opal_atomic_cmpset_rel_64( volatile int64_t *addr, + int64_t oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_cmpset_64(addr, oldval, newval); @@ -190,5 +190,42 @@ #endif /* OPAL_GCC_INLINE_ASSEMBLY */ +#if OPAL_GCC_INLINE_ASSEMBLY +#define OPAL_HAVE_ATOMIC_SWAP_32 1 + +static inline int32_t +opal_atomic_swap_32( volatile int32_t *addr, + int32_t newval) +{ + int32_t prev; + __asm__ __volatile__( + "swap %0, %1" + : "=m" (*addr), "=r" (prev) + : "1" (newval), "m" (*addr) + : "memory"); + return prev; +} + +#if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 + +#undef OPAL_HAVE_ATOMIC_SWAP_64 +#define OPAL_HAVE_ATOMIC_SWAP_64 1 + +static inline int64_t +opal_atomic_swap_64( volatile int64_t *addr, + int64_t newval) +{ + int64_t prev; + __asm__ __volatile__( + "swap %0, %1" + : "=m" (*addr), "=r" (prev) + : "1" (newval), "m" (*addr) + : "memory"); + return prev; +} +#endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ + +#endif /* OPAL_GCC_INLINE_ASSEMBLY */ + #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ Index: opal/include/opal/sys/sync_builtin/atomic.h =================================================================== --- opal/include/opal/sys/sync_builtin/atomic.h (revision 32401) +++ opal/include/opal/sys/sync_builtin/atomic.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -28,7 +28,7 @@ #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 static inline void opal_atomic_mb(void) -{ +{ __sync_synchronize(); } @@ -38,7 +38,7 @@ } static inline void opal_atomic_wmb(void) -{ +{ __sync_synchronize(); } @@ -55,68 +55,52 @@ *********************************************************************/ #define OPAL_HAVE_ATOMIC_CMPSET_32 1 -static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr, + +static inline int32_t opal_atomic_cmpset_32( volatile int32_t *addr, int32_t oldval, int32_t newval) { - return __sync_bool_compare_and_swap(addr, oldval, newval); + return __sync_val_compare_and_swap(addr, oldval, newval); } +#define opal_atomic_cmpset_acq_32 opal_atomic_cmpset_32 +#define opal_atomic_cmpset_rel_32 opal_atomic_cmpset_32 -static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) -{ - return __sync_bool_compare_and_swap(addr, oldval, newval);} - -static inline int opal_atomic_cmpset_32( volatile int32_t *addr, - int32_t oldval, int32_t newval) -{ - return __sync_bool_compare_and_swap(addr, oldval, newval); -} - #define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1 static inline int32_t opal_atomic_add_32(volatile int32_t *addr, int32_t delta) { - return __sync_add_and_fetch(addr, delta); + return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_SUB_32 1 static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta) { - return __sync_sub_and_fetch(addr, delta); + return __sync_fetch_and_sub(addr, delta); } #define OPAL_HAVE_ATOMIC_CMPSET_64 1 -static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr, + +static inline int64_t opal_atomic_cmpset_64( volatile int64_t *addr, int64_t oldval, int64_t newval) { - return __sync_bool_compare_and_swap(addr, oldval, newval); + return __sync_val_compare_and_swap(addr, oldval, newval); } -static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) -{ - return __sync_bool_compare_and_swap(addr, oldval, newval);} +#define opal_atomic_cmpset_acq_64 opal_atomic_cmpset_64 +#define opal_atomic_cmpset_rel_64 opal_atomic_cmpset_64 - -static inline int opal_atomic_cmpset_64( volatile int64_t *addr, - int64_t oldval, int64_t newval) -{ - return __sync_bool_compare_and_swap(addr, oldval, newval); -} - #define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1 static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta) { - return __sync_add_and_fetch(addr, delta); + return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_SUB_64 1 static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta) { - return __sync_sub_and_fetch(addr, delta); + return __sync_fetch_and_sub(addr, delta); } #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ Index: opal/mca/btl/openib/btl_openib_endpoint.c =================================================================== --- opal/mca/btl/openib/btl_openib_endpoint.c (revision 32401) +++ opal/mca/btl/openib/btl_openib_endpoint.c (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2007 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -411,8 +411,8 @@ * was not in "connect" or "bad" flow (failed to allocate memory) * and changed the pointer back to NULL */ - if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL, - (void*)1)) { + if(NULL != opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL, + (void*)1)) { if ((void*)1 != endpoint->eager_rdma_local.base.pval && NULL != endpoint->eager_rdma_local.base.pval) { endpoint->endpoint_btl->super.btl_mpool->mpool_free(endpoint->endpoint_btl->super.btl_mpool, @@ -927,8 +927,8 @@ /* Set local rdma pointer to 1 temporarily so other threads will not try * to enter the function */ - if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL, - (void*)1)) + if(NULL != opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL, + (void*)1)) return; headers_buf = (mca_btl_openib_recv_frag_t*) @@ -998,8 +998,8 @@ endpoint->eager_rdma_local.rd_win?endpoint->eager_rdma_local.rd_win:1; /* set local rdma pointer to real value */ - opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, (void*)1, - buf); + (void)opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, (void*)1, + buf); if(mca_btl_openib_endpoint_send_eager_rdma(endpoint) == OPAL_SUCCESS) { mca_btl_openib_device_t *device = endpoint->endpoint_btl->device; @@ -1008,7 +1008,7 @@ assert(((opal_object_t*)endpoint)->obj_reference_count == 2); do { p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count]; - } while(!opal_atomic_cmpset_ptr(p, NULL, endpoint)); + } while(NULL != opal_atomic_cmpset_ptr(p, NULL, endpoint)); OPAL_THREAD_ADD32(&openib_btl->eager_rdma_channels, 1); /* from this point progress function starts to poll new buffer */ @@ -1022,8 +1022,8 @@ free(headers_buf); unlock_rdma_local: /* set local rdma pointer back to zero. Will retry later */ - opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, - endpoint->eager_rdma_local.base.pval, NULL); + (void)opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, + endpoint->eager_rdma_local.base.pval, NULL); endpoint->eager_rdma_local.frags = NULL; } Index: opal/mca/btl/openib/connect/btl_openib_connect_rdmacm.c =================================================================== --- opal/mca/btl/openib/connect/btl_openib_connect_rdmacm.c (revision 32401) +++ opal/mca/btl/openib/connect/btl_openib_connect_rdmacm.c (working copy) @@ -7,6 +7,9 @@ * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013 Intel, Inc. All rights reserved + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * * $COPYRIGHT$ * @@ -1195,7 +1198,7 @@ OBJ_RELEASE(context); /* Tell the main thread that we're done */ - opal_atomic_add(&disconnect_callbacks, 1); + (void)opal_atomic_add(&disconnect_callbacks, 1); OPAL_OUTPUT((-1, "SERVICE Service thread disconnect on ID %p done; count=%d", (void*) tmp, disconnect_callbacks)); return NULL; Index: opal/mca/btl/sm/btl_sm.c =================================================================== --- opal/mca/btl/sm/btl_sm.c (revision 32401) +++ opal/mca/btl/sm/btl_sm.c (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, @@ -569,7 +569,7 @@ /* Sync with other local procs. Force the FIFO initialization to always * happens before the readers access it. */ - opal_atomic_add_32(&mca_btl_sm_component.sm_seg->module_seg->seg_inited, 1); + (void)opal_atomic_add_32(&mca_btl_sm_component.sm_seg->module_seg->seg_inited, 1); while( n_local_procs > mca_btl_sm_component.sm_seg->module_seg->seg_inited) { opal_progress(); Index: opal/mca/btl/smcuda/btl_smcuda.c =================================================================== --- opal/mca/btl/smcuda/btl_smcuda.c (revision 32401) +++ opal/mca/btl/smcuda/btl_smcuda.c (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2011 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2013 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, @@ -589,7 +589,7 @@ /* Sync with other local procs. Force the FIFO initialization to always * happens before the readers access it. */ - opal_atomic_add_32(&mca_btl_smcuda_component.sm_seg->module_seg->seg_inited, 1); + (void)opal_atomic_add_32(&mca_btl_smcuda_component.sm_seg->module_seg->seg_inited, 1); while( n_local_procs > mca_btl_smcuda_component.sm_seg->module_seg->seg_inited) { opal_progress(); Index: opal/mca/btl/ugni/btl_ugni_smsg.c =================================================================== --- opal/mca/btl/ugni/btl_ugni_smsg.c (revision 32401) +++ opal/mca/btl/ugni/btl_ugni_smsg.c (working copy) @@ -3,6 +3,9 @@ * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -61,7 +64,7 @@ uint32_t len; int count = 0; - if (!opal_atomic_cmpset_32 (&ep->smsg_progressing, 0, 1)) { + if (0 != opal_atomic_cmpset_32 (&ep->smsg_progressing, 0, 1)) { /* already progressing (we can't support reentry here) */ return 0; } Index: opal/mca/btl/vader/btl_vader_fifo.h =================================================================== --- opal/mca/btl/vader/btl_vader_fifo.h (revision 32401) +++ opal/mca/btl/vader/btl_vader_fifo.h (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2009 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -126,7 +126,7 @@ if (OPAL_UNLIKELY(VADER_FIFO_FREE == hdr->next)) { opal_atomic_rmb(); - if (!vader_item_cmpset (&fifo->fifo_tail, value, VADER_FIFO_FREE)) { + if (value != vader_item_cmpset (&fifo->fifo_tail, value, VADER_FIFO_FREE)) { while (VADER_FIFO_FREE == hdr->next) { opal_atomic_rmb (); } Index: opal/mca/btl/vader/btl_vader_xpmem.c =================================================================== --- opal/mca/btl/vader/btl_vader_xpmem.c (revision 32401) +++ opal/mca/btl/vader/btl_vader_xpmem.c (working copy) @@ -2,6 +2,9 @@ /* * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -40,7 +43,7 @@ rc = rcache->rcache_find_all (rcache, (void *) base, bound - base, regs, 10); for (i = 0 ; i < rc ; ++i) { if (bound <= (uintptr_t)regs[i]->bound && base >= (uintptr_t)regs[i]->base) { - opal_atomic_add (®s[i]->ref_count, 1); + (void)opal_atomic_add (®s[i]->ref_count, 1); reg = regs[i]; goto reg_found; } @@ -60,7 +63,7 @@ /* start the new segment from the lower of the two bases */ base = (uintptr_t) regs[i]->base < base ? (uintptr_t) regs[i]->base : base; - opal_atomic_add (®s[i]->ref_count, -1); + (void)opal_atomic_add (®s[i]->ref_count, -1); if (OPAL_LIKELY(0 == regs[i]->ref_count)) { /* this pointer is not in use */ @@ -109,7 +112,7 @@ { struct mca_rcache_base_module_t *rcache = endpoint->rcache; - opal_atomic_add (®->ref_count, -1); + (void)opal_atomic_add (®->ref_count, -1); if (OPAL_UNLIKELY(0 == reg->ref_count && !(reg->flags & MCA_MPOOL_FLAGS_PERSIST))) { rcache->rcache_delete (rcache, reg); opal_memchecker_base_mem_noaccess (reg->alloc_base, (uintptr_t)(reg->bound - reg->base)); Index: opal/runtime/opal_progress.c =================================================================== --- opal/runtime/opal_progress.c (revision 32401) +++ opal/runtime/opal_progress.c (working copy) @@ -3,7 +3,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -215,8 +215,7 @@ void opal_progress_event_users_increment(void) { - int32_t val; - val = opal_atomic_add_32(&num_event_users, 1); + int32_t val = 1 + opal_atomic_add_32(&num_event_users, 1); OPAL_OUTPUT((debug_output, "progress: event_users_increment setting count to %d", val)); @@ -233,8 +232,7 @@ void opal_progress_event_users_decrement(void) { - int32_t val; - val = opal_atomic_sub_32(&num_event_users, 1); + int32_t val = opal_atomic_sub_32(&num_event_users, 1) - 1; OPAL_OUTPUT((debug_output, "progress: event_users_decrement setting count to %d", val)); Index: opal/threads/mutex.h =================================================================== --- opal/threads/mutex.h (revision 32401) +++ opal/threads/mutex.h (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -271,7 +271,7 @@ int32_t ret; if (opal_using_threads()) { - ret = opal_atomic_add_32(addr, delta); + ret = delta + opal_atomic_add_32(addr, delta); } else { ret = (*addr += delta); } @@ -286,7 +286,7 @@ int64_t ret; if (opal_using_threads()) { - ret = opal_atomic_add_64(addr, delta); + ret = delta + opal_atomic_add_64(addr, delta); } else { ret = (*addr += delta); } @@ -301,7 +301,7 @@ size_t ret; if (opal_using_threads()) { - ret = opal_atomic_add_size_t(addr, delta); + ret = delta += opal_atomic_add_size_t(addr, delta); } else { ret = (*addr += delta); } @@ -310,19 +310,21 @@ } /* BWB: FIX ME: remove if possible */ -#define OPAL_CMPSET(x, y, z) ((*(x) == (y)) ? ((*(x) = (z)), 1) : 0) +#define OPAL_CMPSET(x, y, z) ((*(x) == (y)) ? ((*(x) = (z)), (y)) : (*(x))) #if OPAL_HAVE_ATOMIC_CMPSET_32 #define OPAL_ATOMIC_CMPSET_32(x, y, z) \ - (opal_using_threads() ? opal_atomic_cmpset_32(x, y, z) : OPAL_CMPSET(x, y, z)) + (((int32_t)(y)) == (opal_using_threads() ? opal_atomic_cmpset_32(x, (int32_t)(y), (int32_t)(z)) \ + : OPAL_CMPSET(x, (int32_t)(y), (int32_t)(z)))) #endif #if OPAL_HAVE_ATOMIC_CMPSET_64 #define OPAL_ATOMIC_CMPSET_64(x, y, z) \ - (opal_using_threads() ? opal_atomic_cmpset_64(x, y, z) : OPAL_CMPSET(x, y, z)) + (((int64_t)(y)) == (opal_using_threads() ? opal_atomic_cmpset_64(x, (int64_t)(y), (int64_t)(z)) \ + : OPAL_CMPSET(x, (int64_t)(y), (int64_t)(z)))) #endif #if OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64 #define OPAL_ATOMIC_CMPSET(x, y, z) \ - (opal_using_threads() ? opal_atomic_cmpset(x, y, z) : OPAL_CMPSET(x, y, z)) + ((y) == (opal_using_threads() ? opal_atomic_cmpset(x, (y), (z)) : OPAL_CMPSET(x, (y), (z)))) #endif END_C_DECLS Index: oshmem/runtime/oshmem_shmem_finalize.c =================================================================== --- oshmem/runtime/oshmem_shmem_finalize.c (revision 32401) +++ oshmem/runtime/oshmem_shmem_finalize.c (working copy) @@ -1,6 +1,9 @@ /* * Copyright (c) 2013 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * * $COPYRIGHT$ * @@ -63,8 +66,8 @@ int ret = OSHMEM_SUCCESS; static int32_t finalize_has_already_started = 0; - if (opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1) - && oshmem_shmem_initialized && !oshmem_shmem_aborted) { + if ((0 == opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1)) + && oshmem_shmem_initialized && !oshmem_shmem_aborted) { /* Should be called first because ompi_mpi_finalize makes orte and opal finalization */ ret = _shmem_finalize(); Index: test/asm/atomic_cmpset.c =================================================================== --- test/asm/atomic_cmpset.c (revision 32401) +++ test/asm/atomic_cmpset.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -71,11 +71,11 @@ /* thread tests */ for (i = 0; i < nreps; i++) { - opal_atomic_add_32(&val32, 5); + (void)opal_atomic_add_32(&val32, 5); #if OPAL_HAVE_ATOMIC_MATH_64 - opal_atomic_add_64(&val64, 5); + (void)opal_atomic_add_64(&val64, 5); #endif - opal_atomic_add(&valint, 5); + (void)opal_atomic_add(&valint, 5); } return (void *) (unsigned long) (rank + 1000); @@ -102,30 +102,30 @@ /* -- cmpset 32-bit tests -- */ vol32 = 42, old32 = 42, new32 = 50; - assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 1); + assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 42); opal_atomic_rmb(); assert(vol32 == new32); vol32 = 42, old32 = 420, new32 = 50; - assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 0); + assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 42); opal_atomic_rmb(); assert(vol32 == 42); vol32 = 42, old32 = 42, new32 = 50; - assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 1); + assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 42); assert(vol32 == new32); vol32 = 42, old32 = 420, new32 = 50; - assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 0); + assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 42); assert(vol32 == 42); vol32 = 42, old32 = 42, new32 = 50; - assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 1); + assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 42); opal_atomic_rmb(); assert(vol32 == new32); vol32 = 42, old32 = 420, new32 = 50; - assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 0); + assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 42); opal_atomic_rmb(); assert(vol32 == 42); @@ -133,30 +133,30 @@ #if OPAL_HAVE_ATOMIC_MATH_64 vol64 = 42, old64 = 42, new64 = 50; - assert(1 == opal_atomic_cmpset_64(&vol64, old64, new64)); + assert(opal_atomic_cmpset_64(&vol64, old64, new64) == 42); opal_atomic_rmb(); assert(new64 == vol64); vol64 = 42, old64 = 420, new64 = 50; - assert(opal_atomic_cmpset_64(&vol64, old64, new64) == 0); + assert(opal_atomic_cmpset_64(&vol64, old64, new64) == 42); opal_atomic_rmb(); assert(vol64 == 42); vol64 = 42, old64 = 42, new64 = 50; - assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 1); + assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 42); assert(vol64 == new64); vol64 = 42, old64 = 420, new64 = 50; - assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 0); + assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 42); assert(vol64 == 42); vol64 = 42, old64 = 42, new64 = 50; - assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 1); + assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 42); opal_atomic_rmb(); assert(vol64 == new64); vol64 = 42, old64 = 420, new64 = 50; - assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 0); + assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 42); opal_atomic_rmb(); assert(vol64 == 42); #endif @@ -163,30 +163,30 @@ /* -- cmpset int tests -- */ volint = 42, oldint = 42, newint = 50; - assert(opal_atomic_cmpset(&volint, oldint, newint) == 1); + assert(opal_atomic_cmpset(&volint, oldint, newint) == 42); opal_atomic_rmb(); assert(volint ==newint); volint = 42, oldint = 420, newint = 50; - assert(opal_atomic_cmpset(&volint, oldint, newint) == 0); + assert(opal_atomic_cmpset(&volint, oldint, newint) == 42); opal_atomic_rmb(); assert(volint == 42); volint = 42, oldint = 42, newint = 50; - assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 1); + assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 42); assert(volint == newint); volint = 42, oldint = 420, newint = 50; - assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 0); + assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 42); assert(volint == 42); volint = 42, oldint = 42, newint = 50; - assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 1); + assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 42); opal_atomic_rmb(); assert(volint == newint); volint = 42, oldint = 420, newint = 50; - assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 0); + assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 42); opal_atomic_rmb(); assert(volint == 42); @@ -194,30 +194,30 @@ /* -- cmpset ptr tests -- */ volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; - assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 1); + assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == (void*)42); opal_atomic_rmb(); assert(volptr == newptr); volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; - assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 0); + assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == (void*)42); opal_atomic_rmb(); assert(volptr == (void *) 42); volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; - assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 1); + assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == (void*)42); assert(volptr == newptr); volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; - assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 0); + assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == (void*)42); assert(volptr == (void *) 42); volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; - assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 1); + assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == (void*)42); opal_atomic_rmb(); assert(volptr == newptr); volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; - assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 0); + assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == (void*)42); opal_atomic_rmb(); assert(volptr == (void *) 42); @@ -224,7 +224,7 @@ /* -- add_32 tests -- */ val32 = 42; - assert(opal_atomic_add_32(&val32, 5) == (42 + 5)); + assert(opal_atomic_add_32(&val32, 5) == (42)); opal_atomic_rmb(); assert((42 + 5) == val32); @@ -231,7 +231,7 @@ /* -- add_64 tests -- */ #if OPAL_HAVE_ATOMIC_MATH_64 val64 = 42; - assert(opal_atomic_add_64(&val64, 5) == (42 + 5)); + assert(opal_atomic_add_64(&val64, 5) == (42)); opal_atomic_rmb(); assert((42 + 5) == val64); #endif Index: test/asm/atomic_math.c =================================================================== --- test/asm/atomic_math.c (revision 32401) +++ test/asm/atomic_math.c (working copy) @@ -2,7 +2,7 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, @@ -42,11 +42,11 @@ int i; for (i = 0 ; i < count ; ++i) { - opal_atomic_add_32(&val32, 5); + (void)opal_atomic_add_32(&val32, 5); #if OPAL_HAVE_ATOMIC_MATH_64 - opal_atomic_add_64(&val64, 6); + (void)opal_atomic_add_64(&val64, 6); #endif - opal_atomic_add(&valint, 4); + (void)opal_atomic_add(&valint, 4); } return NULL; Index: test/threads/opal_thread.c =================================================================== --- test/threads/opal_thread.c (revision 32401) +++ test/threads/opal_thread.c (working copy) @@ -1,5 +1,8 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. * $COPYRIGHT$ */ @@ -33,13 +36,13 @@ static void* thr1_run(opal_object_t* obj) { - opal_atomic_add(&count, 1); + (void)opal_atomic_add(&count, 1); return NULL; } static void* thr2_run(opal_object_t* obj) { - opal_atomic_add(&count, 2); + (void)opal_atomic_add(&count, 2); return NULL; }