[pypy-commit] pypy default: fix test

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: 
Changeset: r70751:85e7dc6a1289
Date: 2014-04-18 09:44 +0200
http://bitbucket.org/pypy/pypy/changeset/85e7dc6a1289/

Log:fix test

diff --git a/rpython/rlib/test/test_rlocale.py 
b/rpython/rlib/test/test_rlocale.py
--- a/rpython/rlib/test/test_rlocale.py
+++ b/rpython/rlib/test/test_rlocale.py
@@ -40,5 +40,8 @@
 if sys.platform != "darwin" and not sys.platform.startswith("linux"):
 py.test.skip("there is (maybe) no libintl here")
 _gettext = external('gettext', [rffi.CCHARP], rffi.CCHARP)
-res = _gettext("1234")
+p = rffi.str2charp("1234")
+res = _gettext(p)
+assert res == p
 assert rffi.charp2str(res) == "1234"
+rffi.free_charp(p)
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy stmgc-c7: redo this optimization

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r70752:2712ec3d1a46
Date: 2014-04-18 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/2712ec3d1a46/

Log:redo this optimization

diff --git a/rpython/translator/stm/src_stm/stmgcintf.c 
b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -136,8 +136,8 @@
 stm_thread_local.shadowstack;
 #endif
 
+STM_PUSH_ROOT(stm_thread_local, STM_STACK_MARKER_NEW);
 STM_PUSH_ROOT(stm_thread_local, arg);
-/*STM_PUSH_ROOT(END_MARKER_OFF); XXX redo this optimization */
 
 while (1) {
 
@@ -165,7 +165,7 @@
 
 /* invoke the callback in the new transaction */
 STM_POP_ROOT(stm_thread_local, arg);
-assert(v_old_shadowstack == stm_thread_local.shadowstack);
+assert(v_old_shadowstack == stm_thread_local.shadowstack - 1);
 STM_PUSH_ROOT(stm_thread_local, arg);
 
 long result = v_callback(arg, v_counter);
@@ -200,9 +200,9 @@
 assert(pypy_stm_nursery_low_fill_mark == (uintptr_t) -1);
 }
 
-//gcptr x = stm_pop_root();   /* pop the END_MARKER */
-//assert(x == END_MARKER_OFF || x == END_MARKER_ON);
 STM_POP_ROOT_RET(stm_thread_local); /* pop the 'arg' */
+uintptr_t x = (uintptr_t)STM_POP_ROOT_RET(stm_thread_local);
+assert(x == STM_STACK_MARKER_NEW || x == STM_STACK_MARKER_OLD);
 assert(v_old_shadowstack == stm_thread_local.shadowstack);
 }
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] stmgc marker: Get started

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1161:e24c66f0b2b4
Date: 2014-04-18 11:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/e24c66f0b2b4/

Log:Get started

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -630,6 +630,7 @@
value before the transaction start */
 stm_thread_local_t *tl = pseg->pub.running_thread;
 assert(tl->shadowstack >= pseg->shadowstack_at_start_of_transaction);
+pseg->shadowstack_at_abort = tl->shadowstack;
 tl->shadowstack = pseg->shadowstack_at_start_of_transaction;
 tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
 tl->last_abort__bytes_in_nursery = bytes_in_nursery;
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -152,6 +152,7 @@
'thread_local_obj' field. */
 struct stm_shadowentry_s *shadowstack_at_start_of_transaction;
 object_t *threadlocal_at_start_of_transaction;
+struct stm_shadowentry_s *shadowstack_at_abort;
 
 /* For debugging */
 #ifndef NDEBUG
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -379,7 +379,7 @@
 struct stm_shadowentry_s *current = tl->shadowstack;
 struct stm_shadowentry_s *base = tl->shadowstack_base;
 while (current-- != base) {
-if (((uintptr_t)current->ss) > STM_STACK_MARKER_OLD)
+if uintptr_t)current->ss) & 3) == 0)
 mark_visit_object(current->ss, segment_base);
 }
 mark_visit_object(tl->thread_local_obj, segment_base);
diff --git a/c7/stm/marker.c b/c7/stm/marker.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/marker.c
@@ -0,0 +1,32 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+
+void (*stmcb_expand_marker)(uintptr_t odd_number,
+object_t *following_object,
+char *outputbuf, size_t outputbufsize);
+
+
+void marker_fetch(stm_thread_local_t *tl,
+  enum stm_time_e attribute_to, double time)
+{
+tl->longest_marker_state = attribute_to;
+tl->longest_marker_time = time;
+
+if (stmcb_expand_marker != NULL) {
+struct stm_shadowentry_s *current = tl->shadowstack - 1;
+struct stm_shadowentry_s *base = tl->shadowstack_base;
+while (--current >= base) {
+uintptr_t x = (uintptr_t)current->ss;
+if (x & 1) {
+/* the stack entry is an odd number */
+tl->longest_marker_self[0] = 0;
+stmcb_expand_marker(x, current[1].ss,
+tl->longest_marker_self,
+sizeof(tl->longest_marker_self));
+break;
+}
+}
+}
+}
diff --git a/c7/stm/marker.h b/c7/stm/marker.h
new file mode 100644
--- /dev/null
+++ b/c7/stm/marker.h
@@ -0,0 +1,3 @@
+
+void marker_fetch(stm_thread_local_t *tl,
+  enum stm_time_e attribute_to, double time);
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -160,28 +160,26 @@
 --current;
 OPT_ASSERT(current >= base);
 
-switch ((uintptr_t)current->ss) {
+uintptr_t x = (uintptr_t)current->ss;
 
-case 0:   /* NULL */
-continue;
-
-case STM_STACK_MARKER_NEW:
+if ((x & 3) == 0) {
+/* the stack entry is a regular pointer (possibly NULL) */
+minor_trace_if_young(¤t->ss);
+}
+else if (x == STM_STACK_MARKER_NEW) {
 /* the marker was not already seen: mark it as seen,
but continue looking more deeply in the shadowstack */
 current->ss = (object_t *)STM_STACK_MARKER_OLD;
-continue;
-
-case STM_STACK_MARKER_OLD:
+}
+else if (x == STM_STACK_MARKER_OLD) {
 /* the marker was already seen: we can stop the
root stack tracing at this point */
-goto interrupt;
-
-default:
-/* the stack entry is a regular pointer */
-minor_trace_if_young(¤t->ss);
+break;
+}
+else {
+/* it is an odd-valued marker, ignore */
 }
 }
- interrupt:
 minor_trace_if_young(&tl->thread_local_obj);
 }
 
diff --git a/c7/stm/timing.c b/c7/stm/timing.c
--- a/c7/stm/timing.c
+++ b/c7/stm/timing.c
@@ -35,8 +35,18 @@
 {
 stm_thread_local_t *tl = STM_SEGMENT->running_thread;
 TIMING_CHANGE(tl, STM_TIME_OUTSIDE_TRANSACTION);
-add_timing(tl, attribute_to, tl->timing[STM_TIME_RUN_CURRENT]);
+double time_this_transaction = tl->timing[STM_TIME_RUN_CURRENT];
+add_timing(tl, attribute_to, time_this_transaction);
 tl->timing[STM_TIME_RUN_CURRENT] = 0.0f;
+
+if (attribute_to != STM_TIME_RUN_COMMITTED &&
+time_this_transaction > tl->longest_marker_time) {
+assert(tl->shadowstack ==
+   STM_PSEGMENT->s

[pypy-commit] stmgc marker: "Markers", which are a lightweight but ad-hoc way to mark the current position

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1160:4d1f8449c75a
Date: 2014-04-18 11:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/4d1f8449c75a/

Log:"Markers", which are a lightweight but ad-hoc way to mark the
current position

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] stmgc marker: Fixes, see comment in marker.c

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1162:80d12fdc0b89
Date: 2014-04-18 11:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/80d12fdc0b89/

Log:Fixes, see comment in marker.c

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -620,6 +620,9 @@
(int)pseg->transaction_state);
 }
 
+/* look up and preserve the marker information as a string */
+marker_fetch_expand(pseg);
+
 /* throw away the content of the nursery */
 long bytes_in_nursery = throw_away_nursery(pseg);
 
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -158,6 +158,9 @@
 #ifndef NDEBUG
 pthread_t running_pthread;
 #endif
+
+/* Temporarily stores the marker information */
+char marker_self[_STM_MARKER_LEN];
 };
 
 enum /* safe_point */ {
diff --git a/c7/stm/marker.c b/c7/stm/marker.c
--- a/c7/stm/marker.c
+++ b/c7/stm/marker.c
@@ -8,25 +8,39 @@
 char *outputbuf, size_t outputbufsize);
 
 
-void marker_fetch(stm_thread_local_t *tl,
-  enum stm_time_e attribute_to, double time)
+static void marker_fetch_expand(struct stm_priv_segment_info_s *pseg)
 {
-tl->longest_marker_state = attribute_to;
-tl->longest_marker_time = time;
+pseg->marker_self[0] = 0;
 
 if (stmcb_expand_marker != NULL) {
+stm_thread_local_t *tl = pseg->pub.running_thread;
 struct stm_shadowentry_s *current = tl->shadowstack - 1;
 struct stm_shadowentry_s *base = tl->shadowstack_base;
 while (--current >= base) {
 uintptr_t x = (uintptr_t)current->ss;
 if (x & 1) {
 /* the stack entry is an odd number */
-tl->longest_marker_self[0] = 0;
 stmcb_expand_marker(x, current[1].ss,
-tl->longest_marker_self,
-sizeof(tl->longest_marker_self));
+pseg->marker_self, _STM_MARKER_LEN);
 break;
 }
 }
 }
 }
+
+static void marker_copy(stm_thread_local_t *tl,
+struct stm_priv_segment_info_s *pseg,
+enum stm_time_e attribute_to, double time)
+{
+/* Copies the marker information from pseg to tl.  This is called
+   indirectly from abort_with_mutex(), but only if the lost time is
+   greater than that of the previous recorded marker.  By contrast,
+   pseg->marker_self has been filled already in all cases.  The
+   reason for the two steps is that we must fill pseg->marker_self
+   earlier than now (some objects may be GCed), but we only know
+   here the total time it gets attributed.
+*/
+tl->longest_marker_state = attribute_to;
+tl->longest_marker_time = time;
+memcpy(tl->longest_marker_self, pseg->marker_self, _STM_MARKER_LEN);
+}
diff --git a/c7/stm/marker.h b/c7/stm/marker.h
--- a/c7/stm/marker.h
+++ b/c7/stm/marker.h
@@ -1,3 +1,5 @@
 
-void marker_fetch(stm_thread_local_t *tl,
-  enum stm_time_e attribute_to, double time);
+static void marker_fetch_expand(struct stm_priv_segment_info_s *pseg);
+static void marker_copy(stm_thread_local_t *tl,
+struct stm_priv_segment_info_s *pseg,
+enum stm_time_e attribute_to, double time);
diff --git a/c7/stm/timing.c b/c7/stm/timing.c
--- a/c7/stm/timing.c
+++ b/c7/stm/timing.c
@@ -40,12 +40,10 @@
 tl->timing[STM_TIME_RUN_CURRENT] = 0.0f;
 
 if (attribute_to != STM_TIME_RUN_COMMITTED &&
-time_this_transaction > tl->longest_marker_time) {
-assert(tl->shadowstack ==
-   STM_PSEGMENT->shadowstack_at_start_of_transaction);
-tl->shadowstack = STM_PSEGMENT->shadowstack_at_abort;
-marker_fetch(tl, attribute_to, time_this_transaction);
-tl->shadowstack = STM_PSEGMENT->shadowstack_at_start_of_transaction;
+time_this_transaction * 0.99 > tl->longest_marker_time) {
+struct stm_priv_segment_info_s *pseg =
+get_priv_segment(STM_SEGMENT->segment_num);
+marker_copy(tl, pseg, attribute_to, time_this_transaction);
 }
 }
 
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -73,6 +73,8 @@
 _STM_TIME_N
 };
 
+#define _STM_MARKER_LEN  80
+
 typedef struct stm_thread_local_s {
 /* every thread should handle the shadow stack itself */
 struct stm_shadowentry_s *shadowstack, *shadowstack_base;
@@ -93,8 +95,8 @@
 /* the marker with the longest associated time so far */
 enum stm_time_e longest_marker_state;
 double longest_marker_time;
-char longest_marker_self[80];
-char longest_marker_other[80];
+char longest_marker_self[_STM_MARKER_LEN];
+char longest_marker_other[_STM_MARKER_LEN];
 /* the next fields are handled internally by the library */
 int associated_segment_num;
 struct stm_thread_local_s *prev,

[pypy-commit] stmgc marker: Tweaks, and check with demo2.c that it's usable from C code

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1163:9568cd489776
Date: 2014-04-18 12:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/9568cd489776/

Log:Tweaks, and check with demo2.c that it's usable from C code

diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -44,6 +44,14 @@
 visit((object_t **)&n->next);
 }
 
+static void expand_marker(uintptr_t odd_number,
+  object_t *following_object,
+  char *outputbuf, size_t outputbufsize)
+{
+assert(following_object == NULL);
+snprintf(outputbuf, outputbufsize, "<%lu>", odd_number);
+}
+
 
 nodeptr_t global_chained_list;
 
@@ -198,8 +206,18 @@
 
 STM_PUSH_ROOT(stm_thread_local, global_chained_list);  /* remains forever 
in the shadow stack */
 
+int loops = 0;
+
 while (check_sorted() == -1) {
+
+STM_PUSH_ROOT(stm_thread_local, (uintptr_t)(2 * loops + 1));
+STM_PUSH_ROOT(stm_thread_local, NULL);
+
 bubble_run();
+
+STM_POP_ROOT_RET(stm_thread_local);
+STM_POP_ROOT_RET(stm_thread_local);
+loops++;
 }
 
 STM_POP_ROOT(stm_thread_local, global_chained_list);
@@ -246,6 +264,7 @@
 
 stm_setup();
 stm_register_thread_local(&stm_thread_local);
+stmcb_expand_marker = expand_marker;
 
 
 setup_list();
diff --git a/c7/stm/marker.c b/c7/stm/marker.c
--- a/c7/stm/marker.c
+++ b/c7/stm/marker.c
@@ -10,7 +10,8 @@
 
 static void marker_fetch_expand(struct stm_priv_segment_info_s *pseg)
 {
-pseg->marker_self[0] = 0;
+if (pseg->marker_self[0] != 0)
+return;   /* already collected an entry */
 
 if (stmcb_expand_marker != NULL) {
 stm_thread_local_t *tl = pseg->pub.running_thread;
@@ -22,6 +23,11 @@
 /* the stack entry is an odd number */
 stmcb_expand_marker(x, current[1].ss,
 pseg->marker_self, _STM_MARKER_LEN);
+
+if (pseg->marker_self[0] == 0) {
+pseg->marker_self[0] = '?';
+pseg->marker_self[1] = 0;
+}
 break;
 }
 }
@@ -40,7 +46,10 @@
earlier than now (some objects may be GCed), but we only know
here the total time it gets attributed.
 */
-tl->longest_marker_state = attribute_to;
-tl->longest_marker_time = time;
-memcpy(tl->longest_marker_self, pseg->marker_self, _STM_MARKER_LEN);
+if (time * 0.99 > tl->longest_marker_time) {
+tl->longest_marker_state = attribute_to;
+tl->longest_marker_time = time;
+memcpy(tl->longest_marker_self, pseg->marker_self, _STM_MARKER_LEN);
+}
+pseg->marker_self[0] = 0;
 }
diff --git a/c7/stm/timing.c b/c7/stm/timing.c
--- a/c7/stm/timing.c
+++ b/c7/stm/timing.c
@@ -39,8 +39,7 @@
 add_timing(tl, attribute_to, time_this_transaction);
 tl->timing[STM_TIME_RUN_CURRENT] = 0.0f;
 
-if (attribute_to != STM_TIME_RUN_COMMITTED &&
-time_this_transaction * 0.99 > tl->longest_marker_time) {
+if (attribute_to != STM_TIME_RUN_COMMITTED) {
 struct stm_priv_segment_info_s *pseg =
 get_priv_segment(STM_SEGMENT->segment_num);
 marker_copy(tl, pseg, attribute_to, time_this_transaction);
@@ -81,6 +80,10 @@
 fprintf(stderr, "%-24s %9u %8.3f s\n",
 timer_names[i], tl->events[i], (double)tl->timing[i]);
 }
+fprintf(stderr, "%-24s %6s %11.6f s\n",
+"longest recorded marker", "", tl->longest_marker_time);
+fprintf(stderr, "\"%.*s\"\n",
+(int)_STM_MARKER_LEN, tl->longest_marker_self);
 s_mutex_unlock();
 }
 }
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] stmgc default: Fix for release-demo2

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: 
Changeset: r1164:876e03efa95e
Date: 2014-04-18 12:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/876e03efa95e/

Log:Fix for release-demo2

diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -203,7 +203,7 @@
 }
 
 STM_POP_ROOT(stm_thread_local, global_chained_list);
-assert(org == (char *)stm_thread_local.shadowstack);
+OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
 
 unregister_thread_local();
 status = sem_post(&done); assert(status == 0);
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] stmgc marker: hg merge default

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1165:61e5e75b84f9
Date: 2014-04-18 12:09 +0200
http://bitbucket.org/pypy/stmgc/changeset/61e5e75b84f9/

Log:hg merge default

diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -221,7 +221,7 @@
 }
 
 STM_POP_ROOT(stm_thread_local, global_chained_list);
-assert(org == (char *)stm_thread_local.shadowstack);
+OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
 
 unregister_thread_local();
 status = sem_post(&done); assert(status == 0);
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] stmgc marker: Move this into macros. Should allow future optimizations or

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: marker
Changeset: r1166:79b97025b1ba
Date: 2014-04-18 12:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/79b97025b1ba/

Log:Move this into macros. Should allow future optimizations or pushing
in some other stack etc.

diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -210,13 +210,11 @@
 
 while (check_sorted() == -1) {
 
-STM_PUSH_ROOT(stm_thread_local, (uintptr_t)(2 * loops + 1));
-STM_PUSH_ROOT(stm_thread_local, NULL);
+STM_PUSH_MARKER(stm_thread_local, 2 * loops + 1, NULL);
 
 bubble_run();
 
-STM_POP_ROOT_RET(stm_thread_local);
-STM_POP_ROOT_RET(stm_thread_local);
+STM_POP_MARKER(stm_thread_local);
 loops++;
 }
 
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -274,6 +274,19 @@
 #define STM_STACK_MARKER_NEW   2
 #define STM_STACK_MARKER_OLD   6
 
+#define STM_PUSH_MARKER(tl, odd_num, p)   do {  \
+uintptr_t _odd_num = (odd_num); \
+assert(_odd_num & 1);   \
+STM_PUSH_ROOT(tl, _odd_num);\
+STM_PUSH_ROOT(tl, p);   \
+} while (0)
+
+#define STM_POP_MARKER(tl)   ({ \
+object_t *_popped = STM_POP_ROOT_RET(tl);   \
+STM_POP_ROOT_RET(tl);   \
+_popped;\
+})
+
 
 /* Every thread needs to have a corresponding stm_thread_local_t
structure.  It may be a "__thread" global variable or something else.
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: "flushing forward"?

2014-04-18 Thread arigo 
Author: Armin Rigo 
Branch: 
Changeset: r70753:17a4ad744f46
Date: 2014-04-18 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/17a4ad744f46/

Log:"flushing forward"?

diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst
--- a/pypy/doc/stm.rst
+++ b/pypy/doc/stm.rst
@@ -40,7 +40,7 @@
 ``pypy-stm`` project is to improve what is so far the state-of-the-art
 for using multiple CPUs, which for cases where separate processes don't
 work is done by writing explicitly multi-threaded programs.  Instead,
-``pypy-stm`` is flushing forward an approach to *hide* the threads, as
+``pypy-stm`` is pushing forward an approach to *hide* the threads, as
 described below in `atomic sections`_.
 
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy numpy-searchsorted: do error checking, add app-level searchsort, which needs getitem().

2014-04-18 Thread mattip 
Author: Matti Picus 
Branch: numpy-searchsorted
Changeset: r70754:3ec459c1d040
Date: 2014-04-18 15:34 +0300
http://bitbucket.org/pypy/pypy/changeset/3ec459c1d040/

Log:do error checking, add app-level searchsort, which needs getitem().

diff --git a/pypy/module/micronumpy/app_numpy.py 
b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -22,3 +22,34 @@
 arr[j] = i
 i += step
 return arr
+
+# How to call this from descr_searchsorted??
+def searchsort(space, arr, v, side, result):
+def left_find_index(a, val):
+imin = 0
+imax = a.size
+while imin < imax:
+imid = imin + ((imax - imin) >> 1)
+if a[imid] <= val:
+imin = imid +1
+else:
+imax = imid
+return imin
+def right_find_index(a, val):
+imin = 0
+imax = a.size
+while imin < imax:
+imid = imin + ((imax - imin) >> 1)
+if a[imid] < val:
+imin = imid +1
+else:
+imax = imid
+return imin
+if side == 'l':
+func = left_find_index
+else:
+func = right_find_index
+for i in range(v.get_size()):
+result[i] = func(self, v[i])
+return result
+
diff --git a/pypy/module/micronumpy/ndarray.py 
b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -693,9 +693,32 @@
 loop.round(space, self, calc_dtype, self.get_shape(), decimals, out)
 return out
 
-def descr_searchsorted(self, space, w_v, w_side='left'):
-raise OperationError(space.w_NotImplementedError, space.wrap(
-"searchsorted not implemented yet"))
+@unwrap_spec(side=str, w_sorter=WrappedDefault(None))
+def descr_searchsorted(self, space, w_v, side='left', w_sorter=None):
+from pypy.module.micronumpy.sort import searchsort
+if not space.is_none(w_sorter):
+raise OperationError(space.w_NotImplementedError, space.wrap(
+'sorter not supported in searchsort'))
+if not side or len(side) < 1:
+raise OperationError(space.w_ValueError, space.wrap(
+"expected nonempty string for keyword 'side'"))
+elif side[0] == 'l' or side[0] == 'L':
+side = 'l'
+elif side[0] == 'r' or side[0] == 'R':
+side = 'r'
+else:
+raise oefmt(space.w_ValueError,
+ "'%s' is an invalid value for keyword 'side'", side)
+ret = W_NDimArray.from_shape(space, self.get_shape(),
+   descriptor.get_dtype_cache(space).w_longdtype)
+if len(self.get_shape()) > 1:
+raise OperationError(space.w_ValueError, space.wrap(
+"a must be a 1-d array"))
+v = convert_to_array(space, w_v)
+if len(v.get_shape()) >1:
+raise OperationError(space.w_ValueError, space.wrap(
+"v must be a 1-d array-like"))
+return searchsort(self, space, v, side, ret)
 
 def descr_setasflat(self, space, w_v):
 raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -1351,6 +1374,7 @@
 dot = interp2app(W_NDimArray.descr_dot),
 var = interp2app(W_NDimArray.descr_var),
 std = interp2app(W_NDimArray.descr_std),
+searchsorted = interp2app(W_NDimArray.descr_searchsorted),
 
 cumsum = interp2app(W_NDimArray.descr_cumsum),
 cumprod = interp2app(W_NDimArray.descr_cumprod),
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: fix validation of missing r/w op_flag to nditer

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70755:9f3d775f07b7
Date: 2014-04-18 11:58 -0400
http://bitbucket.org/pypy/pypy/changeset/9f3d775f07b7/

Log:fix validation of missing r/w op_flag to nditer

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -80,7 +80,7 @@
 
 class OpFlag(object):
 def __init__(self):
-self.rw = 'r'
+self.rw = ''
 self.broadcast = True
 self.force_contig = False
 self.force_align = False
@@ -145,7 +145,11 @@
 else:
 raise OperationError(space.w_ValueError, space.wrap(
 'op_flags must be a tuple or array of per-op flag-tuples'))
-if op_flag.rw == 'r':
+if op_flag.rw == '':
+raise oefmt(space.w_ValueError,
+"None of the iterator flags READWRITE, READONLY, or "
+"WRITEONLY were specified for an operand")
+elif op_flag.rw == 'r':
 op_flag.get_it_item = (get_readonly_item, get_readonly_slice)
 elif op_flag.rw == 'rw':
 op_flag.get_it_item = (get_readwrite_item, get_readwrite_slice)
diff --git a/pypy/module/micronumpy/test/test_nditer.py 
b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -143,21 +143,17 @@
 a = arange(6).reshape(2,3) - 3
 exc = raises(TypeError, nditer, a, op_dtypes=['complex'])
 assert str(exc.value).startswith("Iterator operand required copying or 
buffering")
+exc = raises(ValueError, nditer, a, op_flags=['copy'], 
op_dtypes=['complex128'])
+assert str(exc.value) == "None of the iterator flags READWRITE, 
READONLY, or WRITEONLY were specified for an operand"
 r = []
 for x in nditer(a, op_flags=['readonly','copy'],
 op_dtypes=['complex128']):
 r.append(sqrt(x))
 assert abs((array(r) - [1.73205080757j, 1.41421356237j, 1j, 0j,
-1+0j, 1.41421356237+0j]).sum()) < 1e-5
-r = []
-for x in nditer(a, op_flags=['copy'],
-op_dtypes=['complex128']):
-r.append(sqrt(x))
-assert abs((array(r) - [1.73205080757j, 1.41421356237j, 1j, 0j,
-1+0j, 1.41421356237+0j]).sum()) < 1e-5
+1+0j, 1.41421356237+0j]).sum()) < 1e-5
 multi = nditer([None, array([2, 3], dtype='int64'), array(2., 
dtype='double')],
-   op_dtypes = ['int64', 'int64', 'float64'],
-   op_flags = [['writeonly', 'allocate'], ['readonly'], 
['readonly']])
+   op_dtypes=['int64', 'int64', 'float64'],
+   op_flags=[['writeonly', 'allocate'], ['readonly'], 
['readonly']])
 for a, b, c in multi:
 a[...] = b * c
 assert (multi.operands[0] == [4, 6]).all()
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: fix nditer getitem return types

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70756:f2b222d84fb1
Date: 2014-04-18 12:29 -0400
http://bitbucket.org/pypy/pypy/changeset/f2b222d84fb1/

Log:fix nditer getitem return types

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -2,9 +2,8 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.module.micronumpy import ufuncs, support
+from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
-from pypy.module.micronumpy.concrete import SliceArray
 from pypy.module.micronumpy.descriptor import decode_w_dtype
 from pypy.module.micronumpy.iterators import ArrayIter, SliceIterator
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
@@ -25,7 +24,8 @@
 class IteratorMixin(object):
 _mixin_ = True
 
-def __init__(self, it, op_flags):
+def __init__(self, nditer, it, op_flags):
+self.nditer = nditer
 self.it = it
 self.st = it.reset()
 self.op_flags = op_flags
@@ -37,7 +37,7 @@
 self.st = self.it.next(self.st)
 
 def getitem(self, space, array):
-return self.op_flags.get_it_item[self.index](space, array, self.it, 
self.st)
+return self.op_flags.get_it_item[self.index](space, self.nditer, 
self.it, self.st)
 
 def setitem(self, space, array, val):
 xxx
@@ -90,14 +90,17 @@
 self.get_it_item = (get_readonly_item, get_readonly_slice)
 
 
-def get_readonly_item(space, array, it, st):
-return space.wrap(it.getitem(st))
+def get_readonly_item(space, nditer, it, st):
+res = concrete.ConcreteNonWritableArrayWithBase(
+[], it.array.dtype, it.array.order, [], [], it.array.storage, nditer)
+res.start = st.offset
+return W_NDimArray(res)
 
 
-def get_readwrite_item(space, array, it, st):
-#create a single-value view (since scalars are not views)
-res = SliceArray(it.array.start + st.offset, [0], [0], [1], it.array, 
array)
-#it.dtype.setitem(res, 0, it.getitem())
+def get_readwrite_item(space, nditer, it, st):
+res = concrete.ConcreteArrayWithBase(
+[], it.array.dtype, it.array.order, [], [], it.array.storage, nditer)
+res.start = st.offset
 return W_NDimArray(res)
 
 
@@ -398,12 +401,14 @@
 if self.external_loop:
 for i in range(len(self.seq)):
 self.iters.append(ExternalLoopIterator(
+self,
 get_external_loop_iter(
 space, self.order, self.seq[i], iter_shape),
 self.op_flags[i]))
 else:
 for i in range(len(self.seq)):
 self.iters.append(BoxIterator(
+self,
 get_iter(
 space, self.order, self.seq[i], iter_shape, 
self.dtypes[i]),
 self.op_flags[i]))
diff --git a/pypy/module/micronumpy/test/test_nditer.py 
b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -4,14 +4,20 @@
 
 class AppTestNDIter(BaseNumpyAppTest):
 def test_basic(self):
-from numpy import arange, nditer
+from numpy import arange, nditer, ndarray
 a = arange(6).reshape(2,3)
+i = nditer(a)
 r = []
-for x in nditer(a):
+for x in i:
+assert type(x) is ndarray
+assert x.base is i
+assert x.shape == ()
+assert x.strides == ()
+exc = raises(ValueError, "x[()] = 42")
+assert str(exc.value) == 'assignment destination is read-only'
 r.append(x)
 assert r == [0, 1, 2, 3, 4, 5]
 r = []
-
 for x in nditer(a.T):
 r.append(x)
 assert r == [0, 1, 2, 3, 4, 5]
@@ -29,9 +35,14 @@
 assert r == [0, 3, 1, 4, 2, 5]
 
 def test_readwrite(self):
-from numpy import arange, nditer
+from numpy import arange, nditer, ndarray
 a = arange(6).reshape(2,3)
-for x in nditer(a, op_flags=['readwrite']):
+i = nditer(a, op_flags=['readwrite'])
+for x in i:
+assert type(x) is ndarray
+assert x.base is i
+assert x.shape == ()
+assert x.strides == ()
 x[...] = 2 * x
 assert (a == [[0, 2, 4], [6, 8, 10]]).all()
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: more nditer cleanups

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70758:9afd3cb41ad2
Date: 2014-04-18 12:44 -0400
http://bitbucket.org/pypy/pypy/changeset/9afd3cb41ad2/

Log:more nditer cleanups

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -10,9 +10,10 @@
 shape_agreement, 
shape_agreement_multiple)
 
 
-class BaseIterator(object):
-def __init__(self, nditer, it, op_flags):
+class Iterator(object):
+def __init__(self, nditer, index, it, op_flags):
 self.nditer = nditer
+self.index = index
 self.it = it
 self.st = it.reset()
 self.op_flags = op_flags
@@ -30,14 +31,6 @@
 xxx
 
 
-class BoxIterator(BaseIterator):
-index = 0
-
-
-class ExternalLoopIterator(BaseIterator):
-index = 1
-
-
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
 ret = []
 if space.is_w(w_op_flags, space.w_None):
@@ -243,15 +236,6 @@
 return SliceIterator(arr, imp.strides, imp.backstrides, shape, 
order=order, backward=backward)
 
 
-def convert_to_array_or_none(space, w_elem):
-'''
-None will be passed through, all others will be converted
-'''
-if space.is_none(w_elem):
-return None
-return convert_to_array(space, w_elem)
-
-
 class IndexIterator(object):
 def __init__(self, shape, backward=False):
 self.shape = shape
@@ -301,7 +285,9 @@
 if space.isinstance_w(w_seq, space.w_tuple) or \
space.isinstance_w(w_seq, space.w_list):
 w_seq_as_list = space.listview(w_seq)
-self.seq = [convert_to_array_or_none(space, w_elem) for w_elem in 
w_seq_as_list]
+self.seq = [convert_to_array(space, w_elem)
+if not space.is_none(w_elem) else None
+for w_elem in w_seq_as_list]
 else:
 self.seq = [convert_to_array(space, w_seq)]
 
@@ -375,8 +361,9 @@
 self.dtypes[i] = seq_d
 elif selfd != seq_d:
 if not 'r' in self.op_flags[i].tmp_copy:
-raise OperationError(space.w_TypeError, space.wrap(
-"Iterator operand required copying or buffering 
for operand %d" % i))
+raise oefmt(space.w_TypeError,
+"Iterator operand required copying or "
+"buffering for operand %d", i)
 impl = self.seq[i].implementation
 new_impl = impl.astype(space, selfd)
 self.seq[i] = W_NDimArray(new_impl)
@@ -387,22 +374,23 @@
 # create an iterator for each operand
 if self.external_loop:
 for i in range(len(self.seq)):
-self.iters.append(ExternalLoopIterator(
-self,
+self.iters.append(Iterator(
+self, 1,
 get_external_loop_iter(
 space, self.order, self.seq[i], iter_shape),
 self.op_flags[i]))
 else:
 for i in range(len(self.seq)):
-self.iters.append(BoxIterator(
-self,
+self.iters.append(Iterator(
+self, 0,
 get_iter(
 space, self.order, self.seq[i], iter_shape, 
self.dtypes[i]),
 self.op_flags[i]))
 
 def set_op_axes(self, space, w_op_axes):
 if space.len_w(w_op_axes) != len(self.seq):
-raise OperationError(space.w_ValueError, space.wrap("op_axes must 
be a tuple/list matching the number of ops"))
+raise oefmt(space.w_ValueError,
+"op_axes must be a tuple/list matching the number of 
ops")
 op_axes = space.listview(w_op_axes)
 l = -1
 for w_axis in op_axes:
@@ -411,10 +399,14 @@
 if l == -1:
 l = axis_len
 elif axis_len != l:
-raise OperationError(space.w_ValueError, space.wrap("Each 
entry of op_axes must have the same size"))
-self.op_axes.append([space.int_w(x) if not space.is_none(x) 
else -1 for x in space.listview(w_axis)])
+raise oefmt(space.w_ValueError,
+"Each entry of op_axes must have the same 
size")
+self.op_axes.append([space.int_w(x) if not space.is_none(x) 
else -1
+ for x in space.listview(w_axis)])
 if l == -1:
-raise OperationError(space.w_ValueError, space.wrap("If op_axes is 
provided, at least one list of axes must be contained within it"))
+raise oefmt(space.w_ValueError,
+"If op_axes is provided, at least one list of axes "
+"must be contained within it"

[pypy-commit] pypy default: simplify

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70757:aba89d1880eb
Date: 2014-04-18 12:35 -0400
http://bitbucket.org/pypy/pypy/changeset/aba89d1880eb/

Log:simplify

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -10,20 +10,7 @@
 shape_agreement, 
shape_agreement_multiple)
 
 
-class AbstractIterator(object):
-def done(self):
-raise NotImplementedError("Abstract Class")
-
-def next(self):
-raise NotImplementedError("Abstract Class")
-
-def getitem(self, space, array):
-raise NotImplementedError("Abstract Class")
-
-
-class IteratorMixin(object):
-_mixin_ = True
-
+class BaseIterator(object):
 def __init__(self, nditer, it, op_flags):
 self.nditer = nditer
 self.it = it
@@ -43,11 +30,11 @@
 xxx
 
 
-class BoxIterator(IteratorMixin, AbstractIterator):
+class BoxIterator(BaseIterator):
 index = 0
 
 
-class ExternalLoopIterator(IteratorMixin, AbstractIterator):
+class ExternalLoopIterator(BaseIterator):
 index = 1
 
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3k-fix-strategies: fix BytesStrategy iter to wrapbytes

2014-04-18 Thread pjenvey 
Author: Philip Jenvey 
Branch: py3k-fix-strategies
Changeset: r70760:724161bbeca1
Date: 2014-04-18 11:13 -0700
http://bitbucket.org/pypy/pypy/changeset/724161bbeca1/

Log:fix BytesStrategy iter to wrapbytes

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1439,7 +1439,7 @@
 
 def next_entry(self):
 for key in self.iterator:
-return self.space.wrap(key)
+return self.space.wrapbytes(key)
 else:
 return None
 
diff --git a/pypy/objspace/std/test/test_setobject.py 
b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -89,6 +89,7 @@
 from pypy.objspace.std.floatobject import W_FloatObject
 
 w = self.space.wrap
+wb = self.space.wrapbytes
 intstr = self.space.fromcache(IntegerSetStrategy)
 tmp_func = intstr.get_storage_from_list
 # test if get_storage_from_list is no longer used
@@ -100,10 +101,10 @@
 assert w_set.strategy is intstr
 assert intstr.unerase(w_set.sstorage) == {1:None, 2:None, 3:None}
 
-w_list = W_ListObject(self.space, [w("1"), w("2"), w("3")])
+w_list = W_ListObject(self.space, [wb("1"), wb("2"), wb("3")])
 w_set = W_SetObject(self.space)
 _initialize_set(self.space, w_set, w_list)
-assert w_set.strategy is self.space.fromcache(UnicodeSetStrategy)
+assert w_set.strategy is self.space.fromcache(BytesSetStrategy)
 assert w_set.strategy.unerase(w_set.sstorage) == {"1":None, "2":None, 
"3":None}
 
 w_list = self.space.iter(W_ListObject(self.space, [w(u"1"), w(u"2"), 
w(u"3")]))
@@ -1005,6 +1006,13 @@
 # gives us 1, but 1 is not in the set any longer.
 raises(RuntimeError, list, it)
 
+def test_iter_bytes_strategy(self):
+l = [b'a', b'b']
+s = set(l)
+n = next(iter(s))
+assert type(n) is bytes
+assert n in l
+
 def test_unicodestrategy(self):
 s = 'àèìòù'
 myset = set([s])
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3k-fix-strategies: we now need bytes_w too

2014-04-18 Thread pjenvey 
Author: Philip Jenvey 
Branch: py3k-fix-strategies
Changeset: r70759:9953e025bce7
Date: 2014-04-18 11:13 -0700
http://bitbucket.org/pypy/pypy/changeset/9953e025bce7/

Log:we now need bytes_w too

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py 
b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1079,6 +1079,10 @@
 assert isinstance(string, str)
 return string
 
+def bytes_w(self, string):
+assert isinstance(string, str)
+return string
+
 def unicode_w(self, string):
 assert isinstance(string, unicode)
 return string
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: simplify nditer

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70761:667ad75d7ce9
Date: 2014-04-18 14:51 -0400
http://bitbucket.org/pypy/pypy/changeset/667ad75d7ce9/

Log:simplify nditer

diff --git a/pypy/module/micronumpy/iterators.py 
b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -164,38 +164,6 @@
 self.array.setitem(state.offset, elem)
 
 
-class SliceIterator(ArrayIter):
-def __init__(self, arr, strides, backstrides, shape, order="C",
-backward=False, dtype=None):
-if dtype is None:
-dtype = arr.implementation.dtype
-self.dtype = dtype
-self.arr = arr
-if backward:
-self.slicesize = shape[0]
-self.gap = [support.product(shape[1:]) * dtype.elsize]
-strides = strides[1:]
-backstrides = backstrides[1:]
-shape = shape[1:]
-strides.reverse()
-backstrides.reverse()
-shape.reverse()
-size = support.product(shape)
-else:
-shape = [support.product(shape)]
-strides, backstrides = calc_strides(shape, dtype, order)
-size = 1
-self.slicesize = support.product(shape)
-self.gap = strides
-ArrayIter.__init__(self, arr.implementation, size, shape, strides, 
backstrides)
-
-def getslice(self):
-from pypy.module.micronumpy.concrete import SliceArray
-return SliceArray(self.offset, self.gap, self.backstrides,
-  [self.slicesize], self.arr.implementation,
-  self.arr, self.dtype)
-
-
 def AxisIter(array, shape, axis, cumulative):
 strides = array.get_strides()
 backstrides = array.get_backstrides()
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -5,56 +5,33 @@
 from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter, SliceIterator
+from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
 shape_agreement, 
shape_agreement_multiple)
 
 
-class Iterator(object):
-def __init__(self, nditer, index, it, op_flags):
-self.nditer = nditer
-self.index = index
-self.it = it
-self.st = it.reset()
-self.op_flags = op_flags
-
-def done(self):
-return self.it.done(self.st)
-
-def next(self):
-self.st = self.it.next(self.st)
-
-def getitem(self, space, array):
-return self.op_flags.get_it_item[self.index](space, self.nditer, 
self.it, self.st)
-
-def setitem(self, space, array, val):
-xxx
-
-
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
-ret = []
 if space.is_w(w_op_flags, space.w_None):
-for i in range(n):
-ret.append(OpFlag())
-elif not space.isinstance_w(w_op_flags, space.w_tuple) and not \
+w_op_flags = space.newtuple([space.wrap('readonly')])
+if not space.isinstance_w(w_op_flags, space.w_tuple) and not \
 space.isinstance_w(w_op_flags, space.w_list):
 raise oefmt(space.w_ValueError,
 '%s must be a tuple or array of per-op flag-tuples',
 name)
+ret = []
+w_lst = space.listview(w_op_flags)
+if space.isinstance_w(w_lst[0], space.w_tuple) or \
+   space.isinstance_w(w_lst[0], space.w_list):
+if len(w_lst) != n:
+raise oefmt(space.w_ValueError,
+'%s must be a tuple or array of per-op flag-tuples',
+name)
+for item in w_lst:
+ret.append(parse_one_arg(space, space.listview(item)))
 else:
-w_lst = space.listview(w_op_flags)
-if space.isinstance_w(w_lst[0], space.w_tuple) or \
-   space.isinstance_w(w_lst[0], space.w_list):
-if len(w_lst) != n:
-raise oefmt(space.w_ValueError,
-'%s must be a tuple or array of per-op 
flag-tuples',
-name)
-for item in w_lst:
-ret.append(parse_one_arg(space, space.listview(item)))
-else:
-op_flag = parse_one_arg(space, w_lst)
-for i in range(n):
-ret.append(op_flag)
+op_flag = parse_one_arg(space, w_lst)
+for i in range(n):
+ret.append(op_flag)
 return ret
 
 
@@ -67,29 +44,6 @@
 self.native_byte_order = False
 self.tmp_copy = ''
 self.allocate = False
-self.get_it_item = (get_readonly_item, get_readonly_slice)
-
-
-def get_readonly_item(space

[pypy-commit] pypy default: support more variations of ndarray.item()

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70763:142661a32d5b
Date: 2014-04-18 17:05 -0400
http://bitbucket.org/pypy/pypy/changeset/142661a32d5b/

Log:support more variations of ndarray.item()

diff --git a/pypy/module/micronumpy/ndarray.py 
b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -18,7 +18,7 @@
 multi_axis_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.flatiter import W_FlatIterator
-from pypy.module.micronumpy.strides import get_shape_from_iterable, to_coords, 
\
+from pypy.module.micronumpy.strides import get_shape_from_iterable, \
 shape_agreement, shape_agreement_multiple
 
 
@@ -469,29 +469,33 @@
 def descr_get_flatiter(self, space):
 return space.wrap(W_FlatIterator(self))
 
-def to_coords(self, space, w_index):
-coords, _, _ = to_coords(space, self.get_shape(),
- self.get_size(), self.get_order(),
- w_index)
-return coords
-
-def descr_item(self, space, w_arg=None):
-if space.is_none(w_arg):
+def descr_item(self, space, __args__):
+args_w, kw_w = __args__.unpack()
+if len(args_w) == 1 and space.isinstance_w(args_w[0], space.w_tuple):
+args_w = space.fixedview(args_w[0])
+shape = self.get_shape()
+coords = [0] * len(shape)
+if len(args_w) == 0:
 if self.get_size() == 1:
 w_obj = self.get_scalar_value()
 assert isinstance(w_obj, boxes.W_GenericBox)
 return w_obj.item(space)
 raise oefmt(space.w_ValueError,
 "can only convert an array of size 1 to a Python scalar")
-if space.isinstance_w(w_arg, space.w_int):
-if self.is_scalar():
-raise oefmt(space.w_IndexError, "index out of bounds")
-i = self.to_coords(space, w_arg)
-item = self.getitem(space, i)
-assert isinstance(item, boxes.W_GenericBox)
-return item.item(space)
-raise OperationError(space.w_NotImplementedError, space.wrap(
-"non-int arg not supported"))
+elif len(args_w) == 1 and len(shape) != 1:
+value = support.index_w(space, args_w[0])
+value = support.check_and_adjust_index(space, value, 
self.get_size(), -1)
+for idim in range(len(shape) - 1, -1, -1):
+coords[idim] = value % shape[idim]
+value //= shape[idim]
+elif len(args_w) == len(shape):
+for idim in range(len(shape)):
+coords[idim] = support.index_w(space, args_w[idim])
+else:
+raise oefmt(space.w_ValueError, "incorrect number of indices for 
array")
+item = self.getitem(space, coords)
+assert isinstance(item, boxes.W_GenericBox)
+return item.item(space)
 
 def descr_itemset(self, space, args_w):
 if len(args_w) == 0:
diff --git a/pypy/module/micronumpy/strides.py 
b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -233,30 +233,6 @@
 return dtype
 
 
-def to_coords(space, shape, size, order, w_item_or_slice):
-'''Returns a start coord, step, and length.
-'''
-start = lngth = step = 0
-if not (space.isinstance_w(w_item_or_slice, space.w_int) or
-space.isinstance_w(w_item_or_slice, space.w_slice)):
-raise OperationError(space.w_IndexError,
- space.wrap('unsupported iterator index'))
-
-start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-
-coords = [0] * len(shape)
-i = start
-if order == 'C':
-for s in range(len(shape) -1, -1, -1):
-coords[s] = i % shape[s]
-i //= shape[s]
-else:
-for s in range(len(shape)):
-coords[s] = i % shape[s]
-i //= shape[s]
-return coords, step, lngth
-
-
 @jit.unroll_safe
 def shape_agreement(space, shape1, w_arr2, broadcast_down=True):
 if w_arr2 is None:
diff --git a/pypy/module/micronumpy/support.py 
b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -25,3 +25,18 @@
 for x in s:
 i *= x
 return i
+
+
+def check_and_adjust_index(space, index, size, axis):
+if index < -size or index >= size:
+if axis >= 0:
+raise oefmt(space.w_IndexError,
+"index %d is out of bounds for axis %d with size %d",
+index, axis, size)
+else:
+raise oefmt(space.w_IndexError,
+"index %d is out of bounds for size %d",
+index, size)
+if index < 0:
+index += size
+return index
diff --git a/pypy/module/micronumpy/test/test_ndarray.py 
b/pypy/module/micronump

[pypy-commit] pypy default: small cleanups for flatiter

2014-04-18 Thread bdkearns 
Author: Brian Kearns 
Branch: 
Changeset: r70762:acdd090d79d7
Date: 2014-04-18 16:28 -0400
http://bitbucket.org/pypy/pypy/changeset/acdd090d79d7/

Log:small cleanups for flatiter

diff --git a/pypy/module/micronumpy/flatiter.py 
b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -35,7 +35,7 @@
 self.iter, self.state = self.base.create_iter()
 
 def descr_len(self, space):
-return space.wrap(self.base.get_size())
+return space.wrap(self.iter.size)
 
 def descr_next(self, space):
 if self.iter.done(self.state):
@@ -48,8 +48,7 @@
 return space.wrap(self.state.index)
 
 def descr_coords(self, space):
-coords = self.base.to_coords(space, space.wrap(self.state.index))
-return space.newtuple([space.wrap(c) for c in coords])
+return space.newtuple([space.wrap(c) for c in self.state.indices])
 
 def descr_getitem(self, space, w_idx):
 if not (space.isinstance_w(w_idx, space.w_int) or
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: Persist the file descriptor used by os.urandom, which speeds up individual calls to it

2014-04-18 Thread alex_gaynor 
Author: Alex Gaynor 
Branch: 
Changeset: r70764:6810f401d08e
Date: 2014-04-18 18:03 -0700
http://bitbucket.org/pypy/pypy/changeset/6810f401d08e/

Log:Persist the file descriptor used by os.urandom, which speeds up
individual calls to it

diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -5,11 +5,13 @@
 import os, sys
 import errno
 
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
 if sys.platform == 'win32':
 from rpython.rlib import rwin32
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform
-from rpython.rtyper.lltypesystem import lltype, rffi
 
 eci = ExternalCompilationInfo(
 includes = ['windows.h', 'wincrypt.h'],
@@ -81,25 +83,28 @@
 return buf.str(n)
 else:  # Posix implementation
 def init_urandom():
-pass
+"""NOT_RPYTHON
+Return an array of one int, initialized to 0.
+It is filled automatically the first time urandom() is called.
+"""
+return lltype.malloc(rffi.CArray(lltype.Signed), 1,
+ immortal=True, zero=True)
 
 def urandom(context, n):
 "Read n bytes from /dev/urandom."
 result = ''
 if n == 0:
 return result
-fd = os.open("/dev/urandom", os.O_RDONLY, 0777)
-try:
-while n > 0:
-try:
-data = os.read(fd, n)
-except OSError, e:
-if e.errno != errno.EINTR:
-raise
-data = ''
-result += data
-n -= len(data)
-finally:
-os.close(fd)
+if not context[0]:
+context[0] = os.open("/dev/urandom", os.O_RDONLY, 0777)
+while n > 0:
+try:
+data = os.read(context[0], n)
+except OSError, e:
+if e.errno != errno.EINTR:
+raise
+data = ''
+result += data
+n -= len(data)
 return result
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy reflex-support: rejig test skipping and use of dummy backend

2014-04-18 Thread wlav 
Author: Wim Lavrijsen 
Branch: reflex-support
Changeset: r70766:2a3916aaa1b8
Date: 2014-04-18 19:38 -0700
http://bitbucket.org/pypy/pypy/changeset/2a3916aaa1b8/

Log:rejig test skipping and use of dummy backend

diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -5,8 +5,10 @@
 
 ifneq (${REFLEXHOME},)
   ROOTSYS := ${REFLEXHOME}
+else ifneq (${ROOTSYS},)
+  ROOTSYS := ${ROOTSYS}
 else
-  ROOTSYS := ${ROOTSYS}
+  DUMMY := t
 endif
 
 ifeq ($(DUMMY),t)
diff --git a/pypy/module/cppyy/test/conftest.py 
b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -2,32 +2,38 @@
 
 @py.test.mark.tryfirst
 def pytest_runtest_setup(item):
-print item
 if py.path.local.sysfind('genreflex') is None:
-#py.test.skip("genreflex is not installed")
-
-# build the dummy CAPI
-
-import os
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.translator import platform
-
-from rpython.rtyper.lltypesystem import rffi
-
-pkgpath = py.path.local(__file__).dirpath().join(os.pardir)
-srcpath = pkgpath.join('src')
-incpath = pkgpath.join('include')
-
-eci = ExternalCompilationInfo(
-separate_module_files=[srcpath.join('dummy_backend.cxx')],
-include_dirs=[incpath],
-use_cpp_linker=True,
-)
-
-soname = platform.platform.compile(
-[], eci,
-outputfilename='libcppyy_backend',
-standalone=False)
+if not item.location[0] in ['test_helper.py', 'test_cppyy.py'] or \
+   (item.location[0] == 'test_cppyy.py' and not 
'TestCPPYYImplementation' in item.location[2]):
+py.test.skip("genreflex is not installed")
 
 import pypy.module.cppyy.capi.loadable_capi as lcapi
-lcapi.reflection_library = str(soname)
+try:
+import ctypes
+ctypes.CDLL(lcapi.reflection_library)
+except Exception, e:
+import os
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.translator import platform
+
+from rpython.rtyper.lltypesystem import rffi
+
+pkgpath = py.path.local(__file__).dirpath().join(os.pardir)
+srcpath = pkgpath.join('src')
+incpath = pkgpath.join('include')
+
+eci = ExternalCompilationInfo(
+separate_module_files=[srcpath.join('dummy_backend.cxx')],
+include_dirs=[incpath],
+use_cpp_linker=True,
+)
+
+soname = platform.platform.compile(
+[], eci,
+outputfilename='libcppyy_backend',
+standalone=False)
+
+lcapi.reflection_library = str(soname)
+
+lcapi.isdummy = True
+
diff --git a/pypy/module/cppyy/test/test_aclassloader.py 
b/pypy/module/cppyy/test/test_aclassloader.py
--- a/pypy/module/cppyy/test/test_aclassloader.py
+++ b/pypy/module/cppyy/test/test_aclassloader.py
@@ -1,7 +1,5 @@
 import py, os, sys
 
-if py.path.local.sysfind('genreflex') is None:
-py.test.skip("genreflex is not installed")
 
 currpath = py.path.local(__file__).dirpath()
 
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py 
b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -1,8 +1,5 @@
 import py, os, sys
 
-if py.path.local.sysfind('genreflex') is None:
-py.test.skip("genreflex is not installed")
-
 from pypy.module.cppyy import capi
 
 
diff --git a/pypy/module/cppyy/test/test_cppyy.py 
b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -1,9 +1,5 @@
 import py, os, sys
 
-isdummy = ''
-if py.path.local.sysfind('genreflex') is None:
-isdummy = 'DUMMY=t'
-
 from pypy.module.cppyy import interp_cppyy, executor
 
 
@@ -13,7 +9,8 @@
 def setup_module(mod):
 if sys.platform == 'win32':
 py.test.skip("win32 not supported so far")
-err = os.system("cd '%s' && make %s example01Dict.so" % (currpath, 
isdummy))
+import pypy.module.cppyy.capi.loadable_capi as lcapi
+err = os.system("cd '%s' && make example01Dict.so" % currpath)
 if err:
 raise OSError("'make' failed (see stderr)")
 
@@ -36,9 +33,6 @@
 spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
 def setup_class(cls):
-if isdummy:
-py.test.skip('skipping further tests in dummy mode')
-
 cls.w_example01, cls.w_payload = 
cls.space.unpackiterable(cls.space.appexec([], """():
 import cppyy
 cppyy.load_reflection_info(%r)
diff --git a/pypy/module/cppyy/test/test_crossing.py 
b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/modu