Re: Add prefetch switch stack hook in scheduler function

[Eric Dumazet]

Ingo Molnar a écrit :

unroll prefetch_range() loops manually.

Signed-off-by: Ingo Molnar <[EMAIL PROTECTED]>

 include/linux/prefetch.h |   31 +++++++++++++++++++++++++++++--
 1 files changed, 29 insertions(+), 2 deletions(-)

Index: linux/include/linux/prefetch.h
===================================================================
--- linux.orig/include/linux/prefetch.h
+++ linux/include/linux/prefetch.h

@@ -58,11 +58,38 @@ static inline void prefetchw(const void static inline void prefetch_range(void *addr, size_t len)

 {
 #ifdef ARCH_HAS_PREFETCH
-       char *cp;
+       char *cp = addr;
        char *end = addr + len;

- for (cp = addr; cp < end; cp += PREFETCH_STRIDE)

+       /*
+        * Unroll agressively:
+        */
+       if (len <= PREFETCH_STRIDE)
                prefetch(cp);
+       else if (len <= 2*PREFETCH_STRIDE) {
+               prefetch(cp);
+               prefetch(cp + PREFETCH_STRIDE);
+       }
+       else if (len <= 3*PREFETCH_STRIDE) {
+               prefetch(cp);
+               prefetch(cp + PREFETCH_STRIDE);
+               prefetch(cp + 2*PREFETCH_STRIDE);
+       }
+       else if (len <= 4*PREFETCH_STRIDE) {
+               prefetch(cp);
+               prefetch(cp + PREFETCH_STRIDE);
+               prefetch(cp + 2*PREFETCH_STRIDE);
+               prefetch(cp + 3*PREFETCH_STRIDE);
+       }
+       else if (len <= 5*PREFETCH_STRIDE) {
+               prefetch(cp);
+               prefetch(cp + PREFETCH_STRIDE);
+               prefetch(cp + 2*PREFETCH_STRIDE);
+               prefetch(cp + 3*PREFETCH_STRIDE);
+               prefetch(cp + 4*PREFETCH_STRIDE);
+       } else
+               for (; cp < end; cp += PREFETCH_STRIDE)
+                       prefetch(cp);
 #endif
 }

-

Please test that len is a constant, or else the inlining is too large for the 
non constant case.

Thank you

static inline void prefetch_range(void *addr, size_t len)
{
        char *cp;
        char *end = addr + len;

        if (__builtin_constant_p(len) && (len <= 5*PREFETCH_STRIDE)) {
                if (len <= PREFETCH_STRIDE)
                        prefetch(cp);
                else if (len <= 2*PREFETCH_STRIDE) {
                        prefetch(cp);
                        prefetch(cp + PREFETCH_STRIDE);
                }
                else if (len <= 3*PREFETCH_STRIDE) {
                        prefetch(cp);
                        prefetch(cp + PREFETCH_STRIDE);
                        prefetch(cp + 2*PREFETCH_STRIDE);
                }
                else if (len <= 4*PREFETCH_STRIDE) {
                        prefetch(cp);
                        prefetch(cp + PREFETCH_STRIDE);
                        prefetch(cp + 2*PREFETCH_STRIDE);
                        prefetch(cp + 3*PREFETCH_STRIDE);
                }
                else if (len <= 5*PREFETCH_STRIDE) {
                        prefetch(cp);
                        prefetch(cp + PREFETCH_STRIDE);
                        prefetch(cp + 2*PREFETCH_STRIDE);
                        prefetch(cp + 3*PREFETCH_STRIDE);
                        prefetch(cp + 4*PREFETCH_STRIDE);
                }
        }
        else
                for (; cp < end; cp += PREFETCH_STRIDE)
                        prefetch(cp);
}