[PATCH v3 2/4] pipe: avoid round_pipe_size() nr_pages overflow on 32-bit

[Joe Lawrence]

The round_pipe_size() function contains a right-bit-shift expression
which may overflow, which would cause undefined results in a subsequent
roundup_pow_of_two() call.

  static inline unsigned int round_pipe_size(unsigned int size)
  {
          unsigned long nr_pages;

          nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
          return roundup_pow_of_two(nr_pages) << PAGE_SHIFT;
  }

PAGE_SIZE is defined as (1UL << PAGE_SHIFT), so:
  - 4 bytes wide on 32-bit (0 to 0xffffffff)
  - 8 bytes wide on 64-bit (0 to 0xffffffffffffffff)

That means that 32-bit round_pipe_size(), nr_pages may overflow to 0:

  size=0x00000000    nr_pages=0x0
  size=0x00000001    nr_pages=0x1
  size=0xfffff000    nr_pages=0xfffff
  size=0xfffff001    nr_pages=0x0         << !
  size=0xffffffff    nr_pages=0x0         << !

This is bad because roundup_pow_of_two(n) is undefined when n == 0!

64-bit is not a problem as the unsigned int size is 4 bytes wide
(similar to 32-bit) and the larger, 8 byte wide unsigned long, is
sufficient to handle the largest value of the bit shift expression:

  size=0xffffffff    nr_pages=100000

Modify round_pipe_size() to return 0 if n == 0 and updates its callers
to handle accordingly.

Reported-by: Mikulas Patocka <mpato...@redhat.com>
Signed-off-by: Joe Lawrence <joe.lawre...@redhat.com>
---
 fs/pipe.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/fs/pipe.c b/fs/pipe.c
index a21ad26de557..8cbc97d97753 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -1017,13 +1017,19 @@ static int fifo_open(struct inode *inode, struct file 
*filp)
 
 /*
  * Currently we rely on the pipe array holding a power-of-2 number
- * of pages.
+ * of pages. Returns 0 on error.
  */
 static inline unsigned int round_pipe_size(unsigned int size)
 {
        unsigned long nr_pages;
 
+       if (size < pipe_min_size)
+               size = pipe_min_size;
+
        nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+       if (nr_pages == 0)
+               return 0;
+
        return roundup_pow_of_two(nr_pages) << PAGE_SHIFT;
 }
 
@@ -1039,6 +1045,8 @@ static long pipe_set_size(struct pipe_inode_info *pipe, 
unsigned long arg)
        long ret = 0;
 
        size = round_pipe_size(arg);
+       if (size == 0)
+               return -EINVAL;
        nr_pages = size >> PAGE_SHIFT;
 
        if (!nr_pages)
@@ -1122,13 +1130,18 @@ static long pipe_set_size(struct pipe_inode_info *pipe, 
unsigned long arg)
 int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf,
                 size_t *lenp, loff_t *ppos)
 {
+       unsigned int rounded_pipe_max_size;
        int ret;
 
        ret = proc_douintvec_minmax(table, write, buf, lenp, ppos);
        if (ret < 0 || !write)
                return ret;
 
-       pipe_max_size = round_pipe_size(pipe_max_size);
+       rounded_pipe_max_size = round_pipe_size(pipe_max_size);
+       if (rounded_pipe_max_size == 0)
+               return -EINVAL;
+
+       pipe_max_size = rounded_pipe_max_size;
        return ret;
 }
 
-- 
1.8.3.1