Hello again,
After releasing 2.2b3, and I'd like to get an idea on how much the new
integrated Blosc can accelerate internal computations in PyTables.
Until now, I'm testing Blosc speed mainly in a couple of computers, so if
anybody wants to help me in seeing the performance in other computers, please
run the attached benchmark and return the output back, together with their
processor information (speed, cache sizes, model), compiler version and
operating system info.
Also, I'm attaching the output for my computer (Core2 @ 2GHz, 32 KB L1 cache,
6 MB L2 shared cache, Intel E8400, gcc 4.4.1, Linux) for comparison purposes,
in case anyone is interested.
Thanks,
--
Francesc Alted
#######################################################################
# This script compares the speed of the computation of a polynomial
# for different (numpy.memmap and tables.Expr) out-of-memory paradigms.
#
# Author: Francesc Alted
# Date: 2010-02-24
#######################################################################
import os
import sys
from time import time
import numpy as np
import tables as tb
from tables import numexpr as ne
expr = ".25*x**3 + .75*x**2 - 1.5*x - 2" # the polynomial to compute
N = 10*1000*1000 # the number of points to compute expression (80 MB)
step = 100*1000 # perform calculation in slices of `step` elements
dtype = np.dtype('f8') # the datatype
# Global variable for the x values for pure numpy & numexpr
x = None
# *** The next variables do not need to be changed ***
# Filenames for numpy.memmap
fprefix = "numpy.memmap" # the I/O file prefix
mpfnames = [fprefix+"-x.bin", fprefix+"-r.bin"]
# Filename for tables.Expr
h5fname = "tablesExpr.h5" # the I/O file
MB = 1024*1024. # a MegaByte
def print_filesize(filename, clib=None, clevel=0):
"""Print some statistics about file sizes."""
#os.system("sync") # make sure that all data has been flushed to disk
if type(filename) is list:
filesize_bytes = 0
for fname in filename:
filesize_bytes += os.stat(fname)[6]
else:
filesize_bytes = os.stat(filename)[6]
filesize_MB = round(filesize_bytes / MB, 1)
print "\t\tTotal file sizes: %d -- (%s MB)" % (filesize_bytes, filesize_MB),
if clevel > 0:
print "(using %s lvl%s)" % (clib, clevel)
else:
print
def populate_x_numpy():
"""Populate the values in x axis for numpy."""
global x
# Populate x in range [-1, 1]
x = np.linspace(-1, 1, N)
def populate_x_memmap():
"""Populate the values in x axis for numpy.memmap."""
# Create container for input
x = np.memmap(mpfnames[0], dtype=dtype, mode="w+", shape=(N,))
# Populate x in range [-1, 1]
for i in xrange(0, N, step):
chunk = np.linspace((2*i-N)/float(N), (2*(i+step)-N)/float(N), step)
x[i:i+step] = chunk
del x # close x memmap
def populate_x_tables(clib, clevel):
"""Populate the values in x axis for pytables."""
f = tb.openFile(h5fname, "w")
# Create container for input
atom = tb.Atom.from_dtype(dtype)
filters = tb.Filters(complib=clib, complevel=clevel)
x = f.createCArray(f.root, "x", atom=atom, shape=(N,), filters=filters)
# Populate x in range [-1, 1]
for i in xrange(0, N, step):
chunk = np.linspace((2*i-N)/float(N), (2*(i+step)-N)/float(N), step)
x[i:i+step] = chunk
f.close()
def compute_numpy():
"""Compute the polynomial with pure numpy."""
y = eval(expr)
def compute_numexpr():
"""Compute the polynomial with pure numexpr."""
y = ne.evaluate(expr)
def compute_memmap():
"""Compute the polynomial with numpy.memmap."""
# Reopen inputs in read-only mode
x = np.memmap(mpfnames[0], dtype=dtype, mode='r', shape=(N,))
# Create the array output
r = np.memmap(mpfnames[1], dtype=dtype, mode="w+", shape=(N,))
# Do the computation by chunks and store in output
r[:] = eval(expr) # where is stored the result?
#r = eval(expr) # result is stored in-memory
del x, r # close x and r memmap arrays
print_filesize(mpfnames)
def compute_tables(clib, clevel):
"""Compute the polynomial with tables.Expr."""
f = tb.openFile(h5fname, "a")
x = f.root.x # get the x input
# Create container for output
atom = tb.Atom.from_dtype(dtype)
filters = tb.Filters(complib=clib, complevel=clevel)
r = f.createCArray(f.root, "r", atom=atom, shape=(N,), filters=filters)
# Do the actual computation and store in output
ex = tb.Expr(expr) # parse the expression
ex.setOutput(r) # where is stored the result?
# when commented out, the result goes in-memory
ex.eval() # evaluate!
f.close()
print_filesize(h5fname, clib, clevel)
if __name__ == '__main__':
print "Total size for datasets:", round(2*N*dtype.itemsize/MB, 1), "MB"
# Get the compression libraries supported
supported_clibs = [clib for clib in ("zlib", "lzo", "bzip2", "blosc")
if tb.whichLibVersion(clib)]
# Initialization code
for what in ["numpy", "numexpr", "numpy.memmap", "tables.Expr"]:
print "Populating x using %s with %d points..." % (what, N)
t0 = time()
if what == "numpy":
populate_x_numpy()
compute = compute_numpy
elif what == "numexpr":
populate_x_numpy()
compute = compute_numexpr
elif what == "numpy.memmap":
populate_x_memmap()
compute = compute_memmap
print "*** Time elapsed populating:", round(time() - t0, 3)
first = True # Sentinel
if what == "tables.Expr":
for clib in supported_clibs:
for clevel in (0, 1, 5, 9):
if not first and clevel == 0:
continue
print "Populating x using %s with %d points..." % (what, N)
populate_x_tables(clib, clevel)
print "*** Time elapsed populating:", round(time() - t0, 3)
print "Computing: '%s' using %s" % (expr, what)
t0 = time()
compute_tables(clib, clevel)
print "**** Time elapsed computing:", round(time() - t0, 3)
first = False
else:
t0 = time()
compute()
print "**** Time elapsed computing:", round(time() - t0, 3)
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
PyTables version: 2.2b3
HDF5 version: 1.8.4-pre1
NumPy version: 1.4.0rc1
Zlib version: 1.2.3
LZO version: 2.03 (Apr 30 2008)
BZIP2 version: 1.0.5 (10-Dec-2007)
Blosc version: 0.8.0 (2010-02-24)
Python version: 2.6.1 (r261:67515, Feb 3 2009, 17:34:37)
[GCC 4.3.2 [gcc-4_3-branch revision 141291]]
Platform: linux2-x86_64
Byte-ordering: little
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
Total size for datasets: 152.6 MB
Populating x using numpy with 10000000 points...
*** Time elapsed populating: 0.183
**** Time elapsed computing: 1.836
Populating x using numexpr with 10000000 points...
*** Time elapsed populating: 0.188
**** Time elapsed computing: 0.118
Populating x using numpy.memmap with 10000000 points...
*** Time elapsed populating: 1.547
Total file sizes: 160000000 -- (152.6 MB)
**** Time elapsed computing: 3.472
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 0.0
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 0.268
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 160137296 -- (152.7 MB)
**** Time elapsed computing: 0.345
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 1.149
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 42992457 -- (41.0 MB) (using zlib lvl1)
**** Time elapsed computing: 1.816
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 2.989
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 39380792 -- (37.6 MB) (using zlib lvl5)
**** Time elapsed computing: 2.255
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 3.547
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 39226132 -- (37.4 MB) (using zlib lvl9)
**** Time elapsed computing: 2.386
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 2.787
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 47131193 -- (44.9 MB) (using lzo lvl1)
**** Time elapsed computing: 1.006
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 1.41
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 47131193 -- (44.9 MB) (using lzo lvl5)
**** Time elapsed computing: 1.014
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 1.423
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 47131193 -- (44.9 MB) (using lzo lvl9)
**** Time elapsed computing: 1.016
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 7.676
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 35429882 -- (33.8 MB) (using bzip2 lvl1)
**** Time elapsed computing: 9.547
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 16.343
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 35429882 -- (33.8 MB) (using bzip2 lvl5)
**** Time elapsed computing: 9.729
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 16.491
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 35429882 -- (33.8 MB) (using bzip2 lvl9)
**** Time elapsed computing: 9.663
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 9.928
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 103591654 -- (98.8 MB) (using blosc lvl1)
**** Time elapsed computing: 0.36
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 0.741
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 64574012 -- (61.6 MB) (using blosc lvl5)
**** Time elapsed computing: 0.586
Populating x using tables.Expr with 10000000 points...
*** Time elapsed populating: 0.883
Computing: '.25*x**3 + .75*x**2 - 1.5*x - 2' using tables.Expr
Total file sizes: 48148116 -- (45.9 MB) (using blosc lvl9)
**** Time elapsed computing: 0.585
------------------------------------------------------------------------------
Download Intel® Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev
_______________________________________________
Pytables-users mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/pytables-users
