Re: [PATCH] IOzone test: Introduce postprocessing module v2
only thing that strikes me is whether the gnuplot support
should be abstracted out a bit. See tko/plotgraph.py ?
On Mon, May 3, 2010 at 2:52 PM, Lucas Meneghel Rodrigues l...@redhat.com
wrote:
This module contains code to postprocess IOzone data
in a convenient way so we can generate performance graphs
and condensed data. The graph generation part depends
on gnuplot, but if the utility is not present,
functionality will gracefully degrade.
Use the postprocessing module introduced on the previous
patch, use it to analyze results and write performance
graphs and performance tables.
Also, in order for other tests to be able to use the
postprocessing code, added the right __init__.py
files, so a simple
from autotest_lib.client.tests.iozone import postprocessing
will work
Note: Martin, as patch will ignore and not create the
zero-sized files (high time we move to git), if the changes
look good to you I can commit them all at once, making sure
all files are created.
Signed-off-by: Lucas Meneghel Rodrigues l...@redhat.com
---
client/tests/iozone/common.py | 8 +
client/tests/iozone/iozone.py | 25 ++-
client/tests/iozone/postprocessing.py | 487
+
3 files changed, 515 insertions(+), 5 deletions(-)
create mode 100644 client/tests/__init__.py
create mode 100644 client/tests/iozone/__init__.py
create mode 100644 client/tests/iozone/common.py
create mode 100755 client/tests/iozone/postprocessing.py
diff --git a/client/tests/__init__.py b/client/tests/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/__init__.py b/client/tests/iozone/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/common.py b/client/tests/iozone/common.py
new file mode 100644
index 000..ce78b85
--- /dev/null
+++ b/client/tests/iozone/common.py
@@ -0,0 +1,8 @@
+import os, sys
+dirname = os.path.dirname(sys.modules[__name__].__file__)
+client_dir = os.path.abspath(os.path.join(dirname, .., ..))
+sys.path.insert(0, client_dir)
+import setup_modules
+sys.path.pop(0)
+setup_modules.setup(base_path=client_dir,
+ root_module_name=autotest_lib.client)
diff --git a/client/tests/iozone/iozone.py b/client/tests/iozone/iozone.py
index fa3fba4..03c2c04 100755
--- a/client/tests/iozone/iozone.py
+++ b/client/tests/iozone/iozone.py
@@ -1,5 +1,6 @@
import os, re
from autotest_lib.client.bin import test, utils
+import postprocessing
class iozone(test.test):
@@ -63,17 +64,19 @@ class iozone(test.test):
self.results = utils.system_output('%s %s' % (cmd, args))
self.auto_mode = (-a in args)
- path = os.path.join(self.resultsdir, 'raw_output_%s' %
self.iteration)
- raw_output_file = open(path, 'w')
- raw_output_file.write(self.results)
- raw_output_file.close()
+ self.results_path = os.path.join(self.resultsdir,
+ 'raw_output_%s' % self.iteration)
+ self.analysisdir = os.path.join(self.resultsdir,
+ 'analysis_%s' % self.iteration)
+
+ utils.open_write_close(self.results_path, self.results)
def __get_section_name(self, desc):
return desc.strip().replace(' ', '_')
- def postprocess_iteration(self):
+ def generate_keyval(self):
keylist = {}
if self.auto_mode:
@@ -150,3 +153,15 @@ class iozone(test.test):
keylist[key_name] = result
self.write_perf_keyval(keylist)
+
+
+ def postprocess_iteration(self):
+ self.generate_keyval()
+ if self.auto_mode:
+ a = postprocessing.IOzoneAnalyzer(list_files=[self.results_path],
+ output_dir=self.analysisdir)
+ a.analyze()
+ p = postprocessing.IOzonePlotter(results_file=self.results_path,
+ output_dir=self.analysisdir)
+ p.plot_all()
+
diff --git a/client/tests/iozone/postprocessing.py
b/client/tests/iozone/postprocessing.py
new file mode 100755
index 000..c995aea
--- /dev/null
+++ b/client/tests/iozone/postprocessing.py
@@ -0,0 +1,487 @@
+#!/usr/bin/python
+
+Postprocessing module for IOzone. It is capable to pick results from an
+IOzone run, calculate the geometric mean for all throughput results for
+a given file size or record size, and then generate a series of 2D and 3D
+graphs. The graph generation functionality depends on gnuplot, and if it
+is not present, functionality degrates gracefully.
+
+...@copyright: Red Hat 2010
+
+import os, sys, optparse, logging, math, time
+import common
+from autotest_lib.client.common_lib import logging_config, logging_manager
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.bin import utils, os_dep
+
+
+_LABELS =
Re: [PATCH] IOzone test: Introduce postprocessing module v2
On Mon, 2010-05-03 at 16:52 -0700, Martin Bligh wrote:
only thing that strikes me is whether the gnuplot support
should be abstracted out a bit. See tko/plotgraph.py ?
I thought about it. Ideally, we would do all the plotting using a python
library, such as matplotlib, which has a decent API. However, I spent
quite some time trying to figure out how to draw the surface graphs
using matplot lib and in the end, I gave up (3d support on that lib is
just starting). There are some other libs, such as mayavi
(http://mayavi.sourceforge.net) that I would like to try out on the near
future. Your code in plotgraph.py is aimed to 2D graphs, a good
candidate for replacement using matplotlib (their support to 2D is
excellent).
So, instead of spending much time encapsulating gnuplot on a nice API,
I'd prefer to have this intermediate work (anyway it does the job) and
when possible, get back to this subject. What do you think?
On Mon, May 3, 2010 at 2:52 PM, Lucas Meneghel Rodrigues l...@redhat.com
wrote:
This module contains code to postprocess IOzone data
in a convenient way so we can generate performance graphs
and condensed data. The graph generation part depends
on gnuplot, but if the utility is not present,
functionality will gracefully degrade.
Use the postprocessing module introduced on the previous
patch, use it to analyze results and write performance
graphs and performance tables.
Also, in order for other tests to be able to use the
postprocessing code, added the right __init__.py
files, so a simple
from autotest_lib.client.tests.iozone import postprocessing
will work
Note: Martin, as patch will ignore and not create the
zero-sized files (high time we move to git), if the changes
look good to you I can commit them all at once, making sure
all files are created.
Signed-off-by: Lucas Meneghel Rodrigues l...@redhat.com
---
client/tests/iozone/common.py |8 +
client/tests/iozone/iozone.py | 25 ++-
client/tests/iozone/postprocessing.py | 487
+
3 files changed, 515 insertions(+), 5 deletions(-)
create mode 100644 client/tests/__init__.py
create mode 100644 client/tests/iozone/__init__.py
create mode 100644 client/tests/iozone/common.py
create mode 100755 client/tests/iozone/postprocessing.py
diff --git a/client/tests/__init__.py b/client/tests/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/__init__.py
b/client/tests/iozone/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/common.py b/client/tests/iozone/common.py
new file mode 100644
index 000..ce78b85
--- /dev/null
+++ b/client/tests/iozone/common.py
@@ -0,0 +1,8 @@
+import os, sys
+dirname = os.path.dirname(sys.modules[__name__].__file__)
+client_dir = os.path.abspath(os.path.join(dirname, .., ..))
+sys.path.insert(0, client_dir)
+import setup_modules
+sys.path.pop(0)
+setup_modules.setup(base_path=client_dir,
+root_module_name=autotest_lib.client)
diff --git a/client/tests/iozone/iozone.py b/client/tests/iozone/iozone.py
index fa3fba4..03c2c04 100755
--- a/client/tests/iozone/iozone.py
+++ b/client/tests/iozone/iozone.py
@@ -1,5 +1,6 @@
import os, re
from autotest_lib.client.bin import test, utils
+import postprocessing
class iozone(test.test):
@@ -63,17 +64,19 @@ class iozone(test.test):
self.results = utils.system_output('%s %s' % (cmd, args))
self.auto_mode = (-a in args)
-path = os.path.join(self.resultsdir, 'raw_output_%s' %
self.iteration)
-raw_output_file = open(path, 'w')
-raw_output_file.write(self.results)
-raw_output_file.close()
+self.results_path = os.path.join(self.resultsdir,
+ 'raw_output_%s' % self.iteration)
+self.analysisdir = os.path.join(self.resultsdir,
+'analysis_%s' % self.iteration)
+
+utils.open_write_close(self.results_path, self.results)
def __get_section_name(self, desc):
return desc.strip().replace(' ', '_')
-def postprocess_iteration(self):
+def generate_keyval(self):
keylist = {}
if self.auto_mode:
@@ -150,3 +153,15 @@ class iozone(test.test):
keylist[key_name] = result
self.write_perf_keyval(keylist)
+
+
+def postprocess_iteration(self):
+self.generate_keyval()
+if self.auto_mode:
+a =
postprocessing.IOzoneAnalyzer(list_files=[self.results_path],
+ output_dir=self.analysisdir)
+a.analyze()
+p =
postprocessing.IOzonePlotter(results_file=self.results_path,
+
Re: [PATCH] IOzone test: Introduce postprocessing module v2
yup, fair enough. Go ahead and check it in. If we end up doing
this in another test, we should make an abstraction
On Mon, May 3, 2010 at 5:39 PM, Lucas Meneghel Rodrigues l...@redhat.com
wrote:
On Mon, 2010-05-03 at 16:52 -0700, Martin Bligh wrote:
only thing that strikes me is whether the gnuplot support
should be abstracted out a bit. See tko/plotgraph.py ?
I thought about it. Ideally, we would do all the plotting using a python
library, such as matplotlib, which has a decent API. However, I spent
quite some time trying to figure out how to draw the surface graphs
using matplot lib and in the end, I gave up (3d support on that lib is
just starting). There are some other libs, such as mayavi
(http://mayavi.sourceforge.net) that I would like to try out on the near
future. Your code in plotgraph.py is aimed to 2D graphs, a good
candidate for replacement using matplotlib (their support to 2D is
excellent).
So, instead of spending much time encapsulating gnuplot on a nice API,
I'd prefer to have this intermediate work (anyway it does the job) and
when possible, get back to this subject. What do you think?
On Mon, May 3, 2010 at 2:52 PM, Lucas Meneghel Rodrigues l...@redhat.com
wrote:
This module contains code to postprocess IOzone data
in a convenient way so we can generate performance graphs
and condensed data. The graph generation part depends
on gnuplot, but if the utility is not present,
functionality will gracefully degrade.
Use the postprocessing module introduced on the previous
patch, use it to analyze results and write performance
graphs and performance tables.
Also, in order for other tests to be able to use the
postprocessing code, added the right __init__.py
files, so a simple
from autotest_lib.client.tests.iozone import postprocessing
will work
Note: Martin, as patch will ignore and not create the
zero-sized files (high time we move to git), if the changes
look good to you I can commit them all at once, making sure
all files are created.
Signed-off-by: Lucas Meneghel Rodrigues l...@redhat.com
---
client/tests/iozone/common.py | 8 +
client/tests/iozone/iozone.py | 25 ++-
client/tests/iozone/postprocessing.py | 487
+
3 files changed, 515 insertions(+), 5 deletions(-)
create mode 100644 client/tests/__init__.py
create mode 100644 client/tests/iozone/__init__.py
create mode 100644 client/tests/iozone/common.py
create mode 100755 client/tests/iozone/postprocessing.py
diff --git a/client/tests/__init__.py b/client/tests/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/__init__.py
b/client/tests/iozone/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/client/tests/iozone/common.py b/client/tests/iozone/common.py
new file mode 100644
index 000..ce78b85
--- /dev/null
+++ b/client/tests/iozone/common.py
@@ -0,0 +1,8 @@
+import os, sys
+dirname = os.path.dirname(sys.modules[__name__].__file__)
+client_dir = os.path.abspath(os.path.join(dirname, .., ..))
+sys.path.insert(0, client_dir)
+import setup_modules
+sys.path.pop(0)
+setup_modules.setup(base_path=client_dir,
+ root_module_name=autotest_lib.client)
diff --git a/client/tests/iozone/iozone.py b/client/tests/iozone/iozone.py
index fa3fba4..03c2c04 100755
--- a/client/tests/iozone/iozone.py
+++ b/client/tests/iozone/iozone.py
@@ -1,5 +1,6 @@
import os, re
from autotest_lib.client.bin import test, utils
+import postprocessing
class iozone(test.test):
@@ -63,17 +64,19 @@ class iozone(test.test):
self.results = utils.system_output('%s %s' % (cmd, args))
self.auto_mode = (-a in args)
- path = os.path.join(self.resultsdir, 'raw_output_%s' %
self.iteration)
- raw_output_file = open(path, 'w')
- raw_output_file.write(self.results)
- raw_output_file.close()
+ self.results_path = os.path.join(self.resultsdir,
+ 'raw_output_%s' % self.iteration)
+ self.analysisdir = os.path.join(self.resultsdir,
+ 'analysis_%s' % self.iteration)
+
+ utils.open_write_close(self.results_path, self.results)
def __get_section_name(self, desc):
return desc.strip().replace(' ', '_')
- def postprocess_iteration(self):
+ def generate_keyval(self):
keylist = {}
if self.auto_mode:
@@ -150,3 +153,15 @@ class iozone(test.test):
keylist[key_name] = result
self.write_perf_keyval(keylist)
+
+
+ def postprocess_iteration(self):
+ self.generate_keyval()
+ if self.auto_mode:
+ a =
postprocessing.IOzoneAnalyzer(list_files=[self.results_path],
+
