On Thu, Sep 23, 2010 at 2:50 PM, Lucas Meneghel Rodrigues <[email protected]> wrote: > > On Thu, 2010-09-23 at 14:46 -0700, Akshay Lal wrote: > > Updated the wb_kupdate test to only perform disk initializations when an > > fs_type has been provided. > > Updated the nightly kernel_qual_fs to run the wb_kupdate test outside the > > default parallel and partitions framework since it doesn't use either. > > I just double checked, and there's no such test upstream, it's a test of > google's internal branch. As allways, if possible, you guys might want > to contribute it upstream :)
Our magic patch mailer seems to have had some trouble of late. Here's the wb_kupdate addition patch that Akshay told it to mail out a few weeks ago: =========================== Added the wb_kupdate code to autotest. This should catch scenarios wherein writeback doesn't flush data to disk that is older than 30 seconds. Signed-off-by: Akshay Lal <[email protected]> --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/client/tests/wb_kupdate/README 2010-08-18 19:28:47.000000000 -0700 @@ -0,0 +1,9 @@ +Description: +------------ +This tests checks the wb_kupdate code path by writting data to a sparse file +mounted on a loop back device formated with a user specified filesystem, +and waiting a maximum of 'max_flush_time' for writeback to flush the dirty +datat from the cache to disk. + +At the end of the test a keyval is generated which states per iteration the time +taken to write the user specified amount of data to disk. --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/client/tests/wb_kupdate/common.py 2010-08-18 19:28:47.000000000 -0700 @@ -0,0 +1,8 @@ +import os, sys +dirname = os.path.dirname(sys.modules[__name__].__file__) +autotest_dir = os.path.abspath(os.path.join(dirname, "..", "..", "..")) +client_dir = os.path.join(autotest_dir, "client") +sys.path.insert(0, client_dir) +import setup_modules +sys.path.pop(0) +setup_modules.setup(base_path=autotest_dir, root_module_name="autotest_lib") --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/client/tests/wb_kupdate/control 2010-08-18 19:28:47.000000000 -0700 @@ -0,0 +1,42 @@ +AUTHOR = "Akshay Lal <[email protected]>" +NAME = "wb_kupdate" +TEST_CATEGORY = "Functional" +TEST_CLASS = "General" +TEST_TYPE = "client" +TIME = 'MEDIUM' +DOC=''' +This tests checks the wb_kupdate code path by writting data to a sparse file +and waiting at max of `max_flush_time` for the file to be flushed from the +cache to disk. +''' + +import os +# Required Parameters: +# -------------------- +duration=2 # In minutes. +mount_point='/export/wb_update' # Absolute path. +file_count=1 +write_size=1 # In MB. + +# Optional Parameters: +# -------------------- +max_flush_time=1 # In minutes. +file_system='ext4' # mkfs.<file_system> must already exist on + # the machine +remove_previous=False # Boolean. +sparse_file=os.path.join( # Absolute path to the sparse file. + os.getcwd(), + 'sparse_file') + +# Beginning execution of the xfstests: +# ------------------------------------ +job.run_test('wb_kupdate', + duration=int(duration), + mount_point=mount_point, + file_count=int(file_count), + write_size=int(write_size), + max_flush_time=int(max_flush_time), + file_system=file_system, + remove_previous=remove_previous, + sparse_file=sparse_file, + tag='wb_kupdate_execution') --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/client/tests/wb_kupdate/wb_kupdate.py 2010-08-18 19:28:47.000000000 -0700 @@ -0,0 +1,288 @@ +import commands +import datetime +import logging +import os +from autotest_lib.client.bin import test +from autotest_lib.client.bin import utils +from autotest_lib.client.common_lib import error + +class wb_kupdate(test.test): + version = 1 + + + def _execute(self, cmd, error_check_skip=False): + """Execute any command on the console. + + @param cmd: the command to execute. + @param error_check_skip: skip proactively checking for errors. Default + is false. + + @returns: a list containing the return status and message of the + command. + + @raises error.TestError: if an exception is caught during command + execution. + """ + # Log the command. + logging.debug('Command to execute: %s', cmd) + try: + # Execute the command. + ret, output = commands.getstatusoutput(cmd) + + # Check the output from the command. + if error_check_skip: + return output + + if ret: + logging.error('Error seen on stderr PIPE. Return Code: %s. ' + 'Message: %s', ret, output) + raise OSError(output) + + # Log the output. + logging.debug('Command output: %s', output) + except OSError, err: + raise error.TestError('Exception caught while executing: %s. ' + 'Error msg: %s' % (cmd, err.strerror)) + + return ret,output + + + def _check_parameters(self, mount_point, write_size, file_count): + """Check all test parameters. + + @param mount_point: the path to the desired mount_point. + @param write_size: the size of data in MB to write. + @param file_count: the number of files to write. + """ + # Check mount_point. + if not os.path.exists(mount_point): + logging.info('%s does not exist. Creating directory.', mount_point) + self._execute('mkdir %s' % mount_point) + + # Check write_size > 0. + if not (write_size > 0): + raise error.TestError('Write size should be a positive integer.') + + # Check file_count > 0. + if not (file_count > 0) : + raise error.TestError('File count shoulde be a positive integer.') + + + def _reset_device(self, mount_point, file_system): + """Reset the test. Reinitialize sparse file. + + @param mount_point: the absolute path to the mount_point. + @param file_system: the type of file_system. + """ + # Umount device. + self._execute('umount %s' % mount_point, error_check_skip=True) + + # Remove sparse_file. + self._execute('rm -rf %s' % self.sparse_file) + + # Recreate sparse_file. + self._execute('dd if=/dev/zero of=%s bs=1M seek=10000 count=1' % + self.sparse_file) + + # Format sparse_file. + self._execute('echo "y" | mkfs -t %s %s' % + (file_system, self.sparse_file), error_check_skip=True) + + # Mount sparse_file. + self._execute('mount -o loop -t %s %s %s' % (file_system, + self.sparse_file, + mount_point)) + + # Flush cache. + self._flush_cache() + + + def _flush_cache(self): + """Flush both read and write cache. + """ + # Flushing read and write cache. + self._execute(self._SYSTEM_FLUSH) + + + def _needs_more_time(self, start_time, duration, _now=None): + """Checks to see if the test has run its course. + + @param start_time: a datetime object specifying the start time of the + test. + @param duration: test duration in minutes. + @param _now: used mostly for testing - ensures that the function returns + pass/fail depnding on the value of _now. + + @return: True if the test still needs to run longer. + False if the test has run for 'duration' minutes. + """ + if not _now: + time_diff = datetime.datetime.now() - start_time + else: + time_diff = _now - start_time + return time_diff <= datetime.timedelta(seconds=duration*60) + + + def _write_data(self, destination, counter, write_size): + """Writes data to the cache/memory. + + @param destination: the absolute path to where the data needs to be + written. + @param counter: the file counter. + @param write_size: the size of data to be written. + + @returns: the time when the write completed as a datetime object. + """ + # Write data to disk. + file_name = os.path.join(destination, 'test_file_%s' % counter) + write_cmd = ('dd if=/dev/zero of=%s bs=1M count=%s' % + (file_name, write_size)) + self._execute(write_cmd) + + # Time the write operation. + write_completion_time = datetime.datetime.now() + + # Return write completion time. + return write_completion_time + + + def _get_disk_usage(self, file_name): + """Returns the disk usage of given file. + + @param file_name: the name of the file. + + @returns: the disk usage as an integer. + """ + # Check du stats. + cmd = '%s %s' % (self._DU_CMD, file_name) + + # Expected value for output = '1028\tfoo' + ret,output = self._execute(cmd) + + # Desired ouput = (1028, foo) + output = output.split('\t') + + return int(output[0]) + + + def _wait_until_data_flushed(self, start_time, max_wait_time): + """Check to see if the sparse file size increases. + + @param start_time: the time when data was actually written into the + cache. + @param max_wait_time: the max amount of time to wait. + + @returns: time waited as a datetime.timedelta object. + """ + current_size = self._get_disk_usage(self.sparse_file) + flushed_size = current_size + + # Keep checking until du value changes. + while current_size == flushed_size: + # Get flushed_size. + flushed_size = self._get_disk_usage(self.sparse_file) + + # Check if data has been synced to disk. + if not self._needs_more_time(start_time, max_wait_time): + raise error.TestError('Data not flushed. Waited for %s minutes ' + 'for data to flush out.' % max_wait_time) + + # Return time waited. + return datetime.datetime.now() - start_time + + + def initialize(self): + """Initialize all private and global member variables. + """ + self._SYSTEM_FLUSH = 'sync; echo 3 > /proc/sys/vm/drop_caches' + self._DU_CMD = 'du' + self.mount_point = '' + self.sparse_file = '' + self.result_map = {} + + + def run_once(self, duration, mount_point, file_count, write_size, + max_flush_time=1, file_system='ext4', remove_previous=False, + sparse_file=os.path.join(os.getcwd(),'sparse_file')): + """Control execution of the test. + + @param duration: an integer defining the duration of test test in + minutes. + @param mount_point: the absolute path to the mount point. + @param file_count: the number of files to write. + @param write_size: the size of each file in MB. + @param max_flush_time: the maximum time to wait for the writeback to + flush dirty data to disk. Default = 1 minute. + @param file_system: the file system to be used. Default = ext4. + @param remove_previous: boolean that allows the removal of previous + files before creating a new one. Default = False. + @param sparse_file: the absolute path to the sparse file. + """ + # Check validity of parameters. + self._check_parameters(mount_point, write_size, file_count) + self.mount_point = mount_point + self.sparse_file = sparse_file + + # Start iterations. + logging.info('Starting test operations.') + test_start_time = datetime.datetime.now() + counter = 1 + + # Continue testing until options.duration runs out. + while self._needs_more_time(test_start_time, int(duration)): + # Check the current file_count. + if counter % file_count == 0 or counter == 1: + # Setting up device. + logging.info('Re-initializing device.') + self._reset_device(self.mount_point, file_system) + + logging.info('Iteration %s.', counter) + + # Write data to disk. + write_completion_time = self._write_data(self.mount_point, counter, + write_size) + logging.debug('Write time:%s', + write_completion_time.strftime("%H:%M:%S")) + + # Wait until data get synced to disk. + time_taken = self._wait_until_data_flushed(write_completion_time, + max_flush_time) + + # Log time statistics. + logging.info('Time taken to flush data: %s seconds.', + time_taken.seconds) + + # Check if there is a need to remove the previously written file. + if remove_previous: + logging.debug('Removing previous file instance.') + rm_cmd = str('rm -rf %s;' % file_name) + self._execute(rm_cmd) + else: + logging.debug('Not removing previous file instance.') + + # Flush cache. + logging.debug('Flush cache between iterations.') + self._flush_cache() + + # Update the result map. + self.result_map[counter] = time_taken.seconds + + # Increment the counter. + counter += 1 + + + def postprocess(self): + """Cleanup routine. + """ + # Write out keyval map. + self.write_perf_keyval(self.result_map) + + # Unmount partition. + logging.debug('Cleanup - unmounting loopback device.') + self._execute('umount %s' % self.mount_point, error_check_skip=True) + + # Removing the sparse file. + logging.debug('Cleanup - removing sparse file.') + self._execute('rm -rf %s' % self.sparse_file) + + logging.info('Test operations completed.') --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/client/tests/wb_kupdate/wb_kupdate_unittest.py 2010-08-18 19:28:47.000000000 -0700 @@ -0,0 +1,105 @@ +#!/usr/bin/python + +import common +import datetime +import logging +import os +import time +import unittest +from autotest_lib.client.bin import test +from autotest_lib.client.bin import utils +from autotest_lib.client.common_lib import error +from autotest_lib.client.common_lib.test_utils import mock +from autotest_lib.client.tests.wb_kupdate import wb_kupdate + +class WbKupdateUnitTest(unittest.TestCase): + def setUp(self): + """Set up all required variables for the Unittest. + """ + self._logger = logging.getLogger() + self._wbkupdate_obj = WbKupdateSubclass() + self._god = mock.mock_god() + + def test_needs_more_time(self): + """Tests the _needs_more_time method. + """ + self._logger.info('Testing the "_needs_more_time" method.') + + # Obvious failure - since start_time < start_time + 1. + self.assertTrue(self._wbkupdate_obj._needs_more_time( + start_time=datetime.datetime.now(), + duration=1)) + + # Check if 1 minute has elapsed since start_time. + self.assertFalse(self._wbkupdate_obj._needs_more_time( + start_time=datetime.datetime.now(), + duration=1, + _now=datetime.datetime.now() + datetime.timedelta(seconds=60))) + + def test_wait_until_data_flushed_pass(self): + """Tests the _wait_until_data_flushed method. + + This tests the "success" code path. + """ + self._logger.info('Testing the "_wait_until_data_flushed" method - ' + 'Success code path.') + + # Creating stubs for required methods. + self._god.stub_function(self._wbkupdate_obj, + "_get_disk_usage") + + # Setting default return values for stub functions. + # Setting the initial size of the file. + self._wbkupdate_obj._get_disk_usage.expect_call('').and_return(10) + # Returning the same file size - forcing code path to enter loop. + self._wbkupdate_obj._get_disk_usage.expect_call('').and_return(10) + # Returning a greater file size - exiting the while loop. + self._wbkupdate_obj._get_disk_usage.expect_call('').and_return(11) + + # Call the method. + self._wbkupdate_obj._wait_until_data_flushed(datetime.datetime.now(), + 1) + + # Ensure all stubbed methods called. + self._god.check_playback() + + + def test_wait_until_data_flushed_fail(self): + """Tests the _wait_until_data_flushed method. + + This tests the "failure" code path. + """ + self._logger.info('Testing the "_wait_until_data_flushed" method - ' + 'Failure code path.') + # Creating stubs for required methods. + self._god.stub_function(self._wbkupdate_obj, + "_get_disk_usage") + + # Setting default return values for stub functions. + # Setting the initial size of the file. + self._wbkupdate_obj._get_disk_usage.expect_call('').and_return(10) + # Returning the same file size - forcing code path to enter loop. + self._wbkupdate_obj._get_disk_usage.expect_call('').and_return(10) + + # Call the method. + self.assertRaises(error.TestError, + self._wbkupdate_obj._wait_until_data_flushed, + start_time=datetime.datetime.now(), + max_wait_time=0) + + # Ensure all stubbed methods called. + self._god.check_playback() + + +class WbKupdateSubclass(wb_kupdate.wb_kupdate): + """Sub-classing the wb_kupdate class. + """ + def __init__(self): + """Empty constructor. + """ + # Create all test defaults. + self.initialize() + + +if __name__ == '__main__': + unittest.main() _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
