already there! :) On Fri, Mar 2, 2012 at 6:36 PM, Jeffrey Squyres <jsquy...@cisco.com> wrote:
> Sweet! > > Can you document this on the wiki in the listing of all the INI fields? > > > On Mar 2, 2012, at 10:27 AM, mi...@osl.iu.edu wrote: > > > Author: miked > > Date: 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > > New Revision: 1440 > > URL: https://svn.open-mpi.org/trac/mtt/changeset/1440 > > > > Log: > > Now it is possible to shuffle tests in mtt. The parameter that enables > this feature is: > > shuffle_tests= > > it can be assigned a comma separated list of allowed values: sections, > tests, execs, nps, args, all. > > > > Notations: > > sections - shuffle test_run sections found in mtt > > tests - shuffle test executables if any (we have single executable > per test_run) > > execs - shuffle exec command line present in main [MTT] section > > nps - shuffle runs with different nps > > args - shuffle executable arguments > > all - enable all the above. > > > > > > Text files modified: > > trunk/lib/MTT/Globals.pm | 65 > +++++++++++---------------------------- > > trunk/lib/MTT/Reporter/Email.pm | 6 ++- > > trunk/lib/MTT/Reporter/TextFile.pm | 2 > > trunk/lib/MTT/Test/Run.pm | 7 +++ > > trunk/lib/MTT/Test/RunEngine.pm | 61 > +++++++++++++++++++++++++++++-------- > > trunk/lib/MTT/Util.pm | 14 ++++++- > > 6 files changed, 89 insertions(+), 66 deletions(-) > > > > Modified: trunk/lib/MTT/Globals.pm > > > ============================================================================== > > --- trunk/lib/MTT/Globals.pm (original) > > +++ trunk/lib/MTT/Globals.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > > @@ -80,8 +80,7 @@ > > > > save_intermediate_report => undef, > > save_intermediate_report_enable => undef, > > - ini_value_run_for => undef, > > - start_time => localtime, > > + shuffle_tests => undef, > > description => undef, > > }; > > > > @@ -181,50 +180,24 @@ > > $Values->{save_intermediate_report_enable} = $val; > > } > > > > -# $val = MTT::Values::Value($ini, "MTT", "finish_at"); > > -# if (defined($val)) { > > -# #finish_at format: hh:mm[dd/MM] > > -# $val =~ m/(\d\d:\d\d)(\[(\d\d)\/(\d\d)\])*/; > > -# my $stop_time = $1.":00"; > > -# my $stop_day = $3; > > -# my $stop_month =$4; > > -# my @timeData = localtime(time); > > -# #@timedata : [0]secs [1]minutes [2]hours [3]days [4]month-1 > > -# my $finish_time = > MTT::Util::parse_time_to_seconds($stop_time); > > -# > > -# > > -# my > $local_time=$timeData[2].":".$timeData[1].":".$timeData[0]; > > -# my $current_time = > MTT::Util::parse_time_to_seconds($local_time); > > -# > > -# > > -# > > -# my $secs_left; > > -# if ($current_time < $finish_time){ > > -# $secs_left = $finish_time-$current_time; > > -# } else { > > -# $secs_left = $finish_time+24*3600-$current_time; > > -# } > > -# $Values->{ini_value_run_for} = $secs_left; > > -# > > -# if (!$stop_month){ > > -# if ($stop_day){ > > -# if ($stop_day < $timeData[3]){ > > -# MTT::Messages::Warning("Stop date is less > than start date: disabling stop_at feature"); > > -# $Values->{ini_value_run_for} = undef; > > -# }else if ($stop_day == $timeData[3]){ > > -# if ($finish_time < $current_time){ > > -# MTT::Messages::Warning("The stop > is less than start time: disabling stop_at feature"); > > -# } > > -# }else{ > > -# if ($finish_time > $current_time){ > > -# $secs_left += 24*3600; > > -# } > > -# } > > -# } > > -# } > > -# printf "secs_left = ",$secs_left, "hours_left = > ",$secs_left/3600,"\n"; > > -# > > -# } > > + > > + $val = MTT::Values::Value($ini, "MTT", "shuffle_tests"); > > + if (defined($val)) { > > + my @shuffles = grep length, split(/\s*,\s*/,$val); > > + my @allowed_shuffles = ('sections', 'tests', 'execs', 'nps', > 'args', 'all'); > > + my %allowed_map = map { $_ => 1} @allowed_shuffles; > > + foreach my $sh (@shuffles){ > > + if (!exists($allowed_map{$sh})){ > > + MTT::Messages::Error("<$sh> is not allowed > shuffle_tests value. Allowed are:", join(',',@allowed_shuffles),"."); > > + } > > + $Values->{shuffle_tests}->{$sh} = 1; > > + } > > + if ($Values->{shuffle_tests}->{all}){ > > + foreach my $allow (@allowed_shuffles){ > > + $Values->{shuffle_tests}->{$allow} = 1; > > + } > > + } > > + } > > } > > > > > #-------------------------------------------------------------------------- > > > > Modified: trunk/lib/MTT/Reporter/Email.pm > > > ============================================================================== > > --- trunk/lib/MTT/Reporter/Email.pm (original) > > +++ trunk/lib/MTT/Reporter/Email.pm 2012-03-02 10:27:34 EST (Fri, 02 > Mar 2012) > > @@ -148,8 +148,10 @@ > > $to = Value( $Ini, $Section, "email_to" ); > > } > > } > > - my $body = $subject."\n".$footer."\n"; > > - MTT::Mail::Send($subject, $to, $from, $body ); > > + my $body = $footer."\n"; > > + if ($to){ > > + MTT::Mail::Send($subject, $to, $from, $body ); > > + } > > } > > > > 1; > > > > Modified: trunk/lib/MTT/Reporter/TextFile.pm > > > ============================================================================== > > --- trunk/lib/MTT/Reporter/TextFile.pm (original) > > +++ trunk/lib/MTT/Reporter/TextFile.pm 2012-03-02 10:27:34 EST > (Fri, 02 Mar 2012) > > @@ -142,7 +142,7 @@ > > sub Flush{ > > my ($info, $entries) = @_; > > my @results_to_flush = @results; > > - push(@results_to_flush, $entries); > > + push(@results_to_flush, $entries) if $entries; > > _summary_report(\@results_to_flush, "yes") > > if (@results_to_flush); > > > > > > Modified: trunk/lib/MTT/Test/Run.pm > > > ============================================================================== > > --- trunk/lib/MTT/Test/Run.pm (original) > > +++ trunk/lib/MTT/Test/Run.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > > @@ -77,7 +77,12 @@ > > > > # Go through all the sections in the ini file looking for section > > # names that begin with "Test run:" > > - foreach my $section ($ini->Sections()) { > > + my @sects = $ini->Sections(); > > + if ($MTT::Globals::Values->{shuffle_tests}->{sections}){ > > + MTT::Util::shuffle(\@sects); > > + } > > + > > + foreach my $section (@sects) { > > # See if we're supposed to terminate. Only check in the > > # outtermost and innermost loops (even though we *could* check > > # at every loop level); that's good enough. > > > > Modified: trunk/lib/MTT/Test/RunEngine.pm > > > ============================================================================== > > --- trunk/lib/MTT/Test/RunEngine.pm (original) > > +++ trunk/lib/MTT/Test/RunEngine.pm 2012-03-02 10:27:34 EST (Fri, 02 > Mar 2012) > > @@ -39,7 +39,7 @@ > > my $mpi_details_name; > > my $test_run_full_name; > > my $break_threshold; > > - > > +my $shuffle; > > # Keep track of how many tests have passed, failed, skipped, and timed > out > > my $test_results_count; > > > > @@ -52,7 +52,6 @@ > > my $report_after_n_results; > > my $prev_section_name = undef; > > my $group_reports = 0; > > -my $tests_performed = 0; > > > > > #-------------------------------------------------------------------------- > > > > @@ -96,6 +95,10 @@ > > $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section, > "break_threshold_timeout"); > > $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section, > "break_threshold_skipped"); > > $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini, > "mtt", "break_threshold_timeout_and_fail"); > > + > > + $shuffle = undef; > > + _register_shuffles($ini,$section); > > + > > my $reports_per_job = Value($ini, "mtt", "reports_per_job"); > > # This boolean value defaults to 0, and allows the user to submit > results > > # after each test to ensure at least *some* results are submitted > (in case > > @@ -167,6 +170,10 @@ > > $verbose_out = 0; > > my $test_count = 0; > > my $printed = 0; > > + if ($shuffle->{tests}){ > > + MTT::Util::shuffle(\@{$ret->{tests}}); > > + } > > + > > foreach my $run (@{$ret->{tests}}) { > > > > # See if we're supposed to terminate. > > @@ -221,6 +228,9 @@ > > if (ref($all_np) eq "") { > > $test_results->{$all_np} = _run_one_np($install_dir, $run, > $mpi_details, $all_np, $force); > > } else { > > + if ($shuffle->{nps}){ > > + MTT::Util::shuffle($all_np); > > + } > > foreach my $this_np (@$all_np) { > > # See if we're supposed to terminate. > > last > > @@ -288,13 +298,19 @@ > > if (ref($all_argv) eq "") { > > $all_argv = [$all_argv]; > > } > > - > > + if ($shuffle->{args}){ > > + MTT::Util::shuffle($all_argv); > > + } > > foreach my $this_argv (@$all_argv) { > > $MTT::Test::Run::test_argv = $this_argv; > > > > # Get all the exec's for this one np > > my $execs = > MTT::Values::EvaluateString($mpi_details->{exec}, $ini, $mpi_details_name); > > - > > + if (ref($execs) ne "" && $shuffle->{execs}){ > > + MTT::Util::shuffle($execs); > > + } > > + > > + > > # If we just got one, run it. Otherwise, loop over running > them. > > if (ref($execs) eq "") { > > _run_one_test($install_dir, $run, $mpi_details, $execs, > $name, 1, > > @@ -497,14 +513,7 @@ > > if (exists($report->{test_result}) && > > (MTT::Values::FAIL == $report->{test_result} || > MTT::Values::TIMED_OUT == $report->{test_result})); > > > > - $tests_performed++; > > > > - #print "\t\t--> Test #",$tests_performed," done\n"; > > - > > - > > - #MTT::Reporter::Flush(); > > - #MTT::Reporter::TextFile::deSubmit(); > > - > > # If there is an after_each step, run it > > $ENV{MTT_TEST_RUN_RESULT_MESSAGE} = > > (MTT::Values::PASS == $report->{test_result} ? "passed" : > > @@ -558,6 +567,32 @@ > > } > > } > > } > > - > > - > > +sub _register_shuffles{ > > + my ($ini,$section) = @_; > > + > > + my $val = MTT::Values::Value($ini, $section, "shuffle_tests"); > > + if (defined($val)) { > > + my @shuffles = grep length, split(/\s*,\s*/,$val); > > + my @allowed_shuffles = ('tests', 'execs', 'nps', 'args', > 'all', 'none'); > > + my %allowed_map = map { $_ => 1} @allowed_shuffles; > > + foreach my $sh (@shuffles){ > > + if (!exists($allowed_map{$sh})){ > > + MTT::Messages::Error("<$sh> is not allowed section > specific shuffle_tests value. Allowed are:", > join(',',@allowed_shuffles),"."); > > + } > > + $shuffle->{$sh} = 1; > > + } > > + if ($shuffle->{none}){ > > + $shuffle = undef; > > + } > > + if ($shuffle->{all}){ > > + foreach my $allow (@allowed_shuffles){ > > + $shuffle->{$allow} = 1; > > + } > > + } > > + > > + } > > + else{ > > + $shuffle = $MTT::Globals::Values->{shuffle_tests}; > > + } > > +} > > 1; > > > > Modified: trunk/lib/MTT/Util.pm > > > ============================================================================== > > --- trunk/lib/MTT/Util.pm (original) > > +++ trunk/lib/MTT/Util.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > > @@ -208,9 +208,6 @@ > > $MTT::Globals::Internals->{is_stopped_on_break_threshold} = > "true"; > > > $MTT::Globals::Internals->{stopped_on_break_threshold_message} = "--> > Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of > $total.\n"; > > print STDOUT "--> Threshold ($per) exceeded for > \"$result_label\": $count->{$result} out of $total.\n"; > > - if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){ > > - print STDOUT "0xdeadbeef: it works"; > > - } > > return 1; > > } > > } > > @@ -395,4 +392,15 @@ > > } > > } > > > > > +#-------------------------------------------------------------------------- > > +sub shuffle{ > > + # Shuffle an array given via reference randomly > > + my $array = shift; > > + my $i; > > + for ($i = @$array; --$i; ) { > > + my $j = int rand ($i+1); > > + next if $i == $j; > > + @$array[$i,$j] = @$array[$j,$i]; > > + } > > +} > > 1; > > _______________________________________________ > > mtt-svn mailing list > > mtt-...@open-mpi.org > > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn > > > -- > Jeff Squyres > jsquy...@cisco.com > For corporate legal information go to: > http://www.cisco.com/web/about/doing_business/legal/cri/ > > > _______________________________________________ > mtt-devel mailing list > mtt-de...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-devel >
