done credit goes to Valentin as a creator of this feature! On Wed, Jan 25, 2012 at 3:11 PM, Jeff Squyres <jsquy...@cisco.com> wrote:
> Yummy -- thanks Mike! > > Can you add this field (and any others you have added) to the wiki? > > https://svn.open-mpi.org/trac/mtt/wiki/MTTINIFileFields > > This is our *only* documentation; it's important to keep it up to date. > > Thanks! > > > On Jan 25, 2012, at 6:02 AM, mi...@osl.iu.edu wrote: > > > Author: miked > > Date: 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012) > > New Revision: 1433 > > URL: https://svn.open-mpi.org/trac/mtt/changeset/1433 > > > > Log: > > * Introduce mtt.break_threshold_timeout_and_fail ini param to specify % > of overall failed tests to trigger stop execution. > > > > > > Text files modified: > > trunk/lib/MTT/Reporter/TextFile.pm | 9 ++++- > > trunk/lib/MTT/Test/Run.pm | 30 ++++++++++++++++---- > > trunk/lib/MTT/Test/RunEngine.pm | 60 > +++++++++++++++++++++++++++++++-------- > > trunk/lib/MTT/Util.pm | 6 ++++ > > trunk/lib/MTT/Values.pm | 3 + > > 5 files changed, 86 insertions(+), 22 deletions(-) > > > > Modified: trunk/lib/MTT/Reporter/TextFile.pm > > > ============================================================================== > > --- trunk/lib/MTT/Reporter/TextFile.pm (original) > > +++ trunk/lib/MTT/Reporter/TextFile.pm 2012-01-25 06:02:47 EST > (Wed, 25 Jan 2012) > > @@ -230,8 +230,13 @@ > > my $filename = "All_phase-summary.txt"; > > my $file = "$dirname/" . MTT::Files::make_safe_filename("$filename"); > > > > - my $body = join("\n", ($summary_header, $table->render, $perf_stat, > $summary_footer)); > > - > > + my $body; > > + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){ > > + $body = join("\n", ($summary_header, $table->render, > $perf_stat, $MTT::Globals::Internals->{stopped_on_break_threshold_message}, > $summary_footer)); > > + } > > + else{ > > + $body = join("\n", ($summary_header, $table->render, > $perf_stat, $summary_footer)); > > + } > > print $body; > > _output_results($file, $body); > > > > > > Modified: trunk/lib/MTT/Test/Run.pm > > > ============================================================================== > > --- trunk/lib/MTT/Test/Run.pm (original) > > +++ trunk/lib/MTT/Test/Run.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012) > > @@ -64,8 +64,11 @@ > > > #-------------------------------------------------------------------------- > > > > sub Run { > > - my ($ini, $ini_full, $install_dir, $runs_data_dir, $force) = @_; > > + my ($ini, $ini_full, $install_dir, $runs_data_dir, $force, > $count_total_tests_number) = @_; > > > > + if ($count_total_tests_number ne "yes"){ > > + Run($ini, $ini_full, $install_dir, $runs_data_dir, > $force,"yes"); > > + } > > # Save the environment > > my %ENV_SAVE = %ENV; > > > > @@ -188,7 +191,7 @@ > > _do_run($ini, $section, $test_build, > > $mpi_get, $mpi_install, > > $install_dir, $runs_data_dir, > > - $force); > > + > $force,$count_total_tests_number); > > delete > $MTT::Globals::Internals->{mpi_get_name}; > > delete > $MTT::Globals::Internals->{mpi_install_name}; > > delete > $MTT::Globals::Internals->{test_get_name}; > > @@ -196,12 +199,24 @@ > > delete > $MTT::Globals::Internals->{test_run_name}; > > %ENV = %ENV_SAVE; > > } > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > } > > + last > > + if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > > > Verbose("*** Run test phase complete\n"); > > @@ -211,7 +226,7 @@ > > > > sub _do_run { > > my ($ini, $section, $test_build, $mpi_get, $mpi_install, > $install_dir, > > - $runs_data_dir, $force) = @_; > > + $runs_data_dir, $force, $count_total_tests_number) = @_; > > > > # Simple section name > > my $simple_section = GetSimpleSection($section); > > @@ -528,9 +543,13 @@ > > # If we got a list of tests to run, invoke the run engine to > > # actually run them. > > if ($ret && $ret->{test_result}) { > > - MTT::Test::RunEngine::RunEngine($ini, $section, $install_dir, > > + my $return_value; > > + $return_value = MTT::Test::RunEngine::RunEngine($ini, $section, > $install_dir, > > $runs_data_dir, $mpi_details, > > - $test_build, $force, $ret); > > + $test_build, $force, $ret, > $count_total_tests_number); > > + if ($count_total_tests_number eq "yes"){ > > + $MTT::Globals::Internals->{total_tests_counter} += > $return_value; > > + } > > } > > > > # Unload any loaded environment modules > > @@ -545,7 +564,6 @@ > > # means reverting to an env snapshot. > > MTT::EnvImporter::unload(reverse @env_importers); > > } > > - > > } > > > > > #-------------------------------------------------------------------------- > > > > Modified: trunk/lib/MTT/Test/RunEngine.pm > > > ============================================================================== > > --- trunk/lib/MTT/Test/RunEngine.pm (original) > > +++ trunk/lib/MTT/Test/RunEngine.pm 2012-01-25 06:02:47 EST (Wed, 25 > Jan 2012) > > @@ -38,10 +38,14 @@ > > my $section; > > my $mpi_details_name; > > my $test_run_full_name; > > +my $break_threshold; > > > > # Keep track of how many tests have passed, failed, skipped, and timed > out > > my $test_results_count; > > > > +#same as test_results_count but global over all envokations of RunEngine > > +my $test_results_count_global; > > + > > # Submit results after each run or after *all* the runs > > my $report_after_each_result = 0; > > my $report_results_count = 0; > > @@ -56,7 +60,7 @@ > > my ($install_dir, $runs_data_dir, $mpi_details, $test_build, $force, > $ret); > > > > # Make sure though, that the $ini remains a global > > - ($ini, $section, $install_dir, $runs_data_dir, $mpi_details, > $test_build, $force, $ret) = @_; > > + ($ini, $section, $install_dir, $runs_data_dir, $mpi_details, > $test_build, $force, $ret, my $count_total_tests_number) = @_; > > > > my $test_results; > > $group_reports = MTT::Values::Value($ini, "mtt", > "submit_group_results"); > > @@ -81,12 +85,16 @@ > > my $variants_count_total = > > $test_count_total * $np_count_total * $argv_count_total * > $exec_count_total; > > > > + if ($count_total_tests_number eq "yes"){ > > + return $variants_count_total; > > + } > > # Set some thresholds for an early exit > > - my $break_threshold; > > + $break_threshold = undef; > > $break_threshold->{MTT::Values::PASS} = Value($ini, $section, > "break_threshold_pass"); > > $break_threshold->{MTT::Values::FAIL} = Value($ini, $section, > "break_threshold_fail"); > > $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section, > "break_threshold_timeout"); > > $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section, > "break_threshold_skipped"); > > + $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini, > "mtt", "break_threshold_timeout_and_fail"); > > > > # This boolean value defaults to 0, and allows the user to submit > results > > # after each test to ensure at least *some* results are submitted > (in case > > @@ -208,25 +216,29 @@ > > $run->{wdir} = $wdir; > > } > > # Just one np, or an array of np values? > > + > > if (ref($all_np) eq "") { > > - $test_results->{$all_np} = > > - _run_one_np($install_dir, $run, $mpi_details, $all_np, > $force); > > + $test_results->{$all_np} = _run_one_np($install_dir, $run, > $mpi_details, $all_np, $force); > > } else { > > foreach my $this_np (@$all_np) { > > # See if we're supposed to terminate. > > last > > - if (MTT::Util::time_to_terminate()); > > + if (MTT::Util::time_to_terminate()); > > + > > + $test_results->{$all_np} = _run_one_np($install_dir, > $run, $mpi_details, $this_np,$force); > > > > - $test_results->{$this_np} = > > - _run_one_np($install_dir, $run, $mpi_details, > $this_np, > > - $force); > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > } > > } > > + > > + last > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}); > > ++$test_count; > > > > # Write out the "to be saved" test run results > > MTT::Test::SaveRuns($runs_data_dir); > > - > > + > > $MTT::Test::Run::mpi_details = $save_run_mpi_details; > > > > # Output a progress bar > > @@ -247,6 +259,7 @@ > > > > MTT::Reporter::QueueSubmit(); > > } > > + > > } > > > > sub _run_one_np { > > @@ -290,16 +303,30 @@ > > foreach my $e (@$execs) { > > # See if we're supposed to terminate. > > last > > - if (MTT::Util::time_to_terminate()); > > + if (MTT::Util::time_to_terminate()); > > + > > _run_one_test($install_dir, $run, $mpi_details, $e, > $name, > > - $variant++, $force); > > + $variant++, $force); > > + > > + last > > + if (MTT::Util::check_break_threshold( > > + $test_results_count_global, > > + $break_threshold, > > + > $MTT::Globals::Internals->{total_tests_counter}) > > + ); > > } > > } > > - > > + last > > + if (MTT::Util::check_break_threshold( > > + $test_results_count_global, > > + $break_threshold, > > + $MTT::Globals::Internals->{total_tests_counter}) > > + ); > > + > > $MTT::Test::Run::test_argv = undef; > > } > > } > > - > > + > > $MTT::Test::Run::test_np = undef; > > } > > > > @@ -457,6 +484,13 @@ > > $test_results_count->{$report->{test_result}}++ > > if (exists($report->{test_result})); > > > > + $test_results_count_global->{$report->{test_result}}++ > > + if (exists($report->{test_result})); > > + > > + $test_results_count_global->{MTT::Values::TIMED_OUT_OR_FAIL}++ > > + if (exists($report->{test_result}) && > > + (MTT::Values::FAIL == $report->{test_result} || > MTT::Values::TIMED_OUT == $report->{test_result})); > > + > > # If there is an after_each step, run it > > $ENV{MTT_TEST_RUN_RESULT_MESSAGE} = > > (MTT::Values::PASS == $report->{test_result} ? "passed" : > > > > Modified: trunk/lib/MTT/Util.pm > > > ============================================================================== > > --- trunk/lib/MTT/Util.pm (original) > > +++ trunk/lib/MTT/Util.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012) > > @@ -205,6 +205,12 @@ > > > > if (($count->{$result} / $total) > $threshold->{$result}) { > > Verbose("--> Threshold ($per) exceeded for > \"$result_label\": $count->{$result} out of $total.\n"); > > + $MTT::Globals::Internals->{is_stopped_on_break_threshold} = > "true"; > > + > $MTT::Globals::Internals->{stopped_on_break_threshold_message} = "--> > Threshold ($per) exceeded for \"$result_label\": $count->{$result} out of > $total.\n"; > > + print STDOUT "--> Threshold ($per) exceeded for > \"$result_label\": $count->{$result} out of $total.\n"; > > + if > ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){ > > + print STDOUT "0xdeadbeef: it works"; > > + } > > return 1; > > } > > } > > > > Modified: trunk/lib/MTT/Values.pm > > > ============================================================================== > > --- trunk/lib/MTT/Values.pm (original) > > +++ trunk/lib/MTT/Values.pm 2012-01-25 06:02:47 EST (Wed, 25 Jan 2012) > > @@ -45,6 +45,7 @@ > > PASS => 1, > > SKIPPED => 2, > > TIMED_OUT => 3, > > + TIMED_OUT_OR_FAIL =>4, > > }; > > > > # Map to human-readable English labels > > @@ -53,7 +54,7 @@ > > $result_messages->{MTT::Values::PASS} = "pass"; > > $result_messages->{MTT::Values::TIMED_OUT} = "timeout"; > > $result_messages->{MTT::Values::SKIPPED} = "skipped"; > > - > > +$result_messages->{MTT::Values::TIMED_OUT_OR_FAIL} = "timeout_and_fail"; > > # current $ini and $section parameters (we use it in funclets) > > our $evaluate_string_ini; > > our $evaluate_string_section; > > _______________________________________________ > > mtt-svn mailing list > > mtt-...@open-mpi.org > > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn > > > -- > Jeff Squyres > jsquy...@cisco.com > For corporate legal information go to: > http://www.cisco.com/web/about/doing_business/legal/cri/ > > > _______________________________________________ > mtt-devel mailing list > mtt-de...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-devel >
