From f80802cb1fbd7f441dd1d04db77d314adb7ccb37 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 7 Mar 2011 13:18:47 -0500 Subject: [PATCH 1/9] ktest: Print logfile name on failure If the test fails and a logfile was specified. Print the name to let the user know where to look for more information on the failure. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index e1c62eeb88f5..3e3615a7d968 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -462,6 +462,10 @@ sub dodie { `$power_off`; } + if (defined($opt{"LOG_FILE"})) { + print " See $opt{LOG_FILE} for more info.\n"; + } + die @_, "\n"; } From 10abf118f090b9a70ea05296a981fa27196dc7c6 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 7 Mar 2011 13:21:00 -0500 Subject: [PATCH 2/9] ktest: Start failure timeout on panic too Currently we just look for a Call Trace to start the time out when to reboot the box. But if the kernel panics and does not show a Call Trace, the test will not reboot the box after the specified timeout. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 3e3615a7d968..6620fe4b364f 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -820,6 +820,7 @@ sub monitor { } if ($full_line =~ /Kernel panic -/) { + $failure_start = time; $bug = 1; } From 612b9e9bd33af80f525a38c05a1663f7126b3d3c Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 7 Mar 2011 13:27:43 -0500 Subject: [PATCH 3/9] ktest: Handle kernels before make oldnoconfig When bisecting, one may come across a kernel that does not have make oldnoconfig. In this case, we need to run the command "yes" into a make oldconfig. This will select defaults instead of 'n' into each command, but it works as a work around. Note, "yes n" will not work because a config may have a value that "n" is not acceptable for. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 6620fe4b364f..c95209bac65a 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -930,6 +930,18 @@ sub check_buildlog { return 1; } +sub make_oldconfig { + my ($defconfig) = @_; + + if (!run_command "$defconfig $make oldnoconfig") { + # Perhaps oldnoconfig doesn't exist in this version of the kernel + # try a yes '' | oldconfig + doprint "oldnoconfig failed, trying yes '' | make oldconfig\n"; + run_command "yes '' | $defconfig $make oldconfig" or + dodie "failed make config oldconfig"; + } +} + sub build { my ($type) = @_; my $defconfig = ""; @@ -975,8 +987,12 @@ sub build { $defconfig = "KCONFIG_ALLCONFIG=$minconfig"; } - run_command "$defconfig $make $type" or - dodie "failed make config"; + if ($type eq "oldnoconfig") { + make_oldconfig $defconfig; + } else { + run_command "$defconfig $make $type" or + dodie "failed make config"; + } $redirect = "$buildlog"; if (!run_command "$make $build_options") { @@ -1395,9 +1411,7 @@ sub create_config { close(OUT); # exit; - run_command "$make oldnoconfig" or - dodie "failed make config oldconfig"; - + make_oldconfig ""; } sub compare_configs { @@ -1599,8 +1613,7 @@ sub config_bisect { close(IN); # Now run oldconfig with the minconfig (and addconfigs) - run_command "$defconfig $make oldnoconfig" or - dodie "failed make config oldconfig"; + make_oldconfig $defconfig; # check to see what we lost (or gained) open (IN, $output_config) From c960bb9f59c0ed2aded535222cfe461ec6c22c95 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:22:39 -0500 Subject: [PATCH 4/9] ktest: Add manual bisect For both git bisect and config bisect, if BISECT_MANUAL is set to 1, then bisect will stop between iterations and ask the user for the result. The actual result is ignored. This makes it possible to use ktest.pl for bisecting configs and git and let the user examine the results themselves and enter their own results. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 33 +++++++++++++++++++++++++++++++-- tools/testing/ktest/sample.conf | 16 ++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index c95209bac65a..e55bd52367a6 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -37,6 +37,7 @@ $default{"POWEROFF_ON_SUCCESS"} = 0; $default{"BUILD_OPTIONS"} = ""; $default{"BISECT_SLEEP_TIME"} = 60; # sleep time between bisects $default{"CLEAR_LOG"} = 0; +$default{"BISECT_MANUAL"} = 0; $default{"SUCCESS_LINE"} = "login:"; $default{"BOOTED_TIMEOUT"} = 1; $default{"DIE_ON_FAILURE"} = 1; @@ -81,6 +82,7 @@ my $addconfig; my $in_bisect = 0; my $bisect_bad = ""; my $reverse_bisect; +my $bisect_manual; my $in_patchcheck = 0; my $run_test; my $redirect; @@ -1046,6 +1048,21 @@ sub get_version { doprint "$version\n"; } +sub answer_bisect { + for (;;) { + doprint "Pass or fail? [p/f]"; + my $ans = ; + chomp $ans; + if ($ans eq "p" || $ans eq "P") { + return 1; + } elsif ($ans eq "f" || $ans eq "F") { + return 0; + } else { + print "Please answer 'P' or 'F'\n"; + } + } +} + sub child_run_test { my $failed = 0; @@ -1214,6 +1231,9 @@ sub run_bisect { my $ret = run_bisect_test $type, $buildtype; + if ($bisect_manual) { + $ret = answer_bisect; + } # Are we looking for where it worked, not failed? if ($reverse_bisect) { @@ -1524,7 +1544,9 @@ sub run_config_bisect { } $ret = run_config_bisect_test $type; - + if ($bisect_manual) { + $ret = answer_bisect; + } if ($ret) { process_passed %current_config; return 0; @@ -1555,7 +1577,13 @@ sub run_config_bisect { $half = int($#start_list / 2); } while ($half > 0); - # we found a single config, try it again + # we found a single config, try it again unless we are running manually + + if ($bisect_manual) { + process_failed $start_list[0]; + return 1; + } + my @tophalf = @start_list[0 .. 0]; $ret = run_config_bisect_test $type; @@ -1925,6 +1953,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i); $sleep_time = set_test_option("SLEEP_TIME", $i); $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i); + $bisect_manual = set_test_option("BISECT_MANUAL", $i); $store_failures = set_test_option("STORE_FAILURES", $i); $timeout = set_test_option("TIMEOUT", $i); $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i); diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 3408c594b2de..af82ac289779 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -528,6 +528,15 @@ # With BISECT_REVERSE = 1, The test will consider failures as # good, and success as bad. # +# BISECT_MANUAL = 1 (optional, default 0) +# +# In case there's a problem with automating the bisect for +# whatever reason. (Can't reboot, want to inspect each iteration) +# Doing a BISECT_MANUAL will have the test wait for you to +# tell it if the test passed or failed after each iteration. +# This is basicall the same as running git bisect yourself +# but ktest will rebuild and install the kernel for you. +# # BISECT_CHECK = 1 (optional, default 0) # # Just to be sure the good is good and bad is bad, setting @@ -613,10 +622,17 @@ # # CONFIG_BISECT is the config that failed to boot # +# If BISECT_MANUAL is set, it will pause between iterations. +# This is useful to use just ktest.pl just for the config bisect. +# If you set it to build, it will run the bisect and you can +# control what happens in between iterations. It will ask you if +# the test succeeded or not and continue the config bisect. +# # Example: # TEST_START # TEST_TYPE = config_bisect # CONFIG_BISECT_TYPE = build # CONFIG_BISECT = /home/test/˘onfig-bad # MIN_CONFIG = /home/test/config-min +# BISECT_MANUAL = 1 # From c23dca7cd28eb1bf90053ce4ba30d65d22d2a81e Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:26:31 -0500 Subject: [PATCH 5/9] ktest: Add BISECT_SKIP If a during a git bisect, ktest fails on something other than what it is testing (if BISECT_TYPE is test but it fails on build), if BISECT_SKIP is set, then it will do a "git bisect skip" instead of just failing the bisect and letting the user find a good commit to test. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 36 +++++++++++++++++++++++++-------- tools/testing/ktest/sample.conf | 9 +++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index e55bd52367a6..0f62916a43f7 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -38,6 +38,7 @@ $default{"BUILD_OPTIONS"} = ""; $default{"BISECT_SLEEP_TIME"} = 60; # sleep time between bisects $default{"CLEAR_LOG"} = 0; $default{"BISECT_MANUAL"} = 0; +$default{"BISECT_SKIP"} = 1; $default{"SUCCESS_LINE"} = "login:"; $default{"BOOTED_TIMEOUT"} = 1; $default{"DIE_ON_FAILURE"} = 1; @@ -83,6 +84,7 @@ my $in_bisect = 0; my $bisect_bad = ""; my $reverse_bisect; my $bisect_manual; +my $bisect_skip; my $in_patchcheck = 0; my $run_test; my $redirect; @@ -1169,7 +1171,15 @@ sub run_git_bisect { return 1; } -# returns 1 on success, 0 on failure +sub bisect_reboot { + doprint "Reboot and sleep $bisect_sleep_time seconds\n"; + reboot; + start_monitor; + wait_for_monitor $bisect_sleep_time; + end_monitor; +} + +# returns 1 on success, 0 on failure, -1 on skip sub run_bisect_test { my ($type, $buildtype) = @_; @@ -1183,6 +1193,10 @@ sub run_bisect_test { build $buildtype or $failed = 1; if ($type ne "build") { + if ($failed && $bisect_skip) { + $in_bisect = 0; + return -1; + } dodie "Failed on build" if $failed; # Now boot the box @@ -1194,6 +1208,12 @@ sub run_bisect_test { monitor or $failed = 1; if ($type ne "boot") { + if ($failed && $bisect_skip) { + end_monitor; + bisect_reboot; + $in_bisect = 0; + return -1; + } dodie "Failed on boot" if $failed; do_run_test or $failed = 1; @@ -1206,11 +1226,7 @@ sub run_bisect_test { # reboot the box to a good kernel if ($type ne "build") { - doprint "Reboot and sleep $bisect_sleep_time seconds\n"; - reboot; - start_monitor; - wait_for_monitor $bisect_sleep_time; - end_monitor; + bisect_reboot; } } else { $result = 1; @@ -1240,10 +1256,13 @@ sub run_bisect { $ret = !$ret; } - if ($ret) { + if ($ret > 0) { return "good"; - } else { + } elsif ($ret == 0) { return "bad"; + } elsif ($bisect_skip) { + doprint "HIT A BAD COMMIT ... SKIPPING\n"; + return "skip"; } } @@ -1954,6 +1973,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $sleep_time = set_test_option("SLEEP_TIME", $i); $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i); $bisect_manual = set_test_option("BISECT_MANUAL", $i); + $bisect_skip = set_test_option("BISECT_SKIP", $i); $store_failures = set_test_option("STORE_FAILURES", $i); $timeout = set_test_option("TIMEOUT", $i); $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i); diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index af82ac289779..4c46b7ef8e46 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -519,6 +519,15 @@ # git bisect good, git bisect bad, and running the git bisect replay # if the BISECT_REPLAY is set. # +# BISECT_SKIP = 1 (optional, default 0) +# +# If BISECT_TYPE is set to test but the build fails, ktest will +# simply fail the test and end their. You could use BISECT_REPLAY +# and BISECT_START to resume after you found a new starting point, +# or you could set BISECT_SKIP to 1. If BISECT_SKIP is set to 1, +# when something other than the BISECT_TYPE fails, ktest.pl will +# run "git bisect skip" and try again. +# # BISECT_REVERSE = 1 (optional, default 0) # # In those strange instances where it was broken forever From 3410f6fd5e3e96de4b557a62b7a2fbf19d74b1cd Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:38:12 -0500 Subject: [PATCH 6/9] ktest: Add BISECT_FILES to run git bisect on paths Add the config option BISECT_FILES that allows the user to specify what path in the kernel to run the git bisect on. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 9 ++++++++- tools/testing/ktest/sample.conf | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 0f62916a43f7..65c5c5515b47 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1280,6 +1280,13 @@ sub bisect { my $type = $opt{"BISECT_TYPE[$i]"}; my $start = $opt{"BISECT_START[$i]"}; my $replay = $opt{"BISECT_REPLAY[$i]"}; + my $start_files = $opt{"BISECT_FILES[$i]"}; + + if (defined($start_files)) { + $start_files = " -- " . $start_files; + } else { + $start_files = ""; + } # convert to true sha1's $good = get_sha1($good); @@ -1333,7 +1340,7 @@ sub bisect { die "Failed to checkout $head"; } - run_command "git bisect start" or + run_command "git bisect start$start_files" or dodie "could not start bisect"; run_command "git bisect good $good" or diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 4c46b7ef8e46..4c83ae77926c 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -528,6 +528,15 @@ # when something other than the BISECT_TYPE fails, ktest.pl will # run "git bisect skip" and try again. # +# BISECT_FILES = (optional, default undefined) +# +# To just run the git bisect on a specific path, set BISECT_FILES. +# For example: +# +# BISECT_FILES = arch/x86 kernel/time +# +# Will run the bisect with "git bisect start -- arch/x86 kernel/time" +# # BISECT_REVERSE = 1 (optional, default 0) # # In those strange instances where it was broken forever From 4651920e0ac7a596c1123a8efdcc2f6d2249deb3 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:40:31 -0500 Subject: [PATCH 7/9] ktest: Fix bug where the test would not end after failure The config STOP_AFTER_FAILURE is the number of seconds to continue the test when a failure is detected. This lets the monitor record more data to the logs and console that may be helpful in solving the bug that was found. But the test had a bug. If the failure caused multiple "Call Trace" stack dumps, the start time to compare the STOP_AFTER_FAILURE would constantly be reset. Only update the start time at the first "Call Trace" instance. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 65c5c5515b47..125ab94e7294 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -804,7 +804,7 @@ sub monitor { } if ($full_line =~ /call trace:/i) { - if (!$skip_call_trace) { + if (!$bug && !$skip_call_trace) { $bug = 1; $failure_start = time; } From 8ea0e063a78df91b1ca4d23ea8d4e2e3047f6950 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:44:35 -0500 Subject: [PATCH 8/9] ktest: Monitor kernel while running of user tests Record the console of tests to both the console and the log. Also, record the bug reports afte the test has completed. Currently, if a kernel bug happens while running the userland test, the test stops and will not record the kernel bug. This makes it difficult to solve what happened. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 125ab94e7294..8bbfceedfdb2 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1110,6 +1110,7 @@ sub do_run_test { # we are not guaranteed to get a full line $full_line .= $line; + doprint $line; if ($full_line =~ /call trace:/i) { $bug = 1; @@ -1126,6 +1127,19 @@ sub do_run_test { } while (!$child_done && !$bug); if ($bug) { + my $failure_start = time; + my $now; + do { + $line = wait_for_input($monitor_fp, 1); + if (defined($line)) { + doprint $line; + } + $now = time; + if ($now - $failure_start >= $stop_after_failure) { + last; + } + } while (defined($line)); + doprint "Detected kernel crash!\n"; # kill the child with extreme prejudice kill 9, $child_pid; From 2d01b26a1b14ab248f850d858f2ff3de3ead6d33 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Tue, 8 Mar 2011 09:47:54 -0500 Subject: [PATCH 9/9] ktest: Add STOP_TEST_AFTER to stop the test after a period of time Currently, if a test causes constant output but never reaches a boot prompt, or crashes, the test will never stop. Add STOP_TEST_AFTER to create a variable that will stop (and fail) the test after it has run for this amount of time. The default is 10 minutes. Setting this variable to -1 will disable it. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 13 ++++++++++++- tools/testing/ktest/sample.conf | 8 ++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 8bbfceedfdb2..a8b3292d0639 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -47,6 +47,7 @@ $default{"SCP_TO_TARGET"} = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE"; $default{"REBOOT"} = "ssh \$SSH_USER\@\$MACHINE reboot"; $default{"STOP_AFTER_SUCCESS"} = 10; $default{"STOP_AFTER_FAILURE"} = 60; +$default{"STOP_TEST_AFTER"} = 600; $default{"LOCALVERSION"} = "-test"; my $ktest_config; @@ -102,6 +103,7 @@ my $console; my $success_line; my $stop_after_success; my $stop_after_failure; +my $stop_test_after; my $build_target; my $target_image; my $localversion; @@ -768,8 +770,10 @@ sub monitor { my $success_start; my $failure_start; + my $monitor_start = time; + my $done = 0; - for (;;) { + while (!$done) { if ($booted) { $line = wait_for_input($monitor_fp, $booted_timeout); @@ -831,6 +835,12 @@ sub monitor { if ($line =~ /\n/) { $full_line = ""; } + + if ($stop_test_after > 0 && !$booted && !$bug) { + if (time - $monitor_start > $stop_test_after) { + $done = 1; + } + } } close(DMESG); @@ -2002,6 +2012,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $success_line = set_test_option("SUCCESS_LINE", $i); $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i); $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i); + $stop_test_after = set_test_option("STOP_TEST_AFTER", $i); $build_target = set_test_option("BUILD_TARGET", $i); $ssh_exec = set_test_option("SSH_EXEC", $i); $scp_to_target = set_test_option("SCP_TO_TARGET", $i); diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 4c83ae77926c..4c5d6bd74a02 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -306,6 +306,14 @@ # (default 60) #STOP_AFTER_FAILURE = 60 +# In case the console constantly fills the screen, having +# a specified time to stop the test if it never succeeds nor fails +# is recommended. +# Note: this is ignored if a success or failure is detected. +# (in seconds) +# (default 600, -1 is to never stop) +#STOP_TEST_AFTER = 600 + # Stop testing if a build fails. If set, the script will end if # a failure is detected, otherwise it will save off the .config, # dmesg and bootlog in a directory called