ktest: Added better console, add test build

Better reading of the console. Added running a script to do testing after build succeeds. Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

ktest: Added better console, add test build
Better reading of the console. Added running a script to do testing after build succeeds. Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Steven Rostedt · Steven Rostedt
1 parent 5f9b6ced04
Showing 1 changed file with 159 additions and 48 deletions Side-by-side Diff
tools/testing/ktest/ktest.pl
@@ -23,6 +23,7 @@
 $opt{"POWEROFF_ON_ERROR"}	= 0;
 $opt{"POWEROFF_ON_SUCCESS"}	= 0;
 $opt{"BUILD_OPTIONS"}		= "";
+$opt{"BISECT_SLEEP_TIME"}	= 10;   # sleep time between bisects
  
 my $version;
 my $grub_number;
@@ -32,6 +33,7 @@
 my $minconfig;
 my $in_bisect = 0;
 my $bisect_bad = "";
+my $run_test;
  
 sub read_config {
     my ($config) = @_;
@@ -68,7 +70,7 @@
 }
  
 sub dodie {
-    doprint "CRITICAL FAILURE... ", @_;
+    doprint "CRITICAL FAILURE... ", @_, "\n";
  
     if ($opt{"REBOOT_ON_ERROR"}) {
 	doprint "REBOOTING\n";
  
  
@@ -84,14 +86,14 @@
  
 sub run_command {
     my ($command) = @_;
-    my $redirect = "";
+    my $redirect_log = "";
  
     if (defined($opt{"LOG_FILE"})) {
-	$redirect = " >> $opt{LOG_FILE} 2>&1";
+	$redirect_log = " >> $opt{LOG_FILE} 2>&1";
     }
  
     doprint "$command ... ";
-    `$command $redirect`;
+    `$command $redirect_log`;
  
     my $failed = $?;
  
@@ -106,7 +108,7 @@
  
 sub get_grub_index {
  
-    return if ($grub_number >= 0);
+    return if (defined($grub_number));
  
     doprint "Find grub menu ... ";
     $grub_number = -1;
  
  
  
  
  
  
  
@@ -164,43 +166,55 @@
     run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 }
  
-sub monitor {
+sub open_console {
+    my ($fp) = @_;
+
     my $flags;
+
+    my $pid = open($fp, "$opt{CONSOLE}|") or
+	dodie "Can't open console $opt{CONSOLE}";
+
+    $flags = fcntl($fp, F_GETFL, 0) or
+	dodie "Can't get flags for the socket: $!\n";
+    $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
+	dodie "Can't set flags for the socket: $!\n";
+
+    return $pid;
+}
+
+sub close_console {
+    my ($fp, $pid) = @_;
+
+    doprint "kill child process $pid\n";
+    kill 2, $pid;
+
+    print "closing!\n";
+    close($fp);
+}
+
+sub monitor {
     my $booted = 0;
     my $bug = 0;
     my $pid;
-    my $doopen2 = 0;
     my $skip_call_trace = 0;
+    my $fp = \*IN;
  
-    if ($doopen2) {
-	$pid = open2(\*IN, \*OUT, $opt{"CONSOLE"});
-	if ($pid < 0) {
-	    dodie "Failed to connect to the console";
-	}
-    } else {
-	$pid = open(IN, "$opt{CONSOLE} |");
-    }
+    $pid = open_console($fp);
  
-    $flags = fcntl(IN, F_GETFL, 0) or
-	dodie "Can't get flags for the socket: $!\n";
-
-    $flags = fcntl(IN, F_SETFL, $flags | O_NONBLOCK) or
-	dodie "Can't set flags for the socket: $!\n";
-
     my $line;
     my $full_line = "";
  
     doprint "Wait for monitor to settle down.\n";
     # read the monitor and wait for the system to calm down
     do {
-	$line = wait_for_input(\*IN, 5);
+	$line = wait_for_input($fp, 5);
     } while (defined($line));
  
     reboot_to;
  
     for (;;) {
  
-	$line = wait_for_input(\*IN);
+	$line = wait_for_input($fp);
  
 	last if (!defined($line));
  
  
  
  
@@ -234,19 +248,15 @@
 	}
     }
  
-    doprint "kill child process $pid\n";
-    kill 2, $pid;
+    close_console($fp, $pid);
  
-    print "closing!\n";
-    close(IN);
-
     if (!$booted) {
-	return 1 if (!$in_bisect);
+	return 1 if ($in_bisect);
 	dodie "failed - never got a boot prompt.\n";
     }
  
     if ($bug) {
-	return 1 if (!$in_bisect);
+	return 1 if ($in_bisect);
 	dodie "failed - got a bug report\n";
     }
  
@@ -395,6 +405,84 @@
     doprint "$version\n";
 }
  
+sub child_run_test {
+    my $failed;
+
+    $failed = !run_command $run_test;
+    exit $failed;
+}
+
+my $child_done;
+
+sub child_finished {
+    $child_done = 1;
+}
+
+sub do_run_test {
+    my $child_pid;
+    my $child_exit;
+    my $pid;
+    my $line;
+    my $full_line;
+    my $bug = 0;
+    my $fp = \*IN;
+
+    $pid = open_console($fp);
+
+    # read the monitor and wait for the system to calm down
+    do {
+	$line = wait_for_input($fp, 1);
+    } while (defined($line));
+
+    $child_done = 0;
+
+    $SIG{CHLD} = qw(child_finished);
+
+    $child_pid = fork;
+
+    child_run_test if (!$child_pid);
+
+    $full_line = "";
+
+    do {
+	$line = wait_for_input($fp, 1);
+	if (defined($line)) {
+
+	    # we are not guaranteed to get a full line
+	    $full_line .= $line;
+
+	    if ($full_line =~ /call trace:/i) {
+		$bug = 1;
+	    }
+
+	    if ($full_line =~ /Kernel panic -/) {
+		$bug = 1;
+	    }
+
+	    if ($line =~ /\n/) {
+		$full_line = "";
+	    }
+	}
+    } while (!$child_done && !$bug);
+
+    if ($bug) {
+	doprint "Detected kernel crash!\n";
+	# kill the child with extreme prejudice
+	kill 9, $child_pid;
+    }
+
+    waitpid $child_pid, 0;
+    $child_exit = $?;
+
+    close_console($fp, $pid);
+
+    if ($bug || $child_exit) {
+	return 1 if $in_bisect;
+	dodie "test failed";
+    }
+    return 0;
+}
+
 sub run_bisect {
     my ($type) = @_;
  
  
@@ -422,11 +510,20 @@
  
 	if ($type ne "boot") {
 	    dodie "Failed on boot" if $failed;
+
+	    $failed = do_run_test;
 	}
     }
  
     if ($failed) {
 	$result = "bad";
+
+	# reboot the box to a good kernel
+	if ($type eq "boot") {
+	    reboot;
+	    doprint "sleep a little for reboot\n";
+	    sleep $opt{"BISECT_SLEEP_TIME"};
+	}
     } else {
 	$result = "good";
     }
  
@@ -443,12 +540,15 @@
     }
  
     doprint "SUCCESS\n";
-    if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\)) \[([[:xdigit:]]+)\]/) {
+    if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 	doprint "$1 [$2]\n";
     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
 	$bisect_bad = $1;
 	doprint "Found bad commit... $1\n";
 	return 0;
+    } else {
+	# we already logged it, just print it now.
+	print $output;
     }
  
  
@@ -479,6 +579,11 @@
     run_command "git bisect bad $bad" or
 	dodie "could not set bisect good to $bad";
  
+    # Can't have a test without having a test to run
+    if ($type eq "test" && !defined($run_test)) {
+	$type = "boot";
+    }
+
     do {
 	$result = run_bisect $type;
     } while ($result);
  
  
  
  
@@ -519,29 +624,30 @@
  
 $make = "$opt{MAKE_CMD} O=$opt{OUTPUT_DIR}";
  
-# First we need to do is the builds
-for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
-    my $type = "BUILD_TYPE[$i]";
+sub set_build_option {
+    my ($name, $i) = @_;
  
-    if (defined($opt{"BUILD_NOCLEAN[$i]"}) &&
-	$opt{"BUILD_NOCLEAN[$i]"} != 0) {
-	$noclean = 1;
-    } else {
-	$noclean = $opt{"BUILD_NOCLEAN"};
-    }
+    my $option = "$name\[$i\]";
  
-    if (defined($opt{"MIN_CONFIG[$i]"})) {
-	$minconfig = $opt{"MIN_CONFIG[$i]"};
-    } elsif (defined($opt{"MIN_CONFIG"})) {
-	$minconfig = $opt{"MIN_CONFIG"};
-    } else {
-	undef $minconfig;
+    if (defined($opt{$option})) {
+	return $opt{$option};
     }
  
-    if (!defined($opt{$type})) {
-	$opt{$type} = $opt{"DEFAULT_BUILD_TYPE"};
+    if (defined($opt{$name})) {
+	return $opt{$name};
     }
  
+    return undef;
+}
+
+# First we need to do is the builds
+for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
+    my $type = "BUILD_TYPE[$i]";
+
+    $noclean = set_build_option("BUILD_NOCLEAN", $i);
+    $minconfig = set_build_option("MIN_CONFIG", $i);
+    $run_test = set_build_option("TEST", $i);
+
     doprint "\n\n";
     doprint "RUNNING TEST $i of $opt{NUM_BUILDS} with option $opt{$type}\n\n";
  
@@ -558,6 +664,11 @@
     get_version;
     install;
     monitor;
+
+    if (defined($run_test)) {
+	do_run_test;
+    }
+
     success $i;
 }
...	...	@@ -23,6 +23,7 @@
23	23	$opt{"POWEROFF_ON_ERROR"} = 0;
24	24	$opt{"POWEROFF_ON_SUCCESS"} = 0;
25	25	$opt{"BUILD_OPTIONS"} = "";
	26	+$opt{"BISECT_SLEEP_TIME"} = 10; # sleep time between bisects
26	27
27	28	my $version;
28	29	my $grub_number;
...	...	@@ -32,6 +33,7 @@
32	33	my $minconfig;
33	34	my $in_bisect = 0;
34	35	my $bisect_bad = "";
	36	+my $run_test;
35	37
36	38	sub read_config {
37	39	my ($config) = @_;
...	...	@@ -68,7 +70,7 @@
68	70	}
69	71
70	72	sub dodie {
71		- doprint "CRITICAL FAILURE... ", @_;
	73	+ doprint "CRITICAL FAILURE... ", @_, "\n";
72	74
73	75	if ($opt{"REBOOT_ON_ERROR"}) {
74	76	doprint "REBOOTING\n";
75	77
76	78
...	...	@@ -84,14 +86,14 @@
84	86
85	87	sub run_command {
86	88	my ($command) = @_;
87		- my $redirect = "";
	89	+ my $redirect_log = "";
88	90
89	91	if (defined($opt{"LOG_FILE"})) {
90		- $redirect = " >> $opt{LOG_FILE} 2>&1";
	92	+ $redirect_log = " >> $opt{LOG_FILE} 2>&1";
91	93	}
92	94
93	95	doprint "$command ... ";
94		- `$command $redirect`;
	96	+ `$command $redirect_log`;
95	97
96	98	my $failed = $?;
97	99
...	...	@@ -106,7 +108,7 @@
106	108
107	109	sub get_grub_index {
108	110
109		- return if ($grub_number >= 0);
	111	+ return if (defined($grub_number));
110	112
111	113	doprint "Find grub menu ... ";
112	114	$grub_number = -1;
113	115
114	116
115	117
116	118
117	119
118	120
119	121
...	...	@@ -164,43 +166,55 @@
164	166	run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" \| grub --batch; reboot)'";
165	167	}
166	168
167		-sub monitor {
	169	+sub open_console {
	170	+ my ($fp) = @_;
	171	+
168	172	my $flags;
	173	+
	174	+ my $pid = open($fp, "$opt{CONSOLE}\|") or
	175	+ dodie "Can't open console $opt{CONSOLE}";
	176	+
	177	+ $flags = fcntl($fp, F_GETFL, 0) or
	178	+ dodie "Can't get flags for the socket: $!\n";
	179	+ $flags = fcntl($fp, F_SETFL, $flags \| O_NONBLOCK) or
	180	+ dodie "Can't set flags for the socket: $!\n";
	181	+
	182	+ return $pid;
	183	+}
	184	+
	185	+sub close_console {
	186	+ my ($fp, $pid) = @_;
	187	+
	188	+ doprint "kill child process $pid\n";
	189	+ kill 2, $pid;
	190	+
	191	+ print "closing!\n";
	192	+ close($fp);
	193	+}
	194	+
	195	+sub monitor {
169	196	my $booted = 0;
170	197	my $bug = 0;
171	198	my $pid;
172		- my $doopen2 = 0;
173	199	my $skip_call_trace = 0;
	200	+ my $fp = \*IN;
174	201
175		- if ($doopen2) {
176		- $pid = open2(\IN, \OUT, $opt{"CONSOLE"});
177		- if ($pid < 0) {
178		- dodie "Failed to connect to the console";
179		- }
180		- } else {
181		- $pid = open(IN, "$opt{CONSOLE} \|");
182		- }
	202	+ $pid = open_console($fp);
183	203
184		- $flags = fcntl(IN, F_GETFL, 0) or
185		- dodie "Can't get flags for the socket: $!\n";
186		-
187		- $flags = fcntl(IN, F_SETFL, $flags \| O_NONBLOCK) or
188		- dodie "Can't set flags for the socket: $!\n";
189		-
190	204	my $line;
191	205	my $full_line = "";
192	206
193	207	doprint "Wait for monitor to settle down.\n";
194	208	# read the monitor and wait for the system to calm down
195	209	do {
196		- $line = wait_for_input(\*IN, 5);
	210	+ $line = wait_for_input($fp, 5);
197	211	} while (defined($line));
198	212
199	213	reboot_to;
200	214
201	215	for (;;) {
202	216
203		- $line = wait_for_input(\*IN);
	217	+ $line = wait_for_input($fp);
204	218
205	219	last if (!defined($line));
206	220
207	221
208	222
209	223
...	...	@@ -234,19 +248,15 @@
234	248	}
235	249	}
236	250
237		- doprint "kill child process $pid\n";
238		- kill 2, $pid;
	251	+ close_console($fp, $pid);
239	252
240		- print "closing!\n";
241		- close(IN);
242		-
243	253	if (!$booted) {
244		- return 1 if (!$in_bisect);
	254	+ return 1 if ($in_bisect);
245	255	dodie "failed - never got a boot prompt.\n";
246	256	}
247	257
248	258	if ($bug) {
249		- return 1 if (!$in_bisect);
	259	+ return 1 if ($in_bisect);
250	260	dodie "failed - got a bug report\n";
251	261	}
252	262
...	...	@@ -395,6 +405,84 @@
395	405	doprint "$version\n";
396	406	}
397	407
	408	+sub child_run_test {
	409	+ my $failed;
	410	+
	411	+ $failed = !run_command $run_test;
	412	+ exit $failed;
	413	+}
	414	+
	415	+my $child_done;
	416	+
	417	+sub child_finished {
	418	+ $child_done = 1;
	419	+}
	420	+
	421	+sub do_run_test {
	422	+ my $child_pid;
	423	+ my $child_exit;
	424	+ my $pid;
	425	+ my $line;
	426	+ my $full_line;
	427	+ my $bug = 0;
	428	+ my $fp = \*IN;
	429	+
	430	+ $pid = open_console($fp);
	431	+
	432	+ # read the monitor and wait for the system to calm down
	433	+ do {
	434	+ $line = wait_for_input($fp, 1);
	435	+ } while (defined($line));
	436	+
	437	+ $child_done = 0;
	438	+
	439	+ $SIG{CHLD} = qw(child_finished);
	440	+
	441	+ $child_pid = fork;
	442	+
	443	+ child_run_test if (!$child_pid);
	444	+
	445	+ $full_line = "";
	446	+
	447	+ do {
	448	+ $line = wait_for_input($fp, 1);
	449	+ if (defined($line)) {
	450	+
	451	+ # we are not guaranteed to get a full line
	452	+ $full_line .= $line;
	453	+
	454	+ if ($full_line =~ /call trace:/i) {
	455	+ $bug = 1;
	456	+ }
	457	+
	458	+ if ($full_line =~ /Kernel panic -/) {
	459	+ $bug = 1;
	460	+ }
	461	+
	462	+ if ($line =~ /\n/) {
	463	+ $full_line = "";
	464	+ }
	465	+ }
	466	+ } while (!$child_done && !$bug);
	467	+
	468	+ if ($bug) {
	469	+ doprint "Detected kernel crash!\n";
	470	+ # kill the child with extreme prejudice
	471	+ kill 9, $child_pid;
	472	+ }
	473	+
	474	+ waitpid $child_pid, 0;
	475	+ $child_exit = $?;
	476	+
	477	+ close_console($fp, $pid);
	478	+
	479	+ if ($bug \|\| $child_exit) {
	480	+ return 1 if $in_bisect;
	481	+ dodie "test failed";
	482	+ }
	483	+ return 0;
	484	+}
	485	+
398	486	sub run_bisect {
399	487	my ($type) = @_;
400	488
401	489
...	...	@@ -422,11 +510,20 @@
422	510
423	511	if ($type ne "boot") {
424	512	dodie "Failed on boot" if $failed;
	513	+
	514	+ $failed = do_run_test;
425	515	}
426	516	}
427	517
428	518	if ($failed) {
429	519	$result = "bad";
	520	+
	521	+ # reboot the box to a good kernel
	522	+ if ($type eq "boot") {
	523	+ reboot;
	524	+ doprint "sleep a little for reboot\n";
	525	+ sleep $opt{"BISECT_SLEEP_TIME"};
	526	+ }
430	527	} else {
431	528	$result = "good";
432	529	}
433	530
...	...	@@ -443,12 +540,15 @@
443	540	}
444	541
445	542	doprint "SUCCESS\n";
446		- if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\)) \[([[:xdigit:]]+)\]/) {
	543	+ if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
447	544	doprint "$1 [$2]\n";
448	545	} elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
449	546	$bisect_bad = $1;
450	547	doprint "Found bad commit... $1\n";
451	548	return 0;
	549	+ } else {
	550	+ # we already logged it, just print it now.
	551	+ print $output;
452	552	}
453	553
454	554
...	...	@@ -479,6 +579,11 @@
479	579	run_command "git bisect bad $bad" or
480	580	dodie "could not set bisect good to $bad";
481	581
	582	+ # Can't have a test without having a test to run
	583	+ if ($type eq "test" && !defined($run_test)) {
	584	+ $type = "boot";
	585	+ }
	586	+
482	587	do {
483	588	$result = run_bisect $type;
484	589	} while ($result);
485	590
486	591
487	592
488	593
...	...	@@ -519,29 +624,30 @@
519	624
520	625	$make = "$opt{MAKE_CMD} O=$opt{OUTPUT_DIR}";
521	626
522		-# First we need to do is the builds
523		-for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
524		- my $type = "BUILD_TYPE[$i]";
	627	+sub set_build_option {
	628	+ my ($name, $i) = @_;
525	629
526		- if (defined($opt{"BUILD_NOCLEAN[$i]"}) &&
527		- $opt{"BUILD_NOCLEAN[$i]"} != 0) {
528		- $noclean = 1;
529		- } else {
530		- $noclean = $opt{"BUILD_NOCLEAN"};
531		- }
	630	+ my $option = "$name\[$i\]";
532	631
533		- if (defined($opt{"MIN_CONFIG[$i]"})) {
534		- $minconfig = $opt{"MIN_CONFIG[$i]"};
535		- } elsif (defined($opt{"MIN_CONFIG"})) {
536		- $minconfig = $opt{"MIN_CONFIG"};
537		- } else {
538		- undef $minconfig;
	632	+ if (defined($opt{$option})) {
	633	+ return $opt{$option};
539	634	}
540	635
541		- if (!defined($opt{$type})) {
542		- $opt{$type} = $opt{"DEFAULT_BUILD_TYPE"};
	636	+ if (defined($opt{$name})) {
	637	+ return $opt{$name};
543	638	}
544	639
	640	+ return undef;
	641	+}
	642	+
	643	+# First we need to do is the builds
	644	+for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
	645	+ my $type = "BUILD_TYPE[$i]";
	646	+
	647	+ $noclean = set_build_option("BUILD_NOCLEAN", $i);
	648	+ $minconfig = set_build_option("MIN_CONFIG", $i);
	649	+ $run_test = set_build_option("TEST", $i);
	650	+
545	651	doprint "\n\n";
546	652	doprint "RUNNING TEST $i of $opt{NUM_BUILDS} with option $opt{$type}\n\n";
547	653
...	...	@@ -558,6 +664,11 @@
558	664	get_version;
559	665	install;
560	666	monitor;
	667	+
	668	+ if (defined($run_test)) {
	669	+ do_run_test;
	670	+ }
	671	+
561	672	success $i;
562	673	}
563	674