Unverified Commit 06168bad authored by boklm's avatar boklm
Browse files

BrowserUnitTests: add diff with previous results

parent c2925586
......@@ -7,6 +7,7 @@ use File::Spec;
use File::Find;
use File::Copy;
use TBBTestSuite::Common qw(exit_error get_nbcpu run_to_file);
use TBBTestSuite::Reports qw(load_report);
my $test_types = {
xpcshell => \&xpcshell_test,
......@@ -17,6 +18,7 @@ our %testsuite = (
test_types => $test_types,
pre_tests => \&pre_tests,
post_tests => \&post_tests,
pre_makereport => \&pre_makereport,
);
sub get_tbbinfos {
......@@ -62,6 +64,72 @@ sub pre_tests {
sub post_tests {
}
sub tests_by_name {
my ($tests) = @_;
my %res = map { $_->{name} => $_ } @$tests;
return \%res;
}
sub xpcshell_subtests_diff {
my ($t1, $t2) = @_;
my (@fail, @fixed);
my %f1 = map { $_ => 1 } @{$t1->{results}{failed}};
my %f2 = map { $_ => 1 } @{$t2->{results}{failed}};
my %f = ( %f1, %f2);
foreach my $t (keys %f) {
if ($f2{$t} && !$f1{$t}) {
push @fail, $t;
}
if (!$f2{$t} && $f1{$t}) {
push @fixed, $t;
}
}
if (@fail or @fixed) {
return { fail => \@fail, fixed => \@fixed };
}
return undef;
}
sub diff_results {
my ($r1, $r2) = @_;
my %res;
$res{run_time} = $r2->{run_time} - $r1->{run_time};
$res{fail_tests} = [];
$res{fixed_tests} = [];
$res{tests_time} = {};
$res{subtests} = {};
my $r1t = tests_by_name($r1->{tests});
my $r2t = tests_by_name($r2->{tests});
foreach my $test (keys %$r2t) {
my ($t1, $t2) = ($r1t->{$test}, $r2t->{$test});
$res{tests_time}->{$test} = $t2->{run_time} - $t1->{run_time};
next unless defined $t1->{results};
next unless defined $t2->{results};
if (!$t2->{results}{success} && $t1->{results}{success}) {
push @{$res{fail_tests}}, $test;
}
if ($t2->{results}{success} && !$t1->{results}{success}) {
push @{$res{fixed_tests}}, $test;
}
if ($t1->{type} eq 'xpcshell') {
my $s = xpcshell_subtests_diff($t1, $t2);
$res{subtests}{$test} = $s if $s;
}
}
return \%res;
}
sub pre_makereport {
my ($report, $tbbfile) = @_;
my $tbbinfos = $report->{tbbfiles}{$tbbfile};
return unless $tbbinfos->{parent_results};
my $r = TBBTestSuite::Reports::load_report($tbbinfos->{parent_results}[0]);
return unless $r;
my $parent = $r->{tbbfiles}{$tbbinfos->{parent_results}[1]};
return unless $parent;
$tbbinfos->{parent_diff} = diff_results($parent, $tbbinfos);
}
sub find_xpcshell_tests {
my ($tbbinfos) = @_;
my @res;
......
......@@ -15,6 +15,13 @@ use TBBTestSuite::Tests;
use Email::Simple;
use Email::Sender::Simple qw(try_to_sendmail);
our (@ISA, @EXPORT_OK);
BEGIN {
require Exporter;
@ISA = qw(Exporter);
@EXPORT_OK = qw(load_report);
}
my %template_functions = (
is_test_error => \&TBBTestSuite::Tests::is_test_error,
is_test_warning => \&TBBTestSuite::Tests::is_test_warning,
......@@ -55,6 +62,12 @@ sub make_report {
INCLUDE_PATH => "$FindBin::Bin/tmpl:$report->{options}{'report-dir'}",
OUTPUT_PATH => $report->{options}{'report-dir'},
);
foreach my $tbbfile (keys %{$report->{tbbfiles}}) {
my $type = $report->{tbbfiles}{$tbbfile}{type};
my $testsuite = $TBBTestSuite::Tests::testsuite_types{$type};
$testsuite->{pre_makereport}($report, $tbbfile)
if $testsuite->{pre_makereport};
}
my %r = ( %template_functions, %$report );
$template->process('screenshots.html', \%r, 'screenshots.html',
binmode => ':utf8');
......@@ -148,4 +161,11 @@ sub email_report {
}
}
sub load_report {
my ($report_name) = @_;
my $reportfile = "$options->{'reports-dir'}/r/$report_name/report.yml";
return undef unless -f $reportfile;
return YAML::LoadFile($reportfile);
}
1;
......@@ -15,7 +15,7 @@ use TBBTestSuite::Options qw($options);
use TBBTestSuite::BrowserBundleTests qw(tbb_filename_infos);
use TBBTestSuite::BrowserUnitTests;
my %testsuite_types = (
our %testsuite_types = (
browserunit => \%TBBTestSuite::BrowserUnitTests::testsuite,
browserbundle => \%TBBTestSuite::BrowserBundleTests::testsuite,
);
......
......@@ -43,6 +43,103 @@
<li><b>Total run time:</b> [% tbbfiles.$tbbfile.run_time %] seconds</li>
</ul>
[% IF tbbfiles.$tbbfile.type == 'browserunit' && tbbfiles.$tbbfile.parent_diff %]
[% SET diff = tbbfiles.$tbbfile.parent_diff %]
<h1>Diff with previous results</h1>
<ul>
[% SET parent_results = tbbfiles.$tbbfile.parent_results %]
<li>Previous results: <a href="../[% parent_results.0 %]/[% parent_results.1 %].html">[% parent_results.0 %]</a></li>
<li><a href="javascript:togglecontent('run_time_diff')"><b>Total run time diff:</b> [% diff.run_time %]</a>
<div id="run_time_diff" style="display:none">
<ul>
[% FOREACH t IN diff.tests_time.nsort.reverse %]
<li>[% t %]: [% diff.tests_time.$t %]</li>
[% END %]
</ul>
</div>
</li>
[% IF ! diff.fail_tests.size %]
<li>No new failing tests</li>
[% ELSE %]
<li><a href="javascript:togglecontent('fail_tests_diff')"><b>[% diff.fail_tests.size %] new failing tests</b></a>
<div id="fail_tests_diff" style="display:none">
<ul>
[% FOREACH t IN diff.fail_tests.sort %]
<li>[% t %]</li>
[% END %]
</ul>
</div>
</li>
[% END %]
[%
SET nb_js_fail = 0;
SET nb_js_fixed = 0;
FOREACH t IN diff.subtests.keys;
SET nb_js_fail = nb_js_fail + diff.subtests.$t.fail.size;
SET nb_js_fixed = nb_js_fixed + diff.subtests.$t.fixed.size;
END
%]
[% IF nb_js_fail %]
<li><a href="javascript:togglecontent('fail_js_diff')"><b>[% nb_js_fail %] new js files failing</b></a>
<div id="fail_js_diff" style="display:none">
<ul>
[% FOREACH t IN diff.subtests.keys %]
[% IF ! diff.subtests.$t.fail.size; NEXT; END %]
<li>[% t %]:
<ul>
[% FOREACH js IN diff.subtests.$t.fail %]
<li>[% js %]</li>
[% END %]
</ul>
</li>
[% END %]
</ul>
</div>
</li>
[% ELSE %]
<li>No new js files failing</li>
[% END %]
[% IF ! diff.fixed_tests.size %]
<li>No fixed tests</li>
[% ELSE %]
<li><a href="javascript:togglecontent('fixed_tests_diff')"><b>[% diff.fixed_tests.size %] fixed tests</b></a>
<div id="fixed_tests_diff" style="display:none">
<ul>
[% FOREACH t IN diff.fixed_tests.sort %]
<li>[% t %]</li>
[% END %]
</ul>
</div>
</li>
[% END %]
[% IF nb_js_fixed %]
<li><a href="javascript:togglecontent('fixed_js_diff')"><b>[% nb_js_fixed %] new js files fixed</b></a>
<div id="fixed_js_diff" style="display:none">
<ul>
[% FOREACH t IN diff.subtests.keys %]
[% IF ! diff.subtests.$t.fixed.size; NEXT; END %]
<li>[% t %]:
<ul>
[% FOREACH js IN diff.subtests.$t.fixed %]
<li>[% js %]</li>
[% END %]
</ul>
</li>
[% END %]
</ul>
</div>
</li>
[% ELSE %]
<li>No js files fixed</li>
[% END %]
</ul>
[% END %]
<h1>Results</h1>
[% FOREACH test IN tbbfiles.$tbbfile.tests %]
[% IF ! test.results; NEXT; END %]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment