GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Unverified Commit 4073e787 authored by boklm's avatar boklm
Browse files

Add a TestSuite object

And change all test suites to use this TestSuite object.
parent d2a841ef
......@@ -2,6 +2,9 @@ package TBBTestSuite::BrowserBundleTests;
use warnings;
use strict;
use parent 'TBBTestSuite::TestSuite';
use English;
use FindBin;
use File::Slurp;
......@@ -19,13 +22,6 @@ use TBBTestSuite::Tests::Command qw(command_run);
use TBBTestSuite::Tests::TorBootstrap;
use TBBTestSuite::XServer qw(start_X stop_X set_Xmode);
our (@ISA, @EXPORT_OK);
BEGIN {
require Exporter;
@ISA = qw(Exporter);
@EXPORT_OK = qw(tbb_filename_infos);
}
my $screenshot_thumbnail;
BEGIN {
# For some reason that I did not understand yet, Image::Magick does
......@@ -40,25 +36,23 @@ BEGIN {
}
}
my $test_types = {
tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
mozmill => \&mozmill_run,
selenium => \&selenium_run,
virustotal => \&virustotal_run,
command => \&command_run,
};
our %testsuite = (
description => 'Tor Browser Bundle integration tests',
test_types => $test_types,
pre_tests => \&pre_tests,
post_tests => \&post_tests,
);
sub test_types {
return {
tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
mozmill => \&mozmill_run,
selenium => \&selenium_run,
virustotal => \&virustotal_run,
command => \&command_run,
};
}
our %testsuite_virustotal = (
%testsuite,
description => 'Tor Browser Bundle Virustotal checks',
);
sub type {
'browserbundle';
}
sub description {
'Tor Browser Bundle integration tests';
}
our @tests = (
{
......@@ -418,36 +412,6 @@ sub get_tbbfile {
unless -f $tbbinfos->{tbbfile};
}
sub tbb_filename_infos {
my ($tbbfile) = @_;
my (undef, undef, $file) = File::Spec->splitpath($tbbfile);
my %res = (filename => $file, tbbfile => $tbbfile);
if ($file =~ m/^tor-browser-linux(..)-([^_]+)_(.+)\.tar\.xz$/) {
@res{qw(os version language)} = ('Linux', $2, $3);
$res{arch} = $1 eq '64' ? 'x86_64' : 'x86';
} elsif ($file =~ m/^torbrowser-install-([^_]+)_(.+)\.exe$/) {
@res{qw(os arch version language)} = ('Windows', 'x86', $1, $2);
} elsif ($file =~ m/^TorBrowserBundle-(.+)-osx32_(.+)\.zip$/) {
@res{qw(os arch version language)} = ('MacOSX', 'x86', $1, $2);
} else {
return undef;
}
if ($options->{virustotal}) {
$res{type} = 'browserbundle_virustotal';
$res{tests} = [
{
name => 'virustotal',
type => 'virustotal',
descr => 'Analyze files on virustotal.com',
},
];
} else {
$res{type} = 'browserbundle';
$res{tests} = [ map { { %$_ } } @tests ];
}
return \%res;
}
sub extract_tbb {
my ($tbbinfos) = @_;
exit_error "Can't open file $tbbinfos->{tbbfile}" unless -f $tbbinfos->{tbbfile};
......@@ -650,6 +614,13 @@ sub set_tbbpaths {
$tbbinfos->{ffprofiledir} = "$tbbinfos->{datadir}/Browser/profile.default";
}
sub new {
my ($ts, $testsuite) = @_;
$testsuite->{type} = 'browserbundle';
$testsuite->{tests} = [ map { { %$_ } } @tests ];
return bless $testsuite, $ts;
}
sub pre_tests {
my ($tbbinfos) = @_;
get_tbbfile($tbbinfos);
......
package TBBTestSuite::BrowserBundleVirusTotal;
use parent 'TBBTestSuite::BrowserBundleTests';
sub description {
'Tor Browser Bundle Virustotal checks';
}
sub type {
'browserbundle_virustotal';
}
sub new {
my ($ts, $testsuite) = @_;
$testsuite->{type} = 'browserbundle_virustotal';
$testsuite->{tests} = [
{
name => 'virustotal',
type => 'virustotal',
descr => 'Analyze files on virustotal.com',
},
];
return bless $testsuite;
}
1;
package TBBTestSuite::BrowserRebaseTests;
use strict;
use parent 'TBBTestSuite::TestSuite';
use File::Slurp;
use IO::CaptureOutput qw(capture_exec);
use TBBTestSuite::BrowserGit qw(git_clone_fetch get_commits_by_branch
parent_commit git_cmd git_cmd_ch);
my $test_types = {
cherry_pick => \&cherry_pick,
};
sub test_types {
return {
cherry_pick => \&cherry_pick,
};
}
our %testsuite = (
description => 'Tor Browser rebase tests',
test_types => $test_types,
pre_tests => \&pre_tests,
post_tests => \&post_tests,
pre_makereport => \&pre_makereport,
pre_reports_index => \&pre_reports_index,
);
sub type {
'browserrebase';
}
sub description {
'Tor Browser rebase tests';
}
sub test_name {
my ($commit) = @_;
......@@ -26,15 +30,16 @@ sub test_name {
return $res;
}
sub get_tbbinfos {
my ($infos) = @_;
sub new {
my ($ts, $infos) = @_;
git_clone_fetch;
my %tbbinfos = (
my $testsuite = {
%$infos,
type => 'browserrebase',
type => $ts->type(),
filename => 'browser-rebase',
tests => [],
);
};
my @commits = reverse get_commits_by_branch($infos->{tb_branch},
$infos->{esr_branch});
shift @commits;
......@@ -46,9 +51,9 @@ sub get_tbbinfos {
commit => $commit,
retry => 1,
};
push @{$tbbinfos{tests}}, $test;
push @{$testsuite->{tests}}, $test;
}
return \%tbbinfos;
return bless $testsuite, $ts;
}
sub pre_tests {
......@@ -65,16 +70,6 @@ sub pre_tests {
git_cmd('git', 'checkout', '-f', 'rebase-test');
}
sub post_tests {
my ($tbbinfos) = @_;
}
sub pre_makereport {
}
sub pre_reports_index {
}
sub cherry_pick {
my ($tbbinfos, $test) = @_;
print "Rebase $test->{commit}\n";
......
package TBBTestSuite::BrowserUnitTests;
use strict;
use parent 'TBBTestSuite::TestSuite';
use FindBin;
use IO::CaptureOutput qw(capture_exec);
use File::Spec;
......@@ -14,29 +17,30 @@ use TBBTestSuite::Reports qw(load_report);
use TBBTestSuite::Options qw($options);
use TBBTestSuite::XServer qw(start_X stop_X set_Xmode);
my $test_types = {
xpcshell => \&xpcshell_test,
mochitest_plain => sub { mochitest_test('mochitest-plain', @_) },
mochitest_chrome => sub { mochitest_test('mochitest-chrome', @_) },
mochitest_browser => sub { mochitest_test('mochitest-browser', @_) },
mochitest_a11y => sub { mochitest_test('mochitest-a11y', @_) },
build_firefox => \&build_firefox,
};
sub test_types {
return {
xpcshell => \&xpcshell_test,
mochitest_plain => sub { mochitest_test('mochitest-plain', @_) },
mochitest_chrome => sub { mochitest_test('mochitest-chrome', @_) },
mochitest_browser => sub { mochitest_test('mochitest-browser', @_) },
mochitest_a11y => sub { mochitest_test('mochitest-a11y', @_) },
build_firefox => \&build_firefox,
};
}
sub description {
'Tor Browser unit tests';
}
our %testsuite = (
description => 'Tor Browser unit tests',
test_types => $test_types,
pre_tests => \&pre_tests,
post_tests => \&post_tests,
pre_makereport => \&pre_makereport,
pre_reports_index => \&pre_reports_index,
);
sub type {
'browserunit';
}
sub get_tbbinfos {
my ($infos) = @_;
my %tbbinfos = (
sub new {
my ($ts, $infos) = @_;
my $tbbinfos = {
%$infos,
type => 'browserunit',
type => $ts->type(),
filename => "browser-$infos->{commit}",
tests => [
{
......@@ -46,10 +50,10 @@ sub get_tbbinfos {
descr => 'Build Firefox',
},
],
);
push @{$tbbinfos{tests}}, find_xpcshell_tests(\%tbbinfos);
push @{$tbbinfos{tests}}, find_mochitest_tests(\%tbbinfos);
return \%tbbinfos;
};
push @{$tbbinfos->{tests}}, find_xpcshell_tests($tbbinfos);
push @{$tbbinfos->{tests}}, find_mochitest_tests($tbbinfos);
return bless $tbbinfos, $ts;
}
sub pre_tests {
......@@ -148,24 +152,23 @@ sub diff_results {
}
sub pre_makereport {
my ($report, $tbbfile, $r) = @_;
my $tbbinfos = $report->{tbbfiles}{$tbbfile};
foreach my $test (@{$tbbinfos->{tests}}) {
my ($testsuite, $report, $r) = @_;
foreach my $test (@{$testsuite->{tests}}) {
mochitest_error_logs($test);
}
return unless $tbbinfos->{parent_results};
$r //= TBBTestSuite::Reports::load_report($tbbinfos->{parent_results}[0]);
return unless $testsuite->{parent_results};
$r //= TBBTestSuite::Reports::load_report($testsuite->{parent_results}[0]);
return unless $r;
my $parent = $r->{tbbfiles}{$tbbinfos->{parent_results}[1]};
my $parent = $r->{tbbfiles}{$testsuite->{parent_results}[1]};
return unless $parent;
$tbbinfos->{parent_diff} = diff_results($parent, $tbbinfos);
$testsuite->{parent_diff} = diff_results($parent, $testsuite);
}
sub pre_reports_index {
my ($reports, $report) = @_;
my ($testsuite, $reports, $report) = @_;
foreach my $tbbfile (keys %{$report->{tbbfiles}}) {
my $tbbinfos = $report->{tbbfiles}{$tbbfile};
pre_makereport($report, $tbbfile,
$tbbinfos->pre_makereport($report,
$reports->{$tbbinfos->{parent_results}[0]})
if $tbbinfos->{parent_results};
foreach my $test (@{$report->{tbbfiles}{$tbbfile}{tests}}) {
......
......@@ -90,11 +90,8 @@ sub make_report {
INCLUDE_PATH => "$FindBin::Bin/tmpl:" . report_dir($report),
OUTPUT_PATH => report_dir($report),
);
foreach my $tbbfile (keys %{$report->{tbbfiles}}) {
my $type = $report->{tbbfiles}{$tbbfile}{type};
my $testsuite = $TBBTestSuite::Tests::testsuite_types{$type};
$testsuite->{pre_makereport}($report, $tbbfile)
if $testsuite->{pre_makereport};
foreach my $testsuite (values %{$report->{tbbfiles}}) {
$testsuite->pre_makereport($report);
}
my %r = ( %template_functions, %$report );
$template->process('screenshots.html', \%r, 'screenshots.html',
......@@ -179,10 +176,11 @@ sub make_reports_index {
@s = @s[0..19] if @s > 20;
load_reports_for_index(\%pre_reports_index, @s);
my $title = "Last 20 reports";
$template->process("reports_index_$type.html",
my ($t) = values %{$reports{$s[0]}->{tbbfiles}};
my $tmpl_file = $t->reports_index_tmpl();
$template->process($tmpl_file,
{ %$vars, reports_list => \@s, title => $title }, "index-$type.html")
|| exit_error "Template Error:\n" . $template->error;
my ($t) = values %{$reports{$s[0]}->{tbbfiles}};
$template->process('tests_index.html', { %$vars, testsuite_type => $type,
tests => $t->{tests} }, "tests-$type.html")
|| exit_error "Template Error:\n" . $template->error;
......@@ -194,7 +192,9 @@ sub make_reports_index {
@{$reports_by_tag{$type}->{$tag}};
load_reports_for_index(\%pre_reports_index, @s);
my $title = "Reports for $tag";
$template->process("reports_index_$type.html",
my ($t) = values %{$reports{$s[0]}->{tbbfiles}};
my $tmpl_file = $t->reports_index_tmpl();
$template->process($tmpl_file,
{ %$vars, reports_list => \@s, title => $title, },
"index-$type-$tag.html")
|| exit_error "Template Error:\n" . $template->error;
......@@ -207,7 +207,9 @@ sub make_reports_index {
@{$reports_by_month{$type}->{$month}};
load_reports_for_index(\%pre_reports_index, @s);
my $title = "$month reports";
$template->process("reports_index_$type.html",
my ($t) = values %{$reports{$s[0]}->{tbbfiles}};
my $tmpl_file = $t->reports_index_tmpl();
$template->process($tmpl_file,
{ %$vars, reports_list => \@s, title => $title, },
"index-$type-$month.html")
|| exit_error "Template Error:\n" . $template->error;
......@@ -263,17 +265,21 @@ sub load_report {
return $reports{$report_name} if exists $reports{$report_name};
my $reportfile = "$options->{'reports-dir'}/r/$report_name/report.yml";
return undef unless -f $reportfile;
return $reports{$report_name} = YAML::Syck::LoadFile($reportfile);
my $r = YAML::Syck::LoadFile($reportfile);
foreach my $testsuite (values %{$r->{tbbfiles}}) {
TBBTestSuite::TestSuite::load($testsuite);
}
return $reports{$report_name} = $r;
}
sub load_reports_for_index {
my ($pre_reports_index, @reports) = @_;
foreach my $rname (@reports) {
my $r = load_report($rname);
my $testsuite = $TBBTestSuite::Tests::testsuite_types{report_type($r)};
if ($testsuite->{pre_reports_index} && !$pre_reports_index->{$rname}) {
$pre_reports_index->{$rname} = 1;
$testsuite->{pre_reports_index}(\%reports, $reports{$rname});
next if $pre_reports_index->{$rname};
$pre_reports_index->{$rname} = 1;
foreach my $testsuite (values %{$r->{tbbfiles}}) {
$testsuite->pre_reports_index(\%reports, $reports{$rname});
}
}
}
......
package TBBTestSuite::TestSuite;
use TBBTestSuite::TestTestSuite;
use TBBTestSuite::BrowserBundleTests;
use TBBTestSuite::BrowserBundleVirusTotal;
use TBBTestSuite::BrowserUnitTests;
use TBBTestSuite::BrowserRebaseTests;
use Scalar::Util 'blessed';
my @testsuite_list = qw(TestTestSuite BrowserBundleTests BrowserBundleVirusTotal
BrowserUnitTests BrowserRebaseTests);
my %testsuite_types;
sub testsuite_types {
return %testsuite_types if %testsuite_types;
foreach my $ts (@testsuite_list) {
$testsuite_types{"TBBTestSuite::${ts}"->type()} = $ts;
}
return %testsuite_types;
}
sub testsuite_infos {
my %testsuite_infos;
foreach my $ts (@testsuite_list) {
my $n = "TBBTestSuite::${ts}";
$testsuite_infos{$n->type()} = {
name => $n,
type => $n->type(),
description => $n->description(),
};
}
return %testsuite_infos;
}
sub new {
my ($testsuite) = @_;
return bless $testsuite;
}
sub load {
my ($testsuite) = @_;
my %ts = testsuite_types;
return bless $testsuite, "TBBTestSuite::$ts{$testsuite->{type}}";
}
sub type {
die "No type";
}
sub description {
die "No description";
}
sub pre_tests {
}
sub post_tests {
}
sub pre_makereport {
}
sub pre_reports_index {
}
sub reports_index_tmpl {
my ($testsuite) = @_;
my $type = $testsuite->type();
return "reports_index_$type.html";
}
1;
package TBBTestSuite::TestTestSuite;
use strict;
use parent 'TBBTestSuite::TestSuite';
my $test_types = {
test_test => \&test_test,
};
our %testsuite = (
description => 'Tor Browser test testsuite',
test_types => $test_types,
pre_tests => \&pre_tests,
post_tests => \&post_tests,
pre_makereport => \&pre_makereport,
pre_reports_index => \&pre_reports_index,
);
sub description { 'Test Test suite' }
sub get_tbbinfos {
my ($infos) = @_;
my %tbbinfos = (
sub type { 'testtestsuite' };
sub test_types {
$test_types;
}
sub new {
my ($ts, $infos) = @_;
my $testsuite = {
%$infos,
type => 'testtestsuite',
filename => 'testtestsuite',
......@@ -39,8 +39,8 @@ sub get_tbbinfos {
fail_type => 'warning',
},
],
);
return \%tbbinfos;
};
return bless $testsuite, $ts;
}
sub pre_tests {
......
......@@ -12,17 +12,19 @@ use LWP::UserAgent;
use TBBTestSuite::Reports;
use TBBTestSuite::Common qw(exit_error);
use TBBTestSuite::Options qw($options);
use TBBTestSuite::BrowserBundleTests qw(tbb_filename_infos);
use TBBTestSuite::BrowserBundleTests;
use TBBTestSuite::BrowserUnitTests;
use TBBTestSuite::XServer qw(set_Xmode);
use TBBTestSuite::TestSuite;
our %testsuite_types = (
browserunit => \%TBBTestSuite::BrowserUnitTests::testsuite,
browserrebase => \%TBBTestSuite::BrowserRebaseTests::testsuite,
browserbundle => \%TBBTestSuite::BrowserBundleTests::testsuite,
browserbundle_virustotal => \%TBBTestSuite::BrowserBundleTests::testsuite_virustotal,
testtestsuite => \%TBBTestSuite::TestTestSuite::testsuite,
);
our (@ISA, @EXPORT_OK);
BEGIN {
require Exporter;
@ISA = qw(Exporter);
@EXPORT_OK = qw(tbb_filename_infos);
}
our %testsuite_types = TBBTestSuite::TestSuite::testsuite_infos();
sub run_tests {
my ($tbbinfos) = @_;
......@@ -38,7 +40,7 @@ sub run_tests {
@{$options->{'disable-tests'}}
: split(',', $options->{'disable-tests'});
}
my $test_types = $testsuite_types{$tbbinfos->{type}}->{test_types};
my $test_types = $tbbinfos->test_types();
foreach my $test (@{$tbbinfos->{tests}}) {
$test->{fail_type} //= 'error';
}
......@@ -130,6 +132,26 @@ sub test_by_name {
return undef;
}
sub tbb_filename_infos {
my ($tbbfile) = @_;
my (undef, undef, $file) = File::Spec->splitpath($tbbfile);
my %res = (filename => $file, tbbfile => $tbbfile);
if ($file =~ m/^tor-browser-linux(..)-([^_]+)_(.+)\.tar\.xz$/) {
@res{qw(os version language)} = ('Linux', $2, $3);
$res{arch} = $1 eq '64' ? 'x86_64' : 'x86';
} elsif ($file =~ m/^torbrowser-install-([^_]+)_(.+)\.exe$/) {
@res{qw(os arch version language)} = ('Windows', 'x86', $1, $2);
} elsif ($file =~ m/^TorBrowserBundle-(.+)-osx32_(.+)\.zip$/) {
@res{qw(os arch version language)} = ('MacOSX', 'x86', $1, $2);
} else {
return undef;
}
return $options->{virustotal} ?
TBBTestSuite::BrowserBundleVirusTotal->new(\%res)
: TBBTestSuite::BrowserBundleTests->new(\%res);
}
sub matching_tbbfile {
my $o = tbb_filename_infos($_[0]);
return $o && $o->{os} eq $options->{os} && $o->{arch} eq $options->{arch};
......@@ -192,12 +214,12 @@ sub test_start {
"results-$tbbinfos->{filename}");
mkdir $tbbinfos->{'results-dir'};
my $testsuite = $testsuite_types{$tbbinfos->{type}};
$testsuite->{pre_tests}($tbbinfos);
$tbbinfos->pre_tests();
$tbbinfos->{start_time} = time;
run_tests($tbbinfos);
$tbbinfos->{finish_time} = time;
$tbbinfos->{run_time} = $tbbinfos->{finish_time} - $tbbinfos->{start_time};
$testsuite->{post_tests}($tbbinfos);