GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Unverified Commit cd763eb8 authored by boklm's avatar boklm
Browse files

Bug 16009: add support for marionette tests

parent a434afd9
......@@ -2,5 +2,6 @@
*.pyc
clones
virtualenv
virtualenv-marionette
reports
tmp
[submodule "mozmill-tests/mozilla-mozmill-tests"]
path = mozmill-tests/mozilla-mozmill-tests
url = https://github.com/boklm/mozilla-mozmill-tests
[submodule "marionette/firefox-ui-tests"]
path = marionette/firefox-ui-tests
url = https://github.com/mozilla/firefox-ui-tests
......@@ -40,6 +40,7 @@ sub test_types {
return {
tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
mozmill => \&mozmill_run,
marionette => \&marionette_run,
selenium => \&selenium_run,
virustotal => \&virustotal_run,
command => \&command_run,
......@@ -580,6 +581,11 @@ sub parse_strace {
$test->{results}{connections} = {};
my %modified_files;
my %removed_files;
if (-f "$logfile.tmp") {
my $txt = read_file("$logfile.tmp");
write_file($logfile, { append => 1 }, $txt);
unlink "$logfile.tmp";
}
my @lines = read_file($logfile) if -f $logfile;
push @lines, read_file($logfile_tmp) if -f $logfile_tmp;
foreach my $line (@lines) {
......@@ -588,7 +594,9 @@ sub parse_strace {
next if $2 =~ m/O_RDONLY/;
next if $1 =~ m/^$tbbinfos->{tbbdir}/;
next if $ignore_files{$1};
next if $1 =~ m/^$ENV{'MOZMILL_SCREENSHOTS'}/;
if ($ENV{'MOZMILL_SCREENSHOTS'}) {
next if $1 =~ m/^$ENV{'MOZMILL_SCREENSHOTS'}/;
}
$modified_files{$1}++;
}
if ($line =~ m/^\d+ unlink\("((?:[^"\\]++|\\.)*+)"/) {
......@@ -626,6 +634,12 @@ sub ff_strace_wrapper {
my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
my $wrapper = <<EOF;
#!/bin/sh
if [ -f $logfile.tmp ]
then
cat $logfile.tmp >> $logfile
rm $logfile.tmp
fi
echo \$@ >> /tmp/ff_run.log
strace -f -o $logfile.tmp -- \'$ff_wrapper\' "\$@"
exit_code=\$?
cat $logfile.tmp >> $logfile
......@@ -643,7 +657,8 @@ sub ffbin_path {
if ($OSNAME eq 'cygwin') {
return winpath("$tbbinfos->{ffbin}.exe");
}
if ($options->{use_strace} && $test->{type} eq 'mozmill') {
my %t = map { $_ => 1 } qw(mozmill marionette);
if ($options->{use_strace} && $t{$test->{type}}) {
return ff_strace_wrapper($tbbinfos, $test);
}
return ff_wrapper($tbbinfos, $test);
......@@ -667,6 +682,33 @@ EOF
write_file($options_file, $content);
}
sub marionette_run {
my ($tbbinfos, $test) = @_;
if ($test->{tried} && $test->{use_net}) {
TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
}
set_test_prefs($tbbinfos, $test);
my $result_file_html = "$tbbinfos->{'results-dir'}/$test->{name}.html";
my $result_file_txt = "$tbbinfos->{'results-dir'}/$test->{name}.txt";
#--log-unittest ./res.txt --log-html ./res.html
my $bin = $OSNAME eq 'cygwin' ? 'Scripts' : 'bin';
system(xvfb_run($test), "$FindBin::Bin/virtualenv-marionette/$bin/tor-browser-tests",
'--log-unittest', winpath($result_file_txt),
'--log-html', winpath($result_file_html),
'--binary', ffbin_path($tbbinfos, $test),
'--profile', winpath($tbbinfos->{ffprofiledir}),
winpath("$FindBin::Bin/marionette/tor_browser_tests/test_$test->{name}.py"));
my @txt_log = read_file($result_file_txt);
my $res_line = shift @txt_log;
$test->{results}{success} = $res_line eq ".\n" || $res_line eq ".\r\n";
$test->{results}{log} = join '', @txt_log;
reset_test_prefs($tbbinfos, $test);
parse_strace($tbbinfos, $test);
check_opened_connections($tbbinfos, $test);
check_modified_files($tbbinfos, $test);
}
sub mozmill_run {
my ($tbbinfos, $test) = @_;
return unless $options->{mozmill};
......
*.log
*.pyc
*.egg-info
recursive-include tor_browser_tests *
Subproject commit 0258610df1fd2f7233807d338436781f6504f67a
from setuptools import setup, find_packages
PACKAGE_VERSION = '0.2'
deps = [
'marionette-client == 1.0.0',
'marionette-driver == 1.0.0',
'mozfile == 1.2',
'mozinfo == 0.8',
'mozlog == 3.0',
]
setup(name='tor-browser-tests',
version=PACKAGE_VERSION,
description='A collection of Tor Browser tests run with Marionette',
long_description='A collection of Tor Browser tests run with Marionette',
classifiers=['Environment :: Console',
'Intended Audience :: Developers',
'Natural Language :: English',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Topic :: Software Development :: Libraries :: Python Modules',
],
url='https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/',
packages=find_packages(),
include_package_data=True,
zip_safe=False,
install_requires=deps,
entry_points="""
[console_scripts]
tor-browser-tests = tor_browser_tests:cli
""")
import os
import sys
from mozlog import structured
from marionette import BaseMarionetteArguments
from marionette import BaseMarionetteTestRunner
from firefox_ui_harness import FirefoxTestCase
def cli(runner_class=BaseMarionetteTestRunner, parser_class=BaseMarionetteArguments):
parser = parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
structured.commandline.add_logging_group(parser)
args = parser.parse_args()
parser.verify_usage(args)
logger = structured.commandline.setup_logging(
args.logger_name, args, {'mach': sys.stdout})
args.logger = logger
try:
runner = runner_class(**vars(args))
runner.test_handlers = [FirefoxTestCase]
runner.run_tests(args.tests)
if runner.failed > 0:
sys.exit(10)
except Exception:
logger.error('Failure during execution test.', exc_info=True)
sys.exit(1)
......@@ -2,14 +2,24 @@
use strict;
use FindBin;
use IO::CaptureOutput qw(qxx);
use Cwd;
my $virtenv = "$FindBin::Bin/virtualenv";
my $virtenv_marionette = "$FindBin::Bin/virtualenv-marionette";
my $pip = "$virtenv/bin/pip";
sub run {
system(@_) == 0 || die "Error running " . join(' ', @_);
}
sub run_from_dir {
my $old_cwd = getcwd;
chdir shift @_;
my $res = run(@_);
chdir $old_cwd;
return $res;
}
sub pip_install {
my ($pkg) = @_;
my ($output) = qxx($pip, 'show', '--disable-pip-version-check', $pkg);
......@@ -21,3 +31,7 @@ for my $pkg (qw(selenium mozmill)) {
pip_install($pkg);
}
run('virtualenv', $virtenv_marionette) unless -d $virtenv_marionette;
run_from_dir('marionette/firefox-ui-tests',
"$virtenv_marionette/bin/python", 'setup.py', 'develop');
run_from_dir('marionette', "$virtenv_marionette/bin/python", 'setup.py', 'develop');
<a href="results-[% tbbfile %]/[% test.name %].html">details</a><br /><br />
[% IF test.results.connections.size %]
<b>List of opened connections:</b>
<ul>
[% FOREACH con IN test.results.connections.nsort.reverse %]
<li[% IF test.results.bad_connections.$con %] class="text_red"[% END %]>[% con %]: [% test.results.connections.$con %] connections</li>
[% END %]
</ul>
[% END %]
[% IF test.results.modified_files.size %]
<b>List of modified files:</b>
<ul>
[% FOREACH file IN test.results.modified_files.sort %]
<li><a href="results-[% tbbfile %]/[% test.name %].sandbox/[% IF ! file.match('^/'); GET tbbfiles.$tbbfile.tbbdir _ '/'; END; %][% file %]">[% file %]</a></li>
[% END %]
</ul>
[% END %]
[% IF test.results.removed_files.size %]
<b>List of removed files:</b>
<ul>
[% FOREACH file IN test.results.removed_files.sort %]
<li>[% file %]</li>
[% END %]
</ul>
[% END %]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment