Skip to content

GitLab

Unverified Commit a46250cc authored by boklm's avatar boklm
Browse files

Remove support for mozmill and selenium tests

parent a0978d09
Hide whitespace changes
Inline Side-by-side
.gitmodules
View file @ a46250cc
[submodule "mozmill-tests/mozilla-mozmill-tests"]
path = mozmill-tests/mozilla-mozmill-tests
url = https://github.com/boklm/mozilla-mozmill-tests
TBBTestSuite/Options.pm
View file @ a46250cc
......@@ -38,8 +38,6 @@ my %default_options = (
action => 'run_tests',
os => get_os_name(),
arch => get_arch(),
mozmill => $OSNAME ne 'darwin',
selenium => $OSNAME ne 'cygwin',
starttor => 1,
gpgcheck => 1,
clean_browserdir => 1,
......@@ -47,7 +45,6 @@ my %default_options = (
'tor-control-port' => '9551',
'tor-socks-port' => '9550',
'reports-dir' => "$FindBin::Bin/reports",
virtualenv => "$FindBin::Bin/virtualenv",
resolution => '1280x1024',
xvfb => 0,
xdummy => $OSNAME ne 'cygwin',
......@@ -57,7 +54,6 @@ my %default_options = (
'email-to' => [],
'email-from' => 'TBB Test Report <tbbtest@example.com>',
'email-subject' => '[test result: [% success ? "ok" : "failed" %]] [% options.name %]',
'mozmill-dir' => 'c:\tbbtestsuite\mozmill-env',
'http-proxy-port' => '8888',
testrequests_types => 'browserbundle',
test_data_url => 'http://test-data.tbb.torproject.org',
......@@ -69,12 +65,12 @@ my %default_options = (
sub get_options {
my @options = qw(mozmill! selenium! starttor! tor-control-port=i
my @options = qw(starttor! tor-control-port=i
tor-socks-port=i reports-dir=s gpgcheck! keyring=s
virtualenv=s xvfb! name=s download-dir=s config=s
xvfb! name=s download-dir=s config=s
action=s enable-tests=s upload-to=s os=s arch=s
virustotal! email-to=s@ email-from=s email-subject=s
mozmill-dir=s reports-url=s http-proxy-port=i
reports-url=s http-proxy-port=i
xdummy! disable-tests=s testrequests_types=s testsuite=s
cleanup!);
my (%cli, %config);
......
TBBTestSuite/TestSuite/BrowserBundleTests.pm
View file @ a46250cc
......@@ -40,9 +40,7 @@ BEGIN {
sub test_types {
return {
tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
mozmill => \&mozmill_run,
marionette => \&marionette_run,
selenium => \&selenium_run,
virustotal => \&virustotal_run,
command => \&command_run,
};
......@@ -554,13 +552,6 @@ sub xvfb_run {
return ('xvfb-run', '--auto-servernum', '-s', "-screen 0 ${resolution}x24");
}
sub mozmill_cmd {
if ($OSNAME eq 'cygwin') {
return ( "$options->{'mozmill-dir'}\\run.cmd", 'mozmill' );
}
return ("$options->{virtualenv}/bin/mozmill");
}
sub check_opened_connections {
my ($tbbinfos, $test) = @_;
my %bad_connections = %{$test->{results}{connections}};
......@@ -682,7 +673,7 @@ sub ffbin_path {
if ($OSNAME eq 'cygwin') {
return winpath("$tbbinfos->{ffbin}.exe");
}
my %t = map { $_ => 1 } qw(mozmill marionette);
my %t = map { $_ => 1 } qw(marionette);
if ($options->{use_strace} && $t{$test->{type}}) {
return ff_strace_wrapper($tbbinfos, $test);
}
......@@ -690,24 +681,6 @@ sub ffbin_path {
return ff_wrapper($tbbinfos, $test);
}
sub mozmill_export_options {
my ($tbbinfos, $test) = @_;
my $options_file = winpath("$FindBin::Bin/mozmill-tests/lib/testsuite.js");
my $json_opts = encode_json clone_strip_coderef $options;
my $json_test = encode_json clone_strip_coderef $test;
my $json_tbbinfos = encode_json clone_strip_coderef
{ %$tbbinfos, tests => undef };
my $content = <<EOF;
var options = $json_opts;
var test = $json_test;
var tbbinfos = $json_tbbinfos;
exports.options = options;
exports.test = test;
exports.tbbinfos = tbbinfos;
EOF
write_file($options_file, $content);
}
sub marionette_export_options {
my ($tbbinfos, $test) = @_;
my $options_file = File::Temp->new();
......@@ -766,62 +739,6 @@ sub marionette_run {
check_modified_files($tbbinfos, $test);
}
sub mozmill_run {
my ($tbbinfos, $test) = @_;
return unless $options->{mozmill};
if ($test->{tried} && $test->{use_net}) {
TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
}
clean_strace($tbbinfos, $test) if $options->{use_strace};
mozmill_export_options($tbbinfos, $test);
set_test_prefs($tbbinfos, $test);
$test->{screenshots} = [];
my $screenshots_tmp = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
$ENV{'MOZMILL_SCREENSHOTS'} = winpath($screenshots_tmp);
my $results_file = "$tbbinfos->{'results-dir'}/$test->{name}.json";
my $mozmill_test = $test->{mozmill_test} // $test->{name};
system(xvfb_run($test), mozmill_cmd(), '-b', ffbin_path($tbbinfos, $test),
'-p', winpath($tbbinfos->{ffprofiledir}),
'-t', winpath("$FindBin::Bin/mozmill-tests/tbb-tests/$mozmill_test.js"),
'--report', 'file://' . winpath($results_file));
my $i = 0;
for my $screenshot_file (reverse sort glob "$screenshots_tmp/*.png") {
move($screenshot_file, "$tbbinfos->{'results-dir'}/$test->{name}-$i.png");
$screenshot_thumbnail->($tbbinfos->{'results-dir'}, "$test->{name}-$i.png");
push @{$test->{screenshots}}, "$test->{name}-$i.png";
$i++;
}
if (-f $results_file) {
$test->{results} = decode_json(read_file($results_file));
$test->{results}{success} = $test->{results}{results}->[0]->{passed} ?
!$test->{results}{results}->[0]->{failed} : 0;
} else {
$test->{results}{success} = 0;
}
reset_test_prefs($tbbinfos, $test);
if ($options->{use_strace}) {
parse_strace($tbbinfos, $test);
check_opened_connections($tbbinfos, $test);
check_modified_files($tbbinfos, $test);
clean_strace($tbbinfos, $test) if $test->{results}{success};
}
}
sub selenium_run {
my ($tbbinfos, $test) = @_;
return unless $options->{selenium};
if ($test->{tried} && $test->{use_net}) {
TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
}
my $result_file = $ENV{SELENIUM_TEST_RESULT_FILE} =
"$tbbinfos->{'results-dir'}/$test->{name}.json";
$ENV{TBB_BIN} = ffbin_path($tbbinfos, $test);
$ENV{TBB_PROFILE} = $tbbinfos->{ffprofiledir};
system(xvfb_run($test), "$options->{virtualenv}/bin/python",
"$FindBin::Bin/selenium-tests/run_test", $test->{name});
$test->{results} = decode_json(read_file($result_file));
}
sub set_tbbpaths {
my ($tbbinfos) = @_;
$tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/firefox";
......
config/linus-nightly-virustotal
View file @ a46250cc
......@@ -33,8 +33,6 @@ return ( args => [] ) unless $ok;
os => 'Windows',
arch => 'x86',
starttor => 0,
mozmill => 0,
selenium => 0,
'email-from' => 'TBB Nightly Tests <boklm@torproject.org>',
'email-to' => [ 'boklm@mars-attacks.org', ],
'reports-url' => 'http://test-reports.tbb.torproject.org/reports/',
......
config/release-virustotal
View file @ a46250cc
......@@ -16,8 +16,6 @@ foreach my $build (@latest_builds) {
os => 'Windows',
arch => 'x86',
starttor => 0,
mozmill => 0,
selenium => 0,
'reports-url' => 'http://test-reports.tbb.torproject.org/reports/',
'email-from' => 'TBB Tests <boklm@torproject.org>',
'email-to' => [ 'tor-qa@lists.torproject.org', 'boklm@mars-attacks.org', ],
......
doc/install.txt
View file @ a46250cc
......@@ -12,7 +12,7 @@ as root, which will install the needed packages. This script has been
tested on Ubuntu, Debian, Fedora, Centos.
After installing the required dependencies, a python virtualenv needs
to be created with mozmill and selenium. This can be with the 'setup-virtualenv'
to be created with marionette. This can be with the 'setup-virtualenv'
script.
Summary of installation instructions:
......
doc/new-test.txt
View file @ a46250cc
......@@ -14,9 +14,9 @@ There are two categories of tests that we are running:
that we find in the sources tree.
- The tor browser bundle tests. Those tests are run on a complete bundle.
The tests in the browser can be implemented using mozmill or selenium.
We also have other types of tests checking that pluggable transports
are working with and without using an http proxy, or checking that
The tests in the browser can be implemented using marionette. We also
have other types of tests checking that pluggable transports are
working with and without using an http proxy, or checking that
binary files have been compiled with hardenning options.
If the goal is to check that the behavior of the browser is correct
......@@ -39,14 +39,14 @@ The list of tests to be run by the test suite is defined in file
Each test is defined with the following properties:
name::
The name of the test. For 'mozmill', 'selenium' and 'tor_bootstrap'
tests this name is used to find the test filename.
The name of the test. For 'marionette', 'tor_bootstrap' tests
this name is used to find the test filename.
type::
The type of tests. This can be 'mozmill', 'selenium',
'tor_bootstrap', 'command', or something else if you want to
write the test in perl. In the later case, you need to define
it in the '%test_types' variable.
The type of tests. This can be 'marionette', 'tor_bootstrap',
'command', or something else if you want to write the test in
perl. In the later case, you need to define it in the
'%test_types' variable.
descr::
A short description.
......@@ -73,39 +73,40 @@ run_once::
of bundles, this test will only be run on the first bundle.
Adding a mozmill test
~~~~~~~~~~~~~~~~~~~~~
Adding a marionette test
~~~~~~~~~~~~~~~~~~~~~~~~
To add a new test based on mozmill, you will add something like this in
'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
To add a new test based on marionette, you will add something like this
in 'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
----
{
name => 'example-test',
type => 'mozmill',
descr => 'Example mozmill test',
type => 'marionette',
descr => 'Example marionette test',
},
----
The mozmill test files are stored in the directory
'mozmill/mozmill-tests/tbb-tests/'. The name of the file is the name
of the test with '.js' at the end, unless mozmill_test is defined, in
which case it is used instead of the name.
The marionette test files are stored in the directory
'marionette/tor_browser_tests/'. The name of the file is 'test_$name.py'
where '$name' is the name of the test, unless marionette_test is defined,
in which case it is used instead of the name.
It is possible to share a single mozmill file for multiple tests. The
mozmill test can access values from the test definition to do something
It is possible to share a single marionette file for multiple tests. The
marionette test can access values from the test definition to do something
different in each test. It is also possible to access the test suite options.
In this example, we are accessing the "timeout" value from the test
definition, and the "test_data_url" from the the testsuite options:
----
var testsuite = require("../lib/testsuite");
import testsuite
var testThings = function() {
var test_data_url = testsuite.options.test_data_url;
var timeout = testsuite.test.timeout;
...
}
class Test(MarionetteTestCase):
def setUp(self):
MarionetteTestCase.setUp(self)
ts = testsuite.TestSuite()
timeout = ts.t['test']['timeout']
test_data_url = ts.t['options']['test_data_url']
----
A common option is 'test_data_url' which is an URL where test data have
......@@ -159,26 +160,6 @@ The 'timeout' value is the time in milliseconds after which the test
should be considered as failed if the 'test_result' element is still
not present.
Adding a selenium test
~~~~~~~~~~~~~~~~~~~~~~
To add a new test based on selenium webdriver, you will add something
like this in 'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
----
{
name => 'example-test',
type => 'selenium',
descr => 'Example selenium test',
},
----
The selenium test files are stored in the directory 'selenium-tests'.
The name of the file that will be executed is 'test_$name.py' (with
$name replaced by the test name).
Adding a command test
~~~~~~~~~~~~~~~~~~~~~
......
doc/usage.txt
View file @ a46250cc
......@@ -55,12 +55,6 @@ Available options
not given, the files are downloaded in a temporary directory
which is removed at the end of the tests.
--no-mozmill::
Don't run mozmill tests.
--no-selenium::
Don't run the Selenium tests.
--no-starttor::
Don't start a tor daemon. If you use this option, you should
already have a tor daemon running.
......@@ -83,11 +77,6 @@ Available options
save the results inside the reports-dir. If not set, a random
name is used.
--virtualenv=<directory>::
Path to the virtualenv where selenium and mozmill are installed.
The default is the 'virtualenv' directory created by the
'setup-virtualenv' script.
--xvfb::
Run the tests using a virtual frame buffer X server. You should
also disable the X server with --no-xdummy.
......@@ -139,10 +128,6 @@ Available options
--email-subject::
Set the template used to create the subject of the reports emails.
--mozmill-dir::
On windows this sets the directory where mozmill-env has been
installed. The default is c:\tbbtestsuite\mozmill-env.
--http-proxy-port::
Select the port used to start an http proxy, for tor_bootstrap
tests that require an http proxy.
......
mozmill-tests/lib/common.js deleted 100644 → 0
View file @ a0978d09
function load_page(controller, url) {
var retry = 4;
while (retry-- > 0) {
var success = true;
controller.open(url);
try {
controller.waitForPageLoad(10000);
} catch(e) {
success = false;
}
if (success) {
return true;
}
}
controller.open(url);
return controller.waitForPageLoad(50000);
}
exports.load_page = load_page;
mozmill-tests/lib/screenshot.js deleted 100644 → 0
View file @ a0978d09
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Based on:
// http://hg.mozilla.org/qa/mozmill-tests/file/905bc0ba7a1e/firefox/lib/screenshot.js
Cu.import("resource://gre/modules/Downloads.jsm");
Cu.import("resource://gre/modules/Services.jsm");
/**
* This function creates a screenshot of the window provided in the given
* controller and highlights elements from the coordinates provided in the
* given boxes-array.
*
* @param {array of array of int} boxes
* @param {MozmillController} controller
*/
function create(controller, boxes) {
var doc = controller.window.document;
var maxWidth = doc.documentElement.boxObject.width;
var maxHeight = doc.documentElement.boxObject.height;
var rect = [];
for (var i = 0, j = boxes.length; i < j; ++i) {
rect = boxes[i];
if (rect[0] + rect[2] > maxWidth) maxWidth = rect[0] + rect[2];
if (rect[1] + rect[3] > maxHeight) maxHeight = rect[1] + rect[3];
}
var canvas = doc.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
var width = doc.documentElement.boxObject.width;
var height = doc.documentElement.boxObject.height;
canvas.width = maxWidth;
canvas.height = maxHeight;
var ctx = canvas.getContext("2d");
ctx.clearRect(0,0, canvas.width, canvas.height);
ctx.save();
ctx.drawWindow(controller.window, 0, 0, width, height, "rgb(0,0,0)");
ctx.restore();
ctx.save();
ctx.fillStyle = "rgba(255,0,0,0.4)";
for (var i = 0, j = boxes.length; i < j; ++i) {
rect = boxes[i];
ctx.fillRect(rect[0], rect[1], rect[2], rect[3]);
}
ctx.restore();
_saveCanvas(canvas);
}
/**
* Saves a given Canvas object to a file.
* The path to save the file under should be given in the
* MOZMILL_SCREENSHOTS environment variable. If not, it will be saved
* in the temporary folder of the system.
*
* @param {canvas} canvas
*/
function _saveCanvas(canvas) {
var env = Cc["@mozilla.org/process/environment;1"]
.getService(Ci.nsIEnvironment);
var file = null;
if (env.exists("MOZMILL_SCREENSHOTS")) {
file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
file.initWithPath(env.get("MOZMILL_SCREENSHOTS"));
}
else {
file = Services.dirsvc.get("TmpD", Ci.nsIFile);
}
file.append("screenshot.png");
// if a file already exists, don't overwrite it and create a new name
file.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0666", 8));
var taskCompleted = false;
Downloads.fetch(canvas.toDataURL("image/png", ""), file).then(() => {
taskCompleted = true;
}, (aReason) => {
taskCompleted = true;
expect.fail("Failed to save the screenshot " + file + ", with: " + aReason);
});
expect.waitFor(() => taskCompleted,
"Saving the screenshot to the disk action has finished.");
}
// Export of functions
exports.create = create;
mozilla-mozmill-tests @ ddfe2275
Compare ddfe2275...ddfe2275
Subproject commit ddfe2275f8dc5b863096c7d1601be2740cd65193
selenium-tests/run_test deleted 100755 → 0
View file @ a0978d09
#!/usr/bin/python
import unittest, time, re
import sys
import json
import os
def save_result(testres):
if not os.environ.get('SELENIUM_TEST_RESULT_FILE'):
return
res = {}
res['success'] = testres.wasSuccessful()
res['total_tests'] = testres.testsRun
res['failed'] = []
failures = testres.failures + testres.errors
for f in failures:
r = {}
r['id'] = f[0].id()
r['error'] = f[1]
res['failed'].append(r)
output = open(os.environ.get('SELENIUM_TEST_RESULT_FILE'), 'w')
output.write(json.dumps(res))
output.close()
if len(sys.argv) != 2:
sys.exit('Usage: run_test <test_name>')
t = __import__('test_%s' % sys.argv[1])
suite = unittest.TestLoader().loadTestsFromTestCase(t.Test)
res = unittest.TextTestRunner(verbosity=2).run(suite)
save_result(res)
selenium-tests/tbbtest.py deleted 100755 → 0
View file @ a0978d09
#!/usr/bin/python
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
import unittest, time, re
import os
class TBBTest(unittest.TestCase):
def setUp(self):
ffbinary = webdriver.firefox.firefox_binary.FirefoxBinary(firefox_path=os.environ['TBB_BIN'])
ffprofile = webdriver.firefox.firefox_profile.FirefoxProfile(profile_directory=os.environ['TBB_PROFILE'])
self.driver = webdriver.Firefox(firefox_binary=ffbinary, firefox_profile=ffprofile)
self.driver.implicitly_wait(30)
self.base_url = "about:tor"
self.verificationErrors = []
self.accept_next_alert = True
def tearDown(self):
self.driver.quit()
self.assertEqual([], self.verificationErrors)
tmpl/details_mozmill.html deleted 100644 → 0
View file @ a0978d09
<ul>
<li><b>Passed tests:</b>
[% GET test.results.results.0.passed %]/[% GET test.results.results.0.passed + test.results.results.0.failed %]
</li>
<li><a href="results-[% tbbfile %]/[% test.name %].json">json results</a></li>
</ul>
[% IF test.results.results.0.failed %]
[% FOREACH f IN test.results.results.0.fails %]
[% FOREACH v IN f.values %]
<b>Error:</b> [% v.name %]<br />
<pre>[% FILTER html_entity %][% v.message %][% END %]</pre><br />
[% END %]
[% END %]
[% END %]
[% IF test.results.connections.size %]
<b>List of opened connections:</b>
<ul>
[% FOREACH con IN test.results.connections.nsort.reverse %]
<li[% IF test.results.bad_connections.$con %] class="text_red"[% END %]>[% con %]: [% test.results.connections.$con %] connections</li>
[% END %]
</ul>
[% END %]
[% IF test.results.modified_files.size %]
<b>List of modified files:</b>
<ul>
[% FOREACH file IN test.results.modified_files.sort %]
<li><a href="results-[% tbbfile %]/[% test.name %].sandbox/[% IF ! file.match('^/'); GET tbbfiles.$tbbfile.tbbdir _ '/'; END; %][% file %]">[% file %]</a></li>
[% END %]
</ul>
[% END %]
[% IF test.results.removed_files.size %]
<b>List of removed files:</b>
<ul>
[% FOREACH file IN test.results.removed_files.sort %]
<li>[% file %]</li>
[% END %]
</ul>
[% END %]
tmpl/details_selenium.html deleted 100644 → 0
View file @ a0978d09
<ul>
[% SET stests = test.results.total_tests - test.results.failed.size %]
<li><b>Passed tests:</b> [% stests %]/[% test.results.total_tests %]</li>
<li><a href="results-[% tbbfile %]/[% test.name %].json">json results</a></li>
</ul>
[% IF test.results.failed %]
[% FOREACH f IN test.results.failed %]
<b>Failed test:</b> [% f.id %]<br />
<pre>[% f.error %]</pre><br />
[% END %]
[% END %]
[% IF test.results.connections.size %]
<b>List of opened connections:</b>
<ul>
[% FOREACH con IN test.results.connections.nsort.reverse %]
<li[% IF test.results.bad_connections.$con %] class="text_red"[% END %]>[% con %]: [% test.results.connections.$con %] connections</li>
[% END %]
</ul>
[% END %]
tmpl/tests_index.html
View file @ a46250cc
......@@ -9,10 +9,8 @@
[% FOREACH test IN tests %]
<h2>[% test.name %] ([% test.type %])</h2>
<p>[% test.descr %]</p>
[% IF test.type == 'mozmill' %]
<p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/mozmill-tests/tbb-tests/[% test.name %].js">test source</a></p>
[% ELSIF test.type == 'selenium' %]
<p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/selenium-tests/test_[% test.name %].py">test source</a></p>
[% IF test.type == 'marionette' %]
<p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/marionette/tor_browser_tests/test_[% test.name %].py">test source</a></p>
[% END %]
[% END %]
</body>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment