Remove support for mozmill and selenium tests

.gitmodules

-[submodule "mozmill-tests/mozilla-mozmill-tests"]
-	path = mozmill-tests/mozilla-mozmill-tests
-	url = https://github.com/boklm/mozilla-mozmill-tests

TBBTestSuite/Options.pm

@@ -38,8 +38,6 @@ my %default_options = (
     action   => 'run_tests',
     os       => get_os_name(),
     arch     => get_arch(),
-    mozmill  => $OSNAME ne 'darwin',
-    selenium => $OSNAME ne 'cygwin',
     starttor => 1,
     gpgcheck => 1,
     clean_browserdir => 1,
@@ -47,7 +45,6 @@ my %default_options = (
     'tor-control-port' => '9551',
     'tor-socks-port'   => '9550',
     'reports-dir'      => "$FindBin::Bin/reports",
-    virtualenv => "$FindBin::Bin/virtualenv",
     resolution => '1280x1024',
     xvfb       => 0,
     xdummy     => $OSNAME ne 'cygwin',
@@ -57,7 +54,6 @@ my %default_options = (
     'email-to' => [],
     'email-from' => 'TBB Test Report <tbbtest@example.com>',
     'email-subject' => '[test result: [% success ? "ok" : "failed" %]] [% options.name %]',
-    'mozmill-dir' => 'c:\tbbtestsuite\mozmill-env',
     'http-proxy-port' => '8888',
     testrequests_types => 'browserbundle',
     test_data_url => 'http://test-data.tbb.torproject.org',
@@ -69,12 +65,12 @@ my %default_options = (
 
 
 sub get_options {
-    my @options = qw(mozmill! selenium! starttor! tor-control-port=i
+    my @options = qw(starttor! tor-control-port=i
                      tor-socks-port=i reports-dir=s gpgcheck! keyring=s
-                     virtualenv=s xvfb! name=s download-dir=s config=s
+                     xvfb! name=s download-dir=s config=s
                      action=s enable-tests=s upload-to=s os=s arch=s
                      virustotal! email-to=s@ email-from=s email-subject=s
-                     mozmill-dir=s reports-url=s http-proxy-port=i
+                     reports-url=s http-proxy-port=i
                      xdummy! disable-tests=s testrequests_types=s testsuite=s
                      cleanup!);
     my (%cli, %config);

TBBTestSuite/TestSuite/BrowserBundleTests.pm

@@ -40,9 +40,7 @@ BEGIN {
 sub test_types {
     return {
         tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
-        mozmill       => \&mozmill_run,
         marionette    => \&marionette_run,
-        selenium      => \&selenium_run,
         virustotal    => \&virustotal_run,
         command       => \&command_run,
     };
@@ -554,13 +552,6 @@ sub xvfb_run {
     return ('xvfb-run', '--auto-servernum', '-s', "-screen 0 ${resolution}x24");
 }
 
-sub mozmill_cmd {
-    if ($OSNAME eq 'cygwin') {
-        return ( "$options->{'mozmill-dir'}\\run.cmd", 'mozmill' );
-    }
-    return ("$options->{virtualenv}/bin/mozmill");
-}
-
 sub check_opened_connections {
     my ($tbbinfos, $test) = @_;
     my %bad_connections =  %{$test->{results}{connections}};
@@ -682,7 +673,7 @@ sub ffbin_path {
     if ($OSNAME eq 'cygwin') {
         return winpath("$tbbinfos->{ffbin}.exe");
     }
-    my %t = map { $_ => 1 } qw(mozmill marionette);
+    my %t = map { $_ => 1 } qw(marionette);
     if ($options->{use_strace} && $t{$test->{type}}) {
         return ff_strace_wrapper($tbbinfos, $test);
     }
@@ -690,24 +681,6 @@ sub ffbin_path {
     return ff_wrapper($tbbinfos, $test);
 }
 
-sub mozmill_export_options {
-    my ($tbbinfos, $test) = @_;
-    my $options_file = winpath("$FindBin::Bin/mozmill-tests/lib/testsuite.js");
-    my $json_opts = encode_json clone_strip_coderef $options;
-    my $json_test = encode_json clone_strip_coderef $test;
-    my $json_tbbinfos = encode_json clone_strip_coderef
-                                { %$tbbinfos, tests => undef };
-    my $content = <<EOF;
-var options = $json_opts;
-var test = $json_test;
-var tbbinfos = $json_tbbinfos;
-exports.options = options;
-exports.test = test;
-exports.tbbinfos = tbbinfos;
-EOF
-    write_file($options_file, $content);
-}
-
 sub marionette_export_options {
     my ($tbbinfos, $test) = @_;
     my $options_file = File::Temp->new();
@@ -766,62 +739,6 @@ sub marionette_run {
     check_modified_files($tbbinfos, $test);
 }
 
-sub mozmill_run {
-    my ($tbbinfos, $test) = @_;
-    return unless $options->{mozmill};
-    if ($test->{tried} && $test->{use_net}) {
-        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
-    }
-    clean_strace($tbbinfos, $test) if $options->{use_strace};
-    mozmill_export_options($tbbinfos, $test);
-    set_test_prefs($tbbinfos, $test);
-    $test->{screenshots} = [];
-    my $screenshots_tmp = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
-    $ENV{'MOZMILL_SCREENSHOTS'} = winpath($screenshots_tmp);
-    my $results_file = "$tbbinfos->{'results-dir'}/$test->{name}.json";
-    my $mozmill_test = $test->{mozmill_test} // $test->{name};
-    system(xvfb_run($test), mozmill_cmd(), '-b', ffbin_path($tbbinfos, $test),
-        '-p', winpath($tbbinfos->{ffprofiledir}),
-        '-t', winpath("$FindBin::Bin/mozmill-tests/tbb-tests/$mozmill_test.js"),
-        '--report', 'file://' . winpath($results_file));
-    my $i = 0;
-    for my $screenshot_file (reverse sort glob "$screenshots_tmp/*.png") {
-        move($screenshot_file, "$tbbinfos->{'results-dir'}/$test->{name}-$i.png");
-        $screenshot_thumbnail->($tbbinfos->{'results-dir'}, "$test->{name}-$i.png");
-        push @{$test->{screenshots}}, "$test->{name}-$i.png";
-        $i++;
-    }
-    if (-f $results_file) {
-        $test->{results} = decode_json(read_file($results_file));
-        $test->{results}{success} = $test->{results}{results}->[0]->{passed} ?
-                                !$test->{results}{results}->[0]->{failed} : 0;
-    } else {
-        $test->{results}{success} = 0;
-    }
-    reset_test_prefs($tbbinfos, $test);
-    if ($options->{use_strace}) {
-        parse_strace($tbbinfos, $test);
-        check_opened_connections($tbbinfos, $test);
-        check_modified_files($tbbinfos, $test);
-        clean_strace($tbbinfos, $test) if $test->{results}{success};
-    }
-}
-
-sub selenium_run {
-    my ($tbbinfos, $test) = @_;
-    return unless $options->{selenium};
-    if ($test->{tried} && $test->{use_net}) {
-        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
-    }
-    my $result_file = $ENV{SELENIUM_TEST_RESULT_FILE} =
-        "$tbbinfos->{'results-dir'}/$test->{name}.json";
-    $ENV{TBB_BIN} = ffbin_path($tbbinfos, $test);
-    $ENV{TBB_PROFILE} = $tbbinfos->{ffprofiledir};
-    system(xvfb_run($test), "$options->{virtualenv}/bin/python",
-        "$FindBin::Bin/selenium-tests/run_test", $test->{name});
-    $test->{results} = decode_json(read_file($result_file));
-}
-
 sub set_tbbpaths {
     my ($tbbinfos) = @_;
     $tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/firefox";

config/linus-nightly-virustotal

@@ -33,8 +33,6 @@ return ( args => [] ) unless $ok;
     os      => 'Windows',
     arch    => 'x86',
     starttor => 0,
-    mozmill  => 0,
-    selenium => 0,
     'email-from' => 'TBB Nightly Tests <boklm@torproject.org>',
     'email-to'   => [ 'boklm@mars-attacks.org', ],
     'reports-url' => 'http://test-reports.tbb.torproject.org/reports/',

config/release-virustotal

@@ -16,8 +16,6 @@ foreach my $build (@latest_builds) {
         os         => 'Windows',
         arch       => 'x86',
         starttor   => 0,
-        mozmill    => 0,
-        selenium   => 0,
         'reports-url' => 'http://test-reports.tbb.torproject.org/reports/',
         'email-from'  => 'TBB Tests <boklm@torproject.org>',
         'email-to'    => [ 'tor-qa@lists.torproject.org', 'boklm@mars-attacks.org', ],

doc/install.txt

@@ -12,7 +12,7 @@ as root, which will install the needed packages. This script has been
 tested on Ubuntu, Debian, Fedora, Centos.
 
 After installing the required dependencies, a python virtualenv needs
-to be created with mozmill and selenium. This can be with the 'setup-virtualenv'
+to be created with marionette. This can be with the 'setup-virtualenv'
 script.
 
 Summary of installation instructions:

doc/new-test.txt

@@ -14,9 +14,9 @@ There are two categories of tests that we are running:
   that we find in the sources tree.
 
 - The tor browser bundle tests. Those tests are run on a complete bundle.
-  The tests in the browser can be implemented using mozmill or selenium.
-  We also have other types of tests checking that pluggable transports
-  are working with and without using an http proxy, or checking that
+  The tests in the browser can be implemented using marionette. We also
+  have other types of tests checking that pluggable transports are
+  working with and without using an http proxy, or checking that
   binary files have been compiled with hardenning options.
 
 If the goal is to check that the behavior of the browser is correct
@@ -39,14 +39,14 @@ The list of tests to be run by the test suite is defined in file
 Each test is defined with the following properties:
 
 name::
-        The name of the test. For 'mozmill', 'selenium' and 'tor_bootstrap'
-        tests this name is used to find the test filename.
+        The name of the test. For 'marionette', 'tor_bootstrap' tests
+        this name is used to find the test filename.
 
 type::
-        The type of tests. This can be 'mozmill', 'selenium',
-        'tor_bootstrap', 'command', or something else if you want to
-        write the test in perl. In the later case, you need to define
-        it in the '%test_types' variable.
+        The type of tests. This can be 'marionette', 'tor_bootstrap',
+        'command', or something else if you want to write the test in
+        perl. In the later case, you need to define it in the
+        '%test_types' variable.
 
 descr::
         A short description.
@@ -73,39 +73,40 @@ run_once::
         of bundles, this test will only be run on the first bundle.
 
 
-Adding a mozmill test
-~~~~~~~~~~~~~~~~~~~~~
+Adding a marionette test
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-To add a new test based on mozmill, you will add something like this in
-'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
+To add a new test based on marionette, you will add something like this
+in 'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
 
 ----
     {
         name  => 'example-test',
-        type  => 'mozmill',
-        descr => 'Example mozmill test',
+        type  => 'marionette',
+        descr => 'Example marionette test',
     },
 ----
 
-The mozmill test files are stored in the directory
-'mozmill/mozmill-tests/tbb-tests/'. The name of the file is the name
-of the test with '.js' at the end, unless mozmill_test is defined, in
-which case it is used instead of the name.
+The marionette test files are stored in the directory
+'marionette/tor_browser_tests/'. The name of the file is 'test_$name.py'
+where '$name' is the name of the test, unless marionette_test is defined,
+in which case it is used instead of the name.
 
-It is possible to share a single mozmill file for multiple tests. The
-mozmill test can access values from the test definition to do something
+It is possible to share a single marionette file for multiple tests. The
+marionette test can access values from the test definition to do something
 different in each test. It is also possible to access the test suite options.
 In this example, we are accessing the "timeout" value from the test
 definition, and the "test_data_url" from the the testsuite options:
 
 ----
-var testsuite = require("../lib/testsuite");
+import testsuite
 
-var testThings = function() {
-    var test_data_url = testsuite.options.test_data_url;
-    var timeout = testsuite.test.timeout;
-    ...
-}
+class Test(MarionetteTestCase):
+   def setUp(self):
+        MarionetteTestCase.setUp(self)
+        ts = testsuite.TestSuite()
+        timeout = ts.t['test']['timeout']
+        test_data_url = ts.t['options']['test_data_url']
 ----
 
 A common option is 'test_data_url' which is an URL where test data have
@@ -159,26 +160,6 @@ The 'timeout' value is the time in milliseconds after which the test
 should be considered as failed if the 'test_result' element is still
 not present.
 
-
-Adding a selenium test
-~~~~~~~~~~~~~~~~~~~~~~
-
-To add a new test based on selenium webdriver, you will add something
-like this in 'TBBTestSuite/TestSuite/BrowserBundleTests.pm':
-
-----
-    {
-        name  => 'example-test',
-        type  => 'selenium',
-        descr => 'Example selenium test',
-    },
-----
-
-The selenium test files are stored in the directory 'selenium-tests'.
-The name of the file that will be executed is 'test_$name.py' (with
-$name replaced by the test name).
-
-
 Adding a command test
 ~~~~~~~~~~~~~~~~~~~~~
 

doc/usage.txt

@@ -55,12 +55,6 @@ Available options
         not given, the files are downloaded in a temporary directory
         which is removed at the end of the tests.
 
---no-mozmill::
-        Don't run mozmill tests.
-
---no-selenium::
-        Don't run the Selenium tests.
-
 --no-starttor::
         Don't start a tor daemon. If you use this option, you should
         already have a tor daemon running.
@@ -83,11 +77,6 @@ Available options
         save the results inside the reports-dir. If not set, a random
         name is used.
 
---virtualenv=<directory>::
-        Path to the virtualenv where selenium and mozmill are installed.
-        The default is the 'virtualenv' directory created by the
-        'setup-virtualenv' script.
-
 --xvfb::
         Run the tests using a virtual frame buffer X server. You should
         also disable the X server with --no-xdummy.
@@ -139,10 +128,6 @@ Available options
 --email-subject::
         Set the template used to create the subject of the reports emails.
 
---mozmill-dir::
-        On windows this sets the directory where mozmill-env has been
-        installed. The default is c:\tbbtestsuite\mozmill-env.
-
 --http-proxy-port::
         Select the port used to start an http proxy, for tor_bootstrap
         tests that require an http proxy.

mozmill-tests/lib/common.js

-function load_page(controller, url) {
-    var retry = 4;
-    while (retry-- > 0) {
-        var success = true;
-        controller.open(url);
-        try {
-            controller.waitForPageLoad(10000);
-        } catch(e) {
-            success = false;
-        }
-        if (success) {
-            return true;
-        }
-    }
-    controller.open(url);
-    return controller.waitForPageLoad(50000);
-}
-
-exports.load_page = load_page;

mozmill-tests/lib/screenshot.js

-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Based on:
-// http://hg.mozilla.org/qa/mozmill-tests/file/905bc0ba7a1e/firefox/lib/screenshot.js
-
-Cu.import("resource://gre/modules/Downloads.jsm");
-Cu.import("resource://gre/modules/Services.jsm");
-
-/**
- * This function creates a screenshot of the window provided in the given
- * controller and highlights elements from the coordinates provided in the
- * given boxes-array.
- *
- * @param {array of array of int} boxes
- * @param {MozmillController} controller
- */
-function create(controller, boxes) {
-  var doc = controller.window.document;
-  var maxWidth = doc.documentElement.boxObject.width;
-  var maxHeight = doc.documentElement.boxObject.height;
-  var rect = [];
-  for (var i = 0, j = boxes.length; i < j; ++i) {
-    rect = boxes[i];
-    if (rect[0] + rect[2] > maxWidth) maxWidth = rect[0] + rect[2];
-    if (rect[1] + rect[3] > maxHeight) maxHeight = rect[1] + rect[3];
-  }
-  var canvas = doc.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
-  var width = doc.documentElement.boxObject.width;
-  var height = doc.documentElement.boxObject.height;
-  canvas.width = maxWidth;
-  canvas.height = maxHeight;
-  var ctx = canvas.getContext("2d");
-  ctx.clearRect(0,0, canvas.width, canvas.height);
-  ctx.save();
-  ctx.drawWindow(controller.window, 0, 0, width, height, "rgb(0,0,0)");
-  ctx.restore();
-  ctx.save();
-  ctx.fillStyle = "rgba(255,0,0,0.4)";
-  for (var i = 0, j = boxes.length; i < j; ++i) {
-    rect = boxes[i];
-    ctx.fillRect(rect[0], rect[1], rect[2], rect[3]);
-  }
-  ctx.restore();
-
-  _saveCanvas(canvas);
-}
-
-/**
- * Saves a given Canvas object to a file.
- * The path to save the file under should be given in the
- * MOZMILL_SCREENSHOTS environment variable. If not, it will be saved
- * in the temporary folder of the system.
- *
- * @param {canvas} canvas
- */
-function _saveCanvas(canvas) {
-  var env = Cc["@mozilla.org/process/environment;1"]
-                .getService(Ci.nsIEnvironment);
-  var file = null;
-  if (env.exists("MOZMILL_SCREENSHOTS")) {
-    file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
-    file.initWithPath(env.get("MOZMILL_SCREENSHOTS"));
-  }
-  else {
-    file = Services.dirsvc.get("TmpD", Ci.nsIFile);
-  }
-
-  file.append("screenshot.png");
-
-  // if a file already exists, don't overwrite it and create a new name
-  file.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0666", 8));
-
-  var taskCompleted = false;
-  Downloads.fetch(canvas.toDataURL("image/png", ""), file).then(() => {
-    taskCompleted = true;
-  }, (aReason) => {
-    taskCompleted = true;
-    expect.fail("Failed to save the screenshot " + file + ", with: " + aReason);
-  });
-
-  expect.waitFor(() => taskCompleted,
-                 "Saving the screenshot to the disk action has finished.");
-}
-
-// Export of functions
-exports.create = create;

mozilla-mozmill-tests @ ddfe2275

-Subproject commit ddfe2275f8dc5b863096c7d1601be2740cd65193

selenium-tests/run_test

-#!/usr/bin/python
-import unittest, time, re
-import sys
-import json
-import os
-
-def save_result(testres):
-    if not os.environ.get('SELENIUM_TEST_RESULT_FILE'):
-        return
-    res = {}
-    res['success'] = testres.wasSuccessful()
-    res['total_tests'] = testres.testsRun
-    res['failed'] = []
-    failures = testres.failures + testres.errors
-    for f in failures:
-        r = {}
-        r['id'] = f[0].id()
-        r['error'] = f[1]
-        res['failed'].append(r)
-    output = open(os.environ.get('SELENIUM_TEST_RESULT_FILE'), 'w')
-    output.write(json.dumps(res))
-    output.close()
-
-
-if len(sys.argv) != 2:
-    sys.exit('Usage: run_test <test_name>')
-
-t = __import__('test_%s' % sys.argv[1])
-suite = unittest.TestLoader().loadTestsFromTestCase(t.Test)
-res = unittest.TextTestRunner(verbosity=2).run(suite)
-save_result(res)

selenium-tests/tbbtest.py

-#!/usr/bin/python
-from selenium import webdriver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support.ui import Select
-from selenium.common.exceptions import NoSuchElementException
-import unittest, time, re
-import os
-
-class TBBTest(unittest.TestCase):
-    def setUp(self):
-        ffbinary = webdriver.firefox.firefox_binary.FirefoxBinary(firefox_path=os.environ['TBB_BIN'])
-        ffprofile = webdriver.firefox.firefox_profile.FirefoxProfile(profile_directory=os.environ['TBB_PROFILE'])
-        self.driver = webdriver.Firefox(firefox_binary=ffbinary, firefox_profile=ffprofile)
-        self.driver.implicitly_wait(30)
-        self.base_url = "about:tor"
-        self.verificationErrors = []
-        self.accept_next_alert = True
-
-    def tearDown(self):
-        self.driver.quit()
-        self.assertEqual([], self.verificationErrors)
-

tmpl/details_mozmill.html

-<ul>
-    <li><b>Passed tests:</b>
-    [% GET test.results.results.0.passed %]/[% GET test.results.results.0.passed + test.results.results.0.failed %]
-    </li>
-    <li><a href="results-[% tbbfile %]/[% test.name %].json">json results</a></li>
-</ul>
-[% IF test.results.results.0.failed %]
-  [% FOREACH f IN test.results.results.0.fails %]
-    [% FOREACH v IN f.values %]
-      <b>Error:</b> [% v.name %]<br />
-      <pre>[% FILTER html_entity %][% v.message %][% END %]</pre><br />
-    [% END %]
-  [% END %]
-[% END %]
-[% IF test.results.connections.size %]
-<b>List of opened connections:</b>
-<ul>
-    [% FOREACH con IN test.results.connections.nsort.reverse %]
-    <li[% IF test.results.bad_connections.$con %] class="text_red"[% END %]>[% con %]: [% test.results.connections.$con %] connections</li>
-    [% END %]
-</ul>
-[% END %]
-[% IF test.results.modified_files.size %]
-<b>List of modified files:</b>
-<ul>
-    [% FOREACH file IN test.results.modified_files.sort %]
-    <li><a href="results-[% tbbfile %]/[% test.name %].sandbox/[% IF ! file.match('^/'); GET tbbfiles.$tbbfile.tbbdir _ '/'; END; %][% file %]">[% file %]</a></li>
-    [% END %]
-</ul>
-[% END %]
-[% IF test.results.removed_files.size %]
-<b>List of removed files:</b>
-<ul>
-    [% FOREACH file IN test.results.removed_files.sort %]
-    <li>[% file %]</li>
-    [% END %]
-</ul>
-[% END %]

tmpl/details_selenium.html

-<ul>
-    [% SET stests = test.results.total_tests - test.results.failed.size %]
-    <li><b>Passed tests:</b> [% stests %]/[% test.results.total_tests %]</li>
-    <li><a href="results-[% tbbfile %]/[% test.name %].json">json results</a></li>
-</ul>
-
-[% IF test.results.failed %]
-  [% FOREACH f IN test.results.failed %]
-    <b>Failed test:</b> [% f.id %]<br />
-    <pre>[% f.error %]</pre><br />
-  [% END %]
-[% END %]
-[% IF test.results.connections.size %]
-<b>List of opened connections:</b>
-<ul>
-    [% FOREACH con IN test.results.connections.nsort.reverse %]
-    <li[% IF test.results.bad_connections.$con %] class="text_red"[% END %]>[% con %]: [% test.results.connections.$con %] connections</li>
-    [% END %]
-</ul>
-[% END %]

tmpl/tests_index.html

@@ -9,10 +9,8 @@
         [% FOREACH test IN tests %]
         <h2>[% test.name %] ([% test.type %])</h2>
         <p>[% test.descr %]</p>
-        [% IF test.type == 'mozmill' %]
-        <p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/mozmill-tests/tbb-tests/[% test.name %].js">test source</a></p>
-        [% ELSIF test.type == 'selenium' %]
-        <p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/selenium-tests/test_[% test.name %].py">test source</a></p>
+        [% IF test.type == 'marionette' %]
+        <p><a href="https://gitweb.torproject.org/boklm/tor-browser-bundle-testsuite.git/blob/HEAD:/marionette/tor_browser_tests/test_[% test.name %].py">test source</a></p>
         [% END %]
         [% END %]
     </body>