GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

BrowserBundleTests.pm 29.2 KB
Newer Older
1
package TBBTestSuite::TestSuite::BrowserBundleTests;
2 3 4

use warnings;
use strict;
boklm's avatar
boklm committed
5 6 7

use parent 'TBBTestSuite::TestSuite';

8 9 10 11 12 13 14 15 16 17
use English;
use FindBin;
use File::Slurp;
use File::Spec;
use File::Find;
use File::Type;
use File::Copy;
use JSON;
use Digest::SHA qw(sha256_hex);
use LWP::UserAgent;
18
use TBBTestSuite::Common qw(exit_error winpath clone_strip_coderef);
19 20
use TBBTestSuite::Options qw($options);
use TBBTestSuite::Tests::VirusTotal qw(virustotal_run);
21
use TBBTestSuite::Tests::Command qw(command_run);
22
use TBBTestSuite::Tests::TorBootstrap;
boklm's avatar
boklm committed
23
use TBBTestSuite::XServer qw(start_X stop_X set_Xmode);
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38

my $screenshot_thumbnail;
BEGIN {
    # For some reason that I did not understand yet, Image::Magick does
    # not work on Windows, so we're not creating thumbnails if we're
    # on Windows. In that case, the thumbnails should be created by the
    # server that receives the results.
    if ($OSNAME ne 'cygwin') {
        require TBBTestSuite::Thumbnail;
        $screenshot_thumbnail = \&TBBTestSuite::Thumbnail::screenshot_thumbnail;
    } else {
        $screenshot_thumbnail = sub { };
    }
}

boklm's avatar
boklm committed
39 40 41 42
sub test_types {
    return {
        tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
        mozmill       => \&mozmill_run,
43
        marionette    => \&marionette_run,
boklm's avatar
boklm committed
44 45 46 47 48
        selenium      => \&selenium_run,
        virustotal    => \&virustotal_run,
        command       => \&command_run,
    };
}
49

boklm's avatar
boklm committed
50 51 52
sub type {
    'browserbundle';
}
53

boklm's avatar
boklm committed
54 55 56
sub description {
    'Tor Browser Bundle integration tests';
}
57

58 59
our @tests = (
    {
boklm's avatar
boklm committed
60 61 62 63 64 65 66 67 68
        name            => 'readelf_RELRO',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check if binaries are RELocation Read-Only',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-ld' ],
        check_output    => sub { ( $_[0] =~ m/GNU_RELRO/ )
                                 && ( $_[0] =~ m/BIND_NOW/ ) },
        enable          => sub { $OSNAME eq 'linux' },
69 70
    },
    {
boklm's avatar
boklm committed
71 72 73 74 75 76 77 78
        name            => 'readelf_stack_canary',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check for stack canary support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-s' ],
        check_output    => sub { $_[0] =~ m/__stack_chk_fail/ },
        enable          => sub { $OSNAME eq 'linux' },
79 80
    },
    {
boklm's avatar
boklm committed
81 82 83 84 85 86 87
        name            => 'readelf_NX',
        type            => 'command',
        descr           => 'Check for NX support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-W', '-l' ],
        check_output    => sub { ! ($_[0] =~ m/GNU_STACK.+RWE/) },
        enable          => sub { $OSNAME eq 'linux' },
88 89
    },
    {
boklm's avatar
boklm committed
90 91 92 93 94 95 96
        name            => 'readelf_PIE',
        type            => 'command',
        descr           => 'Check for PIE support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-h' ],
        check_output    => sub { $_[0] =~ m/Type:\s+DYN/ },
        enable          => sub { $OSNAME eq 'linux' },
97 98
    },
    {
boklm's avatar
boklm committed
99 100 101 102 103 104 105 106
        name            => 'readelf_no_rpath',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check for no rpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/RPATH/ ) },
        enable          => sub { $OSNAME eq 'linux' },
107 108
    },
    {
boklm's avatar
boklm committed
109 110 111 112 113 114 115
        name            => 'readelf_no_runpath',
        type            => 'command',
        descr           => 'Check for no runpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/runpath/ ) },
        enable          => sub { $OSNAME eq 'linux' },
116 117
    },
    {
boklm's avatar
boklm committed
118 119 120 121 122
        name            => 'tor_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
123
        run_once        => 1,
124 125
    },
    {
boklm's avatar
boklm committed
126 127 128
        name            => 'tor_bridge',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge',
129
        enable          => sub { $OSNAME eq 'linux' },
130
        run_once        => 1,
131 132
    },
    {
boklm's avatar
boklm committed
133 134 135 136
        name            => 'tor_bridge_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge and an http proxy',
        httpproxy       => 1,
137
        enable          => sub { $OSNAME eq 'linux' },
138
        run_once        => 1,
139 140
    },
    {
boklm's avatar
boklm committed
141 142 143 144
        name            => 'tor_obfs3',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3',
        enable          => sub { $OSNAME eq 'linux' },
145
        run_once        => 1,
146 147
    },
    {
boklm's avatar
boklm committed
148 149 150 151 152
        name            => 'tor_obfs3_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3 and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
153
        run_once        => 1,
154
    },
155
    {
boklm's avatar
boklm committed
156 157 158 159
        name            => 'tor_obfs4',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4',
        enable          => sub { $OSNAME eq 'linux' && $_[0]->{version} !~ m/^4.0/ },
160
        run_once        => 1,
161 162
    },
    {
boklm's avatar
boklm committed
163 164 165 166 167
        name            => 'tor_obfs4_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4 and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' && $_[0]->{version} !~ m/^4.0/ },
168
        run_once        => 1,
169
    },
170
    {
boklm's avatar
boklm committed
171 172 173 174
        name            => 'tor_fte',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy',
        enable          => sub { $OSNAME eq 'linux' },
175
        run_once        => 1,
176 177
    },
    {
boklm's avatar
boklm committed
178 179 180 181 182
        name            => 'tor_fte_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
183
        run_once        => 1,
184
    },
185 186 187 188 189
    {
        name            => 'tor_scramblesuit',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit',
        enable          => sub { $OSNAME eq 'linux' },
190
        run_once        => 1,
191 192 193 194 195 196 197
    },
    {
        name            => 'tor_scramblesuit_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
198
        run_once        => 1,
199
    },
200 201 202 203 204
    {
        name            => 'tor_meek-google',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-google',
        enable          => sub { $OSNAME eq 'linux' },
205
        run_once        => 1,
206 207 208 209 210 211
    },
    {
        name            => 'tor_meek-amazon',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-amazon',
        enable          => sub { $OSNAME eq 'linux' },
212
        run_once        => 1,
213 214 215 216 217 218
    },
    {
        name            => 'tor_meek-azure',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-azure',
        enable          => sub { $OSNAME eq 'linux' },
219
        run_once        => 1,
220
    },
221
    {
boklm's avatar
boklm committed
222 223 224 225 226
        name            => 'tor_bootstrap',
        type            => 'tor_bootstrap',
        descr           => 'Check that we can bootstrap tor',
        fail_type       => 'fatal',
        no_kill         => 1,
227 228 229
        use_default_config => 1,
    },
    {
boklm's avatar
boklm committed
230 231 232
        name            => 'screenshots',
        type            => 'mozmill',
        descr           => 'Take some screenshots',
233 234
    },
    {
boklm's avatar
boklm committed
235
        name            => 'check',
236
        type            => 'marionette',
boklm's avatar
boklm committed
237 238
        use_net         => 1,
        descr           => 'Check that http://check.torproject.org/ think we are using tor',
239 240
    },
    {
boklm's avatar
boklm committed
241 242 243 244
        name            => 'https-everywhere',
        type            => 'mozmill',
        use_net         => 1,
        descr           => 'Check that https everywhere is enabled and working',
245 246
    },
    {
boklm's avatar
boklm committed
247 248 249 250 251 252
        name            => 'https-everywhere-disabled',
        type            => 'mozmill',
        descr           => 'Check that https everywhere is not doing anything when disabled',
        use_net         => 1,
        pre             => sub { toggle_https_everywhere($_[0], 0) },
        post            => sub { toggle_https_everywhere($_[0], 1) },
253 254
    },
    {
boklm's avatar
boklm committed
255 256 257
        name            => 'settings',
        type            => 'mozmill',
        descr           => 'Check that some important settings are correctly set',
258
    },
boklm's avatar
boklm committed
259
    {
boklm's avatar
boklm committed
260 261 262 263
        name            => 'acid3',
        type            => 'mozmill',
        descr           => 'acid3 tests',
        use_net         => 1,
boklm's avatar
boklm committed
264
        retry           => 4,
boklm's avatar
boklm committed
265
    },
boklm's avatar
boklm committed
266
    {
boklm's avatar
boklm committed
267 268 269 270 271 272 273 274
        name            => 'slider_settings_1',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 1,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
275 276
    },
    {
boklm's avatar
boklm committed
277 278 279 280 281 282 283 284
        name            => 'slider_settings_2',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 2,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
285 286
    },
    {
boklm's avatar
boklm committed
287 288 289 290 291 292 293 294
        name            => 'slider_settings_3',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 3,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
295 296
    },
    {
boklm's avatar
boklm committed
297 298 299 300 301 302 303 304
        name            => 'slider_settings_4',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 4,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
305
    },
boklm's avatar
boklm committed
306
    {
boklm's avatar
boklm committed
307 308 309
        name            => 'dom-objects-enumeration',
        type            => 'mozmill',
        descr           => 'Check the list of DOM Objects exposed in the global namespace',
boklm's avatar
boklm committed
310
    },
311
    {
boklm's avatar
boklm committed
312 313 314 315
        name            => 'navigation-timing',
        type            => 'mozmill',
        descr           => 'Check that the Navigation Timing API is really disabled',
        use_net         => 1,
316
    },
317
    {
boklm's avatar
boklm committed
318 319 320 321
        name            => 'resource-timing',
        type            => 'mozmill',
        descr           => 'Check that the Resource Timing API is really disabled',
        use_net         => 1,
322
    },
323
    {
boklm's avatar
boklm committed
324 325 326
        name            => 'searchengines',
        type            => 'mozmill',
        descr           => 'Check that we have the default search engines set',
327
    },
boklm's avatar
boklm committed
328
    {
boklm's avatar
boklm committed
329 330 331 332 333
        name            => 'noscript',
        type            => 'mozmill',
        descr           => 'Check that noscript options are working',
        use_net         => 1,
        prefs           => {
334
            'extensions.torbutton.security_slider' => 2,
boklm's avatar
boklm committed
335
        },
boklm's avatar
boklm committed
336
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
337
    },
boklm's avatar
boklm committed
338
    {
boklm's avatar
boklm committed
339
        name            => 'fp_screen_dimensions',
340
        type            => 'marionette',
boklm's avatar
boklm committed
341
        descr           => 'Check that screen dimensions are spoofed correctly',
342 343
    },
    {
boklm's avatar
boklm committed
344
        name            => 'fp_screen_coords',
345
        type            => 'marionette',
boklm's avatar
boklm committed
346
        descr           => 'Check that screenX, screenY, screenLeft, screenTop, mozInnerScreenX, mozInnerScreenY are 0',
347 348
    },
    {
boklm's avatar
boklm committed
349
        name            => 'fp_plugins',
350
        type            => 'marionette',
boklm's avatar
boklm committed
351
        descr           => 'Check that plugins are disabled',
352 353
    },
    {
boklm's avatar
boklm committed
354
        name            => 'fp_useragent',
355
        type            => 'marionette',
boklm's avatar
boklm committed
356
        descr           => 'Check that userAgent is as expected',
357 358
    },
    {
boklm's avatar
boklm committed
359
        name            => 'fp_navigator',
360
        type            => 'marionette',
boklm's avatar
boklm committed
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
        descr           => 'Check that navigator properties are as expected',
    },
    {
        name            => 'play_videos',
        type            => 'mozmill',
        descr           => 'Play some videos',
        use_net         => 1,
        mozmill_test    => 'test_page',
        remote          => 1,
        timeout         => 50000,
        interval        => 100,
    },
    {
        name            => 'svg-disable',
        type            => 'mozmill',
        descr           => 'Check if disabling svg is working',
        mozmill_test    => 'svg',
        svg_enabled     => 0,
        use_net         => 1,
        prefs           => {
381 382 383
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'false',
        },
boklm's avatar
boklm committed
384
        enable          => sub { $OSNAME eq 'linux' },
385 386
    },
    {
boklm's avatar
boklm committed
387 388 389 390 391 392 393
        name            => 'svg-enable',
        type            => 'mozmill',
        descr           => 'Check if enabling svg is working',
        mozmill_test    => 'svg',
        use_net         => 1,
        svg_enabled     => 1,
        prefs           => {
394 395 396
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'true',
        },
boklm's avatar
boklm committed
397
        enable          => sub { $OSNAME eq 'linux' },
398
    },
399 400 401 402 403 404
);

sub toggle_https_everywhere {
    my ($tbbinfos, $t) = @_;
    my $prefs = $tbbinfos->{ffprofiledir} . '/extensions/'
        . 'https-everywhere@eff.org/defaults/preferences/preferences.js';
405 406 407
    my $prefs_eff = $tbbinfos->{ffprofiledir} . '/extensions/'
        . 'https-everywhere-eff@eff.org/defaults/preferences/preferences.js';
    $prefs = $prefs_eff unless -f $prefs;
408 409 410 411 412 413 414 415 416 417 418 419 420
    my @f = read_file($prefs);
    foreach (@f) {
        if ($t) {
            s/pref\("extensions\.https_everywhere\.globalEnabled",false\);
             /pref("extensions.https_everywhere.globalEnabled",true);/x;
        } else {
            s/pref\("extensions\.https_everywhere\.globalEnabled",true\);
             /pref("extensions.https_everywhere.globalEnabled",false);/x;
        }
    }
    write_file($prefs, @f);
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
sub set_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.backup";
    my $new_prefs = '';
    foreach my $prefname (sort keys %{$t->{prefs}}) {
        $new_prefs .= "pref(\"$prefname\", $t->{prefs}{$prefname});\n";
    }
    write_file($prefs, {append => 1}, $new_prefs);
    print "prefs file: $prefs\n";
}

sub reset_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.backup", $prefs;
}

boklm's avatar
boklm committed
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
sub set_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.slider_backup";
    write_file($prefs, {append => 1},
      'pref("extensions.torbutton.security_custom", false);' . "\n" .
      "pref(\"extensions.torbutton.security_slider\", $t->{slider_mode});\n");
}

sub reset_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.slider_backup", $prefs;
}

456 457 458
sub tbb_binfiles {
    my ($tbbinfos, $test) = @_;
    return $tbbinfos->{binfiles} if $tbbinfos->{binfiles};
459
    my %binfiles;
460 461 462 463
    my %wanted_types = (
        'application/x-executable-file' => 1,
        'application/x-ms-dos-executable' => 1,
    );
464 465 466
    my $wanted = sub {
        return unless -f $File::Find::name;
        my $type = File::Type->new->checktype_filename($File::Find::name);
467
        return unless $wanted_types{$type};
boklm's avatar
boklm committed
468 469 470
        my $name = $File::Find::name;
        $name =~ s/^$tbbinfos->{tbbdir}\///;
        $binfiles{$name} = 1;
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
    };
    find($wanted, $tbbinfos->{tbbdir});
    return $tbbinfos->{binfiles} = [ keys %binfiles ];
}

sub list_tests {
    foreach my $test (@tests) {
        print "$test->{name} ($test->{type})\n   $test->{descr}\n\n";
    }
}

sub get_tbbfile {
    my ($tbbinfos) = @_;
    $tbbinfos->{tbbfile_orig} = $tbbinfos->{tbbfile};
    if ($tbbinfos->{tbbfile} =~ m/^https?:\/\//) {
        my (undef, undef, $file) = File::Spec->splitpath($tbbinfos->{tbbfile});
        my $output = $options->{'download-dir'} ?
                "$options->{'download-dir'}/$file" : "$tbbinfos->{tmpdir}/$file";
        return $output if -f $output;
        print "Downloading $tbbinfos->{tbbfile}\n";
        my $ua = LWP::UserAgent->new;
        my $resp = $ua->get($tbbinfos->{tbbfile}, ':content_file' => $output);
        exit_error "Error downloading $tbbinfos->{tbbfile}:\n" . $resp->status_line
                unless $resp->is_success;
        $tbbinfos->{tbbfile} = $output;
    }
    exit_error "File $tbbinfos->{tbbfile} does not exist"
                unless -f $tbbinfos->{tbbfile};
}

sub extract_tbb {
    my ($tbbinfos) = @_;
    exit_error "Can't open file $tbbinfos->{tbbfile}" unless -f $tbbinfos->{tbbfile};
    my $tbbfile = File::Spec->rel2abs($tbbinfos->{tbbfile});
    my $tmpdir = $tbbinfos->{tmpdir};
    chdir $tmpdir;
    if ($tbbinfos->{os} eq 'Linux') {
        system('tar', 'xf', $tbbfile);
509 510 511 512 513
        if ($tbbinfos->{language} eq 'ALL') {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser";
        } else {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser_$tbbinfos->{language}";
        }
514
        $tbbinfos->{tbbdir} .= '/Browser';
515 516 517 518
    } elsif ($tbbinfos->{os} eq 'Windows') {
        my (undef, undef, $f) = File::Spec->splitpath($tbbfile);
        copy($tbbfile, "$tmpdir/$f");
        system('7z', 'x', $f);
519
        $tbbinfos->{tbbdir} = "$tmpdir/torbrowser/Browser";
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
        move("$tmpdir/\$_OUTDIR", "$tmpdir/torbrowser");
        move ("$tmpdir/Start Tor Browser.exe", "$tmpdir/torbrowser/");
    }
}

sub xvfb_run {
    my ($test) = @_;
    return () unless $options->{xvfb};
    my $resolution = $test->{resolution} ? $test->{resolution}
                                         : $options->{resolution};
    return ('xvfb-run', '--auto-servernum', '-s', "-screen 0 ${resolution}x24");
}

sub mozmill_cmd {
    if ($OSNAME eq 'cygwin') {
        return ( "$options->{'mozmill-dir'}\\run.cmd", 'mozmill' );
    }
    return ("$options->{virtualenv}/bin/mozmill");
}

sub check_opened_connections {
    my ($tbbinfos, $test) = @_;
542 543 544 545 546 547
    my %bad_connections =  %{$test->{results}{connections}};
    delete $bad_connections{"127.0.0.1:$options->{'tor-control-port'}"};
    delete $bad_connections{"127.0.0.1:$options->{'tor-socks-port'}"};
    # For some reasons, tor-browser creates two connections to the default
    # socks port even when when TOR_SOCKS_PORT is set
    # https://lists.torproject.org/pipermail/tbb-dev/2014-May/000050.html
boklm's avatar
boklm committed
548 549 550 551
    if (defined $bad_connections{'127.0.0.1:9150'}
        && $bad_connections{'127.0.0.1:9150'} <= 2) {
        delete $bad_connections{'127.0.0.1:9150'}
    }
552 553 554 555
    if (%bad_connections) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
556
    $test->{results}{bad_connections} = \%bad_connections;
557 558 559 560
}

sub check_modified_files {
    my ($tbbinfos, $test) = @_;
561
    my @bad_modified_files = @{$test->{results}{modified_files}};
562 563 564 565
    if (@bad_modified_files) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
566 567 568
    $test->{results}{bad_modified_files} = \@bad_modified_files;
}

569 570 571 572
sub clean_strace {
    my ($tbbinfos, $test) = @_;
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
    unlink $logfile;
boklm's avatar
boklm committed
573
    unlink "$logfile.tmp";
574 575
}

576 577 578 579 580
sub parse_strace {
    my ($tbbinfos, $test) = @_;
    my %ignore_files = map { $_ => 1 } qw(/dev/null /dev/tty);
    my %files;
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
boklm's avatar
boklm committed
581
    my $logfile_tmp = "$tbbinfos->{'results-dir'}/$test->{name}.strace.tmp";
582 583 584
    $test->{results}{connections} = {};
    my %modified_files;
    my %removed_files;
585 586 587 588 589
    if (-f "$logfile.tmp") {
        my $txt = read_file("$logfile.tmp");
        write_file($logfile, { append => 1 }, $txt);
        unlink "$logfile.tmp";
    }
590
    my @lines = read_file($logfile) if -f $logfile;
591
    push @lines, read_file($logfile_tmp) if -f $logfile_tmp;
592
    foreach my $line (@lines) {
593 594 595 596 597
        if ($line =~ m/^\d+ open\("((?:[^"\\]++|\\.)*+)", ([^\)]+)/ ||
            $line =~ m/^\d+ openat\([^,]+, "((?:[^"\\]++|\\.)*+)", ([^\)]+)/) {
            next if $2 =~ m/O_RDONLY/;
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
            next if $ignore_files{$1};
598 599 600
            if ($ENV{'MOZMILL_SCREENSHOTS'}) {
                next if $1 =~ m/^$ENV{'MOZMILL_SCREENSHOTS'}/;
            }
601 602 603 604 605 606 607 608 609
            $modified_files{$1}++;
        }
        if ($line =~ m/^\d+ unlink\("((?:[^"\\]++|\\.)*+)"/) {
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
            $removed_files{$1}++;
            delete $modified_files{$1} unless -f $1;
        }
        if ($line =~ m/^\d+ connect\(\d+, {sa_family=AF_INET, sin_port=htons\((\d+)\), sin_addr=inet_addr\("((?:[^"\\]++|\\.)*+)"\)/) {
            $test->{results}{connections}{"$2:$1"}++;
610 611
        }
    }
612 613
    $test->{results}{modified_files} = [ keys %modified_files ];
    $test->{results}{removed_files} = [ keys %removed_files ];
614 615 616 617 618 619 620 621 622
}

sub ff_wrapper {
    my ($tbbinfos, $test) = @_;
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_wrapper";
    return $wrapper_file if -f $wrapper_file;
    my $wrapper = <<EOF;
#!/bin/sh
set -e
623
export HOME="$tbbinfos->{tbbdir}"
624
export LD_LIBRARY_PATH="$tbbinfos->{tbbdir}:$tbbinfos->{tordir}"
625 626 627 628 629 630 631
exec \'$tbbinfos->{ffbin}\' "\$@"
EOF
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

632
sub ff_strace_wrapper {
633 634
    my ($tbbinfos, $test) = @_;
    my $ff_wrapper = ff_wrapper($tbbinfos, $test);
635
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
636 637
    my $wrapper = <<EOF;
#!/bin/sh
638 639 640 641 642 643
if [ -f $logfile.tmp ]
then
   cat $logfile.tmp >> $logfile
   rm $logfile.tmp
fi
echo \$@ >> /tmp/ff_run.log
644 645 646 647 648
strace -f -o $logfile.tmp -- \'$ff_wrapper\' "\$@"
exit_code=\$?
cat $logfile.tmp >> $logfile
rm $logfile.tmp
exit \$?
649 650 651 652 653 654 655 656 657 658 659 660
EOF
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_$test->{name}";
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

sub ffbin_path {
    my ($tbbinfos, $test) = @_;
    if ($OSNAME eq 'cygwin') {
        return winpath("$tbbinfos->{ffbin}.exe");
    }
661 662
    my %t = map { $_ => 1 } qw(mozmill marionette);
    if ($options->{use_strace} && $t{$test->{type}}) {
boklm's avatar
boklm committed
663 664
        return ff_strace_wrapper($tbbinfos, $test);
    }
665
    return ff_wrapper($tbbinfos, $test);
666 667
}

668 669 670
sub mozmill_export_options {
    my ($tbbinfos, $test) = @_;
    my $options_file = winpath("$FindBin::Bin/mozmill-tests/lib/testsuite.js");
671 672
    my $json_opts = encode_json clone_strip_coderef $options;
    my $json_test = encode_json clone_strip_coderef $test;
boklm's avatar
boklm committed
673 674
    my $json_tbbinfos = encode_json clone_strip_coderef
                                { %$tbbinfos, tests => undef };
675 676 677
    my $content = <<EOF;
var options = $json_opts;
var test = $json_test;
boklm's avatar
boklm committed
678
var tbbinfos = $json_tbbinfos;
679 680
exports.options = options;
exports.test = test;
boklm's avatar
boklm committed
681
exports.tbbinfos = tbbinfos;
682 683 684 685
EOF
    write_file($options_file, $content);
}

686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
sub marionette_run {
    my ($tbbinfos, $test) = @_;
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
    set_test_prefs($tbbinfos, $test);

    my $result_file_html = "$tbbinfos->{'results-dir'}/$test->{name}.html";
    my $result_file_txt = "$tbbinfos->{'results-dir'}/$test->{name}.txt";
    #--log-unittest  ./res.txt --log-html ./res.html
    my $bin = $OSNAME eq 'cygwin' ? 'Scripts' : 'bin';
    system(xvfb_run($test), "$FindBin::Bin/virtualenv-marionette/$bin/tor-browser-tests",
        '--log-unittest', winpath($result_file_txt),
        '--log-html', winpath($result_file_html),
        '--binary', ffbin_path($tbbinfos, $test),
        '--profile', winpath($tbbinfos->{ffprofiledir}),
        winpath("$FindBin::Bin/marionette/tor_browser_tests/test_$test->{name}.py"));
    my @txt_log = read_file($result_file_txt);
    my $res_line = shift @txt_log;
    $test->{results}{success} = $res_line eq ".\n" || $res_line eq ".\r\n";
    $test->{results}{log} = join '', @txt_log;
    reset_test_prefs($tbbinfos, $test);
    parse_strace($tbbinfos, $test);
    check_opened_connections($tbbinfos, $test);
    check_modified_files($tbbinfos, $test);
}

713 714 715
sub mozmill_run {
    my ($tbbinfos, $test) = @_;
    return unless $options->{mozmill};
716 717 718
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
719
    clean_strace($tbbinfos, $test) if $options->{use_strace};
720
    mozmill_export_options($tbbinfos, $test);
721
    set_test_prefs($tbbinfos, $test);
722 723 724 725
    $test->{screenshots} = [];
    my $screenshots_tmp = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
    $ENV{'MOZMILL_SCREENSHOTS'} = winpath($screenshots_tmp);
    my $results_file = "$tbbinfos->{'results-dir'}/$test->{name}.json";
726
    my $mozmill_test = $test->{mozmill_test} // $test->{name};
727 728
    system(xvfb_run($test), mozmill_cmd(), '-b', ffbin_path($tbbinfos, $test),
        '-p', winpath($tbbinfos->{ffprofiledir}),
729
        '-t', winpath("$FindBin::Bin/mozmill-tests/tbb-tests/$mozmill_test.js"),
730 731 732 733 734 735 736 737
        '--report', 'file://' . winpath($results_file));
    my $i = 0;
    for my $screenshot_file (reverse sort glob "$screenshots_tmp/*.png") {
        move($screenshot_file, "$tbbinfos->{'results-dir'}/$test->{name}-$i.png");
        $screenshot_thumbnail->($tbbinfos->{'results-dir'}, "$test->{name}-$i.png");
        push @{$test->{screenshots}}, "$test->{name}-$i.png";
        $i++;
    }
738 739 740 741 742 743 744
    if (-f $results_file) {
        $test->{results} = decode_json(read_file($results_file));
        $test->{results}{success} = $test->{results}{results}->[0]->{passed} ?
                                !$test->{results}{results}->[0]->{failed} : 0;
    } else {
        $test->{results}{success} = 0;
    }
745
    reset_test_prefs($tbbinfos, $test);
boklm's avatar
boklm committed
746 747 748 749
    if ($options->{use_strace}) {
        parse_strace($tbbinfos, $test);
        check_opened_connections($tbbinfos, $test);
        check_modified_files($tbbinfos, $test);
750
        clean_strace($tbbinfos, $test) if $test->{results}{success};
boklm's avatar
boklm committed
751
    }
752 753 754 755 756
}

sub selenium_run {
    my ($tbbinfos, $test) = @_;
    return unless $options->{selenium};
757 758 759
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
760 761 762 763 764 765 766 767 768 769 770
    my $result_file = $ENV{SELENIUM_TEST_RESULT_FILE} =
        "$tbbinfos->{'results-dir'}/$test->{name}.json";
    $ENV{TBB_BIN} = ffbin_path($tbbinfos, $test);
    $ENV{TBB_PROFILE} = $tbbinfos->{ffprofiledir};
    system(xvfb_run($test), "$options->{virtualenv}/bin/python",
        "$FindBin::Bin/selenium-tests/run_test", $test->{name});
    $test->{results} = decode_json(read_file($result_file));
}

sub set_tbbpaths {
    my ($tbbinfos) = @_;
771 772 773
    $tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/firefox";
    $tbbinfos->{tordir} = "$tbbinfos->{tbbdir}/TorBrowser/Tor";
    $tbbinfos->{datadir} = "$tbbinfos->{tbbdir}/TorBrowser/Data";
774
    $tbbinfos->{torbin} = "$tbbinfos->{tordir}/tor";
boklm's avatar
boklm committed
775
    $tbbinfos->{ptdir} = winpath("$tbbinfos->{tordir}/PluggableTransports");
776 777 778
    $tbbinfos->{ffprofiledir} = "$tbbinfos->{datadir}/Browser/profile.default";
}

boklm's avatar
boklm committed
779 780 781 782
sub new {
    my ($ts, $testsuite) = @_;
    $testsuite->{type} = 'browserbundle';
    $testsuite->{tests} = [ map { { %$_ } } @tests ];
783 784
    return undef unless $testsuite->{os} eq $options->{os};
    return undef unless $testsuite->{arch} eq $options->{arch};
boklm's avatar
boklm committed
785 786 787
    return bless $testsuite, $ts;
}

788 789 790 791 792 793 794 795 796 797
sub pre_tests {
    my ($tbbinfos) = @_;
    get_tbbfile($tbbinfos);
    if ($tbbinfos->{sha256sum} &&
        $tbbinfos->{sha256sum} ne sha256_hex(read_file($tbbinfos->{tbbfile}))) {
        exit_error "Wrong sha256sum for $tbbinfos->{tbbfile}";
    }
    $tbbinfos->{sha256sum} //= sha256_hex(read_file($tbbinfos->{tbbfile}));
    extract_tbb($tbbinfos);
    set_tbbpaths($tbbinfos);
798 799 800 801
    my $prefs_file = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    open(my $prefs_fh, '>>', $prefs_file);
    print $prefs_fh 'pref("extensions.torbutton.prompted_language", true);', "\n";
    close $prefs_fh;
802
    chdir $tbbinfos->{tbbdir} || exit_error "Can't enter directory $tbbinfos->{tbbdir}";
boklm's avatar
boklm committed
803 804
    copy "$FindBin::Bin/data/cert_override.txt",
          "TorBrowser/Data/Browser/profile.default/cert_override.txt";
805
    $ENV{TOR_SKIP_LAUNCH} = 1;
806 807
    $ENV{TOR_SOCKS_PORT} = $options->{'tor-socks-port'};
    $ENV{TOR_CONTROL_PORT} = $options->{'tor-control-port'};
boklm's avatar
boklm committed
808 809 810
    if ($options->{xdummy}) {
        $tbbinfos->{Xdisplay} = start_X("$tbbinfos->{'results-dir'}/xorg.log");
    }
811 812 813 814 815
}

sub post_tests {
    my ($tbbinfos) = @_;
    TBBTestSuite::Tests::TorBootstrap::stop_tor($tbbinfos);
boklm's avatar
boklm committed
816
    stop_X($tbbinfos->{Xdisplay}) if $options->{xdummy};
817 818 819
}

1;