GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

BrowserBundleTests.pm 29.2 KB
Newer Older
1
package TBBTestSuite::TestSuite::BrowserBundleTests;
2 3 4

use warnings;
use strict;
boklm's avatar
boklm committed
5 6 7

use parent 'TBBTestSuite::TestSuite';

8 9 10 11 12 13 14 15 16 17
use English;
use FindBin;
use File::Slurp;
use File::Spec;
use File::Find;
use File::Type;
use File::Copy;
use JSON;
use Digest::SHA qw(sha256_hex);
use LWP::UserAgent;
18
use TBBTestSuite::Common qw(exit_error winpath clone_strip_coderef);
19 20
use TBBTestSuite::Options qw($options);
use TBBTestSuite::Tests::VirusTotal qw(virustotal_run);
21
use TBBTestSuite::Tests::Command qw(command_run);
22
use TBBTestSuite::Tests::TorBootstrap;
boklm's avatar
boklm committed
23
use TBBTestSuite::XServer qw(start_X stop_X set_Xmode);
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38

my $screenshot_thumbnail;
BEGIN {
    # For some reason that I did not understand yet, Image::Magick does
    # not work on Windows, so we're not creating thumbnails if we're
    # on Windows. In that case, the thumbnails should be created by the
    # server that receives the results.
    if ($OSNAME ne 'cygwin') {
        require TBBTestSuite::Thumbnail;
        $screenshot_thumbnail = \&TBBTestSuite::Thumbnail::screenshot_thumbnail;
    } else {
        $screenshot_thumbnail = sub { };
    }
}

boklm's avatar
boklm committed
39 40 41 42
sub test_types {
    return {
        tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
        mozmill       => \&mozmill_run,
43
        marionette    => \&marionette_run,
boklm's avatar
boklm committed
44 45 46 47 48
        selenium      => \&selenium_run,
        virustotal    => \&virustotal_run,
        command       => \&command_run,
    };
}
49

boklm's avatar
boklm committed
50 51 52
sub type {
    'browserbundle';
}
53

boklm's avatar
boklm committed
54 55 56
sub description {
    'Tor Browser Bundle integration tests';
}
57

58 59
our @tests = (
    {
boklm's avatar
boklm committed
60 61 62 63 64 65 66 67 68
        name            => 'readelf_RELRO',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check if binaries are RELocation Read-Only',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-ld' ],
        check_output    => sub { ( $_[0] =~ m/GNU_RELRO/ )
                                 && ( $_[0] =~ m/BIND_NOW/ ) },
        enable          => sub { $OSNAME eq 'linux' },
69 70
    },
    {
boklm's avatar
boklm committed
71 72 73 74 75 76 77 78
        name            => 'readelf_stack_canary',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check for stack canary support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-s' ],
        check_output    => sub { $_[0] =~ m/__stack_chk_fail/ },
        enable          => sub { $OSNAME eq 'linux' },
79 80
    },
    {
boklm's avatar
boklm committed
81 82 83 84 85 86 87
        name            => 'readelf_NX',
        type            => 'command',
        descr           => 'Check for NX support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-W', '-l' ],
        check_output    => sub { ! ($_[0] =~ m/GNU_STACK.+RWE/) },
        enable          => sub { $OSNAME eq 'linux' },
88 89
    },
    {
boklm's avatar
boklm committed
90 91 92 93 94 95 96
        name            => 'readelf_PIE',
        type            => 'command',
        descr           => 'Check for PIE support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-h' ],
        check_output    => sub { $_[0] =~ m/Type:\s+DYN/ },
        enable          => sub { $OSNAME eq 'linux' },
97 98
    },
    {
boklm's avatar
boklm committed
99 100 101 102 103 104 105 106
        name            => 'readelf_no_rpath',
        fail_type       => 'warning',
        type            => 'command',
        descr           => 'Check for no rpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/RPATH/ ) },
        enable          => sub { $OSNAME eq 'linux' },
107 108
    },
    {
boklm's avatar
boklm committed
109 110 111 112 113 114 115
        name            => 'readelf_no_runpath',
        type            => 'command',
        descr           => 'Check for no runpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/runpath/ ) },
        enable          => sub { $OSNAME eq 'linux' },
116 117
    },
    {
boklm's avatar
boklm committed
118 119 120 121 122
        name            => 'tor_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
123
        run_once        => 1,
124 125
    },
    {
boklm's avatar
boklm committed
126 127 128
        name            => 'tor_bridge',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge',
129
        enable          => sub { $OSNAME eq 'linux' },
130
        run_once        => 1,
131 132
    },
    {
boklm's avatar
boklm committed
133 134 135 136
        name            => 'tor_bridge_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge and an http proxy',
        httpproxy       => 1,
137
        enable          => sub { $OSNAME eq 'linux' },
138
        run_once        => 1,
139 140
    },
    {
boklm's avatar
boklm committed
141 142 143 144
        name            => 'tor_obfs3',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3',
        enable          => sub { $OSNAME eq 'linux' },
145
        run_once        => 1,
146 147
    },
    {
boklm's avatar
boklm committed
148 149 150 151 152
        name            => 'tor_obfs3_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3 and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
153
        run_once        => 1,
154
    },
155
    {
boklm's avatar
boklm committed
156 157 158 159
        name            => 'tor_obfs4',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4',
        enable          => sub { $OSNAME eq 'linux' && $_[0]->{version} !~ m/^4.0/ },
160
        run_once        => 1,
161 162
    },
    {
boklm's avatar
boklm committed
163 164 165 166 167
        name            => 'tor_obfs4_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4 and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' && $_[0]->{version} !~ m/^4.0/ },
168
        run_once        => 1,
169
    },
170
    {
boklm's avatar
boklm committed
171 172 173 174
        name            => 'tor_fte',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy',
        enable          => sub { $OSNAME eq 'linux' },
175
        run_once        => 1,
176 177
    },
    {
boklm's avatar
boklm committed
178 179 180 181 182
        name            => 'tor_fte_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
183
        run_once        => 1,
184
    },
185 186 187 188 189
    {
        name            => 'tor_scramblesuit',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit',
        enable          => sub { $OSNAME eq 'linux' },
190
        run_once        => 1,
191 192 193 194 195 196 197
    },
    {
        name            => 'tor_scramblesuit_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit and an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
198
        run_once        => 1,
199
    },
200 201 202 203 204
    {
        name            => 'tor_meek-google',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-google',
        enable          => sub { $OSNAME eq 'linux' },
205
        run_once        => 1,
206 207 208 209 210 211
    },
    {
        name            => 'tor_meek-amazon',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-amazon',
        enable          => sub { $OSNAME eq 'linux' },
212
        run_once        => 1,
213 214 215 216 217 218
    },
    {
        name            => 'tor_meek-azure',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-azure',
        enable          => sub { $OSNAME eq 'linux' },
219
        run_once        => 1,
220
    },
221
    {
boklm's avatar
boklm committed
222 223 224 225 226
        name            => 'tor_bootstrap',
        type            => 'tor_bootstrap',
        descr           => 'Check that we can bootstrap tor',
        fail_type       => 'fatal',
        no_kill         => 1,
227 228 229
        use_default_config => 1,
    },
    {
boklm's avatar
boklm committed
230 231 232
        name            => 'screenshots',
        type            => 'mozmill',
        descr           => 'Take some screenshots',
233 234
    },
    {
boklm's avatar
boklm committed
235 236 237 238
        name            => 'check',
        type            => 'selenium',
        use_net         => 1,
        descr           => 'Check that http://check.torproject.org/ think we are using tor',
239 240
    },
    {
boklm's avatar
boklm committed
241 242 243 244
        name            => 'https-everywhere',
        type            => 'mozmill',
        use_net         => 1,
        descr           => 'Check that https everywhere is enabled and working',
245 246
    },
    {
boklm's avatar
boklm committed
247 248 249 250 251 252
        name            => 'https-everywhere-disabled',
        type            => 'mozmill',
        descr           => 'Check that https everywhere is not doing anything when disabled',
        use_net         => 1,
        pre             => sub { toggle_https_everywhere($_[0], 0) },
        post            => sub { toggle_https_everywhere($_[0], 1) },
253 254
    },
    {
boklm's avatar
boklm committed
255 256 257
        name            => 'settings',
        type            => 'mozmill',
        descr           => 'Check that some important settings are correctly set',
258
    },
boklm's avatar
boklm committed
259
    {
boklm's avatar
boklm committed
260 261 262 263
        name            => 'acid3',
        type            => 'mozmill',
        descr           => 'acid3 tests',
        use_net         => 1,
boklm's avatar
boklm committed
264
        retry           => 4,
boklm's avatar
boklm committed
265
    },
boklm's avatar
boklm committed
266
    {
boklm's avatar
boklm committed
267 268 269 270 271 272 273 274
        name            => 'slider_settings_1',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 1,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
275 276
    },
    {
boklm's avatar
boklm committed
277 278 279 280 281 282 283 284
        name            => 'slider_settings_2',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 2,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
285 286
    },
    {
boklm's avatar
boklm committed
287 288 289 290 291 292 293 294
        name            => 'slider_settings_3',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 3,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
295 296
    },
    {
boklm's avatar
boklm committed
297 298 299 300 301 302 303 304
        name            => 'slider_settings_4',
        mozmill_test    => 'slider_settings',
        type            => 'mozmill',
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 4,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
305
    },
boklm's avatar
boklm committed
306
    {
boklm's avatar
boklm committed
307 308 309
        name            => 'dom-objects-enumeration',
        type            => 'mozmill',
        descr           => 'Check the list of DOM Objects exposed in the global namespace',
boklm's avatar
boklm committed
310
    },
311
    {
boklm's avatar
boklm committed
312 313 314 315
        name            => 'navigation-timing',
        type            => 'mozmill',
        descr           => 'Check that the Navigation Timing API is really disabled',
        use_net         => 1,
316
    },
317
    {
boklm's avatar
boklm committed
318 319 320 321
        name            => 'resource-timing',
        type            => 'mozmill',
        descr           => 'Check that the Resource Timing API is really disabled',
        use_net         => 1,
322
    },
323
    {
boklm's avatar
boklm committed
324 325 326
        name            => 'searchengines',
        type            => 'mozmill',
        descr           => 'Check that we have the default search engines set',
327
    },
boklm's avatar
boklm committed
328
    {
boklm's avatar
boklm committed
329 330 331 332 333
        name            => 'noscript',
        type            => 'mozmill',
        descr           => 'Check that noscript options are working',
        use_net         => 1,
        prefs           => {
334
            'extensions.torbutton.security_slider' => 2,
boklm's avatar
boklm committed
335
        },
boklm's avatar
boklm committed
336
        enable          => sub { $_[0]->{version} !~ m/^4.0/ },
boklm's avatar
boklm committed
337
    },
boklm's avatar
boklm committed
338
    {
boklm's avatar
boklm committed
339 340 341
        name            => 'fp_screen_dimensions',
        type            => 'selenium',
        descr           => 'Check that screen dimensions are spoofed correctly',
342 343
    },
    {
boklm's avatar
boklm committed
344 345 346
        name            => 'fp_screen_coords',
        type            => 'selenium',
        descr           => 'Check that screenX, screenY, screenLeft, screenTop, mozInnerScreenX, mozInnerScreenY are 0',
347 348
    },
    {
boklm's avatar
boklm committed
349 350 351
        name            => 'fp_plugins',
        type            => 'selenium',
        descr           => 'Check that plugins are disabled',
352 353
    },
    {
boklm's avatar
boklm committed
354 355 356
        name            => 'fp_useragent',
        type            => 'selenium',
        descr           => 'Check that userAgent is as expected',
357
    },        {
boklm's avatar
boklm committed
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
        name            => 'fp_navigator',
        type            => 'selenium',
        descr           => 'Check that navigator properties are as expected',
    },
    {
        name            => 'play_videos',
        type            => 'mozmill',
        descr           => 'Play some videos',
        use_net         => 1,
        mozmill_test    => 'test_page',
        remote          => 1,
        timeout         => 50000,
        interval        => 100,
    },
    {
        name            => 'svg-disable',
        type            => 'mozmill',
        descr           => 'Check if disabling svg is working',
        mozmill_test    => 'svg',
        svg_enabled     => 0,
        use_net         => 1,
        prefs           => {
380 381 382
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'false',
        },
boklm's avatar
boklm committed
383
        enable          => sub { $OSNAME eq 'linux' },
384 385
    },
    {
boklm's avatar
boklm committed
386 387 388 389 390 391 392
        name            => 'svg-enable',
        type            => 'mozmill',
        descr           => 'Check if enabling svg is working',
        mozmill_test    => 'svg',
        use_net         => 1,
        svg_enabled     => 1,
        prefs           => {
393 394 395
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'true',
        },
boklm's avatar
boklm committed
396
        enable          => sub { $OSNAME eq 'linux' },
397
    },
398 399 400 401 402 403
);

sub toggle_https_everywhere {
    my ($tbbinfos, $t) = @_;
    my $prefs = $tbbinfos->{ffprofiledir} . '/extensions/'
        . 'https-everywhere@eff.org/defaults/preferences/preferences.js';
404 405 406
    my $prefs_eff = $tbbinfos->{ffprofiledir} . '/extensions/'
        . 'https-everywhere-eff@eff.org/defaults/preferences/preferences.js';
    $prefs = $prefs_eff unless -f $prefs;
407 408 409 410 411 412 413 414 415 416 417 418 419
    my @f = read_file($prefs);
    foreach (@f) {
        if ($t) {
            s/pref\("extensions\.https_everywhere\.globalEnabled",false\);
             /pref("extensions.https_everywhere.globalEnabled",true);/x;
        } else {
            s/pref\("extensions\.https_everywhere\.globalEnabled",true\);
             /pref("extensions.https_everywhere.globalEnabled",false);/x;
        }
    }
    write_file($prefs, @f);
}

420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
sub set_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.backup";
    my $new_prefs = '';
    foreach my $prefname (sort keys %{$t->{prefs}}) {
        $new_prefs .= "pref(\"$prefname\", $t->{prefs}{$prefname});\n";
    }
    write_file($prefs, {append => 1}, $new_prefs);
    print "prefs file: $prefs\n";
}

sub reset_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.backup", $prefs;
}

boklm's avatar
boklm committed
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
sub set_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.slider_backup";
    write_file($prefs, {append => 1},
      'pref("extensions.torbutton.security_custom", false);' . "\n" .
      "pref(\"extensions.torbutton.security_slider\", $t->{slider_mode});\n");
}

sub reset_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.slider_backup", $prefs;
}

455 456 457
sub tbb_binfiles {
    my ($tbbinfos, $test) = @_;
    return $tbbinfos->{binfiles} if $tbbinfos->{binfiles};
458
    my %binfiles;
459 460 461 462
    my %wanted_types = (
        'application/x-executable-file' => 1,
        'application/x-ms-dos-executable' => 1,
    );
463 464 465
    my $wanted = sub {
        return unless -f $File::Find::name;
        my $type = File::Type->new->checktype_filename($File::Find::name);
466
        return unless $wanted_types{$type};
boklm's avatar
boklm committed
467 468 469
        my $name = $File::Find::name;
        $name =~ s/^$tbbinfos->{tbbdir}\///;
        $binfiles{$name} = 1;
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
    };
    find($wanted, $tbbinfos->{tbbdir});
    return $tbbinfos->{binfiles} = [ keys %binfiles ];
}

sub list_tests {
    foreach my $test (@tests) {
        print "$test->{name} ($test->{type})\n   $test->{descr}\n\n";
    }
}

sub get_tbbfile {
    my ($tbbinfos) = @_;
    $tbbinfos->{tbbfile_orig} = $tbbinfos->{tbbfile};
    if ($tbbinfos->{tbbfile} =~ m/^https?:\/\//) {
        my (undef, undef, $file) = File::Spec->splitpath($tbbinfos->{tbbfile});
        my $output = $options->{'download-dir'} ?
                "$options->{'download-dir'}/$file" : "$tbbinfos->{tmpdir}/$file";
        return $output if -f $output;
        print "Downloading $tbbinfos->{tbbfile}\n";
        my $ua = LWP::UserAgent->new;
        my $resp = $ua->get($tbbinfos->{tbbfile}, ':content_file' => $output);
        exit_error "Error downloading $tbbinfos->{tbbfile}:\n" . $resp->status_line
                unless $resp->is_success;
        $tbbinfos->{tbbfile} = $output;
    }
    exit_error "File $tbbinfos->{tbbfile} does not exist"
                unless -f $tbbinfos->{tbbfile};
}

sub extract_tbb {
    my ($tbbinfos) = @_;
    exit_error "Can't open file $tbbinfos->{tbbfile}" unless -f $tbbinfos->{tbbfile};
    my $tbbfile = File::Spec->rel2abs($tbbinfos->{tbbfile});
    my $tmpdir = $tbbinfos->{tmpdir};
    chdir $tmpdir;
    if ($tbbinfos->{os} eq 'Linux') {
        system('tar', 'xf', $tbbfile);
508 509 510 511 512
        if ($tbbinfos->{language} eq 'ALL') {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser";
        } else {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser_$tbbinfos->{language}";
        }
513
        $tbbinfos->{tbbdir} .= '/Browser';
514 515 516 517
    } elsif ($tbbinfos->{os} eq 'Windows') {
        my (undef, undef, $f) = File::Spec->splitpath($tbbfile);
        copy($tbbfile, "$tmpdir/$f");
        system('7z', 'x', $f);
518
        $tbbinfos->{tbbdir} = "$tmpdir/torbrowser/Browser";
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
        move("$tmpdir/\$_OUTDIR", "$tmpdir/torbrowser");
        move ("$tmpdir/Start Tor Browser.exe", "$tmpdir/torbrowser/");
    }
}

sub xvfb_run {
    my ($test) = @_;
    return () unless $options->{xvfb};
    my $resolution = $test->{resolution} ? $test->{resolution}
                                         : $options->{resolution};
    return ('xvfb-run', '--auto-servernum', '-s', "-screen 0 ${resolution}x24");
}

sub mozmill_cmd {
    if ($OSNAME eq 'cygwin') {
        return ( "$options->{'mozmill-dir'}\\run.cmd", 'mozmill' );
    }
    return ("$options->{virtualenv}/bin/mozmill");
}

sub check_opened_connections {
    my ($tbbinfos, $test) = @_;
541 542 543 544 545 546
    my %bad_connections =  %{$test->{results}{connections}};
    delete $bad_connections{"127.0.0.1:$options->{'tor-control-port'}"};
    delete $bad_connections{"127.0.0.1:$options->{'tor-socks-port'}"};
    # For some reasons, tor-browser creates two connections to the default
    # socks port even when when TOR_SOCKS_PORT is set
    # https://lists.torproject.org/pipermail/tbb-dev/2014-May/000050.html
boklm's avatar
boklm committed
547 548 549 550
    if (defined $bad_connections{'127.0.0.1:9150'}
        && $bad_connections{'127.0.0.1:9150'} <= 2) {
        delete $bad_connections{'127.0.0.1:9150'}
    }
551 552 553 554
    if (%bad_connections) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
555
    $test->{results}{bad_connections} = \%bad_connections;
556 557 558 559
}

sub check_modified_files {
    my ($tbbinfos, $test) = @_;
560
    my @bad_modified_files = @{$test->{results}{modified_files}};
561 562 563 564
    if (@bad_modified_files) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
565 566 567
    $test->{results}{bad_modified_files} = \@bad_modified_files;
}

568 569 570 571
sub clean_strace {
    my ($tbbinfos, $test) = @_;
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
    unlink $logfile;
boklm's avatar
boklm committed
572
    unlink "$logfile.tmp";
573 574
}

575 576 577 578 579
sub parse_strace {
    my ($tbbinfos, $test) = @_;
    my %ignore_files = map { $_ => 1 } qw(/dev/null /dev/tty);
    my %files;
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
boklm's avatar
boklm committed
580
    my $logfile_tmp = "$tbbinfos->{'results-dir'}/$test->{name}.strace.tmp";
581 582 583
    $test->{results}{connections} = {};
    my %modified_files;
    my %removed_files;
584 585 586 587 588
    if (-f "$logfile.tmp") {
        my $txt = read_file("$logfile.tmp");
        write_file($logfile, { append => 1 }, $txt);
        unlink "$logfile.tmp";
    }
589
    my @lines = read_file($logfile) if -f $logfile;
590
    push @lines, read_file($logfile_tmp) if -f $logfile_tmp;
591
    foreach my $line (@lines) {
592 593 594 595 596
        if ($line =~ m/^\d+ open\("((?:[^"\\]++|\\.)*+)", ([^\)]+)/ ||
            $line =~ m/^\d+ openat\([^,]+, "((?:[^"\\]++|\\.)*+)", ([^\)]+)/) {
            next if $2 =~ m/O_RDONLY/;
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
            next if $ignore_files{$1};
597 598 599
            if ($ENV{'MOZMILL_SCREENSHOTS'}) {
                next if $1 =~ m/^$ENV{'MOZMILL_SCREENSHOTS'}/;
            }
600 601 602 603 604 605 606 607 608
            $modified_files{$1}++;
        }
        if ($line =~ m/^\d+ unlink\("((?:[^"\\]++|\\.)*+)"/) {
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
            $removed_files{$1}++;
            delete $modified_files{$1} unless -f $1;
        }
        if ($line =~ m/^\d+ connect\(\d+, {sa_family=AF_INET, sin_port=htons\((\d+)\), sin_addr=inet_addr\("((?:[^"\\]++|\\.)*+)"\)/) {
            $test->{results}{connections}{"$2:$1"}++;
609 610
        }
    }
611 612
    $test->{results}{modified_files} = [ keys %modified_files ];
    $test->{results}{removed_files} = [ keys %removed_files ];
613 614 615 616 617 618 619 620 621
}

sub ff_wrapper {
    my ($tbbinfos, $test) = @_;
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_wrapper";
    return $wrapper_file if -f $wrapper_file;
    my $wrapper = <<EOF;
#!/bin/sh
set -e
622
export HOME="$tbbinfos->{tbbdir}"
623
export LD_LIBRARY_PATH="$tbbinfos->{tbbdir}:$tbbinfos->{tordir}"
624 625 626 627 628 629 630
exec \'$tbbinfos->{ffbin}\' "\$@"
EOF
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

631
sub ff_strace_wrapper {
632 633
    my ($tbbinfos, $test) = @_;
    my $ff_wrapper = ff_wrapper($tbbinfos, $test);
634
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
635 636
    my $wrapper = <<EOF;
#!/bin/sh
637 638 639 640 641 642
if [ -f $logfile.tmp ]
then
   cat $logfile.tmp >> $logfile
   rm $logfile.tmp
fi
echo \$@ >> /tmp/ff_run.log
643 644 645 646 647
strace -f -o $logfile.tmp -- \'$ff_wrapper\' "\$@"
exit_code=\$?
cat $logfile.tmp >> $logfile
rm $logfile.tmp
exit \$?
648 649 650 651 652 653 654 655 656 657 658 659
EOF
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_$test->{name}";
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

sub ffbin_path {
    my ($tbbinfos, $test) = @_;
    if ($OSNAME eq 'cygwin') {
        return winpath("$tbbinfos->{ffbin}.exe");
    }
660 661
    my %t = map { $_ => 1 } qw(mozmill marionette);
    if ($options->{use_strace} && $t{$test->{type}}) {
boklm's avatar
boklm committed
662 663
        return ff_strace_wrapper($tbbinfos, $test);
    }
664
    return ff_wrapper($tbbinfos, $test);
665 666
}

667 668 669
sub mozmill_export_options {
    my ($tbbinfos, $test) = @_;
    my $options_file = winpath("$FindBin::Bin/mozmill-tests/lib/testsuite.js");
670 671
    my $json_opts = encode_json clone_strip_coderef $options;
    my $json_test = encode_json clone_strip_coderef $test;
boklm's avatar
boklm committed
672 673
    my $json_tbbinfos = encode_json clone_strip_coderef
                                { %$tbbinfos, tests => undef };
674 675 676
    my $content = <<EOF;
var options = $json_opts;
var test = $json_test;
boklm's avatar
boklm committed
677
var tbbinfos = $json_tbbinfos;
678 679
exports.options = options;
exports.test = test;
boklm's avatar
boklm committed
680
exports.tbbinfos = tbbinfos;
681 682 683 684
EOF
    write_file($options_file, $content);
}

685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
sub marionette_run {
    my ($tbbinfos, $test) = @_;
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
    set_test_prefs($tbbinfos, $test);

    my $result_file_html = "$tbbinfos->{'results-dir'}/$test->{name}.html";
    my $result_file_txt = "$tbbinfos->{'results-dir'}/$test->{name}.txt";
    #--log-unittest  ./res.txt --log-html ./res.html
    my $bin = $OSNAME eq 'cygwin' ? 'Scripts' : 'bin';
    system(xvfb_run($test), "$FindBin::Bin/virtualenv-marionette/$bin/tor-browser-tests",
        '--log-unittest', winpath($result_file_txt),
        '--log-html', winpath($result_file_html),
        '--binary', ffbin_path($tbbinfos, $test),
        '--profile', winpath($tbbinfos->{ffprofiledir}),
        winpath("$FindBin::Bin/marionette/tor_browser_tests/test_$test->{name}.py"));
    my @txt_log = read_file($result_file_txt);
    my $res_line = shift @txt_log;
    $test->{results}{success} = $res_line eq ".\n" || $res_line eq ".\r\n";
    $test->{results}{log} = join '', @txt_log;
    reset_test_prefs($tbbinfos, $test);
    parse_strace($tbbinfos, $test);
    check_opened_connections($tbbinfos, $test);
    check_modified_files($tbbinfos, $test);
}

712 713 714
sub mozmill_run {
    my ($tbbinfos, $test) = @_;
    return unless $options->{mozmill};
715 716 717
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
718
    clean_strace($tbbinfos, $test) if $options->{use_strace};
719
    mozmill_export_options($tbbinfos, $test);
720
    set_test_prefs($tbbinfos, $test);
721 722 723 724
    $test->{screenshots} = [];
    my $screenshots_tmp = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
    $ENV{'MOZMILL_SCREENSHOTS'} = winpath($screenshots_tmp);
    my $results_file = "$tbbinfos->{'results-dir'}/$test->{name}.json";
725
    my $mozmill_test = $test->{mozmill_test} // $test->{name};
726 727
    system(xvfb_run($test), mozmill_cmd(), '-b', ffbin_path($tbbinfos, $test),
        '-p', winpath($tbbinfos->{ffprofiledir}),
728
        '-t', winpath("$FindBin::Bin/mozmill-tests/tbb-tests/$mozmill_test.js"),
729 730 731 732 733 734 735 736
        '--report', 'file://' . winpath($results_file));
    my $i = 0;
    for my $screenshot_file (reverse sort glob "$screenshots_tmp/*.png") {
        move($screenshot_file, "$tbbinfos->{'results-dir'}/$test->{name}-$i.png");
        $screenshot_thumbnail->($tbbinfos->{'results-dir'}, "$test->{name}-$i.png");
        push @{$test->{screenshots}}, "$test->{name}-$i.png";
        $i++;
    }
737 738 739 740 741 742 743
    if (-f $results_file) {
        $test->{results} = decode_json(read_file($results_file));
        $test->{results}{success} = $test->{results}{results}->[0]->{passed} ?
                                !$test->{results}{results}->[0]->{failed} : 0;
    } else {
        $test->{results}{success} = 0;
    }
744
    reset_test_prefs($tbbinfos, $test);
boklm's avatar
boklm committed
745 746 747 748
    if ($options->{use_strace}) {
        parse_strace($tbbinfos, $test);
        check_opened_connections($tbbinfos, $test);
        check_modified_files($tbbinfos, $test);
749
        clean_strace($tbbinfos, $test) if $test->{results}{success};
boklm's avatar
boklm committed
750
    }
751 752 753 754 755
}

sub selenium_run {
    my ($tbbinfos, $test) = @_;
    return unless $options->{selenium};
756 757 758
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
759 760 761 762 763 764 765 766 767 768 769
    my $result_file = $ENV{SELENIUM_TEST_RESULT_FILE} =
        "$tbbinfos->{'results-dir'}/$test->{name}.json";
    $ENV{TBB_BIN} = ffbin_path($tbbinfos, $test);
    $ENV{TBB_PROFILE} = $tbbinfos->{ffprofiledir};
    system(xvfb_run($test), "$options->{virtualenv}/bin/python",
        "$FindBin::Bin/selenium-tests/run_test", $test->{name});
    $test->{results} = decode_json(read_file($result_file));
}

sub set_tbbpaths {
    my ($tbbinfos) = @_;
770 771 772
    $tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/firefox";
    $tbbinfos->{tordir} = "$tbbinfos->{tbbdir}/TorBrowser/Tor";
    $tbbinfos->{datadir} = "$tbbinfos->{tbbdir}/TorBrowser/Data";
773
    $tbbinfos->{torbin} = "$tbbinfos->{tordir}/tor";
boklm's avatar
boklm committed
774
    $tbbinfos->{ptdir} = winpath("$tbbinfos->{tordir}/PluggableTransports");
775 776 777
    $tbbinfos->{ffprofiledir} = "$tbbinfos->{datadir}/Browser/profile.default";
}

boklm's avatar
boklm committed
778 779 780 781
sub new {
    my ($ts, $testsuite) = @_;
    $testsuite->{type} = 'browserbundle';
    $testsuite->{tests} = [ map { { %$_ } } @tests ];
782 783
    return undef unless $testsuite->{os} eq $options->{os};
    return undef unless $testsuite->{arch} eq $options->{arch};
boklm's avatar
boklm committed
784 785 786
    return bless $testsuite, $ts;
}

787 788 789 790 791 792 793 794 795 796
sub pre_tests {
    my ($tbbinfos) = @_;
    get_tbbfile($tbbinfos);
    if ($tbbinfos->{sha256sum} &&
        $tbbinfos->{sha256sum} ne sha256_hex(read_file($tbbinfos->{tbbfile}))) {
        exit_error "Wrong sha256sum for $tbbinfos->{tbbfile}";
    }
    $tbbinfos->{sha256sum} //= sha256_hex(read_file($tbbinfos->{tbbfile}));
    extract_tbb($tbbinfos);
    set_tbbpaths($tbbinfos);
797 798 799 800
    my $prefs_file = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    open(my $prefs_fh, '>>', $prefs_file);
    print $prefs_fh 'pref("extensions.torbutton.prompted_language", true);', "\n";
    close $prefs_fh;
801
    chdir $tbbinfos->{tbbdir} || exit_error "Can't enter directory $tbbinfos->{tbbdir}";
boklm's avatar
boklm committed
802 803
    copy "$FindBin::Bin/data/cert_override.txt",
          "TorBrowser/Data/Browser/profile.default/cert_override.txt";
804
    $ENV{TOR_SKIP_LAUNCH} = 1;
805 806
    $ENV{TOR_SOCKS_PORT} = $options->{'tor-socks-port'};
    $ENV{TOR_CONTROL_PORT} = $options->{'tor-control-port'};
boklm's avatar
boklm committed
807 808 809
    if ($options->{xdummy}) {
        $tbbinfos->{Xdisplay} = start_X("$tbbinfos->{'results-dir'}/xorg.log");
    }
810 811 812 813 814
}

sub post_tests {
    my ($tbbinfos) = @_;
    TBBTestSuite::Tests::TorBootstrap::stop_tor($tbbinfos);
boklm's avatar
boklm committed
815
    stop_X($tbbinfos->{Xdisplay}) if $options->{xdummy};
816 817 818
}

1;