BrowserBundleTests.pm 35.6 KB
Newer Older
1
package TBBTestSuite::TestSuite::BrowserBundleTests;
2
3
4

use warnings;
use strict;
boklm's avatar
boklm committed
5
6
7

use parent 'TBBTestSuite::TestSuite';

8
9
10
11
12
13
14
use English;
use FindBin;
use File::Slurp;
use File::Spec;
use File::Find;
use File::Type;
use File::Copy;
15
use File::Temp;
16
17
18
use JSON;
use Digest::SHA qw(sha256_hex);
use LWP::UserAgent;
boklm's avatar
boklm committed
19
use IO::CaptureOutput qw(capture_exec);
20
use TBBTestSuite::Common qw(exit_error winpath clone_strip_coderef screenshot_thumbnail);
21
22
use TBBTestSuite::Options qw($options);
use TBBTestSuite::Tests::VirusTotal qw(virustotal_run);
23
use TBBTestSuite::Tests::Command qw(command_run);
24
use TBBTestSuite::Tests::TorBootstrap;
boklm's avatar
boklm committed
25
use TBBTestSuite::XServer qw(start_X stop_X set_Xmode);
26

boklm's avatar
boklm committed
27
28
29
sub test_types {
    return {
        tor_bootstrap => \&TBBTestSuite::Tests::TorBootstrap::start_tor,
30
        marionette    => \&marionette_run,
boklm's avatar
boklm committed
31
32
33
34
        virustotal    => \&virustotal_run,
        command       => \&command_run,
    };
}
35

boklm's avatar
boklm committed
36
37
38
sub type {
    'browserbundle';
}
39

boklm's avatar
boklm committed
40
41
42
sub description {
    'Tor Browser Bundle integration tests';
}
43

44
our @tests = (
boklm's avatar
boklm committed
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    {
        name         => 'win_DEP_ASLR',
        type         => 'command',
        retry        => 1,
        descr        => 'Check DEP/ASLR',
        files        => \&tbb_binfiles,
        command      => [ "$FindBin::Bin/data/check-windows-dep-aslr.sh" ],
        enable       => sub { $_[0]->{os} eq 'Windows' },
        # ticket 16417
        skip_files   => [ qw(
            TorBrowser/Tor/PluggableTransports/_ctypes.pyd
            TorBrowser/Tor/PluggableTransports/_hashlib.pyd
            TorBrowser/Tor/PluggableTransports/_socket.pyd
            TorBrowser/Tor/PluggableTransports/_ssl.pyd
            TorBrowser/Tor/PluggableTransports/bz2.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Cipher._AES.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Hash._SHA256.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Hash._SHA512.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Random.OSRNG.winrandom.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Util._counter.pyd
            TorBrowser/Tor/PluggableTransports/Crypto.Util.strxor.pyd
            TorBrowser/Tor/PluggableTransports/flashproxy-client.exe
            TorBrowser/Tor/PluggableTransports/flashproxy-reg-appspot.exe
            TorBrowser/Tor/PluggableTransports/flashproxy-reg-email.exe
            TorBrowser/Tor/PluggableTransports/flashproxy-reg-http.exe
            TorBrowser/Tor/PluggableTransports/flashproxy-reg-url.exe
            TorBrowser/Tor/PluggableTransports/fte.cDFA.pyd
            TorBrowser/Tor/PluggableTransports/fteproxy.exe
            TorBrowser/Tor/PluggableTransports/M2Crypto.__m2crypto.pyd
            TorBrowser/Tor/PluggableTransports/meek-client-torbrowser.exe
            TorBrowser/Tor/PluggableTransports/meek-client.exe
            TorBrowser/Tor/PluggableTransports/obfs4proxy.exe
            TorBrowser/Tor/PluggableTransports/obfsproxy.exe
            TorBrowser/Tor/PluggableTransports/pyexpat.pyd
            TorBrowser/Tor/PluggableTransports/python27.dll
            TorBrowser/Tor/PluggableTransports/select.pyd
            TorBrowser/Tor/PluggableTransports/terminateprocess-buffer.exe
            TorBrowser/Tor/PluggableTransports/unicodedata.pyd
            TorBrowser/Tor/PluggableTransports/w9xpopen.exe
            TorBrowser/Tor/PluggableTransports/zope.interface._zope_interface_coptimizations.pyd
            ) ],
    },
87
    {
boklm's avatar
boklm committed
88
89
90
91
92
93
94
95
        name            => 'readelf_RELRO',
        type            => 'command',
        descr           => 'Check if binaries are RELocation Read-Only',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-ld' ],
        check_output    => sub { ( $_[0] =~ m/GNU_RELRO/ )
                                 && ( $_[0] =~ m/BIND_NOW/ ) },
        enable          => sub { $OSNAME eq 'linux' },
boklm's avatar
boklm committed
96
97
98
99
100
101
        skip_files   => [ qw(
            TorBrowser/Tor/PluggableTransports/meek-client
            TorBrowser/Tor/PluggableTransports/meek-client-torbrowser
            TorBrowser/Tor/PluggableTransports/meek-client-torbrowser
            TorBrowser/Tor/PluggableTransports/obfs4proxy
            ) ],
102
103
    },
    {
boklm's avatar
boklm committed
104
105
106
107
108
109
110
        name            => 'readelf_stack_canary',
        type            => 'command',
        descr           => 'Check for stack canary support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-s' ],
        check_output    => sub { $_[0] =~ m/__stack_chk_fail/ },
        enable          => sub { $OSNAME eq 'linux' },
boklm's avatar
boklm committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        # ticket 13056
        skip_files   => [ qw(
            libmozalloc.so
            libnssckbi.so
            libplc4.so
            libplds4.so
            TorBrowser/Tor/libstdc++.so.6
            TorBrowser/Tor/PluggableTransports/Crypto/Cipher/_ARC4.so
            TorBrowser/Tor/PluggableTransports/Crypto/Cipher/_XOR.so
            TorBrowser/Tor/PluggableTransports/Crypto/Util/_counter.so
            TorBrowser/Tor/PluggableTransports/fte/cDFA.so
            TorBrowser/Tor/PluggableTransports/meek-client-torbrowser
            TorBrowser/Tor/PluggableTransports/twisted/python/_initgroups.so
            TorBrowser/Tor/PluggableTransports/twisted/runner/portmap.so
            TorBrowser/Tor/PluggableTransports/twisted/test/raiser.so
            TorBrowser/Tor/PluggableTransports/zope/interface/_zope_interface_coptimizations.so
            TorBrowser/Tor/PluggableTransports/meek-client
            TorBrowser/Tor/PluggableTransports/obfs4proxy
            ) ],
130
131
    },
    {
boklm's avatar
boklm committed
132
133
134
135
136
137
138
        name            => 'readelf_NX',
        type            => 'command',
        descr           => 'Check for NX support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-W', '-l' ],
        check_output    => sub { ! ($_[0] =~ m/GNU_STACK.+RWE/) },
        enable          => sub { $OSNAME eq 'linux' },
139
140
    },
    {
boklm's avatar
boklm committed
141
142
143
144
145
146
147
        name            => 'readelf_PIE',
        type            => 'command',
        descr           => 'Check for PIE support',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-h' ],
        check_output    => sub { $_[0] =~ m/Type:\s+DYN/ },
        enable          => sub { $OSNAME eq 'linux' },
boklm's avatar
boklm committed
148
149
150
151
        skip_files   => [ qw(
            TorBrowser/Tor/PluggableTransports/meek-client
            TorBrowser/Tor/PluggableTransports/meek-client-torbrowser
            TorBrowser/Tor/PluggableTransports/obfs4proxy
152
            TorBrowser/Tor/PluggableTransports/snowflake-client
boklm's avatar
boklm committed
153
            ) ],
154
155
    },
    {
boklm's avatar
boklm committed
156
157
158
159
160
161
162
        name            => 'readelf_no_rpath',
        type            => 'command',
        descr           => 'Check for no rpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/RPATH/ ) },
        enable          => sub { $OSNAME eq 'linux' },
163
164
    },
    {
boklm's avatar
boklm committed
165
166
167
168
169
170
171
        name            => 'readelf_no_runpath',
        type            => 'command',
        descr           => 'Check for no runpath',
        files           => \&tbb_binfiles,
        command         => [ 'readelf', '-d' ],
        check_output    => sub { ! ( $_[0] =~ m/runpath/ ) },
        enable          => sub { $OSNAME eq 'linux' },
172
    },
boklm's avatar
boklm committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    {
        name            => 'otool_PIE',
        type            => 'command',
        descr           => 'Check for PIE support',
        files           => \&tbb_osx_executable_files,
        command         => [ 'otool', '-hv' ],
        check_output    => sub {
            my @lines = split("\n", $_[0]);
            my $last_line = pop @lines;
            my ($flags) = $last_line =~ m/^\s*[^\s]+\s+[^\s]+\s+[^\s]+\s+[^\s]+\s+[^\s]+\s+[^\s]+\s+[^\s]+\s+(.*)/;
            my %flags = map { $_ => 1 } split(/\s+/, $flags);
            return $flags{PIE};
        },
        enable          => sub { $OSNAME eq 'darwin' },
        retry           => 1,
boklm's avatar
boklm committed
188
189
190
191
192
        skip_files   => [ qw(
            Contents/MacOS/Tor/PluggableTransports/meek-client
            Contents/MacOS/Tor/PluggableTransports/meek-client-torbrowser
            Contents/MacOS/Tor/PluggableTransports/obfs4proxy
            ) ],
boklm's avatar
boklm committed
193
    },
194
    {
boklm's avatar
boklm committed
195
196
197
198
199
        name            => 'tor_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using an http proxy',
        httpproxy       => 1,
        enable          => sub { $OSNAME eq 'linux' },
200
        run_once        => 1,
201
202
    },
    {
boklm's avatar
boklm committed
203
204
205
        name            => 'tor_bridge',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge',
206
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
207
        run_once        => 1,
208
209
    },
    {
boklm's avatar
boklm committed
210
211
212
213
        name            => 'tor_bridge_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using a bridge and an http proxy',
        httpproxy       => 1,
214
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
215
        run_once        => 1,
216
217
    },
    {
boklm's avatar
boklm committed
218
219
220
        name            => 'tor_obfs3',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3',
221
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
222
        run_once        => 1,
223
224
    },
    {
boklm's avatar
boklm committed
225
226
227
228
        name            => 'tor_obfs3_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs3 and an http proxy',
        httpproxy       => 1,
229
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
230
        run_once        => 1,
231
    },
232
    {
boklm's avatar
boklm committed
233
234
235
        name            => 'tor_obfs4',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4',
236
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
237
        run_once        => 1,
238
239
    },
    {
boklm's avatar
boklm committed
240
241
242
243
        name            => 'tor_obfs4_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using obfs4 and an http proxy',
        httpproxy       => 1,
244
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
245
        run_once        => 1,
246
    },
247
    {
boklm's avatar
boklm committed
248
249
250
        name            => 'tor_fte',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy',
251
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
252
        run_once        => 1,
253
254
    },
    {
boklm's avatar
boklm committed
255
256
257
258
        name            => 'tor_fte_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using fteproxy and an http proxy',
        httpproxy       => 1,
259
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
260
        run_once        => 1,
261
    },
262
263
264
265
    {
        name            => 'tor_scramblesuit',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit',
266
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
267
        run_once        => 1,
268
269
270
271
272
273
    },
    {
        name            => 'tor_scramblesuit_httpproxy',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using scramblesuit and an http proxy',
        httpproxy       => 1,
274
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
275
        run_once        => 1,
276
    },
277
278
279
280
    {
        name            => 'tor_meek-amazon',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-amazon',
281
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
282
        run_once        => 1,
283
284
285
286
287
    },
    {
        name            => 'tor_meek-azure',
        type            => 'tor_bootstrap',
        descr           => 'Access tor using meek-azure',
288
        enable          => sub { $OSNAME eq 'linux' && $options->{PTtests} },
289
        run_once        => 1,
290
    },
291
    {
boklm's avatar
boklm committed
292
293
294
295
296
        name            => 'tor_bootstrap',
        type            => 'tor_bootstrap',
        descr           => 'Check that we can bootstrap tor',
        fail_type       => 'fatal',
        no_kill         => 1,
297
298
299
        use_default_config => 1,
    },
    {
boklm's avatar
boklm committed
300
        name            => 'screenshots',
301
        type            => 'marionette',
boklm's avatar
boklm committed
302
        descr           => 'Take some screenshots',
303
304
    },
    {
boklm's avatar
boklm committed
305
        name            => 'check',
306
        type            => 'marionette',
boklm's avatar
boklm committed
307
308
        use_net         => 1,
        descr           => 'Check that http://check.torproject.org/ think we are using tor',
309
310
    },
    {
boklm's avatar
boklm committed
311
        name            => 'https-everywhere',
312
        type            => 'marionette',
boklm's avatar
boklm committed
313
314
        use_net         => 1,
        descr           => 'Check that https everywhere is enabled and working',
315
316
    },
    {
boklm's avatar
boklm committed
317
        name            => 'https-everywhere-disabled',
318
319
        marionette_test => 'https-everywhere',
        type            => 'marionette',
boklm's avatar
boklm committed
320
321
        descr           => 'Check that https everywhere is not doing anything when disabled',
        use_net         => 1,
boklm's avatar
boklm committed
322
323
324
        prefs           => {
            'extensions.https_everywhere.globalEnabled' => 'false',
        },
325
326
    },
    {
boklm's avatar
boklm committed
327
        name            => 'settings',
328
        type            => 'marionette',
boklm's avatar
boklm committed
329
        descr           => 'Check that some important settings are correctly set',
330
    },
boklm's avatar
boklm committed
331
    {
boklm's avatar
boklm committed
332
        name            => 'acid3',
boklm's avatar
boklm committed
333
        type            => 'marionette',
boklm's avatar
boklm committed
334
335
        descr           => 'acid3 tests',
        use_net         => 1,
boklm's avatar
boklm committed
336
        retry           => 4,
boklm's avatar
boklm committed
337
338
        # the acid3 test is disabled for now
        enable          => sub { 0; },
boklm's avatar
boklm committed
339
    },
boklm's avatar
boklm committed
340
    {
boklm's avatar
boklm committed
341
        name            => 'slider_settings_1',
342
343
        marionette_test => 'slider_settings',
        type            => 'marionette',
boklm's avatar
boklm committed
344
345
346
347
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 1,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
boklm's avatar
boklm committed
348
349
    },
    {
boklm's avatar
boklm committed
350
        name            => 'slider_settings_2',
351
352
        marionette_test => 'slider_settings',
        type            => 'marionette',
boklm's avatar
boklm committed
353
354
355
356
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 2,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
boklm's avatar
boklm committed
357
358
    },
    {
boklm's avatar
boklm committed
359
        name            => 'slider_settings_3',
360
361
        marionette_test => 'slider_settings',
        type            => 'marionette',
boklm's avatar
boklm committed
362
363
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 3,
364
        enable          => sub { $_[0]->{version} =~ m/^6.0/ },
boklm's avatar
boklm committed
365
366
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
boklm's avatar
boklm committed
367
368
    },
    {
boklm's avatar
boklm committed
369
        name            => 'slider_settings_4',
370
371
        marionette_test => 'slider_settings',
        type            => 'marionette',
boklm's avatar
boklm committed
372
373
374
375
        descr           => 'Check that settings are set according to security slider mode',
        slider_mode     => 4,
        pre             => \&set_slider_mode,
        post            => \&reset_slider_mode,
boklm's avatar
boklm committed
376
    },
boklm's avatar
boklm committed
377
    {
boklm's avatar
boklm committed
378
        name            => 'dom-objects-enumeration',
379
        type            => 'marionette',
boklm's avatar
boklm committed
380
        descr           => 'Check the list of DOM Objects exposed in the global namespace',
boklm's avatar
boklm committed
381
    },
382
383
384
385
386
    {
        name            => 'dom-objects-enumeration-worker',
        type            => 'marionette',
        descr           => 'Check the list of DOM Objects exposed in a Worker context',
    },
387
    {
boklm's avatar
boklm committed
388
        name            => 'navigation-timing',
389
        type            => 'marionette',
boklm's avatar
boklm committed
390
391
        descr           => 'Check that the Navigation Timing API is really disabled',
        use_net         => 1,
392
    },
393
    {
boklm's avatar
boklm committed
394
        name            => 'resource-timing',
395
        type            => 'marionette',
396
        descr           => 'Check that the Resource Timing API is really disabled',
boklm's avatar
boklm committed
397
        use_net         => 1,
398
399
        # To check that the test fails when resource timing is enabled,
        # uncomment this:
400
401
        #prefs           => {
        #    'dom.enable_resource_timing' => 'true',
402
403
404
405
406
407
408
409
410
411
        #},
    },
    {
        name            => 'user-timing',
        type            => 'marionette',
        descr           => 'Check that the User Timing API is really disabled',
        use_net         => 1,
        # To check that the test fails when user timing is enabled,
        # uncomment this:
        #prefs           => {
412
        #    'dom.enable_user_timing' => 'true',
413
        #},
414
    },
415
416
417
418
419
420
421
422
423
424
425
426
427
428
    {
        name            => 'user-timing-worker',
        type            => 'marionette',
        marionette_test => 'page',
        remote          => 0,
        timeout         => 500,
        descr           => 'Check that the User Timing API in Worker context is really disabled',
        use_net         => 1,
        # To check that the test fails when user timing is enabled,
        # uncomment this:
        #prefs           => {
        #    'dom.enable_user_timing' => 'true',
        #},
    },
429
430
431
432
433
434
435
436
437
438
439
    {
        name            => 'performance-observer',
        type            => 'marionette',
        descr           => 'Check that the Performance Observer API is really disabled',
        use_net         => 1,
        # To check that the test fails when performance observer is enabled,
        # uncomment this:
        #prefs           => {
        #    'dom.enable_performance_observer' => 'true',
        #},
    },
440
    {
boklm's avatar
boklm committed
441
        name            => 'searchengines',
442
        type            => 'marionette',
boklm's avatar
boklm committed
443
        descr           => 'Check that we have the default search engines set',
444
    },
boklm's avatar
boklm committed
445
    {
boklm's avatar
boklm committed
446
        name            => 'noscript',
447
        type            => 'marionette',
boklm's avatar
boklm committed
448
449
450
        descr           => 'Check that noscript options are working',
        use_net         => 1,
        prefs           => {
451
            'extensions.torbutton.security_slider' => 2,
boklm's avatar
boklm committed
452
453
        },
    },
boklm's avatar
boklm committed
454
    {
boklm's avatar
boklm committed
455
        name            => 'fp_screen_dimensions',
456
        type            => 'marionette',
boklm's avatar
boklm committed
457
        descr           => 'Check that screen dimensions are spoofed correctly',
458
459
    },
    {
boklm's avatar
boklm committed
460
        name            => 'fp_screen_coords',
461
        type            => 'marionette',
boklm's avatar
boklm committed
462
        descr           => 'Check that screenX, screenY, screenLeft, screenTop, mozInnerScreenX, mozInnerScreenY are 0',
463
464
    },
    {
boklm's avatar
boklm committed
465
        name            => 'fp_plugins',
466
        type            => 'marionette',
boklm's avatar
boklm committed
467
        descr           => 'Check that plugins are disabled',
468
469
    },
    {
boklm's avatar
boklm committed
470
        name            => 'fp_useragent',
471
        type            => 'marionette',
boklm's avatar
boklm committed
472
        descr           => 'Check that userAgent is as expected',
473
474
    },
    {
boklm's avatar
boklm committed
475
        name            => 'fp_navigator',
476
        type            => 'marionette',
boklm's avatar
boklm committed
477
478
479
480
        descr           => 'Check that navigator properties are as expected',
    },
    {
        name            => 'play_videos',
481
        type            => 'marionette',
boklm's avatar
boklm committed
482
483
        descr           => 'Play some videos',
        use_net         => 1,
484
        marionette_test => 'page',
boklm's avatar
boklm committed
485
486
487
488
489
        remote          => 1,
        timeout         => 50000,
    },
    {
        name            => 'svg-disable',
boklm's avatar
boklm committed
490
        type            => 'marionette',
boklm's avatar
boklm committed
491
        descr           => 'Check if disabling svg is working',
boklm's avatar
boklm committed
492
        marionette_test => 'svg',
boklm's avatar
boklm committed
493
494
        use_net         => 1,
        prefs           => {
495
496
497
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'false',
        },
boklm's avatar
boklm committed
498
        enable          => sub { $OSNAME eq 'linux' },
499
500
    },
    {
boklm's avatar
boklm committed
501
        name            => 'svg-enable',
boklm's avatar
boklm committed
502
        type            => 'marionette',
boklm's avatar
boklm committed
503
        descr           => 'Check if enabling svg is working',
boklm's avatar
boklm committed
504
        marionette_test => 'svg',
boklm's avatar
boklm committed
505
506
        use_net         => 1,
        prefs           => {
507
508
509
            'extensions.torbutton.security_custom' => 'true',
            'svg.in-content.enabled' => 'true',
        },
boklm's avatar
boklm committed
510
        enable          => sub { $OSNAME eq 'linux' },
511
    },
512
513
514
515
516
517
    {
        name            => 'download_pdf',
        type            => 'marionette',
        descr           => 'Check if download of PDF is working (#19402)',
        use_net         => 1,
    },
boklm's avatar
boklm committed
518
519
520
521
522
523
524
    {
        name            => 'pinning_now',
        type            => 'marionette',
        descr           => 'Check if static public key pinning is working (#20149)',
        marionette_test => 'pinning',
        use_net         => 1,
    },
525
526
);

527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
sub set_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.backup";
    my $new_prefs = '';
    foreach my $prefname (sort keys %{$t->{prefs}}) {
        $new_prefs .= "pref(\"$prefname\", $t->{prefs}{$prefname});\n";
    }
    write_file($prefs, {append => 1}, $new_prefs);
}

sub reset_test_prefs {
    my ($tbbinfos, $t) = @_;
    return unless $t->{prefs};
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.backup", $prefs;
}

boklm's avatar
boklm committed
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
sub set_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    copy $prefs, "$prefs.slider_backup";
    write_file($prefs, {append => 1},
      'pref("extensions.torbutton.security_custom", false);' . "\n" .
      "pref(\"extensions.torbutton.security_slider\", $t->{slider_mode});\n");
}

sub reset_slider_mode {
    my ($tbbinfos, $t) = @_;
    my $prefs = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    move "$prefs.slider_backup", $prefs;
}

561
562
563
sub tbb_binfiles {
    my ($tbbinfos, $test) = @_;
    return $tbbinfos->{binfiles} if $tbbinfos->{binfiles};
564
    my %binfiles;
565
566
567
568
    my %wanted_types = (
        'application/x-executable-file' => 1,
        'application/x-ms-dos-executable' => 1,
    );
569
570
571
    my $wanted = sub {
        return unless -f $File::Find::name;
        my $type = File::Type->new->checktype_filename($File::Find::name);
572
        return unless $wanted_types{$type};
boklm's avatar
boklm committed
573
574
575
        my $name = $File::Find::name;
        $name =~ s/^$tbbinfos->{tbbdir}\///;
        $binfiles{$name} = 1;
576
577
578
579
580
    };
    find($wanted, $tbbinfos->{tbbdir});
    return $tbbinfos->{binfiles} = [ keys %binfiles ];
}

boklm's avatar
boklm committed
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
sub tbb_osx_executable_files {
    my ($tbbinfos, $test) = @_;
    return $tbbinfos->{osx_executable_files} if $tbbinfos->{osx_executable_files};
    my %exec_files;
    my $wanted = sub {
        return unless -f $File::Find::name;
        $ENV{LC_ALL}= 'C';
        my ($out, $err, $success) = capture_exec('otool', '-hv', $File::Find::name);
        return unless $success;
        my @out_lines = split("\n", $out);
        return if $out_lines[0] =~ m/is not an object file/;
        my $last_line = pop @out_lines;
        my ($type) = $last_line =~ m/^\s*[^\s]+\s+[^\s]+\s+[^\s]+\s+[^\s]+\s+([^\s]+)\s+[^\s]+\s+[^\s]+\s+/;
        my $name = $File::Find::name;
        $name =~ s/^$tbbinfos->{tbbdir}\///;
        $exec_files{$name} = 1 if $type eq 'EXECUTE';
    };
    find($wanted, $tbbinfos->{tbbdir});
    return $tbbinfos->{osx_executable_files} = [ keys %exec_files ];
}

602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
sub list_tests {
    foreach my $test (@tests) {
        print "$test->{name} ($test->{type})\n   $test->{descr}\n\n";
    }
}

sub get_tbbfile {
    my ($tbbinfos) = @_;
    $tbbinfos->{tbbfile_orig} = $tbbinfos->{tbbfile};
    if ($tbbinfos->{tbbfile} =~ m/^https?:\/\//) {
        my (undef, undef, $file) = File::Spec->splitpath($tbbinfos->{tbbfile});
        my $output = $options->{'download-dir'} ?
                "$options->{'download-dir'}/$file" : "$tbbinfos->{tmpdir}/$file";
        return $output if -f $output;
        print "Downloading $tbbinfos->{tbbfile}\n";
        my $ua = LWP::UserAgent->new;
        my $resp = $ua->get($tbbinfos->{tbbfile}, ':content_file' => $output);
        exit_error "Error downloading $tbbinfos->{tbbfile}:\n" . $resp->status_line
                unless $resp->is_success;
        $tbbinfos->{tbbfile} = $output;
    }
    exit_error "File $tbbinfos->{tbbfile} does not exist"
                unless -f $tbbinfos->{tbbfile};
}

sub extract_tbb {
    my ($tbbinfos) = @_;
    exit_error "Can't open file $tbbinfos->{tbbfile}" unless -f $tbbinfos->{tbbfile};
    my $tbbfile = File::Spec->rel2abs($tbbinfos->{tbbfile});
    my $tmpdir = $tbbinfos->{tmpdir};
    chdir $tmpdir;
    if ($tbbinfos->{os} eq 'Linux') {
        system('tar', 'xf', $tbbfile);
635
636
637
638
639
        if ($tbbinfos->{language} eq 'ALL') {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser";
        } else {
            $tbbinfos->{tbbdir} = "$tmpdir/tor-browser_$tbbinfos->{language}";
        }
640
        $tbbinfos->{tbbdir} .= '/Browser';
641
642
643
644
    } elsif ($tbbinfos->{os} eq 'Windows') {
        my (undef, undef, $f) = File::Spec->splitpath($tbbfile);
        copy($tbbfile, "$tmpdir/$f");
        system('7z', 'x', $f);
645
        $tbbinfos->{tbbdir} = "$tmpdir/torbrowser/Browser";
646
647
648
649
650
        move("$tmpdir/\$_OUTDIR", "$tmpdir/torbrowser") if -d "$tmpdir/\$_OUTDIR";
        if (-d "$tmpdir/Browser") {
            mkdir "$tmpdir/torbrowser";
            move("$tmpdir/Browser", "$tmpdir/torbrowser/Browser");
        }
651
        move ("$tmpdir/Start Tor Browser.exe", "$tmpdir/torbrowser/");
652
        system('chmod', '-R', '+rx', $tmpdir) if $OSNAME eq 'cygwin';
boklm's avatar
boklm committed
653
654
655
656
657
658
    } elsif ($tbbinfos->{os} eq 'MacOSX') {
        my $mountpoint = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
        system('hdiutil', 'mount', '-mountpoint', $mountpoint, $tbbfile);
        system('cp', '-a', "$mountpoint/TorBrowser.app", "$tmpdir/TorBrowser.app");
        system('hdiutil', 'unmount', $mountpoint);
        $tbbinfos->{tbbdir} = "$tmpdir/TorBrowser.app";
659
660
661
662
663
664
665
666
667
668
669
670
671
    }
}

sub xvfb_run {
    my ($test) = @_;
    return () unless $options->{xvfb};
    my $resolution = $test->{resolution} ? $test->{resolution}
                                         : $options->{resolution};
    return ('xvfb-run', '--auto-servernum', '-s', "-screen 0 ${resolution}x24");
}

sub check_opened_connections {
    my ($tbbinfos, $test) = @_;
672
673
674
675
676
677
    my %bad_connections =  %{$test->{results}{connections}};
    delete $bad_connections{"127.0.0.1:$options->{'tor-control-port'}"};
    delete $bad_connections{"127.0.0.1:$options->{'tor-socks-port'}"};
    # For some reasons, tor-browser creates two connections to the default
    # socks port even when when TOR_SOCKS_PORT is set
    # https://lists.torproject.org/pipermail/tbb-dev/2014-May/000050.html
boklm's avatar
boklm committed
678
679
680
681
    if (defined $bad_connections{'127.0.0.1:9150'}
        && $bad_connections{'127.0.0.1:9150'} <= 2) {
        delete $bad_connections{'127.0.0.1:9150'}
    }
682
683
684
685
    if (%bad_connections) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
686
    $test->{clean_strace} //= !%bad_connections;
687
    $test->{results}{bad_connections} = \%bad_connections;
688
689
690
691
}

sub check_modified_files {
    my ($tbbinfos, $test) = @_;
692
    my @bad_modified_files = @{$test->{results}{modified_files}};
693
694
695
696
    if (@bad_modified_files) {
        $test->{results}{success} = 0;
        $test->{retry} = 0;
    }
697
    $test->{clean_strace} //= !@bad_modified_files;
698
699
700
    $test->{results}{bad_modified_files} = \@bad_modified_files;
}

701
702
sub clean_strace {
    my ($tbbinfos, $test) = @_;
703
    return unless $test->{clean_strace};
704
705
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
    unlink $logfile;
boklm's avatar
boklm committed
706
    unlink "$logfile.tmp";
707
708
}

709
710
711
sub parse_strace {
    my ($tbbinfos, $test) = @_;
    my %ignore_files = map { $_ => 1 } qw(/dev/null /dev/tty);
712
713
    my @ignore_re = ( qr/^\/dev\/dri/ );
    push @ignore_re, qr/^$test->{workspace}/ if $test->{workspace};
714
715
716
717
718
    my %files;
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
    $test->{results}{connections} = {};
    my %modified_files;
    my %removed_files;
719
720
721
722
723
    if (-f "$logfile.tmp") {
        my $txt = read_file("$logfile.tmp");
        write_file($logfile, { append => 1 }, $txt);
        unlink "$logfile.tmp";
    }
724
    my @lines = read_file($logfile) if -f $logfile;
725
    LINE: foreach my $line (@lines) {
726
727
728
729
730
        if ($line =~ m/^\d+ open\("((?:[^"\\]++|\\.)*+)", ([^\)]+)/ ||
            $line =~ m/^\d+ openat\([^,]+, "((?:[^"\\]++|\\.)*+)", ([^\)]+)/) {
            next if $2 =~ m/O_RDONLY/;
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
            next if $ignore_files{$1};
731
732
733
            if ($ENV{'MOZMILL_SCREENSHOTS'}) {
                next if $1 =~ m/^$ENV{'MOZMILL_SCREENSHOTS'}/;
            }
734
735
736
            foreach my $re (@ignore_re) {
                next LINE if $1 =~ m/$re/;
            }
737
738
739
740
            $modified_files{$1}++;
        }
        if ($line =~ m/^\d+ unlink\("((?:[^"\\]++|\\.)*+)"/) {
            next if $1 =~ m/^$tbbinfos->{tbbdir}/;
741
742
743
744
            next if $ignore_files{$1};
            foreach my $re (@ignore_re) {
                next LINE if $1 =~ m/$re/;
            }
745
746
747
            $removed_files{$1}++;
            delete $modified_files{$1} unless -f $1;
        }
boklm's avatar
boklm committed
748
        if ($line =~ m/^\d+ connect\(\d+, \{sa_family=AF_INET, sin_port=htons\((\d+)\), sin_addr=inet_addr\("((?:[^"\\]++|\\.)*+)"\)/) {
749
            $test->{results}{connections}{"$2:$1"}++;
750
751
        }
    }
752
753
    $test->{results}{modified_files} = [ keys %modified_files ];
    $test->{results}{removed_files} = [ keys %removed_files ];
754
755
756
757
758
759
760
761
762
}

sub ff_wrapper {
    my ($tbbinfos, $test) = @_;
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_wrapper";
    return $wrapper_file if -f $wrapper_file;
    my $wrapper = <<EOF;
#!/bin/sh
set -e
763
export HOME="$tbbinfos->{tbbdir}"
764
export LD_LIBRARY_PATH="$tbbinfos->{tbbdir}:$tbbinfos->{tordir}"
765
766
export FONTCONFIG_PATH="\${HOME}/TorBrowser/Data/fontconfig"
export FONTCONFIG_FILE="fonts.conf"
767
768
769
770
771
772
773
exec \'$tbbinfos->{ffbin}\' "\$@"
EOF
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

774
sub ff_strace_wrapper {
775
776
    my ($tbbinfos, $test) = @_;
    my $ff_wrapper = ff_wrapper($tbbinfos, $test);
777
    my $logfile = "$tbbinfos->{'results-dir'}/$test->{name}.strace";
778
779
    my $wrapper = <<EOF;
#!/bin/sh
780
781
782
783
784
785
if [ -f $logfile.tmp ]
then
   cat $logfile.tmp >> $logfile
   rm $logfile.tmp
fi
echo \$@ >> /tmp/ff_run.log
786
787
788
789
790
strace -f -o $logfile.tmp -- \'$ff_wrapper\' "\$@"
exit_code=\$?
cat $logfile.tmp >> $logfile
rm $logfile.tmp
exit \$?
791
792
793
794
795
796
797
798
799
800
801
802
EOF
    my $wrapper_file = "$tbbinfos->{tbbdir}/ff_$test->{name}";
    write_file($wrapper_file, $wrapper);
    chmod 0700, $wrapper_file;
    return $wrapper_file;
}

sub ffbin_path {
    my ($tbbinfos, $test) = @_;
    if ($OSNAME eq 'cygwin') {
        return winpath("$tbbinfos->{ffbin}.exe");
    }
803
    my %t = map { $_ => 1 } qw(marionette);
804
    if ($options->{use_strace} && $t{$test->{type}}) {
boklm's avatar
boklm committed
805
806
        return ff_strace_wrapper($tbbinfos, $test);
    }
boklm's avatar
boklm committed
807
    return $tbbinfos->{ffbin} if $OSNAME eq 'darwin';
808
    return ff_wrapper($tbbinfos, $test);
809
810
}

811
812
813
814
815
816
817
818
819
820
821
822
sub marionette_export_options {
    my ($tbbinfos, $test) = @_;
    my $options_file = File::Temp->new();
    my $json = {
        options  => clone_strip_coderef($options),
        test     => clone_strip_coderef($test),
        tbbinfos => clone_strip_coderef({ %$tbbinfos, tests => undef }),
    };
    write_file($options_file, encode_json($json));
    return $options_file;
}

823
824
825
826
827
828
829
sub marionette_run {
    my ($tbbinfos, $test) = @_;
    if ($test->{tried} && $test->{use_net}) {
        TBBTestSuite::Tests::TorBootstrap::send_newnym($tbbinfos);
    }
    set_test_prefs($tbbinfos, $test);

830
831
    my $options_file = marionette_export_options($tbbinfos, $test);
    $ENV{TESTSUITE_DATA_FILE} = winpath($options_file);
832
833
    my $result_file_html = "$tbbinfos->{'results-dir'}/$test->{name}.html";
    my $result_file_txt = "$tbbinfos->{'results-dir'}/$test->{name}.txt";
834
835
    $test->{workspace} = "$tbbinfos->{'results-dir'}/$test->{name}_ws";
    mkdir $test->{workspace};
836
837
    #--log-unittest  ./res.txt --log-html ./res.html
    my $bin = $OSNAME eq 'cygwin' ? 'Scripts' : 'bin';
838
    my $marionette_test = $test->{marionette_test} // $test->{name};
839
    my $pypath = $ENV{PYTHONPATH};
boklm's avatar
boklm committed
840
841
842
843
    my $old_pypath = $ENV{PYTHONPATH};
    $ENV{PYTHONPATH} = winpath("$FindBin::Bin/marionette/tor_browser_tests/lib");
    my $sep = $OSNAME eq 'cygwin' ? ';' : ':';
    $ENV{PYTHONPATH} .= $sep . $old_pypath if $old_pypath;
844
845
846
    $test->{screenshots} = [];
    my $screenshots_tmp = File::Temp::newdir('XXXXXX', DIR => $options->{tmpdir});
    $ENV{'MARIONETTE_SCREENSHOTS'} = winpath($screenshots_tmp);
847
848
849
850
851
    system(xvfb_run($test), "$FindBin::Bin/virtualenv-marionette/$bin/tor-browser-tests",
        '--log-unittest', winpath($result_file_txt),
        '--log-html', winpath($result_file_html),
        '--binary', ffbin_path($tbbinfos, $test),
        '--profile', winpath($tbbinfos->{ffprofiledir}),
852
        $OSNAME eq 'cygwin' ? () : ('--workspace', $test->{workspace}),
853
        winpath("$FindBin::Bin/marionette/tor_browser_tests/test_${marionette_test}.py"));
854
    $ENV{PYTHONPATH} = $pypath;
855
    my @txt_log = -f $result_file_txt ? read_file($result_file_txt) : ('NoFile');
856
857
858
    my $res_line = shift @txt_log;
    $test->{results}{success} = $res_line eq ".\n" || $res_line eq ".\r\n";
    $test->{results}{log} = join '', @txt_log;
859
860
861
    my $i = 0;
    for my $screenshot_file (sort glob "$screenshots_tmp/*.png") {
        move($screenshot_file, "$tbbinfos->{'results-dir'}/$test->{name}-$i.png");
862
        screenshot_thumbnail($tbbinfos->{'results-dir'}, "$test->{name}-$i.png");
863
864
865
        push @{$test->{screenshots}}, "$test->{name}-$i.png";
        $i++;
    }
866
867
868
869
    reset_test_prefs($tbbinfos, $test);
    parse_strace($tbbinfos, $test);
    check_opened_connections($tbbinfos, $test);
    check_modified_files($tbbinfos, $test);
870
    clean_strace($tbbinfos, $test);
871
872
}

873
874
sub set_tbbpaths {
    my ($tbbinfos) = @_;
875
876
877
    $tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/firefox";
    $tbbinfos->{tordir} = "$tbbinfos->{tbbdir}/TorBrowser/Tor";
    $tbbinfos->{datadir} = "$tbbinfos->{tbbdir}/TorBrowser/Data";
boklm's avatar
boklm committed
878
879
    if ($tbbinfos->{os} eq 'MacOSX') {
        $tbbinfos->{ffbin} = "$tbbinfos->{tbbdir}/Contents/MacOS/firefox";
880
881
882
883
884
885
886
887
888
        unless ($tbbinfos->{version} =~ m/^5./) {
            $tbbinfos->{ffprofiledir} = "$tbbinfos->{tbbdir}/Contents/Resources/distribution";
            $tbbinfos->{tordir} = "$tbbinfos->{tbbdir}/Contents/Resources/TorBrowser/Tor";
            $tbbinfos->{datadir} = "$tbbinfos->{tbbdir}/../TorBrowser-data";
            $tbbinfos->{torrcdefaults} = "$tbbinfos->{tordir}/torrc-defaults";
            $tbbinfos->{torgeoip} = "$tbbinfos->{tordir}/geoip";
            mkdir $tbbinfos->{datadir} unless -d $tbbinfos->{datadir};
            mkdir "$tbbinfos->{datadir}/Tor" unless -d "$tbbinfos->{datadir}/Tor";
        }
boklm's avatar
boklm committed
889
    }
890
891
    $tbbinfos->{torrcdefaults} //= "$tbbinfos->{datadir}/Tor/torrc-defaults";
    $tbbinfos->{torgeoip} //= "$tbbinfos->{datadir}/Tor/geoip";
892
    $tbbinfos->{torbin} = "$tbbinfos->{tordir}/tor";
boklm's avatar
boklm committed
893
    $tbbinfos->{ptdir} = winpath("$tbbinfos->{tordir}/PluggableTransports");
894
    $tbbinfos->{ffprofiledir} //= "$tbbinfos->{datadir}/Browser/profile.default";
895
896
}

boklm's avatar
boklm committed
897
898
899
900
sub new {
    my ($ts, $testsuite) = @_;
    $testsuite->{type} = 'browserbundle';
    $testsuite->{tests} = [ map { { %$_ } } @tests ];
901
902
    return undef unless $testsuite->{os} eq $options->{os};
    return undef unless $testsuite->{arch} eq $options->{arch};
boklm's avatar
boklm committed
903
904
905
    return bless $testsuite, $ts;
}

906
907
908
909
910
911
912
913
914
915
sub pre_tests {
    my ($tbbinfos) = @_;
    get_tbbfile($tbbinfos);
    if ($tbbinfos->{sha256sum} &&
        $tbbinfos->{sha256sum} ne sha256_hex(read_file($tbbinfos->{tbbfile}))) {
        exit_error "Wrong sha256sum for $tbbinfos->{tbbfile}";
    }
    $tbbinfos->{sha256sum} //= sha256_hex(read_file($tbbinfos->{tbbfile}));
    extract_tbb($tbbinfos);
    set_tbbpaths($tbbinfos);
916
917
918
919
    my $prefs_file = "$tbbinfos->{ffprofiledir}/preferences/extension-overrides.js";
    open(my $prefs_fh, '>>', $prefs_file);
    print $prefs_fh 'pref("extensions.torbutton.prompted_language", true);', "\n";
    close $prefs_fh;
920
    chdir $tbbinfos->{tbbdir} || exit_error "Can't enter directory $tbbinfos->{tbbdir}";
boklm's avatar
boklm committed
921
    copy "$FindBin::Bin/data/cert_override.txt",
boklm's avatar
boklm committed
922
          "$tbbinfos->{ffprofiledir}/cert_override.txt";
923
    $ENV{TOR_SKIP_LAUNCH} = 1;
924
925
    $ENV{TOR_SOCKS_PORT} = $options->{'tor-socks-port'};
    $ENV{TOR_CONTROL_PORT} = $options->{'tor-control-port'};
boklm's avatar
boklm committed
926
927
928
    if ($options->{xdummy}) {
        $tbbinfos->{Xdisplay} = start_X("$tbbinfos->{'results-dir'}/xorg.log");
    }
929
930
931
932
933
}

sub post_tests {
    my ($tbbinfos) = @_;
    TBBTestSuite::Tests::TorBootstrap::stop_tor($tbbinfos);
boklm's avatar
boklm committed
934
    stop_X($tbbinfos->{Xdisplay}) if $options->{xdummy};
935
936
937
}

1;