Merge remote-tracking branch 'boklm-gitlab/bug_25102_v13' into master

tools/ansible/Makefile

@@ -6,3 +6,6 @@ fpcentral:
 
 boklm-tbb-nightly-build:
 	ansible-playbook --vault-password-file=~/ansible-vault/boklm-tbb-nightly -i inventory boklm-tbb-nightly-build.yml
+
+tbb-nightly-signing:
+	ANSIBLE_CONFIG='$(@D)/ansible.cfg' ansible-playbook -i inventory tbb-nightly-signing.yml

tools/ansible/README

@@ -25,6 +25,13 @@ boklm-tbb-nightly-build:
   For more details, see also this ticket:
   https://trac.torproject.org/projects/tor/ticket/33948
 
+tbb-nightly-signing:
+  This target is used to deploy a nightly signing machine. The
+  configuration of nightly signing is done in the following files:
+   tools/ansible/roles/tbb-nightly-signing/defaults/main.yml
+   tools/signing/nightly/config.yml
+   tools/signing/nightly/update-responses-base-config.yml
+
 
 Adding, removing, updating users on the Tor Browser team build machine
 ======================================================================

tools/ansible/ansible.cfg

+[ssh_connection]
+; When connecting to a v3 onion, we get the error:
+; "unix_listener: [...] too long for Unix domain socket"
+; We solve this by using %n (The original remote hostname, as given on
+; the command line) instead of %h (The remote hostname) in the control path.
+control_path=%(directory)s/%%r-%%n-%%r

tools/ansible/inventory

 build-sunet-a ansible_ssh_user=root ansible_ssh_host=build-sunet-a.torproject.net
 fpcentral ansible_become=True ansible_become_method=sudo ansible_become_user=fpcentral ansible_ssh_host=forrestii.torproject.org allow_world_readable_tmpfiles=True
 boklm-tbb-nightly-build ansible_ssh_user=root ansible_become_method=su
+tbb-nightly-signing ansible_ssh_user=root ansible_become_method=su
 
 [tbb-build]
 build-sunet-a

tools/ansible/roles/tbb-nightly-signing/defaults/main.yml

+---
+nightly_signing_user: nightly-signing
+nightly_signing_cron_hour: '*'
+nightly_signing_cron_minute: '0,30'
+tor_browser_build_dir: "/home/{{ nightly_signing_user }}/tor-browser-build"
+tor_browser_build_git_url: https://git.torproject.org/builders/tor-browser-build.git
+tor_browser_build_commit: 8d66414b7860751ffec6a83a6bc6dbfbd94f801a

tools/ansible/roles/tbb-nightly-signing/tasks/main.yml

+---
+- name: Install dependencies
+  apt:
+      name: "{{ item }}"
+      state: present
+  with_items:
+      - git
+      - libdatetime-perl
+      - libfindbin-libs-perl
+      - libfile-slurp-perl
+      - libxml-writer-perl
+      - libio-captureoutput-perl
+      - libparallel-forkmanager-perl
+      - libxml-libxml-perl
+      - libwww-perl
+      - libjson-perl
+      - libyaml-libyaml-perl
+      - libyaml-perl
+      - libtemplate-perl
+      - libio-handle-util-perl
+      - libio-all-perl
+      - libio-captureoutput-perl
+      - libpath-tiny-perl
+      - libstring-shellquote-perl
+      - libsort-versions-perl
+      - libdigest-sha-perl
+      - libdata-uuid-perl
+      - libdata-dump-perl
+      - libfile-copy-recursive-perl
+      - libnss3-tools
+      - rsync
+
+- name: create nightly-signing user
+  user:
+      name: "{{ nightly_signing_user }}"
+      comment: "Tor Browser Nightly Signing"
+      createhome: yes
+      home: "/home/{{ nightly_signing_user }}"
+
+- name: clone tor-browser-build
+  become: yes
+  become_user: "{{ nightly_signing_user }}"
+  git:
+      repo: "{{ tor_browser_build_git_url }}"
+      dest: "{{ tor_browser_build_dir }}"
+      version: "{{ tor_browser_build_commit }}"
+
+- name: add cron to sign nighly build
+  cron:
+      name: tbb-sign-nightly-build
+      user: "{{ nightly_signing_user }}"
+      hour: "{{ nightly_signing_cron_hour }}"
+      minute: "{{ nightly_signing_cron_minute }}"
+      job: "torsocks /home/{{ nightly_signing_user }}/tor-browser-build/tools/signing/nightly/sign-nightly"

tools/ansible/tbb-nightly-signing.yml

+---
+
+- hosts: tbb-nightly-signing
+  roles:
+      - role: tbb-nightly-signing
+      - role: unattended-upgrades

tools/signing/README

@@ -5,3 +5,6 @@ omitting specific paths and credential information.
 Additionally, when starting to used them for an own signing setup don't forget
 to adapt the locale list if needed. The entitlement files, however, are kept
 up-to-date.
+
+The scripts in the nightly/ directory are used to sign and publish the
+nightly updates.

tools/signing/nightly/.gitignore

+lock
 nssdb
+martools-*

tools/signing/nightly/config.yml

+---
+martools_version: 9.0.2
+martools_url: https://archive.torproject.org/tor-package-archive/torbrowser/
+martools_gpg_keyring: keyring/torbrowser.gpg
+builds_url: http://f4amtbsowhix7rrf.onion/tor-browser-builds
+builds_url_auth_basic_username: tor-guest
+builds_url_auth_basic_password: tor-guest
+publish_dirs:
+    - nightly-linux-x86_64
+    - nightly-linux-i686
+    - nightly-windows-x86_64
+    - nightly-windows-i686
+    - nightly-osx-x86_64
+nss_db_dir: nssdb
+nss_certname: nightly-marsigner
+gpg_keyring: keyring/torbrowser-nightly.gpg
+rsync_dest: nightly-publish:nightly-updates/
+post_rsync_cmd: ssh nightly-publish-static-update true

tools/signing/nightly/sign-nightly

+#!/usr/bin/perl -w
+use strict;
+use FindBin;
+use YAML::XS qw(LoadFile DumpFile);
+use POSIX qw(setlocale LC_ALL);
+use File::Path qw(make_path);
+use File::Temp;
+use File::Copy;
+use File::Basename;
+use LWP::Simple;
+use Path::Tiny;
+use DateTime;
+use Digest::SHA qw(sha256_hex);
+use JSON;
+
+# Set umask and locale to provide a consistent environment
+umask(0022);
+$ENV{"LC_ALL"} = "C";
+setlocale(LC_ALL, "C");
+
+sub exit_error {
+    print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
+    unlink "$FindBin::Bin/lock";
+    exit (exists $_[1] ? $_[1] : 1);
+}
+
+exit_error "Missing config file: $FindBin::Bin/config.yml"
+           unless -f "$FindBin::Bin/config.yml";
+my $config = LoadFile("$FindBin::Bin/config.yml");
+my $topdir = "$FindBin::Bin/../../..";
+
+{
+    no warnings 'redefine';
+    sub LWP::UserAgent::get_basic_credentials {
+        if ($config->{builds_url_auth_basic_username}
+            && $config->{builds_url_auth_basic_password}) {
+            return ( $config->{builds_url_auth_basic_username},
+                     $config->{builds_url_auth_basic_password} );
+        }
+        return ();
+    }
+}
+
+sub run_alone {
+    my $pidfile = "$FindBin::Bin/lock";
+    if (-f $pidfile) {
+        my $pid = path($pidfile)->slurp_utf8;
+        if (kill(0, $pid)) {
+            print STDERR "Script is already running ($pid). Exiting.\n";
+            exit 0;
+        }
+    }
+    path($pidfile)->spew_utf8($$);
+}
+
+END {
+    unlink "$FindBin::Bin/lock";
+}
+
+sub get_tmpdir {
+    my ($config) = @_;
+    return File::Temp->newdir($config->{tmp_dir} ?
+                                (DIR => $config->{tmp_dir})
+                                : ());
+}
+
+sub basedir_path {
+    my ($path, $basedir) = @_;
+    return ( $path =~ m|^/| ) ? $path : "$basedir/$path";
+}
+
+sub get_last_build_version {
+    my ($config, $publish_dir) = @_;
+    my $today = 'tbb-nightly.' . DateTime->now->ymd('.');
+    my $dt_yesterday = DateTime->now - DateTime::Duration->new(days => 1);
+    my $yesterday = 'tbb-nightly.' . $dt_yesterday->ymd('.');
+    for my $version ($today, $yesterday) {
+        my $url = "$config->{builds_url}/$version/$publish_dir/sha256sums-unsigned-build.incrementals.txt";
+        return $version if get($url);
+    }
+    return undef;
+}
+
+sub get_current_version {
+    my ($publish_dir) = @_;
+    my $file = "$topdir/nightly/$publish_dir-current-version.txt";
+    return undef unless -f $file;
+    return path($file)->slurp_utf8;
+}
+
+sub set_current_version {
+    my ($publish_dir, $version) = @_;
+    my $file = "$topdir/nightly/$publish_dir-current-version.txt";
+    path($file)->spew_utf8($version);
+}
+
+sub get_new_version {
+    my ($config, $publish_dir) = @_;
+    my $today = 'tbb-nightly.' . DateTime->now->ymd('.');
+    my $current_ver = get_current_version($publish_dir);
+    my $last_ver = get_last_build_version($config, $publish_dir);
+    return $last_ver unless defined($current_ver);
+    return undef if $current_ver eq $today;
+    return undef unless defined($last_ver);
+    return undef if $current_ver eq $last_ver;
+    return $last_ver;
+}
+
+sub download_file {
+    my ($url, $file, $sha256sum) = @_;
+    my $retries = 5;
+    while ($retries > 0) {
+        $retries--;
+        print "Downloading $url\n";
+        next unless getstore("$url", "$file.tmp") == 200;
+        next unless $sha256sum eq sha256_hex(path("$file.tmp")->slurp_raw);
+        move("$file.tmp", $file);
+        return 1;
+    }
+    exit_error "Error downloading $url";
+}
+
+sub fetch_version {
+    my ($config, $publish_dir, $version) = @_;
+    my $tmpdir = get_tmpdir($config);
+    my $urldir = "$config->{builds_url}/$version/$publish_dir";
+    my $destdir = "$topdir/nightly/$publish_dir/$version";
+
+    return if -d $destdir;
+    my $gpg_keyring = basedir_path($config->{gpg_keyring}, $topdir);
+    for my $file (qw/sha256sums-unsigned-build.txt sha256sums-unsigned-build.incrementals.txt/) {
+        my $url = "$urldir/$file";
+        exit_error "Error downloading $url"
+                unless getstore($url, "$tmpdir/$file") == 200;
+        exit_error "Error downloading $url.asc"
+                unless getstore("$url.asc", "$tmpdir/$file.asc") == 200;
+        exit_error "Error checking gpg signature for $url"
+                if system('gpg', '--no-default-keyring', '--keyring', $gpg_keyring,
+                          '--verify', "$tmpdir/$file.asc",
+                          "$tmpdir/$file");
+    }
+    my %sums = map { chomp; reverse split '  ', $_ }
+        (
+            path("$tmpdir/sha256sums-unsigned-build.txt")->lines_utf8,
+            path("$tmpdir/sha256sums-unsigned-build.incrementals.txt")->lines_utf8,
+        );
+    my @build_infos_file = grep { $_ =~ m/build-infos-.*\.json/ } keys %sums;
+    exit_error "Missing build-infos.json in $urldir" unless @build_infos_file;
+    download_file("$urldir/$build_infos_file[0]",
+                  "$tmpdir/build-infos.json", $sums{$build_infos_file[0]});
+    foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) {
+        download_file("$urldir/$file", "$tmpdir/$file", $sums{$file});
+    }
+    make_path("$topdir/nightly/$publish_dir");
+    move $tmpdir, $destdir;
+    chmod 0755, $destdir;
+}
+
+sub setup_martools {
+    my ($config) = @_;
+    my $martools_dir = "$FindBin::Bin/mar-tools-$config->{martools_version}";
+    if (! -d $martools_dir) {
+        my $file = "mar-tools-linux64.zip";
+        my $url = "$config->{martools_url}/$config->{martools_version}/$file";
+        my $tmpdir = get_tmpdir($config);
+        exit_error "Error downloading $url"
+                unless getstore($url, "$tmpdir/$file") == 200;
+        exit_error "Error downloading $url.asc"
+                unless getstore("$url.asc", "$tmpdir/$file.asc") == 200;
+        my $gpg_keyring = basedir_path($config->{martools_gpg_keyring}, $topdir);
+        exit_error "Error checking gpg signature for $url"
+                if system('gpg', '--no-default-keyring', '--keyring', $gpg_keyring,
+                          '--verify', "$tmpdir/$file.asc",
+                          "$tmpdir/$file");
+        exit_error "Error extracting martools"
+                unless system('unzip', '-d', $martools_dir, '-x',
+                                       "$tmpdir/$file") == 0;
+    }
+    $ENV{PATH} = "$martools_dir/mar-tools:$ENV{PATH}";
+    if ($ENV{LD_LIBRARY_PATH}) {
+        $ENV{LD_LIBRARY_PATH} = "$martools_dir/mar-tools:$ENV{LD_LIBRARY_PATH}";
+    } else {
+        $ENV{LD_LIBRARY_PATH} = "$martools_dir/mar-tools";
+    }
+}
+
+sub sign_version {
+    my ($config, $publish_dir, $version) = @_;
+    setup_martools($config);
+    my $nss_db_dir = basedir_path($config->{nss_db_dir}, $FindBin::Bin);
+    for my $marfile (path("$topdir/nightly/$publish_dir/$version")->children(qr/\.mar$/)) {
+        print "Signing $marfile\n";
+        exit_error "Error signing $marfile"
+          unless system('signmar', '-d', $nss_db_dir, '-n',
+                        $config->{nss_certname}, '-s', $marfile,
+                        "$marfile-signed") == 0;
+          move("$marfile-signed", $marfile);
+    }
+}
+
+sub get_buildinfos {
+    my ($filename) = @_;
+    exit_error "$filename does not exist" unless -f $filename;
+    return decode_json(path($filename)->slurp_utf8);
+}
+
+sub update_responses {
+    my ($config, $publish_dir, $version) = @_;
+    my $ur_config = LoadFile("$FindBin::Bin/update-responses-base-config.yml");
+    $ur_config->{download}{mars_url} .= "/$publish_dir";
+    $ur_config->{releases_dir} = "$topdir/nightly/$publish_dir";
+    $ur_config->{channels}->{nightly} = $version;
+    $ur_config->{versions}->{$version} = $ur_config->{versions}->{nightly_version};
+    my $buildinfos = get_buildinfos("$topdir/nightly/$publish_dir/$version/build-infos.json");
+    $ur_config->{versions}->{$version}{platformVersion} = $buildinfos->{firefox_platform_version};
+    $ur_config->{versions}->{$version}{buildID} = $buildinfos->{firefox_buildid};
+    DumpFile("$topdir/tools/update-responses/config.yml", $ur_config);
+    my $htdocsdir = "$topdir/tools/update-responses/htdocs/nightly";
+    path($htdocsdir)->remove_tree({ safe => 0 });
+    exit_error "Error running update_responses for $publish_dir" unless
+       system("$topdir/tools/update-responses/update_responses") == 0;
+    path("$topdir/nightly/updates/$publish_dir/nightly")->remove_tree({ safe => 0 });
+    make_path("$topdir/nightly/updates/$publish_dir");
+    move($htdocsdir, "$topdir/nightly/updates/$publish_dir/nightly");
+}
+
+sub remove_oldversions {
+    my ($config, $publish_dir, $version) = @_;
+    for my $dir (path("$topdir/nightly/$publish_dir")->children) {
+        my ($filename) = fileparse($dir);
+        next if $filename eq $version;
+        path($dir)->remove_tree({ safe => 0 });
+    }
+}
+
+sub sync_dest {
+    my ($config) = @_;
+    exit_error "Error running rsync"
+        if system('rsync', '-aH', '--delete-after',
+                  "$topdir/nightly/", "$config->{rsync_dest}/");
+    if ($config->{post_rsync_cmd}) {
+        exit_error "Error running $config->{post_rsync_cmd}"
+                if system($config->{post_rsync_cmd});
+    }
+}
+
+run_alone;
+my $some_updates = 0;
+foreach my $publish_dir (@{$config->{publish_dirs}}) {
+    my $new_version = get_new_version($config, $publish_dir);
+    next unless $new_version;
+    fetch_version($config, $publish_dir, $new_version);
+    sign_version($config, $publish_dir, $new_version);
+    update_responses($config, $publish_dir, $new_version);
+    set_current_version($publish_dir, $new_version);
+    remove_oldversions($config, $publish_dir, $new_version);
+    $some_updates = 1;
+}
+sync_dest($config) if $some_updates;

tools/signing/nightly/update-responses-base-config.yml

+---
+download:
+    mars_url: https://nightlies.tbb.torproject.org/nightly-updates
+appname_marfile: tor-browser
+build_targets:
+    linux32: Linux_x86-gcc3
+    linux64: Linux_x86_64-gcc3
+    win32:
+        - WINNT_x86-gcc3
+        - WINNT_x86-gcc3-x86
+        - WINNT_x86-gcc3-x64
+    win64: WINNT_x86_64-gcc3-x64
+    osx32: Darwin_x86-gcc3
+    osx64: Darwin_x86_64-gcc3
+channels:
+    nightly: nightly_version
+versions:
+    nightly_version:
+        platformVersion: 68.2.0
+        detailsURL: http://f4amtbsowhix7rrf.onion/
+        migrate_langs:
+            pt-PT: pt-BR
+        minSupportedInstructionSet: SSE2
+        osx32:
+            minSupportedOSVersion: 13.0.0
+        osx64:
+            minSupportedOSVersion: 13.0.0
+        win32:
+            minSupportedOSVersion: 6.1
+        win64:
+            minSupportedOSVersion: 6.1
+mar_compression: xz