Skip to content
Snippets Groups Projects
Unverified Commit 49ec15d1 authored by Georg Koppen's avatar Georg Koppen
Browse files

Merge remote-tracking branch 'boklm-gitlab/bug_25102_v13' into master

parents 653f38ee 17804f5a
No related branches found
No related tags found
No related merge requests found
File added
......@@ -6,3 +6,6 @@ fpcentral:
boklm-tbb-nightly-build:
ansible-playbook --vault-password-file=~/ansible-vault/boklm-tbb-nightly -i inventory boklm-tbb-nightly-build.yml
tbb-nightly-signing:
ANSIBLE_CONFIG='$(@D)/ansible.cfg' ansible-playbook -i inventory tbb-nightly-signing.yml
......@@ -25,6 +25,13 @@ boklm-tbb-nightly-build:
For more details, see also this ticket:
https://trac.torproject.org/projects/tor/ticket/33948
tbb-nightly-signing:
This target is used to deploy a nightly signing machine. The
configuration of nightly signing is done in the following files:
tools/ansible/roles/tbb-nightly-signing/defaults/main.yml
tools/signing/nightly/config.yml
tools/signing/nightly/update-responses-base-config.yml
Adding, removing, updating users on the Tor Browser team build machine
======================================================================
......
[ssh_connection]
; When connecting to a v3 onion, we get the error:
; "unix_listener: [...] too long for Unix domain socket"
; We solve this by using %n (The original remote hostname, as given on
; the command line) instead of %h (The remote hostname) in the control path.
control_path=%(directory)s/%%r-%%n-%%r
build-sunet-a ansible_ssh_user=root ansible_ssh_host=build-sunet-a.torproject.net
fpcentral ansible_become=True ansible_become_method=sudo ansible_become_user=fpcentral ansible_ssh_host=forrestii.torproject.org allow_world_readable_tmpfiles=True
boklm-tbb-nightly-build ansible_ssh_user=root ansible_become_method=su
tbb-nightly-signing ansible_ssh_user=root ansible_become_method=su
[tbb-build]
build-sunet-a
......
---
nightly_signing_user: nightly-signing
nightly_signing_cron_hour: '*'
nightly_signing_cron_minute: '0,30'
tor_browser_build_dir: "/home/{{ nightly_signing_user }}/tor-browser-build"
tor_browser_build_git_url: https://git.torproject.org/builders/tor-browser-build.git
tor_browser_build_commit: 8d66414b7860751ffec6a83a6bc6dbfbd94f801a
---
- name: Install dependencies
apt:
name: "{{ item }}"
state: present
with_items:
- git
- libdatetime-perl
- libfindbin-libs-perl
- libfile-slurp-perl
- libxml-writer-perl
- libio-captureoutput-perl
- libparallel-forkmanager-perl
- libxml-libxml-perl
- libwww-perl
- libjson-perl
- libyaml-libyaml-perl
- libyaml-perl
- libtemplate-perl
- libio-handle-util-perl
- libio-all-perl
- libio-captureoutput-perl
- libpath-tiny-perl
- libstring-shellquote-perl
- libsort-versions-perl
- libdigest-sha-perl
- libdata-uuid-perl
- libdata-dump-perl
- libfile-copy-recursive-perl
- libnss3-tools
- rsync
- name: create nightly-signing user
user:
name: "{{ nightly_signing_user }}"
comment: "Tor Browser Nightly Signing"
createhome: yes
home: "/home/{{ nightly_signing_user }}"
- name: clone tor-browser-build
become: yes
become_user: "{{ nightly_signing_user }}"
git:
repo: "{{ tor_browser_build_git_url }}"
dest: "{{ tor_browser_build_dir }}"
version: "{{ tor_browser_build_commit }}"
- name: add cron to sign nighly build
cron:
name: tbb-sign-nightly-build
user: "{{ nightly_signing_user }}"
hour: "{{ nightly_signing_cron_hour }}"
minute: "{{ nightly_signing_cron_minute }}"
job: "torsocks /home/{{ nightly_signing_user }}/tor-browser-build/tools/signing/nightly/sign-nightly"
---
- hosts: tbb-nightly-signing
roles:
- role: tbb-nightly-signing
- role: unattended-upgrades
......@@ -5,3 +5,6 @@ omitting specific paths and credential information.
Additionally, when starting to used them for an own signing setup don't forget
to adapt the locale list if needed. The entitlement files, however, are kept
up-to-date.
The scripts in the nightly/ directory are used to sign and publish the
nightly updates.
lock
nssdb
martools-*
---
martools_version: 9.0.2
martools_url: https://archive.torproject.org/tor-package-archive/torbrowser/
martools_gpg_keyring: keyring/torbrowser.gpg
builds_url: http://f4amtbsowhix7rrf.onion/tor-browser-builds
builds_url_auth_basic_username: tor-guest
builds_url_auth_basic_password: tor-guest
publish_dirs:
- nightly-linux-x86_64
- nightly-linux-i686
- nightly-windows-x86_64
- nightly-windows-i686
- nightly-osx-x86_64
nss_db_dir: nssdb
nss_certname: nightly-marsigner
gpg_keyring: keyring/torbrowser-nightly.gpg
rsync_dest: nightly-publish:nightly-updates/
post_rsync_cmd: ssh nightly-publish-static-update true
#!/usr/bin/perl -w
use strict;
use FindBin;
use YAML::XS qw(LoadFile DumpFile);
use POSIX qw(setlocale LC_ALL);
use File::Path qw(make_path);
use File::Temp;
use File::Copy;
use File::Basename;
use LWP::Simple;
use Path::Tiny;
use DateTime;
use Digest::SHA qw(sha256_hex);
use JSON;
# Set umask and locale to provide a consistent environment
umask(0022);
$ENV{"LC_ALL"} = "C";
setlocale(LC_ALL, "C");
sub exit_error {
print STDERR "Error: ", $_[0], "\n";
chdir '/';
unlink "$FindBin::Bin/lock";
exit (exists $_[1] ? $_[1] : 1);
}
exit_error "Missing config file: $FindBin::Bin/config.yml"
unless -f "$FindBin::Bin/config.yml";
my $config = LoadFile("$FindBin::Bin/config.yml");
my $topdir = "$FindBin::Bin/../../..";
{
no warnings 'redefine';
sub LWP::UserAgent::get_basic_credentials {
if ($config->{builds_url_auth_basic_username}
&& $config->{builds_url_auth_basic_password}) {
return ( $config->{builds_url_auth_basic_username},
$config->{builds_url_auth_basic_password} );
}
return ();
}
}
sub run_alone {
my $pidfile = "$FindBin::Bin/lock";
if (-f $pidfile) {
my $pid = path($pidfile)->slurp_utf8;
if (kill(0, $pid)) {
print STDERR "Script is already running ($pid). Exiting.\n";
exit 0;
}
}
path($pidfile)->spew_utf8($$);
}
END {
unlink "$FindBin::Bin/lock";
}
sub get_tmpdir {
my ($config) = @_;
return File::Temp->newdir($config->{tmp_dir} ?
(DIR => $config->{tmp_dir})
: ());
}
sub basedir_path {
my ($path, $basedir) = @_;
return ( $path =~ m|^/| ) ? $path : "$basedir/$path";
}
sub get_last_build_version {
my ($config, $publish_dir) = @_;
my $today = 'tbb-nightly.' . DateTime->now->ymd('.');
my $dt_yesterday = DateTime->now - DateTime::Duration->new(days => 1);
my $yesterday = 'tbb-nightly.' . $dt_yesterday->ymd('.');
for my $version ($today, $yesterday) {
my $url = "$config->{builds_url}/$version/$publish_dir/sha256sums-unsigned-build.incrementals.txt";
return $version if get($url);
}
return undef;
}
sub get_current_version {
my ($publish_dir) = @_;
my $file = "$topdir/nightly/$publish_dir-current-version.txt";
return undef unless -f $file;
return path($file)->slurp_utf8;
}
sub set_current_version {
my ($publish_dir, $version) = @_;
my $file = "$topdir/nightly/$publish_dir-current-version.txt";
path($file)->spew_utf8($version);
}
sub get_new_version {
my ($config, $publish_dir) = @_;
my $today = 'tbb-nightly.' . DateTime->now->ymd('.');
my $current_ver = get_current_version($publish_dir);
my $last_ver = get_last_build_version($config, $publish_dir);
return $last_ver unless defined($current_ver);
return undef if $current_ver eq $today;
return undef unless defined($last_ver);
return undef if $current_ver eq $last_ver;
return $last_ver;
}
sub download_file {
my ($url, $file, $sha256sum) = @_;
my $retries = 5;
while ($retries > 0) {
$retries--;
print "Downloading $url\n";
next unless getstore("$url", "$file.tmp") == 200;
next unless $sha256sum eq sha256_hex(path("$file.tmp")->slurp_raw);
move("$file.tmp", $file);
return 1;
}
exit_error "Error downloading $url";
}
sub fetch_version {
my ($config, $publish_dir, $version) = @_;
my $tmpdir = get_tmpdir($config);
my $urldir = "$config->{builds_url}/$version/$publish_dir";
my $destdir = "$topdir/nightly/$publish_dir/$version";
return if -d $destdir;
my $gpg_keyring = basedir_path($config->{gpg_keyring}, $topdir);
for my $file (qw/sha256sums-unsigned-build.txt sha256sums-unsigned-build.incrementals.txt/) {
my $url = "$urldir/$file";
exit_error "Error downloading $url"
unless getstore($url, "$tmpdir/$file") == 200;
exit_error "Error downloading $url.asc"
unless getstore("$url.asc", "$tmpdir/$file.asc") == 200;
exit_error "Error checking gpg signature for $url"
if system('gpg', '--no-default-keyring', '--keyring', $gpg_keyring,
'--verify', "$tmpdir/$file.asc",
"$tmpdir/$file");
}
my %sums = map { chomp; reverse split ' ', $_ }
(
path("$tmpdir/sha256sums-unsigned-build.txt")->lines_utf8,
path("$tmpdir/sha256sums-unsigned-build.incrementals.txt")->lines_utf8,
);
my @build_infos_file = grep { $_ =~ m/build-infos-.*\.json/ } keys %sums;
exit_error "Missing build-infos.json in $urldir" unless @build_infos_file;
download_file("$urldir/$build_infos_file[0]",
"$tmpdir/build-infos.json", $sums{$build_infos_file[0]});
foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) {
download_file("$urldir/$file", "$tmpdir/$file", $sums{$file});
}
make_path("$topdir/nightly/$publish_dir");
move $tmpdir, $destdir;
chmod 0755, $destdir;
}
sub setup_martools {
my ($config) = @_;
my $martools_dir = "$FindBin::Bin/mar-tools-$config->{martools_version}";
if (! -d $martools_dir) {
my $file = "mar-tools-linux64.zip";
my $url = "$config->{martools_url}/$config->{martools_version}/$file";
my $tmpdir = get_tmpdir($config);
exit_error "Error downloading $url"
unless getstore($url, "$tmpdir/$file") == 200;
exit_error "Error downloading $url.asc"
unless getstore("$url.asc", "$tmpdir/$file.asc") == 200;
my $gpg_keyring = basedir_path($config->{martools_gpg_keyring}, $topdir);
exit_error "Error checking gpg signature for $url"
if system('gpg', '--no-default-keyring', '--keyring', $gpg_keyring,
'--verify', "$tmpdir/$file.asc",
"$tmpdir/$file");
exit_error "Error extracting martools"
unless system('unzip', '-d', $martools_dir, '-x',
"$tmpdir/$file") == 0;
}
$ENV{PATH} = "$martools_dir/mar-tools:$ENV{PATH}";
if ($ENV{LD_LIBRARY_PATH}) {
$ENV{LD_LIBRARY_PATH} = "$martools_dir/mar-tools:$ENV{LD_LIBRARY_PATH}";
} else {
$ENV{LD_LIBRARY_PATH} = "$martools_dir/mar-tools";
}
}
sub sign_version {
my ($config, $publish_dir, $version) = @_;
setup_martools($config);
my $nss_db_dir = basedir_path($config->{nss_db_dir}, $FindBin::Bin);
for my $marfile (path("$topdir/nightly/$publish_dir/$version")->children(qr/\.mar$/)) {
print "Signing $marfile\n";
exit_error "Error signing $marfile"
unless system('signmar', '-d', $nss_db_dir, '-n',
$config->{nss_certname}, '-s', $marfile,
"$marfile-signed") == 0;
move("$marfile-signed", $marfile);
}
}
sub get_buildinfos {
my ($filename) = @_;
exit_error "$filename does not exist" unless -f $filename;
return decode_json(path($filename)->slurp_utf8);
}
sub update_responses {
my ($config, $publish_dir, $version) = @_;
my $ur_config = LoadFile("$FindBin::Bin/update-responses-base-config.yml");
$ur_config->{download}{mars_url} .= "/$publish_dir";
$ur_config->{releases_dir} = "$topdir/nightly/$publish_dir";
$ur_config->{channels}->{nightly} = $version;
$ur_config->{versions}->{$version} = $ur_config->{versions}->{nightly_version};
my $buildinfos = get_buildinfos("$topdir/nightly/$publish_dir/$version/build-infos.json");
$ur_config->{versions}->{$version}{platformVersion} = $buildinfos->{firefox_platform_version};
$ur_config->{versions}->{$version}{buildID} = $buildinfos->{firefox_buildid};
DumpFile("$topdir/tools/update-responses/config.yml", $ur_config);
my $htdocsdir = "$topdir/tools/update-responses/htdocs/nightly";
path($htdocsdir)->remove_tree({ safe => 0 });
exit_error "Error running update_responses for $publish_dir" unless
system("$topdir/tools/update-responses/update_responses") == 0;
path("$topdir/nightly/updates/$publish_dir/nightly")->remove_tree({ safe => 0 });
make_path("$topdir/nightly/updates/$publish_dir");
move($htdocsdir, "$topdir/nightly/updates/$publish_dir/nightly");
}
sub remove_oldversions {
my ($config, $publish_dir, $version) = @_;
for my $dir (path("$topdir/nightly/$publish_dir")->children) {
my ($filename) = fileparse($dir);
next if $filename eq $version;
path($dir)->remove_tree({ safe => 0 });
}
}
sub sync_dest {
my ($config) = @_;
exit_error "Error running rsync"
if system('rsync', '-aH', '--delete-after',
"$topdir/nightly/", "$config->{rsync_dest}/");
if ($config->{post_rsync_cmd}) {
exit_error "Error running $config->{post_rsync_cmd}"
if system($config->{post_rsync_cmd});
}
}
run_alone;
my $some_updates = 0;
foreach my $publish_dir (@{$config->{publish_dirs}}) {
my $new_version = get_new_version($config, $publish_dir);
next unless $new_version;
fetch_version($config, $publish_dir, $new_version);
sign_version($config, $publish_dir, $new_version);
update_responses($config, $publish_dir, $new_version);
set_current_version($publish_dir, $new_version);
remove_oldversions($config, $publish_dir, $new_version);
$some_updates = 1;
}
sync_dest($config) if $some_updates;
---
download:
mars_url: https://nightlies.tbb.torproject.org/nightly-updates
appname_marfile: tor-browser
build_targets:
linux32: Linux_x86-gcc3
linux64: Linux_x86_64-gcc3
win32:
- WINNT_x86-gcc3
- WINNT_x86-gcc3-x86
- WINNT_x86-gcc3-x64
win64: WINNT_x86_64-gcc3-x64
osx32: Darwin_x86-gcc3
osx64: Darwin_x86_64-gcc3
channels:
nightly: nightly_version
versions:
nightly_version:
platformVersion: 68.2.0
detailsURL: http://f4amtbsowhix7rrf.onion/
migrate_langs:
pt-PT: pt-BR
minSupportedInstructionSet: SSE2
osx32:
minSupportedOSVersion: 13.0.0
osx64:
minSupportedOSVersion: 13.0.0
win32:
minSupportedOSVersion: 6.1
win64:
minSupportedOSVersion: 6.1
mar_compression: xz
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment