Unverified Commit de3f97f3 authored by boklm's avatar boklm
Browse files

Bug 17381: add and adapt the update_responses script

Add the update_responses and incrementals makefile rules.

The update_responses script is identical to what we have in
tor-browser-bundle.git, however the config.yml file it is using is now
generated using the infos from rbm.
parent f3fed13f
......@@ -83,6 +83,26 @@ signtag-release: submodule-update
signtag-alpha: submodule-update
$(rbm) build release --step signtag --target alpha
incrementals-release: submodule-update
$(rbm) build release --step update_responses_config --target release
tools/update-responses/download_missing_versions release
tools/update-responses/gen_incrementals release
$(rbm) build release --step hash_incrementals --target release
incrementals-alpha: submodule-update
$(rbm) build release --step update_responses_config --target alpha
tools/update-responses/download_missing_versions alpha
tools/update-responses/gen_incrementals alpha
$(rbm) build release --step hash_incrementals --target alpha
update_responses-release: submodule-update
$(rbm) build release --step update_responses_config --target release --target signed
tools/update-responses/update_responses release
update_responses-alpha: submodule-update
$(rbm) build release --step update_responses_config --target alpha --target signed
tools/update-responses/update_responses alpha
submodule-update:
git submodule update --init
......
......@@ -77,3 +77,13 @@ signtag-{release,alpha}
Create a git signed tag for the selected channel, using the version and
build number defined as var/torbrowser_version and var/torbrowser_build.
incrementals-{release,alpha}
----------------------------
Create incremental mar files for an unsigned build in the release or
alpha channel.
update_responses-{release,alpha}
--------------------------------
Create update responses xml files for a signed build in the release or
alpha channel.
......@@ -7,7 +7,8 @@ git_url: https://git.torproject.org/tor-browser.git
gpg_keyring: torbutton.gpg
var:
firefox_version: 52.3.0esr
firefox_platform_version: 52.3.0
firefox_version: '[% c("var/firefox_platform_version") %]esr'
torbrowser_branch: 7.5
torbrowser_update_channel: alpha
copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]'
......
# vim: filetype=yaml sw=2
version: '[% c("var/torbrowser_version") %]'
output_dir: 'release/unsigned'
output_dir: release
var:
signed_status: unsigned
today: '[% USE date; date.format(format = "%Y-%m-%d") %]'
publish_dir: '[% c("version") %]-[% c("var/torbrowser_build") %]'
publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]'
targets:
torbrowser-all:
......@@ -50,7 +51,7 @@ targets:
publish_dir: '[% c("var/today") %]'
alpha:
output_dir: 'alpha/unsigned'
output_dir: alpha
var:
build_target: alpha
......@@ -60,6 +61,10 @@ targets:
build_target: torbrowser-testbuild
publish_dir: ''
signed:
var:
signed_status: signed
input_files:
# Release
......@@ -112,3 +117,13 @@ steps:
debug: 0
input_files: []
signtag: '[% INCLUDE signtag %]'
update_responses_config:
build_log: '-'
debug: 0
input_files: []
update_responses_config: '[% INCLUDE update_responses_config %]'
hash_incrementals:
build_log: '-'
debug: 0
input_files: []
hash_incrementals: '[% INCLUDE hash_incrementals %]'
#!/bin/bash
[% c("var/set_default_env") -%]
cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]
sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt
#!/bin/bash
[% c("var/set_default_env") -%]
cat > [% shell_quote(c("basedir")) %]/tools/update-responses/config.yml << 'EOF'
[% INCLUDE update_responses_config.yml -%]
EOF
# Update / create symlink $torbrowser_version -> $torbrowser_version-buildN
versiondir=[% shell_quote(path(dest_dir)) _ '/' _ c("var/signed_status") _ '/' _ c("version") %]
test -L "$versiondir" && rm -f "$versiondir"
test -d "$versiondir" || ln -s [% shell_quote(c("version")) %]-[% shell_quote(c("var/torbrowser_build")) %] "$versiondir"
---
appname_marfile: tor-browser
appname_bundle_osx: TorBrowser
appname_bundle_linux: tor-browser
appname_bundle_win: torbrowser-install
releases_dir: [% path(c('output_dir')) %]/[% c("var/signed_status") %]
download:
archive_url: https://archive.torproject.org/tor-package-archive/torbrowser
gpg_keyring: ../../keyring/torbrowser.gpg
bundles_url: https://dist.torproject.org/torbrowser
mars_url: https://cdn.torproject.org/aus1/torbrowser
build_targets:
linux32: Linux_x86-gcc3
linux64: Linux_x86_64-gcc3
win32:
- WINNT_x86-gcc3
- WINNT_x86-gcc3-x86
- WINNT_x86-gcc3-x64
osx32: Darwin_x86-gcc3
osx64: Darwin_x86_64-gcc3
channels:
[% pc('firefox', 'var/torbrowser_update_channel') %]: [% c("var/torbrowser_version") %]
versions:
[% c("var/torbrowser_version") %]:
platformVersion: [% pc('firefox', 'var/firefox_platform_version') %]
detailsURL: https://blog.torproject.org/blog/tor-browser-[% c("var/torbrowser_version") FILTER remove('\.') %]-released
incremental_from:
[% FOREACH v IN c("var/torbrowser_incremental_from") -%]
- [% v %]
[% END -%]
migrate_archs:
osx32: osx64
migrate_langs:
pt-PT: pt-BR
win32:
minSupportedInstructionSet: SSE2
osx32:
minSupportedOSVersion: 13.0.0
osx64:
minSupportedOSVersion: 13.0.0
......@@ -16,6 +16,9 @@ buildconf:
var:
torbrowser_version: '7.5a4'
torbrowser_build: 'build2'
torbrowser_incremental_from:
- 7.5a2
- 7.5a3
project_name: tor-browser
multi_lingual: 0
build_mar: 1
......
Tor Browser Update Responses script
===================================
This repository contains a script to generate responses for Tor Browser
updater.
See ticket [#12622](https://trac.torproject.org/projects/tor/ticket/12622)
for details.
Dependencies
------------
The following perl modules need to be installed to run the script:
FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp
IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON
On Debian / Ubuntu you can install them with:
```
# apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \
libdigest-sha-perl libxml-writer-perl \
libio-captureoutput-perl libparallel-forkmanager-perl \
libxml-libxml-perl libwww-perl libjson-perl
```
On Red Hat / Fedora you can install them with:
```
# for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \
File::Temp IO::CaptureOutput Parallel::ForkManager \
XML::LibXML LWP JSON
do yum install "perl($module)"; done
```
URL Format
----------
The URL format is:
https://something/$channel/$build_target/$tb_version/$lang?force=1
'build_target' is the OS for which the browser was built. The correspo
ndance between the build target and the OS name that we use in archive
files is defined in the config.yml file.
'tb_version' is the Tor Browser version.
'lang' is the locale.
update_responses
\ No newline at end of file
update_responses
\ No newline at end of file
update_responses
\ No newline at end of file
update_responses
\ No newline at end of file
#!/usr/bin/perl -w
use strict;
use feature "state";
use English;
use FindBin;
use YAML qw(LoadFile);
use File::Slurp;
use Digest::SHA qw(sha256_hex);
use XML::Writer;
use Cwd;
use File::Copy;
use File::Temp;
use File::Find;
use POSIX qw(setlocale LC_ALL);
use IO::CaptureOutput qw(capture_exec);
use Parallel::ForkManager;
use File::Basename;
use XML::LibXML '1.70';
use LWP::Simple;
use JSON;
# Set umask and locale to provide a consistent environment for MAR file
# generation, etc.
umask(0022);
$ENV{"LC_ALL"} = "C";
setlocale(LC_ALL, "C");
my $htdocsdir = "$FindBin::Bin/htdocs";
my $config = LoadFile("$FindBin::Bin/config.yml");
my %htdocsfiles;
my $releases_dir = $config->{releases_dir};
$releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//;
my @check_errors;
my $initPATH = $ENV{PATH};
my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH};
sub exit_error {
print STDERR "Error: ", $_[0], "\n";
chdir '/';
exit (exists $_[1] ? $_[1] : 1);
}
sub build_targets_by_os {
return ($_[0]) unless $config->{build_targets}{$_[0]};
my $r = $config->{build_targets}{$_[0]};
return ref $r eq 'ARRAY' ? @$r : ($r);
}
sub get_nbprocs {
return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
if (-f '/proc/cpuinfo') {
return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
}
return 4;
}
sub write_htdocs {
my ($channel, $file, $content) = @_;
mkdir $htdocsdir unless -d $htdocsdir;
mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel";
write_file("$htdocsdir/$channel/$file", $content);
$htdocsfiles{$channel}->{$file} = 1;
}
sub clean_htdocs {
my (@channels) = @_;
foreach my $channel (@channels) {
opendir(my $d, "$htdocsdir/$channel");
my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d;
closedir $d;
unlink map { "$htdocsdir/$channel/$_" } @files;
}
}
sub get_sha512_hex_of_file {
my ($file) = @_;
my $sha = Digest::SHA->new("512");
$sha->addfile($file);
return $sha->hexdigest;
}
sub get_version_files {
my ($config, $version) = @_;
return if $config->{versions}{$version}{files};
my $appname = $config->{appname_marfile};
my $files = {};
my $vdir = "$releases_dir/$version";
my $download_url = "$config->{download}{mars_url}/$version";
opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
foreach my $file (readdir $d) {
next unless -f "$vdir/$file";
if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) {
my ($os, $lang) = ($1, $2);
$files->{$os}{$lang}{complete} = {
type => 'complete',
URL => "$download_url/$file",
size => -s "$vdir/$file",
hashFunction => 'SHA512',
hashValue => get_sha512_hex_of_file("$vdir/$file"),
};
next;
}
if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) {
my ($os, $from_version, $lang) = ($1, $2, $3);
$files->{$os}{$lang}{partial}{$from_version} = {
type => 'partial',
URL => "$download_url/$file",
size => -s "$vdir/$file",
hashFunction => 'SHA512',
hashValue => get_sha512_hex_of_file("$vdir/$file"),
}
}
}
closedir $d;
$config->{versions}{$version}{files} = $files;
}
sub get_version_downloads {
my ($config, $version) = @_;
my $downloads = {};
my $vdir = "$releases_dir/$version";
my $download_url = "$config->{download}{bundles_url}/$version";
opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
foreach my $file (readdir $d) {
next unless -f "$vdir/$file";
my ($os, $lang);
if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) {
($os, $lang) = ('osx64', $1);
} elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) {
($os, $lang) = ($1, $2);
} elsif ($file =~ m/^$config->{appname_bundle_win}-${version}_(.+).exe$/) {
($os, $lang) = ('win32', $1);
} else {
next;
}
$downloads->{$os}{$lang} = {
binary => "$download_url/$file",
sig => "$download_url/$file.asc",
};
}
closedir $d;
$config->{versions}{$version}{downloads} = $downloads;
}
sub extract_mar {
my ($mar_file, $dest_dir) = @_;
my $old_cwd = getcwd;
mkdir $dest_dir;
chdir $dest_dir or exit_error "Cannot enter $dest_dir";
my $res = system('mar', '-x', $mar_file);
exit_error "Error extracting $mar_file" if $res;
my $bunzip_file = sub {
return unless -f $File::Find::name;
rename $File::Find::name, "$File::Find::name.bz2";
system('bunzip2', "$File::Find::name.bz2") == 0
|| exit_error "Error decompressing $File::Find::name";
};
find($bunzip_file, $dest_dir);
my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest'
: 'updatev2.manifest';
my @lines = read_file($manifest) if -f $manifest;
foreach my $line (@lines) {
if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) {
exit_error "$mar_file: Could not create symlink $1 -> $2"
unless symlink $2, $1;
}
}
chdir $old_cwd;
}
sub mar_filename {
my ($appname, $version, $os, $lang) = @_;
"$releases_dir/$version/$appname-$os-${version}_$lang.mar";
}
sub create_incremental_mar {
my ($config, $pm, $from_version, $new_version, $os, $lang) = @_;
my $appname = $config->{appname_marfile};
my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar";
my $mar_file_path = "$releases_dir/$new_version/$mar_file";
if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) {
print "Skipping $mar_file\n";
return;
}
print "Starting $mar_file\n";
my $download_url = "$config->{download}{mars_url}/$new_version";
my $finished_file = sub {
exit_error "Error creating $mar_file" unless $_[1] == 0;
print "Finished $mar_file\n";
$config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
type => 'partial',
URL => "$download_url/$mar_file",
size => -s $mar_file_path,
hashFunction => 'SHA512',
hashValue => get_sha512_hex_of_file($mar_file_path),
};
};
return if $pm->start($finished_file);
my $tmpdir = File::Temp->newdir();
extract_mar(mar_filename($appname, $from_version, $os, $lang), "$tmpdir/A");
extract_mar(mar_filename($appname, $new_version, $os, $lang), "$tmpdir/B");
if ($ENV{CHECK_CODESIGNATURE_EXISTS}) {
unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources"
&& -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") {
exit_error "Missing code signature while creating $mar_file";
}
}
my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
$mar_file_path, "$tmpdir/A", "$tmpdir/B");
if (!$success) {
unlink $mar_file_path if -f $mar_file_path;
exit_error "making incremental mar:\n" . $err;
}
$pm->finish;
}
sub create_incremental_mars_for_version {
my ($config, $version) = @_;
my $pm = Parallel::ForkManager->new(get_nbprocs);
$pm->run_on_finish(sub { $_[2]->(@_) });
my $v = $config->{versions}{$version};
foreach my $from_version (@{$v->{incremental_from}}) {
$config->{versions}{$from_version} //= {};
get_version_files($config, $from_version);
my $from_v = $config->{versions}{$from_version};
foreach my $os (keys %{$v->{files}}) {
foreach my $lang (keys %{$v->{files}{$os}}) {
next unless defined $from_v->{files}{$os}{$lang}{complete};
create_incremental_mar($config, $pm, $from_version, $version, $os, $lang);
}
}
}
$pm->wait_all_children;
}
sub get_config {
my ($config, $version, $os, $name) = @_;
return $config->{versions}{$version}{$os}{$name}
// $config->{versions}{$version}{$name}
// $config->{$name};
}
sub channel_to_version {
my ($config, @channels) = @_;
return values %{$config->{channels}} unless @channels;
foreach my $channel (@channels) {
exit_error "Unknown channel $channel"
unless $config->{channels}{$channel};
}
return map { $config->{channels}{$_} } @channels;
}
sub get_buildinfos {
my ($config, $version) = @_;
return if exists $config->{versions}{$version}{buildID};
extract_martools($version);
my $files = $config->{versions}{$version}{files};
foreach my $os (keys %$files) {
foreach my $lang (keys %{$files->{$os}}) {
next unless $files->{$os}{$lang}{complete};
my $tmpdir = File::Temp->newdir();
extract_mar(
mar_filename($config->{appname_marfile}, $version, $os, $lang),
"$tmpdir");
my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini";
$appfile = "$tmpdir/Contents/Resources/application.ini"
if -f "$tmpdir/Contents/Resources/application.ini";
exit_error "Could not find application.ini" unless $appfile;
foreach my $line (read_file($appfile)) {
if ($line =~ m/^BuildID=(.*)$/) {
$config->{versions}{$version}{buildID} = $1;
return;
}
}
exit_error "Could not extract buildID from application.ini";
}
}
}
sub get_response {
my ($config, $version, $os, @patches) = @_;
my $res;
my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8');
$writer->xmlDecl;
$writer->startTag('updates');
if (get_config($config, $version, $os, 'unsupported')) {
$writer->startTag('update',
unsupported => 'true',
detailsURL => get_config($config, $version, $os, 'detailsURL'),
);
goto CLOSETAGS;
}
my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion');
my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet');
$writer->startTag('update',
type => 'minor',
displayVersion => $version,
appVersion => $version,
platformVersion => get_config($config, $version, $os, 'platformVersion'),
buildID => get_config($config, $version, $os, 'buildID'),
detailsURL => get_config($config, $version, $os, 'detailsURL'),
actions => 'showURL',
openURL => get_config($config, $version, $os, 'detailsURL'),
defined $minversion ? ( minSupportedOSVersion => $minversion ) : (),
defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (),
);
foreach my $patch (@patches) {
my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch;
$writer->startTag('patch', @sorted_patch);
$writer->endTag('patch');
}
CLOSETAGS:
$writer->endTag('update');
$writer->endTag('updates');
$writer->end;
return $res;
}
sub write_responses {
my ($config, @channels) = @_;
@channels = keys %{$config->{channels}} unless @channels;
foreach my $channel (@channels) {
my $version = $config->{channels}{$channel};
get_version_files($config, $version);
get_buildinfos($config, $version);
my $files = $config->{versions}{$version}{files};
my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {};
foreach my $old_os (keys %$migrate_archs) {
my $new_os = $migrate_archs->{$old_os};
foreach my $lang (keys %{$files->{$new_os}}) {
$files->{$old_os}{$lang}{complete} =
$files->{$new_os}{$lang}{complete};
}
}
foreach my $os (keys %$files) {
foreach my $lang (keys %{$files->{$os}}) {
my $resp = get_response($config, $version, $os,
$files->{$os}{$lang}{complete});
write_htdocs($channel, "$version-$os-$lang.xml", $resp);
foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) {
$resp = get_response($config, $version, $os,
$files->{$os}{$lang}{complete},
$files->{$os}{$lang}{partial}{$from_version});
write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp);
}
}
}
write_htdocs($channel, 'no-update.xml',
'<?xml version="1.0" encoding="UTF-8"?>'
. "\n<updates></updates>\n");
}
}
sub write_htaccess {
my ($config, @channels) = @_;
@channels = keys %{$config->{channels}} unless @channels;
my $flags = "[last]";
foreach my $channel (@channels) {
my $htaccess = "RewriteEngine On\n";
my $version = $config->{channels}{$channel};
my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {};
my $files = $config->{versions}{$version}{files};
$htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n";
foreach my $os (sort keys %$files) {