Bug 17381: add and adapt the update_responses script

Add the update_responses and incrementals makefile rules.

The update_responses script is identical to what we have in
tor-browser-bundle.git, however the config.yml file it is using is now
generated using the infos from rbm.

Makefile

@@ -83,6 +83,26 @@ signtag-release: submodule-update
 signtag-alpha: submodule-update
 	$(rbm) build release --step signtag --target alpha
 
+incrementals-release: submodule-update
+	$(rbm) build release --step update_responses_config --target release
+	tools/update-responses/download_missing_versions release
+	tools/update-responses/gen_incrementals release
+	$(rbm) build release --step hash_incrementals --target release
+
+incrementals-alpha: submodule-update
+	$(rbm) build release --step update_responses_config --target alpha
+	tools/update-responses/download_missing_versions alpha
+	tools/update-responses/gen_incrementals alpha
+	$(rbm) build release --step hash_incrementals --target alpha
+
+update_responses-release: submodule-update
+	$(rbm) build release --step update_responses_config --target release --target signed
+	tools/update-responses/update_responses release
+
+update_responses-alpha: submodule-update
+	$(rbm) build release --step update_responses_config --target alpha --target signed
+	tools/update-responses/update_responses alpha
+
 submodule-update:
 	git submodule update --init
 

README.MAKEFILE

@@ -77,3 +77,13 @@ signtag-{release,alpha}
 Create a git signed tag for the selected channel, using the version and
 build number defined as var/torbrowser_version and var/torbrowser_build.
 
+incrementals-{release,alpha}
+----------------------------
+Create incremental mar files for an unsigned build in the release or
+alpha channel.
+
+update_responses-{release,alpha}
+--------------------------------
+Create update responses xml files for a signed build in the release or
+alpha channel.
+

projects/firefox/config

@@ -7,7 +7,8 @@ git_url: https://git.torproject.org/tor-browser.git
 gpg_keyring: torbutton.gpg
 
 var:
-  firefox_version: 52.3.0esr
+  firefox_platform_version: 52.3.0
+  firefox_version: '[% c("var/firefox_platform_version") %]esr'
   torbrowser_branch: 7.5
   torbrowser_update_channel: alpha
   copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]'

projects/release/config

 # vim: filetype=yaml sw=2
 version: '[% c("var/torbrowser_version") %]'
-output_dir: 'release/unsigned'
+output_dir: release
 
 var:
+  signed_status: unsigned
   today: '[% USE date; date.format(format = "%Y-%m-%d") %]'
-  publish_dir: '[% c("version") %]-[% c("var/torbrowser_build") %]'
+  publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]'
 
 targets:
   torbrowser-all:
@@ -50,7 +51,7 @@ targets:
       publish_dir: '[% c("var/today") %]'
 
   alpha:
-    output_dir: 'alpha/unsigned'
+    output_dir: alpha
     var:
       build_target: alpha
 
@@ -60,6 +61,10 @@ targets:
       build_target: torbrowser-testbuild
       publish_dir: ''
 
+  signed:
+    var:
+      signed_status: signed
+
 input_files:
 
 # Release
@@ -112,3 +117,13 @@ steps:
     debug: 0
     input_files: []
     signtag: '[% INCLUDE signtag %]'
+  update_responses_config:
+    build_log: '-'
+    debug: 0
+    input_files: []
+    update_responses_config: '[% INCLUDE update_responses_config %]'
+  hash_incrementals:
+    build_log: '-'
+    debug: 0
+    input_files: []
+    hash_incrementals: '[% INCLUDE hash_incrementals %]'

projects/release/hash_incrementals

+#!/bin/bash
+[% c("var/set_default_env") -%]
+cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[%  c("version") %]-[% c("var/torbrowser_build") %]
+sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt

projects/release/update_responses_config

+#!/bin/bash
+[% c("var/set_default_env") -%]
+cat > [% shell_quote(c("basedir")) %]/tools/update-responses/config.yml << 'EOF'
+[% INCLUDE update_responses_config.yml -%]
+EOF
+
+# Update / create symlink $torbrowser_version -> $torbrowser_version-buildN
+versiondir=[% shell_quote(path(dest_dir)) _ '/' _ c("var/signed_status") _ '/' _ c("version") %]
+test -L "$versiondir" && rm -f "$versiondir"
+test -d "$versiondir" || ln -s [% shell_quote(c("version")) %]-[% shell_quote(c("var/torbrowser_build")) %] "$versiondir"
+

projects/release/update_responses_config.yml

+---
+appname_marfile: tor-browser
+appname_bundle_osx: TorBrowser
+appname_bundle_linux: tor-browser
+appname_bundle_win: torbrowser-install
+releases_dir: [% path(c('output_dir')) %]/[% c("var/signed_status") %]
+download:
+    archive_url: https://archive.torproject.org/tor-package-archive/torbrowser
+    gpg_keyring: ../../keyring/torbrowser.gpg
+    bundles_url: https://dist.torproject.org/torbrowser
+    mars_url: https://cdn.torproject.org/aus1/torbrowser
+build_targets:
+    linux32: Linux_x86-gcc3
+    linux64: Linux_x86_64-gcc3
+    win32:
+        - WINNT_x86-gcc3
+        - WINNT_x86-gcc3-x86
+        - WINNT_x86-gcc3-x64
+    osx32: Darwin_x86-gcc3
+    osx64: Darwin_x86_64-gcc3
+channels:
+    [% pc('firefox', 'var/torbrowser_update_channel') %]: [% c("var/torbrowser_version") %]
+versions:
+    [% c("var/torbrowser_version") %]:
+        platformVersion: [% pc('firefox', 'var/firefox_platform_version') %]
+        detailsURL: https://blog.torproject.org/blog/tor-browser-[% c("var/torbrowser_version") FILTER remove('\.') %]-released
+        incremental_from:
+[% FOREACH v IN c("var/torbrowser_incremental_from") -%]
+          - [% v %]
+[% END -%]
+        migrate_archs:
+          osx32: osx64
+        migrate_langs:
+            pt-PT: pt-BR
+        win32:
+            minSupportedInstructionSet: SSE2
+        osx32:
+            minSupportedOSVersion: 13.0.0
+        osx64:
+            minSupportedOSVersion: 13.0.0

rbm.conf

@@ -16,6 +16,9 @@ buildconf:
 var:
   torbrowser_version: '7.5a4'
   torbrowser_build: 'build2'
+  torbrowser_incremental_from:
+    - 7.5a2
+    - 7.5a3
   project_name: tor-browser
   multi_lingual: 0
   build_mar: 1

tools/update-responses/.gitignore

+/htdocs
+/config.yml

tools/update-responses/README.md

+Tor Browser Update Responses script
+===================================
+
+This repository contains a script to generate responses for Tor Browser
+updater.
+
+See ticket [#12622](https://trac.torproject.org/projects/tor/ticket/12622)
+for details.
+
+
+Dependencies
+------------
+
+The following perl modules need to be installed to run the script:
+  FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp
+  IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON
+
+On Debian / Ubuntu you can install them with:
+
+```
+  # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \
+                    libdigest-sha-perl libxml-writer-perl \
+                    libio-captureoutput-perl libparallel-forkmanager-perl \
+                    libxml-libxml-perl libwww-perl libjson-perl
+```
+
+On Red Hat / Fedora you can install them with:
+
+```
+  # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \
+                  File::Temp IO::CaptureOutput Parallel::ForkManager \
+                  XML::LibXML LWP JSON
+    do yum install "perl($module)"; done
+```
+
+
+URL Format
+----------
+
+The URL format is:
+  https://something/$channel/$build_target/$tb_version/$lang?force=1
+
+'build_target' is the OS for which the browser was built. The correspo
+ndance between the build target and the OS name that we use in archive
+files is defined in the config.yml file.
+
+'tb_version' is the Tor Browser version.
+
+'lang' is the locale.
+

tools/update-responses/check_update_responses_deployement

+update_responses
\ No newline at end of file

tools/update-responses/download_missing_versions

+update_responses
\ No newline at end of file

tools/update-responses/gen_incrementals

+update_responses
\ No newline at end of file

tools/update-responses/get_channel_version

+update_responses
\ No newline at end of file

tools/update-responses/update_responses

+#!/usr/bin/perl -w
+
+use strict;
+use feature "state";
+use English;
+use FindBin;
+use YAML qw(LoadFile);
+use File::Slurp;
+use Digest::SHA qw(sha256_hex);
+use XML::Writer;
+use Cwd;
+use File::Copy;
+use File::Temp;
+use File::Find;
+use POSIX qw(setlocale LC_ALL);
+use IO::CaptureOutput qw(capture_exec);
+use Parallel::ForkManager;
+use File::Basename;
+use XML::LibXML '1.70';
+use LWP::Simple;
+use JSON;
+
+# Set umask and locale to provide a consistent environment for MAR file
+# generation, etc.
+umask(0022);
+$ENV{"LC_ALL"} = "C";
+setlocale(LC_ALL, "C");
+
+my $htdocsdir = "$FindBin::Bin/htdocs";
+my $config = LoadFile("$FindBin::Bin/config.yml");
+my %htdocsfiles;
+my $releases_dir = $config->{releases_dir};
+$releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//;
+my @check_errors;
+my $initPATH = $ENV{PATH};
+my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH};
+
+sub exit_error {
+    print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
+    exit (exists $_[1] ? $_[1] : 1);
+}
+
+sub build_targets_by_os {
+    return ($_[0]) unless $config->{build_targets}{$_[0]};
+    my $r = $config->{build_targets}{$_[0]};
+    return ref $r eq 'ARRAY' ? @$r : ($r);
+}
+
+sub get_nbprocs {
+    return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
+    if (-f '/proc/cpuinfo') {
+        return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
+    }
+    return 4;
+}
+
+sub write_htdocs {
+    my ($channel, $file, $content) = @_;
+    mkdir $htdocsdir unless -d $htdocsdir;
+    mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel";
+    write_file("$htdocsdir/$channel/$file", $content);
+    $htdocsfiles{$channel}->{$file} = 1;
+}
+
+sub clean_htdocs {
+    my (@channels) = @_;
+    foreach my $channel (@channels) {
+        opendir(my $d, "$htdocsdir/$channel");
+        my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d;
+        closedir $d;
+        unlink map { "$htdocsdir/$channel/$_" } @files;
+    }
+}
+
+sub get_sha512_hex_of_file {
+    my ($file) = @_;
+    my $sha = Digest::SHA->new("512");
+    $sha->addfile($file);
+    return $sha->hexdigest;
+}
+
+sub get_version_files {
+    my ($config, $version) = @_;
+    return if $config->{versions}{$version}{files};
+    my $appname = $config->{appname_marfile};
+    my $files = {};
+    my $vdir = "$releases_dir/$version";
+    my $download_url = "$config->{download}{mars_url}/$version";
+    opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
+    foreach my $file (readdir $d) {
+        next unless -f "$vdir/$file";
+        if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) {
+            my ($os, $lang) = ($1, $2);
+            $files->{$os}{$lang}{complete} = {
+                type => 'complete',
+                URL => "$download_url/$file",
+                size => -s "$vdir/$file",
+                hashFunction => 'SHA512',
+                hashValue => get_sha512_hex_of_file("$vdir/$file"),
+            };
+            next;
+        }
+        if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) {
+            my ($os, $from_version, $lang) = ($1, $2, $3);
+            $files->{$os}{$lang}{partial}{$from_version} = {
+                type => 'partial',
+                URL => "$download_url/$file",
+                size => -s "$vdir/$file",
+                hashFunction => 'SHA512',
+                hashValue => get_sha512_hex_of_file("$vdir/$file"),
+            }
+        }
+    }
+    closedir $d;
+    $config->{versions}{$version}{files} = $files;
+}
+
+sub get_version_downloads {
+    my ($config, $version) = @_;
+    my $downloads = {};
+    my $vdir = "$releases_dir/$version";
+    my $download_url = "$config->{download}{bundles_url}/$version";
+    opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
+    foreach my $file (readdir $d) {
+        next unless -f "$vdir/$file";
+        my ($os, $lang);
+        if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) {
+            ($os, $lang) = ('osx64', $1);
+        } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) {
+            ($os, $lang) = ($1, $2);
+        } elsif ($file =~ m/^$config->{appname_bundle_win}-${version}_(.+).exe$/) {
+            ($os, $lang) = ('win32', $1);
+        } else {
+            next;
+        }
+        $downloads->{$os}{$lang} = {
+            binary => "$download_url/$file",
+            sig => "$download_url/$file.asc",
+        };
+    }
+    closedir $d;
+    $config->{versions}{$version}{downloads} = $downloads;
+}
+
+sub extract_mar {
+    my ($mar_file, $dest_dir) = @_;
+    my $old_cwd = getcwd;
+    mkdir $dest_dir;
+    chdir $dest_dir or exit_error "Cannot enter $dest_dir";
+    my $res = system('mar', '-x', $mar_file);
+    exit_error "Error extracting $mar_file" if $res;
+    my $bunzip_file = sub {
+        return unless -f $File::Find::name;
+        rename $File::Find::name, "$File::Find::name.bz2";
+        system('bunzip2', "$File::Find::name.bz2") == 0
+                || exit_error "Error decompressing $File::Find::name";
+    };
+    find($bunzip_file, $dest_dir);
+    my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest'
+                        : 'updatev2.manifest';
+    my @lines = read_file($manifest) if -f $manifest;
+    foreach my $line (@lines) {
+        if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) {
+            exit_error "$mar_file: Could not create symlink $1 -> $2"
+                unless symlink $2, $1;
+        }
+    }
+    chdir $old_cwd;
+}
+
+sub mar_filename {
+    my ($appname, $version, $os, $lang) = @_;
+    "$releases_dir/$version/$appname-$os-${version}_$lang.mar";
+}
+
+sub create_incremental_mar {
+    my ($config, $pm, $from_version, $new_version, $os, $lang) = @_;
+    my $appname = $config->{appname_marfile};
+    my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar";
+    my $mar_file_path = "$releases_dir/$new_version/$mar_file";
+    if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) {
+        print "Skipping $mar_file\n";
+        return;
+    }
+    print "Starting $mar_file\n";
+    my $download_url = "$config->{download}{mars_url}/$new_version";
+    my $finished_file = sub {
+        exit_error "Error creating $mar_file" unless $_[1] == 0;
+        print "Finished $mar_file\n";
+        $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
+            type => 'partial',
+            URL => "$download_url/$mar_file",
+            size => -s $mar_file_path,
+            hashFunction => 'SHA512',
+            hashValue => get_sha512_hex_of_file($mar_file_path),
+        };
+    };
+    return if $pm->start($finished_file);
+    my $tmpdir = File::Temp->newdir();
+    extract_mar(mar_filename($appname, $from_version, $os, $lang), "$tmpdir/A");
+    extract_mar(mar_filename($appname, $new_version, $os, $lang), "$tmpdir/B");
+    if ($ENV{CHECK_CODESIGNATURE_EXISTS}) {
+        unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources"
+            && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") {
+            exit_error "Missing code signature while creating $mar_file";
+        }
+    }
+    my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
+                                   $mar_file_path, "$tmpdir/A", "$tmpdir/B");
+    if (!$success) {
+        unlink $mar_file_path if -f $mar_file_path;
+        exit_error "making incremental mar:\n" . $err;
+    }
+    $pm->finish;
+}
+
+sub create_incremental_mars_for_version {
+    my ($config, $version) = @_;
+    my $pm = Parallel::ForkManager->new(get_nbprocs);
+    $pm->run_on_finish(sub { $_[2]->(@_) });
+    my $v = $config->{versions}{$version};
+    foreach my $from_version (@{$v->{incremental_from}}) {
+        $config->{versions}{$from_version} //= {};
+        get_version_files($config, $from_version);
+        my $from_v = $config->{versions}{$from_version};
+        foreach my $os (keys %{$v->{files}}) {
+            foreach my $lang (keys %{$v->{files}{$os}}) {
+                next unless defined $from_v->{files}{$os}{$lang}{complete};
+                create_incremental_mar($config, $pm, $from_version, $version, $os, $lang);
+            }
+        }
+    }
+    $pm->wait_all_children;
+}
+
+sub get_config {
+    my ($config, $version, $os, $name) = @_;
+    return $config->{versions}{$version}{$os}{$name}
+        // $config->{versions}{$version}{$name}
+        // $config->{$name};
+}
+
+sub channel_to_version {
+    my ($config, @channels) = @_;
+    return values %{$config->{channels}} unless @channels;
+    foreach my $channel (@channels) {
+        exit_error "Unknown channel $channel"
+                unless $config->{channels}{$channel};
+    }
+    return map { $config->{channels}{$_} } @channels;
+}
+
+sub get_buildinfos {
+    my ($config, $version) = @_;
+    return if exists $config->{versions}{$version}{buildID};
+    extract_martools($version);
+    my $files = $config->{versions}{$version}{files};
+    foreach my $os (keys %$files) {
+        foreach my $lang (keys %{$files->{$os}}) {
+            next unless $files->{$os}{$lang}{complete};
+            my $tmpdir = File::Temp->newdir();
+            extract_mar(
+                mar_filename($config->{appname_marfile}, $version, $os, $lang),
+                "$tmpdir");
+            my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini";
+            $appfile = "$tmpdir/Contents/Resources/application.ini"
+                                if -f "$tmpdir/Contents/Resources/application.ini";
+            exit_error "Could not find application.ini" unless $appfile;
+            foreach my $line (read_file($appfile)) {
+                if ($line =~ m/^BuildID=(.*)$/) {
+                    $config->{versions}{$version}{buildID} = $1;
+                    return;
+                }
+            }
+            exit_error "Could not extract buildID from application.ini";
+        }
+    }
+}
+
+sub get_response {
+    my ($config, $version, $os, @patches) = @_;
+    my $res;
+    my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8');
+    $writer->xmlDecl;
+    $writer->startTag('updates');
+    if (get_config($config, $version, $os, 'unsupported')) {
+        $writer->startTag('update',
+            unsupported => 'true',
+            detailsURL => get_config($config, $version, $os, 'detailsURL'),
+        );
+        goto CLOSETAGS;
+    }
+    my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion');
+    my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet');
+    $writer->startTag('update',
+        type => 'minor',
+        displayVersion => $version,
+        appVersion => $version,
+        platformVersion => get_config($config, $version, $os, 'platformVersion'),
+        buildID => get_config($config, $version, $os, 'buildID'),
+        detailsURL => get_config($config, $version, $os, 'detailsURL'),
+        actions => 'showURL',
+        openURL => get_config($config, $version, $os, 'detailsURL'),
+        defined $minversion ? ( minSupportedOSVersion => $minversion ) : (),
+        defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (),
+    );
+    foreach my $patch (@patches) {
+        my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch;
+        $writer->startTag('patch', @sorted_patch);
+        $writer->endTag('patch');
+    }
+    CLOSETAGS:
+    $writer->endTag('update');
+    $writer->endTag('updates');
+    $writer->end;
+    return $res;
+}
+
+sub write_responses {
+    my ($config, @channels) = @_;
+    @channels = keys %{$config->{channels}} unless @channels;
+    foreach my $channel (@channels) {
+        my $version = $config->{channels}{$channel};
+        get_version_files($config, $version);
+        get_buildinfos($config, $version);
+        my $files = $config->{versions}{$version}{files};
+        my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {};
+        foreach my $old_os (keys %$migrate_archs) {
+            my $new_os = $migrate_archs->{$old_os};
+            foreach my $lang (keys %{$files->{$new_os}}) {
+                $files->{$old_os}{$lang}{complete} =
+                        $files->{$new_os}{$lang}{complete};
+            }
+        }
+        foreach my $os (keys %$files) {
+            foreach my $lang (keys %{$files->{$os}}) {
+                my $resp = get_response($config, $version, $os,
+                                $files->{$os}{$lang}{complete});
+                write_htdocs($channel, "$version-$os-$lang.xml", $resp);
+                foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) {
+                    $resp = get_response($config, $version, $os,
+                                $files->{$os}{$lang}{complete},
+                                $files->{$os}{$lang}{partial}{$from_version});
+                    write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp);
+                }
+            }
+        }
+        write_htdocs($channel, 'no-update.xml',
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            . "\n<updates></updates>\n");
+    }
+}
+
+sub write_htaccess {
+    my ($config, @channels) = @_;
+    @channels = keys %{$config->{channels}} unless @channels;
+    my $flags = "[last]";
+    foreach my $channel (@channels) {
+        my $htaccess = "RewriteEngine On\n";
+        my $version = $config->{channels}{$channel};
+        my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {};
+        my $files = $config->{versions}{$version}{files};
+        $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n";
+        foreach my $os (sort keys %$files) {