Bug 27218: generate multiple bundles in parallel

projects/tor-browser/build

@@ -5,6 +5,9 @@ export TORBROWSER_VERSION='[% c("version") %]'
 
 mkdir -p $distdir
 
+scripts_dir=/var/tmp/build_scripts
+mkdir -p "$scripts_dir"
+
 OUTDIR='[% dest_dir _ "/" _ c("filename") %]'
 mkdir -p $OUTDIR
 
@@ -260,8 +263,6 @@ popd
   mv $distdir/tbb-windows-installer ${TB_STAGE_DIR}
 [% END %]
 
-cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
-
 [% IF c("var/windows") %]
   TBDIR="$distdir/$PKG_DIR/Tor Browser/Browser"
 [% ELSIF c("var/osx") %]
@@ -270,6 +271,11 @@ cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
   TBDIR="$distdir/$PKG_DIR/Browser"
 [% END %]
 
+cat > "$scripts_dir/create-$PKG_DIR" << SCRIPT_EOF
+#!/bin/bash
+set -e
+cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
+
 pushd "$TBDIR[% IF c("var/osx") %]/Contents/Resources/[% END %]"
 rm -f precomplete
 python $MARTOOLS/createprecomplete.py
@@ -279,8 +285,8 @@ cd $distdir
 
 [% IF c("var/build_mar") -%]
   # Create full MAR file and compressed package.
-  MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_${PKG_LOCALE}.mar
-  MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "$TBDIR"
+  [% SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.mar' %]
+  MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "$TBDIR"
 [% END -%]
 
 [% IF c("var/linux") %]
@@ -306,6 +312,7 @@ cd $distdir
   popd
 [% END %]
 rm -rf $distdir/${PKG_DIR}
+SCRIPT_EOF
 
 cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
 [% IF c("var/linux-x86_64") -%]
@@ -322,6 +329,7 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
     [% SET lang = tmpl(lang);
        SET xpi = '$rootdir/' _ c('input_files_by_name/firefox-langpacks') _ '/' _ lang _ '.xpi';
        SET tbdir = '$distdir/tor-browser_' _ lang;
+       SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_' _ lang _ '.mar';
        IF c("var/osx");
          SET browserdir = tbdir _ '/Tor Browser.app';
        ELSIF c("var/windows");
@@ -330,6 +338,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
          SET browserdir = tbdir _ '/Browser';
        END;
     %]
+      cat > "$scripts_dir/create-tor-browser_[% lang %]" << SCRIPT_EOF
+#!/bin/bash
+      set -e
       cp -a ${TB_STAGE_DIR} [% tbdir %]
       cp [% xpi %] "[% browserdir %]/$EXTSPATH/langpack-[% lang %]@firefox.mozilla.org.xpi"
 
@@ -357,9 +368,10 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
       python $MARTOOLS/createprecomplete.py
       popd
 
+      cd $distdir
+
       # Create full MAR file and compressed package for this locale.
-      MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_[% lang %].mar
-      MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "[% browserdir %]"
+      MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "[% browserdir %]"
       [% IF c("var/linux") %]
         [% SET tardir = 'tor-browser_' _ lang;
           c('tar', {
@@ -388,5 +400,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
         popd
       [% END %]
       rm -rf [% tbdir %]
+SCRIPT_EOF
   [% END %]
 [% END %]
+
+chmod 775 $rootdir/run_scripts "$scripts_dir"/*
+$rootdir/run_scripts [% c("buildconf/num_procs") %] "$scripts_dir"

projects/tor-browser/config

@@ -6,6 +6,14 @@ var:
   container:
     use_container: 1
   ddmg: '[% INCLUDE ddmg.sh %]'
+  deps:
+    - python
+    - libparallel-forkmanager-perl
+    - libfile-slurp-perl
+    - zip
+    - unzip
+    - bzip2
+    - xz-utils
 
 targets:
   linux-i686:
@@ -17,14 +25,9 @@ targets:
   osx-x86_64:
     var:
       mar_osname: osx64
-      deps:
+      arch_deps:
         - genisoimage
-        - zip
-        - unzip
-        - python
-        - bzip2
         - faketime
-        - xz-utils
   windows:
     var:
       arch_deps:
@@ -41,6 +44,7 @@ targets:
 
 input_files:
   - project: container-image
+  - filename: run_scripts
   - project: firefox
     name: firefox
   - project: tor

projects/tor-browser/ddmg.sh

@@ -4,8 +4,8 @@ find [% src %] ! -executable -exec chmod 0644 {} \;
 
 find [% src %] -exec [% c("var/touch") %] {} \;
 
-dmg_tmpdir=$(mktemp -d)
-[% SET filelist = '"$dmg_tmpdir/filelist.txt"' %]
+dmg_tmpdir=\$(mktemp -d)
+[% SET filelist = '"\$dmg_tmpdir/filelist.txt"' %]
 pushd [% src %] 
 find . -type f | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" > [% filelist %]
 find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelist %]
@@ -13,9 +13,9 @@ find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelis
 export LD_PRELOAD=[% c("var/faketime_path") %]
 export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
 
-genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
+genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "\$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
 
-dmg dmg "$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
+dmg dmg "\$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
 popd
 
-rm -Rf "$dmg_tmpdir"
+rm -Rf "\$dmg_tmpdir"

projects/tor-browser/run_scripts

+#!/usr/bin/perl -w
+use strict;
+
+use Parallel::ForkManager;
+use File::Slurp;
+
+sub exit_error {
+    print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
+    exit (exists $_[1] ? $_[1] : 1);
+}
+
+exit_error "Wrong number of arguments" unless @ARGV == 2;
+my ($nb_threads, $scripts_dir) = @ARGV;
+exit_error "Could not enter $scripts_dir" unless chdir $scripts_dir;
+my $pm = Parallel::ForkManager->new($nb_threads);
+$pm->run_on_finish(
+    sub {
+        my ($pid, $exit, $id) = @_;
+        exit_error "Error running $id" unless $exit == 0;
+        print "Finished $id\n";
+    }
+);
+foreach my $script (sort(read_dir($scripts_dir))) {
+    $pm->start($script) and next;
+    print "Running $script\n";
+    exit_error "Error running $script" unless system("./$script") == 0;
+    $pm->finish;
+}
+$pm->wait_all_children;