Bug 40596: Switch compress_tar from gz to zst (for builds targeting non-linux)

The version of tar in stretch does not know how to extract tar.zst
files, so we don't switch to zst for linux builds yet.

README

@@ -12,6 +12,9 @@ newuidmap and newgidmap commands.
 The sources of most components are downloaded using git, which needs to
 be installed. The mercurial package is also needed.
 
+Zstandard (zstd) is used to compress some tarballs and needs to be
+installed. You'll also need tar >= 1.31, for zstd support.
+
 You also need a few perl modules installed:
 - YAML::XS
 - File::Basename
@@ -40,7 +43,7 @@ If you are running Debian or Ubuntu, you can install them with:
                   libstring-shellquote-perl libsort-versions-perl \
                   libdigest-sha-perl libdata-uuid-perl libdata-dump-perl \
                   libfile-copy-recursive-perl libfile-slurp-perl git \
-                  mercurial uidmap
+                  mercurial uidmap zstd
 
 If you are running an Arch based system, you should be able to install them with:
 
@@ -49,7 +52,8 @@ If you are running an Arch based system, you should be able to install them with
             perl-file-copy-recursive perl-string-shellquote \
             perl-sort-versions perl-data-uuid perl-data-dump perl-json \
             perl-digest-sha1 perl-io-all perl-file-slurp perl-sys-syscall \
-            perl-parallel-forkmanager perl-xml-libxml perl-lwp-protocol-https
+            perl-parallel-forkmanager perl-xml-libxml perl-lwp-protocol-https \
+            zstd
 
 On Arch based systems you also need to generate some .ph files RBM expects:
 

projects/aiohttp/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/[% project %]
 cd /var/tmp/dist/

projects/aiohttp_socks/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/
 cd /var/tmp/dist/

projects/aiorpcx/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/
 cd /var/tmp/dist/

projects/android-components/build

@@ -31,7 +31,7 @@ mkdir /var/tmp/build
   # according to the Gradle dependencies list.
   cp -rf $distdir/application-services/maven/org $gradle_repo
 [% END %]
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 cd $builddir-[% c('version') %]
 

projects/application-services/build

@@ -30,7 +30,7 @@ export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
   cp -r $gradle_repo/maven2/* $gradle_repo
 [% END %]
 
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 # Prepare the offline build.
 # Move the directory for hardcoding the path in .cargo/config.

projects/async_timeout/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/[% project %]
 cd /var/tmp/dist/

projects/attr/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/[% project %]
 cd /var/tmp/dist/

projects/cbindgen/build

@@ -9,7 +9,7 @@ export PATH="/var/tmp/dist/rust/bin:$PATH"
   export LD_LIBRARY_PATH="/var/tmp/dist/rust/lib:$LD_LIBRARY_PATH"
 [% END -%]
 mkdir -p /var/tmp/build
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 # Now prepare the offline build
 # Move the directory for hardcoding the path in .cargo/config

projects/cctools/build

@@ -7,7 +7,7 @@ tar -C /var/tmp/dist -xf [% c('input_files_by_name/libtapi') %]
 mv /var/tmp/dist/libtapi/* $distdir
 export CLANG_DIR=/var/tmp/dist/clang
 mkdir -p /var/tmp/build
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 cd /var/tmp/build/[% project %]-[% c('version') %]/cctools
 
 export CC=$CLANG_DIR/bin/clang

projects/certifi/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/
 cd /var/tmp/dist/

projects/chardet/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/
 cd /var/tmp/dist/

projects/cmake/build

@@ -6,7 +6,7 @@ distdir=/var/tmp/dist/[% project %]
                               hardened_gcc => 0 }) %]
 [% END -%]
 mkdir -p /var/tmp/build
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 cd /var/tmp/build/[% project %]-[% c('version') %]
 ./bootstrap --prefix=$distdir
 make -j[% c("num_procs") %]

projects/common/list_toolchain_updates

@@ -6,7 +6,7 @@ rootdir="$(pwd)"
 no_updates="$rootdir/no_updates.txt"
 updates="$rootdir/updates.txt"
 
-tar -xf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 cd [% project %]-[% c('version') %]
 
 function check_update_needed() {

projects/conjure/build

@@ -12,7 +12,7 @@ mkdir -p $distdir
 [% END -%]
 
 mkdir -p /var/tmp/build
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 cd /var/tmp/build/[% project %]-[% c('version') %]
 
 tar -xf $rootdir/[% c('input_files_by_name/go_vendor') %]

projects/ecdsa/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 
-tar xvf [% project %]-[% c('version') %].tar.gz
+tar -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p /var/tmp/dist/
 cd /var/tmp/dist/

projects/electrum-nmc/build

 #!/bin/sh
 [% c("var/set_default_env") -%]
 mkdir -p /var/tmp/build/[% project %]
-tar -C /var/tmp/build/[% project %] -xf [% project %]-[% c("version") %].tar.gz
+tar -C /var/tmp/build/[% project %] -xf [% project %]-[% c("version") %].tar.[% c('compress_tar') %]
 
 cd /var/tmp/build/[% project %]/[% project %]-*
 

projects/fenix/build

@@ -30,7 +30,7 @@ mkdir -p /var/tmp/build
   cp $distdir/application-services/nimbus-fml $nimbusdir/
   tar -C $distdir -xf $rootdir/[% c('input_files_by_name/translation-fenix') %]
 [% END %]
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 cd $builddir-[% c("version") %]
 

projects/firefox-l10n/config

@@ -9,7 +9,7 @@ steps:
     version: '[% c("hg_hash") %]'
     fetch_locale: |
       [% c("var/set_default_env") -%]
-      tar xf '[% project %]-[% c("version") %].tar.gz'
+      tar xf '[% project %]-[% c("version") %].tar.[% c("compress_tar") %]'
       mv [% project %]-[% c("version") %] [% c("input_file_var/locale") %]
       tar caf '[% dest_dir %]/[% c("filename") %]' [% c("input_file_var/locale") %]
     input_files: []

projects/firefox/build

@@ -56,7 +56,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bi
   export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
 [% END -%]
 
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 
 mkdir -p $distdir/[% IF ! c("var/macos") %]Browser[% END %]