Add snowflake

Only for linux-x86_64 for now, as it fails to build on linux-i686.

.gitignore

 git_clones
 hg_clones
+gclient
 out
 release
 alpha

projects/depot_tools/build

+#!/bin/bash
+[% c("var/set_default_env") -%]
+tar xf [% project %]-[% c("version") %].tar.gz
+mv [% project %]-[% c("version") %] [% project %]
+[% c('tar', {
+        tar_src => [ project ],
+        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+    }) %]

projects/depot_tools/config

+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_hash: 28216cd14b44716db5c83634afbdc6e90492652b
+git_url: https://chromium.googlesource.com/chromium/tools/depot_tools.git
+filename: '[% project %]-[% c("version") %].tar.gz'

projects/gcc/config

@@ -10,7 +10,7 @@ var:
     - libc6-dev-i386
   setup: |
     mkdir -p /var/tmp/dist
-    tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %]
+    tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %]
     export PATH="/var/tmp/dist/gcc/bin:$PATH"
     [% IF c("var/osname") == "linux-i686" -%]
     export LD_LIBRARY_PATH=/var/tmp/dist/gcc/lib32

projects/go-webrtc/config

+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/keroserene/go-webrtc.git
+git_hash: 69c5ca70533142494d145b43ef2f46ca94715557
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+remote_docker: 1
+
+build: '[% c("projects/go/var/build_go_lib") %]'
+
+var:
+  go_lib: github.com/keroserene/go-webrtc
+  build_go_lib_pre: |
+    [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+    [% IF c("var/linux") %]
+      tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+      export PATH="/var/tmp/dist/binutils/bin:$PATH"
+    [% END -%]
+    tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/webrtc') %]
+    # Replace the prebuilt webrtc library with our own one.
+    rm -rf include/ lib/
+    cp -a /var/tmp/dist/webrtc/{include,lib} .
+    # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with
+    # the prebuilt libwebrtc-magic.a libraries. Since we build our own
+    # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here,
+    # using CGO_CXXFLAGS.
+    export CGO_CXXFLAGS=-D_GLIBCXX_USE_CXX11_ABI=1
+
+targets:
+  master:
+    git_hash: master
+  linux-x86_64:
+    var:
+      arch_deps:
+        - pkg-config
+        - libx11-dev
+  linux-i686:
+    var:
+      arch_deps:
+        - pkg-config:i386
+        - libx11-dev:i386
+        - lib32stdc++6
+
+input_files:
+  - project: docker-image
+  - name: go
+    project: go
+  - name: webrtc
+    project: webrtc
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+  - name: binutils
+    project: binutils
+    enable: '[% c("var/linux") %]'

projects/go/config

@@ -35,6 +35,7 @@ var:
     do
       patch -p1 < $p
     done
+    [% IF c("var/build_go_lib_pre"); GET c("var/build_go_lib_pre"); END; -%]
     [% IF c("var/go_lib_install") -%]
       [% FOREACH inst IN c("var/go_lib_install") %]
         go install [% inst %]

projects/snowflake/build

+#!/bin/bash
+[% c("var/set_default_env") -%]
+[% pc('go', 'var/setup', { go_tarfile => c('input_files_by_name/go') }) %]
+[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% IF c("var/linux") %]
+  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+  export PATH="/var/tmp/dist/binutils/bin:$PATH"
+[% END -%]
+distdir=/var/tmp/dist/[% project %]
+[% IF c("var/osx") %]
+  PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports"
+  DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/snowflake"
+[% ELSE %]
+  PTDIR="$distdir/TorBrowser/Tor/PluggableTransports"
+  DOCSDIR="$distdir/TorBrowser/Docs/snowflake"
+[% END %]
+mkdir -p $PTDIR $DOCSDIR
+
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/go-webrtc') %]
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/uniuri') %]
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/goptlib') %]
+
+mkdir -p /var/tmp/build
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+cd /var/tmp/build/[% project %]-[% c('version') %]
+
+cd client
+go build -ldflags '-s'
+cp -a client[% IF c("var/windows") %].exe[% END %] $PTDIR/snowflake-client[% IF c("var/windows") %].exe[% END %]
+
+cd ..
+cp -a README.md LICENSE $DOCSDIR
+
+cd $distdir
+[% c('tar', {
+        tar_src => [ '.' ],
+        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+    }) %]

projects/snowflake/config

+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://git.torproject.org/pluggable-transports/snowflake.git
+git_hash: 6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+remote_docker: 1
+
+targets:
+  linux-i686:
+    var:
+      arch_deps:
+        - pkg-config
+        - libx11-dev:i386
+        - lib32stdc++6
+  linux-x86_64:
+    var:
+      arch_deps:
+        - pkg-config
+        - libx11-dev
+
+input_files:
+  - project: docker-image
+  - name: go
+    project: go
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+  - name: binutils
+    project: binutils
+    enable: '[% c("var/linux") %]'
+  - name: go-webrtc
+    project: go-webrtc
+  - name: uniuri
+    project: uniuri
+  - name: goptlib
+    project: goptlib

projects/tor-browser/build

@@ -52,6 +52,9 @@ mv [% c('input_files_by_name/noscript') %] $TBDIR/$EXTSPATH/{73a6fe31-595d-460b-
 tar -C $TBDIR -xf [% c('input_files_by_name/obfs4') %]
 tar -C $TBDIR -xf [% c('input_files_by_name/meek') %]
 mv $TBDIR/meek-http-helper@bamsoftware.com.xpi $TBDIR/$MEEKPROFILEPATH/extensions/
+[% IF c("var/snowflake") %]
+  tar -C $TBDIR -xf [% c('input_files_by_name/snowflake') -%]
+[% END -%]
 
 tar -C $TBDIR[% IF c("var/osx") %]/Contents/Resources[% END %] -xf [% c('input_files_by_name/fonts') %]
 

projects/tor-browser/config

@@ -48,6 +48,9 @@ input_files:
     name: meek
   - project: obfs4
     name: obfs4
+  - project: snowflake
+    name: snowflake
+    enable: '[% c("var/snowflake") %]'
   - filename: Bundle-Data
   - URL: https://addons.cdn.mozilla.net/user-media/addons/722/noscript_security_suite-2.9.5.3-fx+sm.xpi
     name: noscript

projects/webrtc/build

+#!/bin/bash
+[% c("var/set_default_env") -%]
+distdir=/var/tmp/dist/[% project %]
+[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
+mkdir -p /var/tmp/dist
+
+[% IF c("var/linux") %]
+  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+  export PATH="/var/tmp/dist/binutils/bin:$PATH"
+[% END -%]
+
+# Setting up depot_tools
+# https://dev.chromium.org/developers/how-tos/install-depot-tools
+tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/depot_tools') %]
+export PATH="$PATH:/var/tmp/dist/depot_tools"
+# Disable automatic updating.
+export DEPOT_TOOLS_UPDATE=0
+
+mkdir -p /var/tmp/build
+tar -C /var/tmp/build -xf webrtc-sources-[% c('version') %].tar.gz
+
+builddir=/var/tmp/build/[% project %]/src
+cd $builddir
+export GYP_CROSSCOMPILE=1
+export GYP_DEFINES="OS=[% c("var/webrtc/os") %] target_arch=[% c("var/webrtc/arch") %]"
+# Do not use bundled utilities.
+GYP_DEFINES+=" clang=0 host_clang=0 use_sysroot=0"
+GYP_DEFINES+=" linux_use_bundled_gold=0 linux_use_gold_flags=0 linux_use_bundled_binutils=0"
+# Avoid some dependencies. Some of these don't seem to work fully; for
+# example even with use_alsa=0 we need libasound2-dev.
+GYP_DEFINES+=" use_x11=0 use_gtk=0 use_gconf=0 use_alsa=0 use_pulseaudio=0 use_libpci=0 include_internal_audio_device=0 enabled_libjingle_device_manager=0"
+# examples and tests also bring in dependencies.
+GYP_DEFINES+=" include_examples=0 include_tests=0"
+# embedded=1 is supposed to turn of various features; see
+# https://bugs.chromium.org/p/chromium/issues/detail?id=318413.
+GYP_DEFINES+=" embedded=1"
+GYP_DEFINES+=" werror="
+[% IF c("var/linux") -%]
+  export CC=gcc
+  export CXX=g++
+[% END -%]
+[% IF c("var/linux-i686") -%]
+  GYP_DEFINES+=" target_arch=ia32"
+  export CC='gcc -m32'
+  export CXX='g++ -m32'
+  export CFLAGS=-m32
+  export CXXFLAGS=-m32
+  export LDFLAGS=-m32
+[% END -%]
+JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py
+ninja -C out/Release
+# Run ninja once more, without include_tests=0, in order to build just the
+# FakeAudioCaptureModule that go-webrtc uses.
+GYP_DEFINES="$(echo "$GYP_DEFINES" | sed -e 's/include_tests=0//g')"
+JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py
+ninja -C out/Release obj/webrtc/api/test/peerconnection_unittests.fakeaudiocapturemodule.o
+# https://github.com/keroserene/go-webrtc/issues/23#issuecomment-175312648
+# dump_syms_regtest.o is actually an executable, not an object file. If not
+# excluded, it results in the error:
+# libwebrtc-linux-386-magic.a(dump_syms_regtest.o): unsupported ELF file type 2
+ar crs libwebrtc-magic.a $(find . -name '*.o' -not -name '*.main.o' -not -name 'dump_syms_regtest.o' | sort)
+
+mkdir -p $distdir
+cd $distdir
+mkdir -p include lib
+cp -f $builddir/libwebrtc-magic.a [% c("var/webrtc/lib_path") %]
+INCLUDE_DIR="$PWD/include"
+cd $builddir
+for h in $(find talk/ webrtc/ -type f -name '*.h');
+do
+  mkdir -p "$INCLUDE_DIR/$(dirname $h)"
+  cp -f "$h" "$INCLUDE_DIR/$h"
+done
+
+cd /var/tmp/dist
+[% c('tar', {
+        tar_src => [ project ],
+        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+        }) %]

projects/webrtc/config

+# vim: filetype=yaml sw=2
+version: '[% c("var/webrtc_tag") %]'
+remote_docker: 1
+filename: 'webrtc-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+var:
+  webrtc_tag: f33698296719f956497d2dbff81b5080864a8804
+
+input_files:
+  - project: docker-image
+  - project: webrtc
+    pkg_type: fetch_sources
+  - project: depot_tools
+    name: depot_tools
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+  - project: binutils
+    name: binutils
+    enable: '[% c("var/linux") %]'
+
+targets:
+  linux:
+    var:
+      webrtc:
+        os: linux
+  linux-i686:
+    var:
+      dockerbuild: "[% pc('docker-image', 'pre') %]"
+      sort_deps: 0
+      arch_deps:
+        - default-jre-headless
+        - default-jdk
+        - lib32asound2-dev
+        - lib64expat1
+        - libexpat1-dev:i386
+        - libudev-dev:i386
+        - libx11-dev:i386
+        - libxext-dev:i386
+        - libxrandr-dev:i386
+        - lib32stdc++6
+        - hardening-wrapper
+      webrtc:
+        arch: ia32
+        lib_arch: 386
+        lib_path: lib/libwebrtc-linux-386-magic.a
+  linux-x86_64:
+    var:
+      arch_deps:
+        - default-jre-headless
+        - default-jdk
+        - libasound2-dev
+        - libexpat1-dev
+        - libudev-dev
+        - libx11-dev
+        - libxext-dev
+        - libxrandr-dev
+      webrtc:
+        arch: x64
+        lib_arch: amd64
+        lib_path: lib/libwebrtc-linux-amd64-magic.a
+
+steps:
+  fetch_sources:
+    remote_docker: 0
+    filename: 'webrtc-sources-[% c("var/webrtc_tag") %].tar.gz'
+    fetch_sources: |
+      #!/bin/bash
+      [% c("var/set_default_env") -%]
+      # WebRTC is special, having its own build system that brings in lots of Chromium dependencies.
+      # https://webrtc.org/native-code/development/
+      tar xf [% c('input_files_by_name/depot_tools') %]
+      export PATH="$rootdir/depot_tools:$PATH"
+      # GYP_CROSSCOMPILE=1 and GYP_DEFINES="use_x11=0" prevent probing for certain dependencies.
+      # Use --no-history because the whole checkout with history is about 12 GB.
+      export GYP_CROSSCOMPILE=1
+      export GYP_DEFINES="use_x11=0"
+      # JAVA_HOME is needed in a hook for libjingle. The readlink line tries to find the current JRE.
+      # default-java comes from the package default-jdk-headless.
+      export JAVA_HOME=/usr/lib/jvm/default-java
+      clone_dir='[% c("basedir") %]/gclient/webrtc'
+      mkdir -p "$clone_dir"
+      cd "$clone_dir"
+      if [ ! -d "src" ];
+      then
+        # "fetch" is part of depot_tools.
+        #fetch --nohooks --no-history webrtc
+        # FIXME: To avoid the unconditional `gclient sync` in the call to fetch,
+        # we inline the result of a `fetch --dry-run`
+        gclient root
+        gclient config --spec 'solutions = [
+          {
+            "managed": False,
+            "name": "src",
+            "url": "https://chromium.googlesource.com/external/webrtc.git",
+            "custom_deps": {},
+            "deps_file": "DEPS",
+            "safesync_url": "",
+          },
+        ]
+        '
+        gclient sync --nohooks --no-history --with_branch_heads -r [% c("var/webrtc_tag") %]
+        cd src
+        git submodule foreach 'git config -f $toplevel/.git/config submodule.$name.ignore all'
+        git config --add remote.origin.fetch '+refs/tags/*:refs/tags/*'
+        git config diff.ignoreSubmodules all
+        cd ..
+      fi
+      # "gclient" is part of depot_tools. This download takes a long time the first time.
+      gclient sync --no-history --with_branch_heads -r [% c("var/webrtc_tag") %]
+      cd ..
+      tar --exclude .git -czf [% dest_dir _ '/' _ c('filename') %] webrtc
+    input_files:
+      - project: depot_tools
+        name: depot_tools
+        pkg_type: build

rbm.conf

@@ -105,6 +105,7 @@ targets:
   linux-x86_64:
     arch: x86_64
     var:
+      linux-x86_64: 1
       osname: linux-x86_64
       deps:
         - build-essential
@@ -115,9 +116,12 @@ targets:
         - libtool
         - zip
         - unzip
+      # We only build snowflake on linux-x86_64 for now
+      snowflake: 1
   linux-i686:
     arch: i686
     var:
+      linux-i686: 1
       osname: linux-i686
       configure_opt: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32'
       pre_pkginst: dpkg --add-architecture i386