Bug 22499: build obfsproxy and fteproxy for Windows

Also finish including the Linux build of obfsproxy and fteproxy.

Based on gitian/descriptors/windows/gitian-pluggable-transports.yml
from tor-browser-bundle.git.

projects/argparse/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% ELSE -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/argparse.py $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  mkdir -p $pydir/Lib/site-packages
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  $PYTHON setup.py install --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  python2 setup.py build --build-lib build
+  cp -a build/argparse.py $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],

projects/argparse/config

@@ -12,8 +12,15 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
   - URL: 'https://pypi.python.org/packages/source/a/argparse/argparse-[% c("version") %].tar.gz'
     sha256sum: ddaf4b0a618335a32b6664d4ae038a1de8fbada3b25033f9021510ed2b3941a4
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/fteproxy/build

@@ -3,11 +3,28 @@
 distdir="/var/tmp/dist/[% project %]"
 [% c("var/set_PTDIR_DOCSDIR") -%]
 mkdir -p $PTDIR $DOCSDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% END -%]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/libfte') %]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
-cp -a bin/fteproxy $PTDIR/fteproxy.bin
-cp -ra fteproxy $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  $PYTHON setup_tbb.py py2exe
+  py2exe_zip_timestomp dist/fteproxy.zip
+  cp -an dist/{*.pyd,*.exe,*.zip} $PTDIR/
+  mkdir -p $PTDIR/fteproxy
+  cp -a fteproxy/VERSION $PTDIR/fteproxy
+  mkdir -p $PTDIR/fteproxy/defs
+  cp -a fteproxy/defs/*.json $PTDIR/fteproxy/defs
+  rm -Rf $pydir
+[% ELSE -%]
+  cp -a bin/fteproxy $PTDIR/fteproxy.bin
+  cp -ra fteproxy $PTDIR/
+[% END -%]
 cp -a {COPYING,README.md} $DOCSDIR/
 cd $distdir
 [% c('tar', {

projects/fteproxy/config

@@ -15,6 +15,15 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
+  - project: libfte
+    name: libfte
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/libfte/build

@@ -10,9 +10,26 @@ tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
 tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/gmp') %]
 ln -s /var/tmp/dist/gmp thirdparty/gmp
-export PYTHON=python2
-make
-cp -ra fte $PTDIR/
+mkdir -p $distdir
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/obfsproxy') %]
+[% IF c("var/windows") -%]
+  pydir=$distdir/python
+  export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
+  export LD_PRELOAD=
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  # FTE only needs libgmp-10.dll and no libgmpxx anymore.
+  cp -a /var/tmp/dist/gmp/bin/libgmp-10.dll .
+  cp -a /var/tmp/dist/gmp/bin/libgmp-10.dll $distdir/TorBrowser/Tor/
+  WINDOWS_BUILD=1 CROSS_COMPILE=1 make
+  $PYTHON setup.py install --prefix=$(winepath -w "$pydir")
+  $PYTHON setup.py install
+[% ELSE -%]
+  export PYTHON=python2
+  make
+  cp -ra fte $PTDIR/
+  mkdir -p $distdir/TorBrowser/Tor
+  cp /var/tmp/dist/gmp/lib/libgmp.so.10 $distdir/TorBrowser/Tor/
+[% END -%]
 cp -a {LICENSE,README.md} $DOCSDIR/
 cp -a thirdparty/re2/LICENSE $DOCSDIR/LICENSE.re2
 cd $distdir

projects/libfte/config

@@ -14,6 +14,12 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
+      arch_deps:
+        - faketime
 
 input_files:
   - project: container-image
@@ -21,3 +27,5 @@ input_files:
     project: '[% c("var/compiler") %]'
   - project: gmp
     name: gmp
+  - project: obfsproxy
+    name: obfsproxy

projects/obfsproxy/build

@@ -3,13 +3,37 @@
 distdir="/var/tmp/dist/[% project %]"
 [% c("var/set_PTDIR_DOCSDIR") -%]
 mkdir -p $PTDIR $DOCSDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% ELSE -%]
+[% END -%]
+mkdir -p $distdir
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pycrypto') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/twisted') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/argparse') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/parsley') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pyptlib') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pyyaml') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/txsocksx') %]
+tar -C $distdir -xf $rootdir/[% c('input_files_by_name/zope.interface') %]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/obfsproxy $PTDIR/
-cp -a bin/obfsproxy $PTDIR/obfsproxy.bin
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  $PYTHON setup_py2exe.py py2exe
+  $PYTHON setup.py install --prefix=$(winepath -w $pydir)
+  py2exe_zip_timestomp py2exe_bundle/dist/obfsproxy.zip
+  cp -an py2exe_bundle/dist/{*.pyd,*.exe,*.zip} $PTDIR/
+  # http://bugs.winehq.org/show_bug.cgi?id=3591
+  cp -a /var/tmp/dist/winpython/python27.dll $PTDIR
+[% ELSE -%]
+  export PYTHON=python2
+  $PYTHON setup.py build --build-lib build
+  cp -a build/obfsproxy $PTDIR/
+  cp -a bin/obfsproxy $PTDIR/obfsproxy.bin
+[% END -%]
 cp -a {LICENSE,README} $DOCSDIR
 cd $distdir
 [% c('tar', {

projects/obfsproxy/config

@@ -7,7 +7,7 @@ gpg_keyring: obfsproxy.gpg
 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
 
 var:
-  DOCSDIR_project: Obsfproxy
+  DOCSDIR_project: Obfsproxy
   container:
     use_container: 1
 
@@ -17,6 +17,29 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
+  - project: pycrypto
+    name: pycrypto
+  - project: twisted
+    name: twisted
+  - project: argparse
+    name: argparse
+  - project: parsley
+    name: parsley
+  - project: pyptlib
+    name: pyptlib
+  - project: pyyaml
+    name: pyyaml
+  - project: txsocksx
+    name: txsocksx
+  - project: zope.interface
+    name: zope.interface
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/parsley/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% ELSE -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/Parsley-[% c('version') %].tar.gz
 cd /var/tmp/build/Parsley-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/parsley.py build/ometa build/terml $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  mkdir -p $pydir/Lib/site-packages
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  $PYTHON setup.py install --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  PYTHON=python2
+  $PYTHON setup.py build --build-lib build
+  cp -a build/parsley.py build/ometa build/terml $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],

projects/parsley/config

@@ -12,8 +12,15 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
   - URL: 'https://pypi.python.org/packages/source/P/Parsley/Parsley-[% c("version") %].tar.gz'
     sha256sum: 50d30cee70770fd44db7cea421cb2fb75af247c3a1cd54885c06b30a7c85dd23
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/pycrypto/build

 #!/bin/bash
+set -e
 [% c("var/set_default_env") -%]
 [% c("var/setarch") -%]
 [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/linux") -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
-export PYTHON=python2
 [% IF c("var/linux-i686") -%]
   export CFLAGS=-m32
   export CXXFLAGS=-m32
   export LDFLAGS=-m32
 [% END -%]
-./configure --build=i686-linux-gnu [% c("var/configure_opt") %]
-$PYTHON setup.py build --build-lib build
-cp -a build/Crypto $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  mkdir -p "$pydir"
+  export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
+  export LD_PRELOAD=[% c("var/faketime_path") %]
+  # This is bogus, that we run the configure script in the build environment,
+  # but it seems to work. https://bugs.launchpad.net/pycrypto/+bug/1096207 for
+  # ac_cv_func_malloc_0_nonnull.
+  ac_cv_func_malloc_0_nonnull=yes sh configure --host=i686-w64-mingw32
+  export LD_PRELOAD=
+  $PYTHON setup.py build_ext -c mingw32
+  $PYTHON setup.py install --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  export PYTHON=python2
+  ./configure --build=i686-linux-gnu [% c("var/configure_opt") %]
+  python2 setup.py build --build-lib build
+  cp -a build/Crypto $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],

projects/pycrypto/config

@@ -12,6 +12,13 @@ targets:
         - python-dev
         - python-setuptools
 
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
+      arch_deps:
+        - faketime
+
 input_files:
   - project: container-image
   - name: '[% c("var/compiler") %]'

projects/pyptlib/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/linux") -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz
 cd /var/tmp/build/[% project %]-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/pyptlib $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  mkdir -p "$pydir"
+  $PYTHON setup.py install --single-version-externally-managed --record /dev/null --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  export PYTHON=python2
+  $PYTHON setup.py build --build-lib build
+  cp -a build/pyptlib $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],

projects/pyptlib/config

@@ -16,6 +16,13 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/pyyaml/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% ELSE -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/PyYAML-[% c('version') %].tar.gz
 cd /var/tmp/build/PyYAML-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/yaml $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  mkdir -p $pydir/Lib/site-packages
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  $PYTHON setup.py install --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  export PYTHON=python2
+  $PYTHON setup.py build --build-lib build
+  cp -a build/yaml $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],

projects/pyyaml/config

@@ -12,8 +12,15 @@ targets:
       arch_deps:
         - python-setuptools
         - python-dev
+  windows-i686:
+    var:
+      compiler: winpython
+      post_pkginst: '[% c("var/install_wine_ppa") %]'
 
 input_files:
   - project: container-image
   - URL: 'https://pypi.python.org/packages/source/P/PyYAML/PyYAML-[% c("version") %].tar.gz'
     sha256sum: c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+    enable: '[% c("var/windows") %]'

projects/tor-browser/build

@@ -56,6 +56,10 @@ mv $TBDIR/meek-http-helper@bamsoftware.com.xpi $TBDIR/$MEEKPROFILEPATH/extension
   tar -C $TBDIR -xf [% c('input_files_by_name/snowflake') -%]
 [% END -%]
 
+[% IF c("var/fteproxy") -%]
+  tar -C $TBDIR -xf [% c('input_files_by_name/fteproxy') %]
+[% END -%]
+
 tar -C $TBDIR[% IF c("var/osx") %]/Contents/Resources[% END %] -xf [% c('input_files_by_name/fonts') %]
 
 [% IF c("var/linux") %]

projects/tor-browser/config

@@ -50,6 +50,9 @@ input_files:
     name: meek
   - project: obfs4
     name: obfs4
+  - project: fteproxy
+    name: fteproxy
+    enable: '[% c("var/fteproxy") %]'
   - project: snowflake
     name: snowflake
     enable: '[% c("var/snowflake") %]'

projects/twisted/build

 #!/bin/bash
 [% c("var/set_default_env") -%]
 distdir="/var/tmp/dist/[% project %]"
-[% c("var/set_PTDIR_DOCSDIR") -%]
-mkdir -p $PTDIR
+[% IF c("var/windows") -%]
+  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% ELSE -%]
+  [% c("var/set_PTDIR_DOCSDIR") -%]
+  mkdir -p $PTDIR
+[% END -%]
 mkdir -p /var/tmp/build
 tar -C /var/tmp/build -xf $rootdir/Twisted-[% c('version') %].tar.bz2
 cd /var/tmp/build/Twisted-[% c('version') %]
-export PYTHON=python2
-$PYTHON setup.py build --build-lib build
-cp -a build/twisted $PTDIR/
+[% IF c("var/windows") -%]
+  pydir="$distdir/python"
+  export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
+  export LD_PRELOAD=[% c("var/faketime_path") %]
+  # twisted/internet/iocpreactor/iocpsupport/iocpsupport.c includes "python.h"
+  # rather than "Python.h".
+  ln -sf Python.h /var/tmp/dist/winpython/include/python.h
+  # We need to set the "mingw32" compiler to avoid an error in build_ext, but
+  # Twisted's "install" command calls build_ext unconditionally, whether the
+  # extensions have been built already or not, so we can't just call build_ext
+  # separately as with other packages. The "install" command doesn't recognize
+  # the -c option, so we set the compiler in a configuration file.
+  echo $'[build_ext]\ncompiler=mingw32' > setup.cfg
+  mkdir -p $pydir/Lib/site-packages
+  export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages"
+  LD_PRELOAD= $PYTHON setup.py install --single-version-externally-managed --record /dev/null --prefix=$(winepath -w $pydir)
+[% ELSE -%]
+  export PYTHON=python2
+  $PYTHON setup.py build --build-lib build
+  cp -a build/twisted $PTDIR/
+[% END -%]
 cd $distdir
 [% c('tar', {
         tar_src => [ '.' ],