Github pr 1448: update .gitlab-ci.yml to remove broken cruft and add a complete test suite

This code is from @eighthave

Closes #32193 (closed)

Github PR at https://github.com/torproject/tor/pull/1448

We'll want to iterate a lot on this to get our gitlab CI into its final form, but let's see what we can use now to get to a better point.

We'll want to have one gitlab-ci script that we keep the same for 0.3.5 and forward.

.gitlab-ci.yml

-before_script:
-    - apt-get update -qq
-    - apt-get upgrade -qy
 
-build:
+.artifacts-template: &artifacts-template
+  artifacts:
+    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
+    expire_in: 1 week
+    when: always
+    paths:
+      - config.log
+      - external
+      - openssl/configdata.pm
+      - src/app/tor
+
+.test-template: &test-template
   script:
-    - apt-get install -qy --fix-missing automake build-essential
-      libevent-dev libssl-dev zlib1g-dev
-      libseccomp-dev liblzma-dev libscrypt-dev
     - ./autogen.sh
-    - ./configure --disable-asciidoc --enable-fatal-warnings
-      --disable-silent-rules
+    - ./configure
+          --enable-fatal-warnings
+          --enable-lzma
+          --disable-asciidoc
+          --disable-silent-rules
+          --prefix=/usr
+    - grep -E '^# *define +HAVE_LZMA +1$' orconfig.h
+    - grep -E '^# *define +ENABLE_OPENSSL +1$' orconfig.h || true
+    - grep -E '^# *define +HAVE_TLS_METHOD +1$' orconfig.h || true
+
     - make check || (e=$?; cat test-suite.log; exit $e)
+    - readelf -d src/app/tor > readelf.txt
+
+    - grep -E '\(NEEDED\) +Shared library. \[libcrypto\.' readelf.txt
+    - grep -E '\(NEEDED\) +Shared library. \[libssl\.' readelf.txt
+    - grep -E '\(NEEDED\) +Shared library. \[liblzma\.' readelf.txt
+    - grep tor_lzma_state_size_precalc src/app/tor
     - make install
 
+.apt-template: &apt-template |
+      export LC_ALL=C.UTF-8
+      echo Etc/UTC > /etc/timezone
+      echo 'quiet "1";' \
+           'APT::Install-Recommends "0";' \
+           'APT::Install-Suggests "0";' \
+           'APT::Acquire::Retries "20";' \
+           'APT::Get::Assume-Yes "true";' \
+           'Dpkg::Use-Pty "0";' \
+        >> /etc/apt/apt.conf.d/99gitlab
+      apt-get update
+      apt-get dist-upgrade
+
+.debian-template: &debian-template
+  <<: *artifacts-template
+  variables:
+    DEBIAN_FRONTEND: noninteractive
+  before_script:
+    - *apt-template
+    - apt-get install
+        automake
+        build-essential
+        libevent-dev
+        liblzma-dev
+        libscrypt-dev
+        libseccomp-dev
+        libssl-dev
+        pkg-config
+        python3
+        zlib1g-dev
+    - if [[ "$CI_JOB_NAME" == *"_clang"* ]]; then
+          export CHUTNEY="yes";
+          export SKIP_MAKE_CHECK="yes";
+          export CHUTNEY_ALLOW_FAILURES="2";
+          apt-get install clang;
+      fi
+    - if [[ "$CI_JOB_NAME" == *"_rust"* ]]; then
+          apt-get install rustc;
+      fi
+  after_script:
+    - echo "Download debug artifacts from https://gitlab.com/${CI_PROJECT_PATH}/-/jobs"
+
+
+.android-template: &android-template
+  image: registry.gitlab.com/fdroid/ci-images-client:latest
+  variables:
+    DEBIAN_FRONTEND: noninteractive
+  before_script:
+    - *apt-template
+    - apt-get install autoconf automake ca-certificates curl gzip make libtinfo5 libtool pkg-config tar unzip wget
+  script:
+    - ndk=android-ndk-`echo $CI_JOB_NAME | awk '{print $2}'`
+    - ndkzip=${ndk}-linux-x86_64.zip
+    - curl --silent http://dl.google.com/android/repository/${ndkzip} > $ndkzip
+    - unzip -q $ndkzip
+    - export ANDROID_NDK_HOME=`pwd`/$ndk
+    - export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
+
+    - export ABI=`echo $CI_JOB_NAME | awk '{print $5}'`
+    - test $ABI = "armeabi-v7a"
+        && { host_triplet=armv7a-linux-androideabi; alt_triplet=arm-linux-androideabi; arch=arm; path_triplet=$alt_triplet; }
+    - test $ABI = "arm64-v8a"
+        && { host_triplet=aarch64-linux-androideabi; alt_triplet=$host_triplet; arch=arm64; path_triplet=$host_triplet; }
+    - test $ABI = "x86"
+        && { host_triplet=i686-linux-android; alt_triplet=$host_triplet; arch=$ABI; path_triplet=$ABI; }
+    - test $ABI = "x86_64"
+        && { host_triplet=$ABI-linux-android; alt_triplet=$host_triplet; arch=$ABI; path_triplet=$ABI; }
+    - platform=21
+    - export CC=${host_triplet}${platform}-clang
+
+    - mkdir -p external
+    - export EXTERNAL_ROOT=`pwd`/external
+
+    # if using an old NDK, then more configuration is needed
+    - if ! which $CC; then
+        export CC="${alt_triplet}-gcc --sysroot=$ANDROID_NDK_HOME/platforms/android-$platform/arch-$arch -isystem $ANDROID_NDK_HOME/sysroot/usr/include/$alt_triplet -isystem $ANDROID_NDK_HOME/sysroot/usr/include";
+        export CFLAGS=-D__ANDROID_API__=$platform
+        export PATH=$ANDROID_NDK_HOME/toolchains/${alt_triplet}-4.9/prebuilt/linux-x86_64/bin:$PATH;
+      fi
+
+    - mkdir openssl
+    - curl --silent --location https://github.com/openssl/openssl/archive/OpenSSL_1_1_1d.tar.gz
+        | tar -xz --directory=openssl --strip-components=1
+    - cd openssl
+    - ./Configure no-shared android-$arch -D__ANDROID_API__=$platform
+        --prefix=$EXTERNAL_ROOT --openssldir=$EXTERNAL_ROOT
+    - make install_dev
+    - cd ..
+
+    - mkdir libevent
+    - curl --silent --location https://github.com/libevent/libevent/archive/release-2.1.11-stable.tar.gz
+        | tar -xz --directory=libevent --strip-components=1
+    - cd libevent
+    - ./autogen.sh
+    - ./configure
+        --host="${alt_triplet}"
+        --disable-libevent-regress
+        --disable-samples
+        --disable-shared
+        --prefix=$EXTERNAL_ROOT
+    - make install
+    - cd ..
+
+    - ./autogen.sh
+    - ./configure
+        --host="$host_triplet"
+        --enable-android
+        --enable-static-libevent --with-libevent-dir=$EXTERNAL_ROOT
+        --enable-static-openssl --with-openssl-dir=$EXTERNAL_ROOT
+        --prefix=$EXTERNAL_ROOT
+
+    - grep -E '^# *define +ENABLE_OPENSSL +1$' orconfig.h
+    - grep -E '^# *define +HAVE_TLS_METHOD +1$' orconfig.h
+
+    - make install
+
+    - export AVD_SDK=`echo $CI_JOB_NAME | awk '{print $3}'`
+    - export AVD_TAG=`echo $CI_JOB_NAME | awk '{print $4}'`
+    - export AVD_PACKAGE="system-images;android-${AVD_SDK};${AVD_TAG};${ABI}"
+    - echo $AVD_PACKAGE
+    - emulator -accel-check || true
+    - rm -rf $ANDROID_HOME/emulator;
+    - wget -q http://dl.google.com/android/repository/emulator-linux-5264690.zip;
+    - echo "48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea  emulator-linux-5264690.zip" | sha1sum -c;
+    - unzip -qq -d $ANDROID_HOME emulator-linux-5264690.zip;
+    - grep -v '^License' $ANDROID_HOME/tools/source.properties
+    - echo y | sdkmanager "platforms;android-$AVD_SDK" > /dev/null
+    - if ! avdmanager list avd | grep "Name. avd$AVD_SDK$"; then
+          set -x;
+          rm -rf ~/.android/avd  $ANDROID_HOME/system-images;
+          echo y | sdkmanager "$AVD_PACKAGE" > /dev/null;
+          echo no | avdmanager create avd --name avd$AVD_SDK --tag "$AVD_TAG" --package "$AVD_PACKAGE" --sdcard 64M --device "Nexus 5";
+          export RAMSIZE="`sed -n 's,^MemAvailable:[^0-9]*\([0-9][0-9]*\)[^0-9]*$,\1,p' /proc/meminfo`";
+          if [ $RAMSIZE -le 2048 ]; then
+              sed -i '/^hw\.ramSize\s*=.*/d' ~/.android/avd/*.avd/config.ini;
+              echo "hw.ramSize=1024" >> ~/.android/avd/*.avd/config.ini;
+          fi;
+          avdmanager list avd;
+          set +x;
+      fi
+    - grep -v '^License' $ANDROID_HOME/system-images/android-$AVD_SDK/$AVD_TAG/$ABI/source.properties
+    - adb start-server
+    - ls -l ~/.android
+    - emulator -version
+    - emulator -avd avd$AVD_SDK
+        -no-audio
+        -no-snapstorage
+        -no-window
+        -skip-adb-auth
+        -verbose
+        -wipe-data
+        &
+    - wait-for-emulator
+    - adb devices
+    - adb shell input keyevent 82 &
+
+    # adb shell on older emulators didn't pass the return value, so we need tricks
+    # https://stackoverflow.com/a/58452689
+    - EXITVALUE=0
+    - cd src/test
+    - for f in test test-memwipe test-slow; do
+        printf "\n\n------------------------------------------\n$f\n";
+        adb -e push $f /data/local/tmp/;
+        adb -e shell "cd /data/local/tmp; ./$f"'; echo -n $? > '"$f.result";
+        test `adb -e shell "cat /data/local/tmp/$f.result"` = "0" || EXITVALUE=1;
+      done
+    - exit $EXITVALUE
+
+android r17b 22 default armeabi-v7a:
+  <<: *android-template
+
+android r20 22 default armeabi-v7a:
+  <<: *android-template
+
+
+asciidoc:
+  image: debian:testing-slim
+  <<: *debian-template
+  script:
+    - apt-get install asciidoc docbook-xml docbook-xsl xmlto
+    - ./autogen.sh
+    - ./configure --enable-asciidoc --disable-module-dirauth --disable-unittests
+    - make install
+    - test -e /usr/local/share/man/man1/tor.1
+    - test -e /usr/local/share/man/man1/tor-gencert.1
+    - test -e /usr/local/share/man/man1/tor-resolve.1
+    - test -e /usr/local/share/man/man1/torify.1
+    - test -e /usr/local/share/man/man1/tor-print-ed-signing-cert.1
+
+
+debian_testing:
+  image: debian:testing
+  <<: *debian-template
+  <<: *test-template
+
+debian_testing_clang:
+  image: debian:testing
+  <<: *debian-template
+  <<: *test-template
+
+debian_testing_rust:
+  image: debian:testing
+  <<: *debian-template
+  <<: *test-template
+
+debian_testing_clang_rust:
+  image: debian:testing
+  <<: *debian-template
+  <<: *test-template
+
+debian_stable:
+  image: debian:stable
+  <<: *debian-template
+  <<: *test-template
+
+ubuntu_rolling:
+  image: ubuntu:rolling
+  <<: *debian-template
+  <<: *test-template
+
+ubuntu_lts:
+  image: ubuntu:latest
+  <<: *debian-template
+  <<: *test-template
+
+ubuntu_xenial:
+  image: ubuntu:xenial
+  <<: *debian-template
+  <<: *test-template
+
+
+archlinux:
+  image: archlinux/base
+  <<: *artifacts-template
+  before_script:
+    - pacman --sync --sysupgrade --refresh --noconfirm
+        autoconf
+        automake
+        gcc
+        glibc
+        grep
+        libevent
+        libseccomp
+        make
+        openssl
+        pkg-config
+        xz
+        zlib
+  <<: *test-template
+
+
+alpine:
+  image: alpine:3.7
+  <<: *artifacts-template
+  before_script:
+    - apk add --no-cache
+        autoconf
+        automake
+        bash
+        build-base
+        gcc
+        libevent-dev
+        libseccomp-dev
+        make
+        openssl-dev
+        python
+        sed
+        xz-dev
+        zlib-dev
+    - sed -i '/<sys\/fcntl.h>/d' src/lib/crypt_ops/crypto_rand.c
+  <<: *test-template
+
+centos_latest:
+  image: centos:latest
+  <<: *artifacts-template
+  before_script:
+    # tricks to hopefully make runs more reliable
+    - echo "timeout=600" >> /etc/yum.conf
+    - echo "retries=50" >> /etc/yum.conf
+    - echo "keepcache=True" >> /etc/yum.conf
+    - cat /etc/yum.conf
+
+    - yum -y update || yum -y update
+    - yum -y groupinstall "Development Tools"
+    - yum -y install
+        automake
+        libevent-devel
+        openssl-devel
+        xz-devel
+        zlib-devel
+  <<: *test-template
+
+fedora_latest:
+  image: fedora:latest
+  <<: *artifacts-template
+  before_script:
+    # tricks to hopefully make runs more reliable
+    - echo "timeout=600" >> /etc/dnf/dnf.conf
+    - echo "retries=50" >> /etc/dnf/dnf.conf
+    - echo "keepcache=True" >> /etc/dnf/dnf.conf
+
+    - dnf -y update || dnf -y update
+    - dnf -y install @development-tools
+        automake
+        diffutils
+        libevent-devel
+        libscrypt-devel
+        libseccomp-devel
+        openssl-devel
+        python
+        xz-devel
+        zlib-devel
+  <<: *test-template