GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

ci-driver.sh 10.3 KB
Newer Older
1 2
#!/bin/bash

3 4 5 6 7
####
# DO NOT EDIT THIS FILE IN MASTER.  ONLY EDIT IT IN THE OLDEST SUPPORTED
# BRANCH, THEN MERGE FORWARD.
####

8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
# This script is used to build Tor for continuous integration.  It should
# be kept the same for all supported Tor versions.
#
# It's subject to the regular Tor license; see LICENSE for copying
# information.

set -o errexit
set -o nounset

# Options for this script.
DEBUG_CI="${DEBUG_CI:-no}"
COLOR_CI="${COLOR_CI:-yes}"

# Options for which CI system this is.
ON_GITLAB="${ON_GITLAB:-yes}"

# Options for how to build Tor.  All should be yes/no.
FATAL_WARNINGS="${FATAL_WARNINGS:-yes}"
HARDENING="${HARDENING:-no}"
COVERAGE="${COVERAGE:-no}"
RUST="${RUST:-no}"
DOXYGEN="${DOXYGEN:-no}"
ASCIIDOC="${ASCIIDOC:-no}"

# Options for which tests to run.   All should be yes/no.
CHECK="${CHECK:-yes}"
STEM="${STEM:-no}"
CHUTNEY="${CHUTNEY:-no}"
DISTCHECK="${DISTCHECK:-no}"

# Options for where the Tor source is.
CI_SRCDIR="${CI_SRCDIR:-.}"

# Options for where to build.
CI_BUILDDIR="${CI_BUILDDIR:-./build}"

# How parallel should we run make?
MAKE_J_OPT="${MAKE_J_OPT:--j4}"
# Should we stop after make finds an error?
MAKE_K_OPT="${MAKE_K_OPT:--k}"

# What make target should we use for chutney?
CHUTNEY_MAKE_TARGET="${CHUTNEY_MAKE_TARGET:-test-network}"

# Where do we find our additional testing tools?
CHUTNEY_PATH="${CHUTNEY_PATH:-}"
STEM_PATH="${STEM_PATH:-}"

#############################################################################
# Preliminary functions.

# Terminal coloring/emphasis stuff.
if [[ "${COLOR_CI}" == "yes" ]]; then
    T_RED=$(tput setaf 1 || true)
    T_GREEN=$(tput setaf 2 || true)
    T_DIM=$(tput dim || true)
    T_BOLD=$(tput bold || true)
    T_RESET=$(tput sgr0 || true)
else
    T_RED=
    T_GREEN=
    T_DIM=
    T_BOLD=
    T_RESET=
fi

function error()
{
    echo "${T_BOLD}${T_RED}ERROR:${T_RESET} $*" 1>&2
}
function die()
{
    echo "${T_BOLD}${T_RED}FATAL ERROR:${T_RESET} $*" 1>&2
    exit 1
}
function hooray()
{
    echo "${T_BOLD}${T_GREEN}$*${T_RESET}"
}

if [[ "${DEBUG_CI}" == "yes" ]]; then
    function debug()
    {
        echo "${T_DIM}(debug): $*${T_RESET}"
    }
else
    function debug()
    {
        :
    }
fi

function yes_or_no()
{
    local varname="$1"
    local value="${!varname}"
    debug "${varname} is ${value}"
    if [[ "${value}" != 'yes' && "${value}" != 'no' ]]; then
        die "${varname} must be 'yes' or 'no'.  Got unexpected value ${value}".
    fi
}

function incompatible()
{
    local varname1="$1"
    local varname2="$2"
    local val1="${!varname1}"
    local val2="${!varname2}"
    if [[ "${val1}" = 'yes' && "${val2}" = 'yes' ]]; then
        die "Cannot set both ${varname1} and ${varname2}: they are incompatible."
    fi
}

function runcmd()
{
    echo "${T_BOLD}\$ $*${T_RESET}"
    if ! "$@" ; then
        error "command '$*' has failed."
        return 1
    fi
}

function show_git_version()
{
    local tool="$1"
    local dir="$2"
    local version="?????"
    if [[ -e "$dir/.git" ]] ; then
        version=$(cd "$dir"; git rev-parse HEAD)
    fi
    echo "${T_BOLD}$tool:${T_RESET} $version"
}

if [[ "${ON_GITLAB}" == "yes" ]]; then
    function start_section()
    {
	local label="$1"
	local stamp
	stamp=$(date +%s)
	printf "section_start:%s:%s\r\e[0K" "$stamp" "$label"
	echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}"
    }
    function end_section()
    {
	local label="$1"
	local stamp
	stamp=$(date +%s)
	printf "section_end:%s:%s\r\e[0K" "$stamp" "$label"
    }
else
    function start_section()
    {
	true
    }
    function end_section()
    {
	true
    }
fi

if [[ "$*" == "" ]]; then
    RUN_STAGE_CONFIGURE="yes"
    RUN_STAGE_BUILD="yes"
    RUN_STAGE_TEST="yes"
else
    RUN_STAGE_CONFIGURE="no"
    RUN_STAGE_BUILD="no"
    RUN_STAGE_TEST="no"

    for stage in "$@"; do
	case "$stage" in
	    configure)
		RUN_STAGE_CONFIGURE="yes"
		;;
	    build)
		RUN_STAGE_BUILD="yes"
		;;
	    test)
		RUN_STAGE_TEST="yes"
		;;
	    *)
		error "Unknown stage $stage"
		;;
	esac
    done
fi

#############################################################################
# Validate inputs.

debug Validating inputs
yes_or_no DEBUG_CI
yes_or_no COLOR_CI
yes_or_no ON_GITLAB
yes_or_no FATAL_WARNINGS
yes_or_no HARDENING
yes_or_no COVERAGE
yes_or_no RUST
yes_or_no DOXYGEN
yes_or_no ASCIIDOC

yes_or_no CHECK
yes_or_no STEM
yes_or_no DISTCHECK

incompatible DISTCHECK CHECK
incompatible DISTCHECK CHUTNEY
incompatible DISTCHECK STEM
incompatible DISTCHECK COVERAGE
incompatible DISTCHECK DOXYGEN

if [[ "${CHUTNEY}" = yes && "${CHUTNEY_PATH}" = '' ]] ; then
    die "CHUTNEY is set to 'yes', but CHUTNEY_PATH was not specified."
fi

if [[ "${STEM}" = yes && "${STEM_PATH}" = '' ]] ; then
    die "STEM is set to 'yes', but STEM_PATH was not specified."
fi

#############################################################################
# Set up options for make and configure.

make_options=()
if [[ "$MAKE_J_OPT" != "" ]]; then
    make_options+=("$MAKE_J_OPT")
fi
if [[ "$MAKE_K_OPT" != "" ]]; then
    make_options+=("$MAKE_K_OPT")
fi

configure_options=()
if [[ "$FATAL_WARNINGS" == "yes" ]]; then
    configure_options+=("--enable-fatal-warnings")
fi
if [[ "$HARDENING" == "yes" ]]; then
    configure_options+=("--enable-fragile-hardening")
fi
if [[ "$COVERAGE" == "yes" ]]; then
    configure_options+=("--enable-coverage")
fi
if [[ "$RUST" == "yes" ]]; then
    configure_options+=("--enable-rust")
fi
if [[ "$ASCIIDOC" != "yes" ]]; then
    configure_options+=("--disable-asciidoc")
fi

#############################################################################
# Tell the user about our versions of different tools and packages.

uname -a
python -V || echo "no 'python' binary."
260
python3 -V || echo "no 'python3' binary."
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380

show_git_version Tor "${CI_SRCDIR}"
if [[ "${STEM}" = "yes" ]]; then
    show_git_version Stem "${STEM_PATH}"
fi
if [[ "${CHUTNEY}" = "yes" ]]; then
    show_git_version Chutney "${CHUTNEY_PATH}"
fi

#############################################################################
# Make sure the directories are all there.

# Make sure CI_SRCDIR exists and has a file we expect.
if [[ ! -d "$CI_SRCDIR" ]] ; then
    die "CI_SRCDIR=${CI_SRCDIR} is not a directory"
fi
if [[ ! -f "$CI_SRCDIR/src/core/or/or.h" ]] ; then
    die "CI_SRCDIR=${CI_SRCDIR} does not look like a Tor directory."
fi

# Make CI_SRCDIR absolute.
CI_SRCDIR=$(cd "$CI_SRCDIR" && pwd)

# Create an "artifacts" directory to copy artifacts into.
mkdir -p ./artifacts

if [[ "$RUN_STAGE_CONFIGURE" = "yes" ]]; then

    start_section "Autogen"
    runcmd cd "${CI_SRCDIR}"
    runcmd ./autogen.sh
    runcmd mkdir -p "${CI_BUILDDIR}"
    runcmd cd "${CI_BUILDDIR}"
    end_section "Autogen"

    # make the builddir absolute too.
    CI_BUILDDIR=$(pwd)

    start_section "Configure"
    if ! runcmd "${CI_SRCDIR}"/configure "${configure_options[@]}" ; then
	error "Here is the end of config.log:"
	runcmd tail config.log
	die "Unable to continue"
    fi
    end_section "Configure"
else
    debug "Skipping configure stage. Making sure that ${CI_BUILDDIR}/config.log exists."
    if [[ ! -d "${CI_BUILDDIR}" ]]; then
	die "Build directory ${CI_BUILDDIR} did not exist!";
    fi
    if [[ ! -f "${CI_BUILDDIR}/config.log" ]]; then
	die "Tor was not configured in ${CI_BUILDDIR}!";
    fi

    cp config.log "${CI_SRCDIR}"/artifacts

    runcmd cd "${CI_BUILDDIR}"
    CI_BUILDDIR=$(pwd)
fi

###############################
# Build Tor.

if [[ "$RUN_STAGE_BUILD" = "yes" ]] ; then
    if [[ "$DISTCHECK" = "no" ]]; then
	start_section "Build"
	runcmd make "${make_options[@]}" all
        cp src/app/tor "${CI_SRCDIR}"/artifacts
	end_section "Build"
    else
	export DISTCHECK_CONFIGURE_FLAGS="${configure_options[*]}"
	# XXXX Set make options?
	start_section Distcheck
	if runcmd make "${make_options[@]}" distcheck ; then
            hooray "Distcheck was successful. Nothing further will be done."
            # We have to exit early here, since we can't do any other tests.
            cp tor-*.tar.gz "${CI_SRCDIR}"/artifacts
            exit 0
	else
            error "Diagnostics:"
            runcmd make show-distdir-testlog || true
            runcmd make show-distdir-core || true
            die "Unable to continue."
	fi
	end_section Distcheck
    fi
fi
##############################
# Run tests.

if [[ "$RUN_STAGE_TEST" == "no" ]]; then
    echo "Skipping tests. Exiting now."
    exit 0
fi

if [[ "$RUN_STAGE_BUILD" = "no" ]] ; then
    debug "Skipped build stage. Making sure that ./src/app/tor exists."
    if [[ ! -f "./src/app/tor" ]]; then
	die "$(pwd)/src/app/tor does not exist"
    fi
fi

FAILED_TESTS=""

if [[ "${DOXYGEN}" = 'yes' ]]; then
    start_section Doxygen
    if runcmd make doxygen; then
	hooray "make doxygen has succeeded."
    else
	FAILED_TESTS="${FAILED_TESTS} doxygen"
    fi
    end_section Doxygen
fi

if [[ "${CHECK}" = "yes" ]]; then
    start_section "Check"
    if runcmd make "${make_options[@]}" check; then
        hooray "make check has succeeded."
    else
        error "Here are the contents of the test suite output:"
381
        runcmd cat test-suite.log || true
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
        FAILED_TESTS="${FAILED_TESTS} check"
    fi
    end_section "Check"
fi

if [[ "${CHUTNEY}" = "yes" ]]; then
    start_section "Chutney"
    if runcmd make "${CHUTNEY_MAKE_TARGET}"; then
        hooray "Chutney tests have succeeded"
    else
        error "Chutney says:"
        runcmd "${CHUTNEY_PATH}"/tools/diagnostics.sh || true
        # XXXX These next two should be part of a make target.
        runcmd ls test_network_log || true
        runcmd cat test_network_log || true
        FAILED_TESTS="${FAILED_TESTS} chutney"
    fi
    end_section "Chutney"
fi

if [[ "${STEM}" = "yes" ]]; then
   start_section "Stem"
   # XXXX This shold probably be part some test-stem make target.
   if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \
         python3 "${STEM_PATH}/run_tests.py" \
         --tor src/app/tor \
         --integ --test control.controller \
         --test control.base_controller \
         --test process \
         --log TRACE \
         --log-file stem.log ; then
       hooray "Stem tests have succeeded"
   else
       error "Stem output:"
       runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log
       runcmd grep -v "SocketClosed" stem.log | tail -1000
       FAILED_TESTS="${FAILED_TESTS} stem"
   fi
   end_section "Stem"
fi

# TODO: Coverage

if [[ "${FAILED_TESTS}" != "" ]]; then
    die "Failed tests: ${FAILED_TESTS}"
fi

hooray "Everything seems fine."