ui-test.sh 3.96 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# This script does the following:
# 1. Retrieves gcloud service account token
# 2. Activates gcloud service account
# 3. Connects to google Firebase (using TestArmada's Flank tool)
# 4. Executes UI tests
# 5. Puts test artifacts into the test_artifacts folder

# NOTE:
# Flank supports sharding across multiple devices at a time, but gcloud API
# only supports 1 defined APK per test run.


# If a command fails then do not proceed and fail this script too.
set -e

#########################
# The command line help #
#########################
display_help() {
    echo "Usage: $0 Build_Variant [Number_Shards...]"
    echo
    echo "Examples:"
    echo "To run UI tests on ARM device shard (1 test / shard)"
29
    echo "$ ui-test.sh arm64-v8a -1"
30
31
    echo
    echo "To run UI tests on X86 device (on 3 shards)"
32
    echo "$ ui-test.sh x86 3"
33
34
35
    echo
}

Johan Lorenzo's avatar
Johan Lorenzo committed
36
37
38
39
40
41
get_abs_filename() {
  relative_filename="$1"
  echo "$(cd "$(dirname "$relative_filename")" && pwd)/$(basename "$relative_filename")"
}


42
43
44
45
46
47
48
# Basic parameter check
if [[ $# -lt 1 ]]; then
    echo "Error: please provide at least one build variant (arm|x86)"
    display_help
    exit 1
fi

49
device_type="$1"  # arm64-v8a | armeabi-v7a | x86_64 | x86
Johan Lorenzo's avatar
Johan Lorenzo committed
50
51
52
53
APK_APP="$2"
APK_TEST="$3"
if [[ ! -z "$4" ]]; then
    num_shards=$4
54
55
56
57
fi

JAVA_BIN="/usr/bin/java"
PATH_TEST="./automation/taskcluster/androidTest"
58
FLANK_BIN="/builds/worker/test-tools/flank.jar"
59
60
ARTIFACT_DIR="/builds/worker/artifacts"
RESULTS_DIR="${ARTIFACT_DIR}/results"
61
62
63
64
65

echo
echo "ACTIVATE SERVICE ACCT"
echo
# this is where the Google Testcloud project ID is set
Johan Lorenzo's avatar
Johan Lorenzo committed
66
gcloud config set project "$GOOGLE_PROJECT"
67
68
echo

Johan Lorenzo's avatar
Johan Lorenzo committed
69
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
70
71
72
echo
echo

73
# Disable exiting on error. If the tests fail we want to continue
74
75
76
# and try to download the artifacts. We will exit with the actual error code later.
set +e

77
78
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
    flank_template="${PATH_TEST}/flank-${device_type}.yml"
79
80
81
82
elif [[ "${device_type}" == "x86-start-test" ]]; then
    flank_template="${PATH_TEST}/flank-x86-start-test.yml"
elif [[ "${device_type}" == "arm-start-test" ]]; then
    flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml"
isabelrios's avatar
isabelrios committed
83
84
elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then
    flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml"
85
else
86
    echo "FAILURE: flank config file not found!"
87
    exitcode=1
88
89
fi

Johan Lorenzo's avatar
Johan Lorenzo committed
90
91
APK_APP="$(get_abs_filename $APK_APP)"
APK_TEST="$(get_abs_filename $APK_TEST)"
92
echo "device_type: ${device_type}"
Johan Lorenzo's avatar
Johan Lorenzo committed
93
echo "APK_APP: ${APK_APP}"
94
echo "APK_TEST: ${APK_TEST}"
95
96
97
98

# function to exit script with exit code from test run.
# (Only 0 if all test executions passed)
function failure_check() {
99
100
    echo
    echo
101
    if [[ $exitcode -ne 0 ]]; then
102
        echo "FAILURE: UI test run failed, please check above URL"
103
    else
104
	      echo "All UI test(s) have passed!"
105
    fi
106
107
108
109

    echo
    echo "RESULTS"
    echo
110
111
112
113
114
115
116
117
118

    mkdir -p /builds/worker/artifacts/github
    chmod +x $PATH_TEST/parse-ui-test.py
    $PATH_TEST/parse-ui-test.py \
        --exit-code "${exitcode}" \
        --log flank.log \
        --results "${RESULTS_DIR}" \
        --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \
	--device-type "${device_type}"
119
120
}

121
122
123
echo
echo "FLANK VERSION"
echo
Johan Lorenzo's avatar
Johan Lorenzo committed
124
$JAVA_BIN -jar $FLANK_BIN --version
125
126
127
echo
echo

128
129
130
echo
echo "EXECUTE TEST(S)"
echo
131
132
# Note that if --local-results-dir is "results", timestamped sub-directory will
# contain the results. For any other value, the directory itself will have the results.
133
set -o pipefail && $JAVA_BIN -jar $FLANK_BIN android run \
134
135
136
137
138
139
140
	--config=$flank_template \
	--max-test-shards=$num_shards \
	--app=$APK_APP --test=$APK_TEST \
	--local-result-dir="${RESULTS_DIR}" \
	--project=$GOOGLE_PROJECT \
	| tee flank.log

141
142
143
exitcode=$?
failure_check

144
exit $exitcode