Unverified Commit a4577553 authored by Gregory Mierzwinski's avatar Gregory Mierzwinski Committed by GitHub
Browse files

Bug 1613483 - Add all Browsertime tests with visual metrics to Fenix repo. (#9087)

* Add visual-metrics docker type.

* Add required browsertime toolchain fetches.

* Add browsertime tests for technical and visual metrics.

* Run browsertime tests in a cron task.

* Run visual metrics on all browsertime tests.

* Use spaces instead of tabs, and resolve visual-metric nits.

* Enable browsertime on pull request for testing.

* Restrict PR tests to amazon on browsertime.

* First attempt using multi_dep.

* Add a primary dependency to browsertime.

* Try by not popping.

* Debug prints.

* Make one grouping per browsertime task.

* Try without the multi_dep transform.

* Delete dependent-tasks in visual-metrics transformer.

* Update setuptools installed and copy run-on-tasks-for.

* Use get when getting run-on-tasks-for.

* Add new pinned requirements.

* Try it.

* Set run-on-tasks-for properly.

* Remove print statement.

* Remove single_dep loader, and print statements.

* Remove run-on-tasks-for testing setting.

* Restart testing, and set user to root in visual-metrics Docker.

* Remove testing settings.

* Remove fetch-content from Docker.

* Change attributes grouping method.

* Run all tests as a check.

* Undo testing changes, and fix a bad test name.
parent 0d974fe2
......@@ -24,3 +24,9 @@ jobs:
treeherder-symbol: raptor-D
target-tasks-method: raptor
when: [{hour: 1, minute: 0}]
- name: browsertime
job:
type: decision-task
treeherder-symbol: btime-D
target-tasks-method: browsertime
when: [{hour: 1, minute: 0}]
---
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.transforms.browsertime:transforms
- fenix_taskgraph.transforms.notify:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- signing
- toolchain
primary-dependency: signing
only-for-build-types:
- performance-test
only-for-abis:
- armeabi-v7a
- arm64-v8a
job-defaults:
dependencies:
geckoview-nightly: geckoview-nightly
notify:
by-level:
'3':
email:
content: This calls for an action of the Performance team. Use the link to view it on Treeherder.
link:
text: Treeherder Job
href: 'https://treeherder.mozilla.org/#/jobs?repo={product_name}&revision={head_rev}&searchStr={task_name}'
on-reasons: [failed]
subject: '[{product_name}] Raptor-Browsertime job "{task_name}" failed'
to-addresses: [perftest-alerts@mozilla.com]
default: {}
run-on-tasks-for: []
treeherder:
kind: test
tier: 2
platform:
by-abi:
arm64-v8a: android-hw-p2-8-0-android-aarch64/opt
armeabi-v7a: android-hw-g5-7-0-arm7-api-16/opt
worker-type:
by-abi:
armeabi-v7a: t-bitbar-gw-perf-g5
arm64-v8a: t-bitbar-gw-perf-p2
worker:
max-run-time: 3600
env:
GECKO_HEAD_REPOSITORY: "https://hg.mozilla.org/mozilla-central"
MOZ_AUTOMATION: "1"
MOZ_HIDE_RESULTS_TABLE: "1"
MOZ_NO_REMOTE: "1"
MOZ_NODE_PATH: "/usr/local/bin/node"
MOZHARNESS_CONFIG: "raptor/android_hw_config.py"
MOZHARNESS_SCRIPT: "raptor_script.py"
NEED_XVFB: "false"
NO_FAIL_ON_TEST_ERRORS: "1"
XPCOM_DEBUG_BREAK: "warn"
artifacts:
- name: public/logs/
path: workspace/logs
type: directory
- name: public/test_info/
path: workspace/build/blobber_upload_dir
type: directory
run-visual-metrics: True
run:
using: run-task
checkout: false
run-as-root: true
command:
- 'bash'
- './test-linux.sh'
- '--cfg=mozharness/configs/raptor/android_hw_config.py'
- '--app=fenix'
- '--browsertime'
- '--cold'
- '--binary=org.mozilla.fenix.performancetest'
- '--activity=org.mozilla.fenix.browser.BrowserPerformanceTestActivity'
- '--download-symbols=ondemand'
- '--browsertime-node=$MOZ_FETCHES_DIR/node/bin/node'
- '--browsertime-geckodriver=$MOZ_FETCHES_DIR/geckodriver'
- '--browsertime-ffmpeg=$MOZ_FETCHES_DIR/ffmpeg-4.1.4-i686-static/bin/ffmpeg'
- '--browsertime-browsertimejs=$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js'
fetches:
toolchain:
- browsertime
- linux64-ffmpeg-4.1.4
- linux64-geckodriver
- linux64-minidump-stackwalk
- linux64-node
jobs:
tp6m-1-cold:
test-name: amazon
treeherder:
symbol: 'Btime(tp6m-1-c)'
tp6m-2-cold:
test-name: google
treeherder:
symbol: 'Btime(tp6m-2-c)'
tp6m-3-cold:
test-name: instagram
treeherder:
symbol: 'Btime(tp6m-3-c)'
tp6m-4-cold:
test-name: bing-search-restaurants
treeherder:
symbol: 'Btime(tp6m-4-c)'
tp6m-5-cold:
test-name: ebay-kleinanzeigen-search
treeherder:
symbol: 'Btime(tp6m-5-c)'
tp6m-6-cold:
test-name: amazon-search
treeherder:
symbol: 'Btime(tp6m-6-c)'
tp6m-7-cold:
test-name: wikipedia
treeherder:
symbol: 'Btime(tp6m-7-c)'
tp6m-8-cold:
test-name: booking
treeherder:
symbol: 'Btime(tp6m-8-c)'
tp6m-9-cold:
test-name: cnn-ampstories
treeherder:
symbol: 'Btime(tp6m-9-c)'
tp6m-10-cold:
test-name: bbc
treeherder:
symbol: 'Btime(tp6m-10-c)'
tp6m-11-cold:
test-name: microsoft-support
treeherder:
symbol: 'Btime(tp6m-11-c)'
tp6m-12-cold:
test-name: imdb
treeherder:
symbol: 'Btime(tp6m-12-c)'
tp6m-13-cold:
test-name: espn
treeherder:
symbol: 'Btime(tp6m-13-c)'
tp6m-14-cold:
test-name: facebook-cristiano
treeherder:
symbol: 'Btime(tp6m-14-c)'
tp6m-15-cold:
test-name: facebook
treeherder:
symbol: 'Btime(tp6m-15-c)'
tp6m-16-cold:
test-name: youtube
treeherder:
symbol: 'Btime(tp6m-16-c)'
tp6m-17-cold:
test-name: bing
treeherder:
symbol: 'Btime(tp6m-17-c)'
tp6m-18-cold:
test-name: ebay-kleinanzeigen
treeherder:
symbol: 'Btime(tp6m-18-c)'
tp6m-19-cold:
test-name: google-maps
treeherder:
symbol: 'Btime(tp6m-19-c)'
tp6m-20-cold:
test-name: youtube-watch
treeherder:
symbol: 'Btime(tp6m-20-c)'
tp6m-21-cold:
test-name: reddit
treeherder:
symbol: 'Btime(tp6m-21-c)'
tp6m-22-cold:
test-name: stackoverflow
treeherder:
symbol: 'Btime(tp6m-22-c)'
tp6m-23-cold:
test-name: jianshu
treeherder:
symbol: 'Btime(tp6m-23-c)'
tp6m-24-cold:
test-name: allrecipes
treeherder:
symbol: 'Btime(tp6m-24-c)'
tp6m-25-cold:
test-name: web-de
treeherder:
symbol: 'Btime(tp6m-25-c)'
tp6m-26-cold:
test-name: aframe
treeherder:
symbol: 'Btime(tp6m-26-c)'
tp6m-27-cold:
test-name: cnn
treeherder:
symbol: 'Btime(tp6m-27-c)'
......@@ -13,6 +13,7 @@ treeherder:
'productionFennec': 'Production-related tasks with same APK configuration as Fennec'
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'
'Btime': 'Raptor-Browsertime tests'
task-priority: highest
......
......@@ -19,3 +19,6 @@ jobs:
ui-tests:
parent: base
symbol: I(ui-tests)
visual-metrics:
parent: base
symbol: I(visual-metrics)
......@@ -10,6 +10,27 @@ job-defaults:
using: index-search
jobs:
browsertime:
description: "Browsertime toolchain"
attributes:
toolchain-artifact: public/build/browsertime.tar.bz2
run:
index-search:
- gecko.cache.level-3.toolchains.v3.browsertime.latest
linux64-ffmpeg-4.1.4:
description: "FFMPEG fetch"
attributes:
toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
run:
index-search:
- gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
linux64-geckodriver:
description: "Geckodriver toolchain"
attributes:
toolchain-artifact: public/build/geckodriver.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
linux64-minidump-stackwalk:
description: "minidump_stackwalk toolchain"
attributes:
......@@ -17,3 +38,17 @@ jobs:
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
linux64-node:
description: "Node.js toolchain"
attributes:
toolchain-artifact: public/build/node.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
visual-metrics:
description: "Browsertime visual metrics analsyis script"
attributes:
toolchain-artifact: public/visualmetrics.py
run:
index-search:
- gecko.cache.level-3.content.v1.visual-metrics.latest
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: fenix_taskgraph.loader.multi_dep:loader
kind-dependencies:
- browsertime
- toolchain
primary-dependency:
- browsertime
group-by: attributes
only-for-attributes:
- run-visual-metrics
transforms:
- fenix_taskgraph.transforms.visual_metrics:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-template:
description: "Run visual metrics calculations on Raptor"
run-on-projects: []
run-on-tasks-for: []
worker-type: b-android
treeherder:
tier: 3
kind: other
worker:
docker-image: {in-tree: visual-metrics}
max-run-time: 900
artifacts:
- type: file
name: public/perfherder-data.json
path: /builds/worker/artifacts/perfherder-data.json
- type: file
name: public/summary.json
path: /builds/worker/artifacts/summary.json
fetches:
toolchain:
- visual-metrics
run:
using: run-task
command: /builds/worker/bin/run-visual-metrics.py -- --orange --perceptual --contentful --force --renderignore 5 --json --viewport
checkout: false
run-as-root: true
# %ARG DOCKER_IMAGE_PARENT
FROM $DOCKER_IMAGE_PARENT
MAINTAINER Gregory Mierzwinski <gmierzwinski@mozilla.com>
# run-task expects to run as root
USER root
RUN apt-get update -qq && \
apt-get install -y \
ffmpeg \
imagemagick \
pyssim \
python \
python-pil \
python3 \
python3-pip
WORKDIR /builds/worker
USER worker:worker
COPY requirements.txt /builds/worker/requirements.txt
RUN pip3 install setuptools==46.0.0
RUN pip3 install --require-hashes -r /builds/worker/requirements.txt && \
rm /builds/worker/requirements.txt
COPY run-visual-metrics.py /builds/worker/bin/run-visual-metrics.py
COPY performance-artifact-schema.json /builds/worker/performance-artifact-schema.json
USER root
RUN chmod +x /builds/worker/bin/run-visual-metrics.py
VOLUME /builds/worker/artifacts/
{
"definitions": {
"application_schema": {
"properties": {
"name": {
"title": "Application under performance test",
"enum": [
"firefox",
"chrome",
"chromium",
"fennec",
"geckoview",
"refbrow",
"fenix"
],
"maxLength": 10,
"type": "string"
},
"version": {
"title": "Application's version",
"maxLength": 40,
"type": "string"
}
},
"required": ["name"],
"type": "object"
},
"framework_schema": {
"properties": {
"name": {
"title": "Framework name",
"type": "string"
}
},
"type": "object"
},
"subtest_schema": {
"properties": {
"name": {
"title": "Subtest name",
"type": "string"
},
"publicName": {
"title": "Public subtest name",
"description": "Allows renaming test's name, without breaking existing performance data series",
"maxLength": 30,
"type": "string"
},
"value": {
"description": "Summary value for subtest",
"title": "Subtest value",
"type": "number",
"minimum": -1000000000000.0,
"maximum": 1000000000000.0
},
"unit": {
"title": "Measurement unit",
"type": "string",
"minLength": 1,
"maxLength": 20
},
"lowerIsBetter": {
"description": "Whether lower values are better for subtest",
"title": "Lower is better",
"type": "boolean"
},
"shouldAlert": {
"description": "Whether we should alert",
"title": "Should alert",
"type": "boolean"
},
"alertThreshold": {
"description": "% change threshold before alerting",
"title": "Alert threshold",
"type": "number",
"minimum": 0.0,
"maximum": 1000.0
},
"minBackWindow": {
"description": "Minimum back window to use for alerting",
"title": "Minimum back window",
"type": "number",
"minimum": 1,
"maximum": 255
},
"maxBackWindow": {
"description": "Maximum back window to use for alerting",
"title": "Maximum back window",
"type": "number",
"minimum": 1,
"maximum": 255
},
"foreWindow": {
"description": "Fore window to use for alerting",
"title": "Fore window",
"type": "number",
"minimum": 1,
"maximum": 255
}
},
"required": [
"name",
"value"
],
"type": "object"
},
"suite_schema": {
"properties": {
"name": {
"title": "Suite name",
"type": "string"
},
"publicName": {
"title": "Public suite name",
"description": "Allows renaming suite's name, without breaking existing performance data series",
"maxLength": 30,
"type": "string"
},
"tags": {
"type": "array",
"title": "Free form tags, which ease the grouping & searching of performance tests",
"description": "Similar to extraOptions, except it does not break existing performance data series",
"items": {
"type": "string",
"pattern": "^[a-zA-Z0-9]{1,24}$"
},
"uniqueItems": true,
"maxItems": 14
},
"extraOptions": {
"type": "array",
"title": "Extra options used in running suite",
"items": {
"type": "string",
"maxLength": 100
},
"uniqueItems": true,
"maxItems": 8
},
"subtests": {
"items": {
"$ref": "#/definitions/subtest_schema"
},
"title": "Subtests",
"type": "array"
},
"value": {
"title": "Suite value",
"type": "number",
"minimum": -1000000000000.0,
"maximum": 1000000000000.0
},
"unit": {
"title": "Measurement unit",
"type": "string",
"minLength": 1,
"maxLength": 20
},
"lowerIsBetter": {
"description": "Whether lower values are better for suite",
"title": "Lower is better",
"type": "boolean"
},
"shouldAlert": {
"description": "Whether we should alert on this suite (overrides default behaviour)",
"title": "Should alert",
"type": "boolean"
},
"alertThreshold": {
"description": "% change threshold before alerting",
"title": "Alert threshold",
"type": "number",
"minimum": 0.0,
"maximum": 1000.0
},
"minBackWindow": {
"description": "Minimum back window to use for alerting",
"title": "Minimum back window",
"type": "integer",
"minimum": 1,
"maximum": 255
},
"maxBackWindow": {
"description": "Maximum back window to use for alerting",
"title": "Maximum back window",
"type": "integer",
"minimum": 1,
"maximum": 255
},
"foreWindow": {
"description": "Fore window to use for alerting",
"title": "Fore window",
"type": "integer",
"minimum": 1,
"maximum": 255
}
},
"required": [
"name",
"subtests"
],
"type": "object"
}
},
"description": "Structure for submitting performance data as part of a job",
"id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#",
"properties": {
"application":{
"$ref": "#/definitions/application_schema"
},
"framework": {
"$ref": "#/definitions/framework_schema"
},