Unverified Commit 88baeceb authored by Gregory Mierzwinski's avatar Gregory Mierzwinski Committed by GitHub
Browse files

Bug 1636589 - Update visual-metrics, and geckodriver archive path. (#10558)

* Fix browsertime failures.

* Run a browsertime test.

* Undo browsertime test.
parent 55a8bf7f
...@@ -27,7 +27,7 @@ linux64-ffmpeg-4.1.4: ...@@ -27,7 +27,7 @@ linux64-ffmpeg-4.1.4:
linux64-geckodriver: linux64-geckodriver:
attributes: attributes:
toolchain-artifact: public/build/geckodriver.tar.xz toolchain-artifact: public/build/geckodriver.tar.gz
description: "Geckodriver toolchain" description: "Geckodriver toolchain"
run: run:
index-search: index-search:
...@@ -20,10 +20,10 @@ WORKDIR /builds/worker ...@@ -20,10 +20,10 @@ WORKDIR /builds/worker
USER worker:worker USER worker:worker
COPY requirements.txt /builds/worker/requirements.txt COPY requirements.txt /builds/worker/requirements.txt
RUN pip3 install setuptools==46.0.0
RUN pip3 install --require-hashes -r /builds/worker/requirements.txt && \ RUN pip3 install --require-hashes -r /builds/worker/requirements.txt && \
rm /builds/worker/requirements.txt rm /builds/worker/requirements.txt
COPY similarity.py /builds/worker/bin/similarity.py
COPY run-visual-metrics.py /builds/worker/bin/run-visual-metrics.py COPY run-visual-metrics.py /builds/worker/bin/run-visual-metrics.py
COPY performance-artifact-schema.json /builds/worker/performance-artifact-schema.json COPY performance-artifact-schema.json /builds/worker/performance-artifact-schema.json
# Dependency hashes must be for python3.6
# Direct dependencies # Direct dependencies
attrs==19.1.0 --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 attrs==19.1.0 --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79
structlog==19.1.0 --hash=sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536 structlog==19.1.0 --hash=sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536
voluptuous==0.11.5 --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1 voluptuous==0.11.5 --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1
jsonschema==3.2.0 --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 jsonschema==3.2.0 --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163
numpy==1.18.3 --hash=sha256:a551d8cc267c634774830086da42e4ba157fa41dd3b93982bc9501b284b0c689
scipy==1.4.1 --hash=sha256:386086e2972ed2db17cebf88610aab7d7f6e2c0ca30042dc9a89cf18dcc363fa
matplotlib==3.0.3 --hash=sha256:e8d1939262aa6b36d0c51f50a50a43a04b9618d20db31e6c0192b1463067aeef
opencv-python== --hash=sha256:dcb8da8c5ebaa6360c8555547a4c7beb6cd983dd95ba895bb78b86cc8cf3de2b
# Transitive dependencies # Transitive dependencies
importlib_metadata==1.1.0 --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742 importlib_metadata==1.1.0 --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742
more_itertools==8.0.0 --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45 more_itertools==8.0.0 --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45
pyrsistent==0.15.6 --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b pyrsistent==0.15.6 --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b
setuptools==46.0.0 --hash=sha256:693e0504490ed8420522bf6bc3aa4b0da6a9f1c80c68acfb4e959275fd04cd82
six==1.12.0 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c six==1.12.0 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c
zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335
cycler==0.10.0 --hash=sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d
kiwisolver==1.1.0 --hash=sha256:400599c0fe58d21522cae0e8b22318e09d9729451b17ee61ba8e1e7c0346565c
pyparsing==2.4.7 --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
python-dateutil==2.8.1 --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
setuptools==46.1.3 --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee
...@@ -47,6 +47,7 @@ JOB_SCHEMA = Schema( ...@@ -47,6 +47,7 @@ JOB_SCHEMA = Schema(
{Required("test_name"): str, Required("browsertime_json_path"): str} {Required("test_name"): str, Required("browsertime_json_path"): str}
], ],
Required("application"): {Required("name"): str, "version": str}, Required("application"): {Required("name"): str, "version": str},
Required("extra_options"): [str],
} }
) )
...@@ -154,13 +155,13 @@ def read_json(json_path, schema): ...@@ -154,13 +155,13 @@ def read_json(json_path, schema):
The contents of the file at ``json_path`` interpreted as JSON. The contents of the file at ``json_path`` interpreted as JSON.
""" """
try: try:
with open(str(json_path), "r") as f: with open(str(json_path), "r", encoding="utf-8", errors="ignore") as f:
data = json.load(f) data = json.load(f)
except Exception: except Exception:
log.error("Could not read JSON file", path=json_path, exc_info=True) log.error("Could not read JSON file", path=json_path, exc_info=True)
raise raise
log.info("Loaded JSON from file", path=json_path, read_json=data) log.info("Loaded JSON from file", path=json_path)
try: try:
schema(data) schema(data)
...@@ -202,9 +203,9 @@ def main(log, args): ...@@ -202,9 +203,9 @@ def main(log, args):
tar.extractall(path=str(fetch_dir)) tar.extractall(path=str(fetch_dir))
except Exception: except Exception:
log.error( log.error(
"Could not read extract browsertime results archive", "Could not read/extract browsertime results archive",
path=browsertime_results_path, path=browsertime_results_path,
exc_info=True, exc_info=True
) )
return 1 return 1
log.info("Extracted browsertime results", path=browsertime_results_path) log.info("Extracted browsertime results", path=browsertime_results_path)
...@@ -213,6 +214,11 @@ def main(log, args): ...@@ -213,6 +214,11 @@ def main(log, args):
jobs_json_path = fetch_dir / "browsertime-results" / "jobs.json" jobs_json_path = fetch_dir / "browsertime-results" / "jobs.json"
jobs_json = read_json(jobs_json_path, JOB_SCHEMA) jobs_json = read_json(jobs_json_path, JOB_SCHEMA)
except Exception: except Exception:
"Could not open the jobs.json file",
return 1 return 1
jobs = [] jobs = []
...@@ -223,6 +229,11 @@ def main(log, args): ...@@ -223,6 +229,11 @@ def main(log, args):
try: try:
browsertime_json = read_json(browsertime_json_path, BROWSERTIME_SCHEMA) browsertime_json = read_json(browsertime_json_path, BROWSERTIME_SCHEMA)
except Exception: except Exception:
"Could not open a browsertime.json file",
return 1 return 1
for site in browsertime_json: for site in browsertime_json:
...@@ -272,6 +283,35 @@ def main(log, args): ...@@ -272,6 +283,35 @@ def main(log, args):
"type": "vismet", "type": "vismet",
"suites": suites, "suites": suites,
} }
for entry in suites:
entry["extraOptions"] = jobs_json["extra_options"]
# Try to get the similarity for all possible tests, this means that we
# will also get a comparison of recorded vs. live sites to check
# the on-going quality of our recordings.
similarity = None
if "android" in os.getenv("TC_PLATFORM", ""):
from similarity import calculate_similarity
similarity = calculate_similarity(jobs_json, fetch_dir, OUTPUT_DIR, log)
except Exception:
log.info("Failed to calculate similarity score", exc_info=True)
if similarity:
"name": "Similarity3D",
"value": similarity[0],
"replicates": [similarity[0]],
"lowerIsBetter": False,
"unit": "a.u.",
"name": "Similarity2D",
"value": similarity[1],
"replicates": [similarity[1]],
"lowerIsBetter": False,
"unit": "a.u.",
# Validates the perf data complies with perfherder schema. # Validates the perf data complies with perfherder schema.
# The perfherder schema uses jsonschema so we can't use voluptuous here. # The perfherder schema uses jsonschema so we can't use voluptuous here.
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import cv2
import json
import numpy as np
import os
import pathlib
import shutil
import socket
import tarfile
import tempfile
import urllib
from functools import wraps
from matplotlib import pyplot as plt
from scipy.stats import spearmanr
def open_data(file):
return cv2.VideoCapture(str(file))
def socket_timeout(value=120):
"""Decorator for socket timeouts."""
def _socket_timeout(func):
def __socket_timeout(*args, **kw):
old = socket.getdefaulttimeout()
return func(*args, **kw)
return __socket_timeout
return _socket_timeout
def query_activedata(query_json, log):
"""Used to run queries on active data."""
active_data_url = "http://activedata.allizom.org/query"
req = urllib.request.Request(active_data_url)
req.add_header("Content-Type", "application/json")
jsondata = json.dumps(query_json)
jsondataasbytes = jsondata.encode("utf-8")
req.add_header("Content-Length", len(jsondataasbytes))
log.info("Querying Active-data...")
response = urllib.request.urlopen(req, jsondataasbytes)
log.info("Status: %s" % {str(response.getcode())})
data = json.loads(response.read().decode("utf8").replace("'", '"'))["data"]
return data
def download(url, loc, log):
"""Downloads from a url (with a timeout)."""
log.info("Downloading %s" % url)
urllib.request.urlretrieve(url, loc)
except Exception as e:
return False
return True
def get_frames(video):
"""Gets all frames from a video into a list."""
allframes = []
while video.isOpened():
ret, frame = video.read()
if ret:
# Convert to gray to simplify the process
allframes.append(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY))
return allframes
def calculate_similarity(jobs_json, fetch_dir, output, log):
"""Calculates the similarity score against the last live site test.
The technique works as follows:
1. Get the last live site test.
2. For each 15x15 video pairings, build a cross-correlation matrix:
1. Get each of the videos and calculate their histograms
across the full videos.
2. Calculate the correlation coefficient between these two.
3. Average the cross-correlation matrix to obtain the score.
The 2D similarity score is the same, except that it builds a histogram
from the final frame instead of the full video.
For finding the last live site, we use active-data. We search for
PGO android builds since this metric is only available for live sites that
run on android in mozilla-cental. Given that live sites currently
run on cron 3 days a week, then it's also reasonable to look for tasks
which have occurred before today and within the last two weeks at most.
But this is a TODO for future work, since we need to determine a better
way of selecting the last task (HG push logs?) - there's a lot that factors
into these choices, so it might require a multi-faceted approach.
jobs_json: The jobs JSON that holds extra information.
fetch_dir: The fetch directory that holds the new videos.
log: The logger.
Two similarity scores (3D, 2D) as a float, or None if there was an issue.
app = jobs_json["application"]["name"]
test = jobs_json["jobs"][0]["test_name"]
splittest = test.split("-cold")
cold = ""
if len(splittest) > 0:
cold = ".*cold"
test = splittest[0]
# PGO vs. OPT shouldn't matter much, but we restrict it to PGO builds here
# for android, and desktop tests have the opt/pgo restriction removed
plat = os.getenv("TC_PLATFORM", "")
if "android" in plat:
plat = plat.replace("/opt", "/pgo")
plat = plat.replace("/opt", "").replace("/pgo", "")
ad_query = {
"from": "task",
"limit": 1000,
"where": {
"and": [
"regexp": {
"run.name": ".*%s.*browsertime.*-live.*%s%s.*%s.*"
% (plat, app, cold, test)
{"not": {"prefix": {"run.name": "test-vismet"}}},
{"in": {"repo.branch.name": ["mozilla-central"]}},
{"gte": {"action.start_time": {"date": "today-week-week"}}},
{"lt": {"action.start_time": {"date": "today-1day"}}},
{"in": {"task.run.state": ["completed"]}},
"select": ["action.start_time", "run.name", "task.artifacts"],
# Run the AD query and find the browsertime videos to download
failed = False
data = query_activedata(ad_query, log)
except Exception as e:
failed = True
if failed or not data:
log.info("Couldn't get activedata data")
return None
log.info("Found %s datums" % str(len(data["action.start_time"])))
maxind = np.argmax([float(t) for t in data["action.start_time"]])
artifacts = data["task.artifacts"][maxind]
btime_artifact = None
for art in artifacts:
if "browsertime-results" in art["name"]:
btime_artifact = art["url"]
if not btime_artifact:
log.info("Can't find an older live site")
return None
# Download the browsertime videos and untar them
tmpdir = tempfile.mkdtemp()
loc = os.path.join(tmpdir, "tmpfile.tgz")
if not download(btime_artifact, loc, log):
return None
tmploc = tempfile.mkdtemp()
with tarfile.open(str(loc)) as tar:
except Exception:
"Could not read/extract old browsertime results archive",
return None
# Find all the videos
oldmp4s = [str(f) for f in pathlib.Path(tmploc).rglob("*.mp4")]
log.info("Found %s old videos" % str(len(oldmp4s)))
newmp4s = [str(f) for f in pathlib.Path(fetch_dir).rglob("*.mp4")]
log.info("Found %s new videos" % str(len(newmp4s)))
# Finally, calculate the 2D/3D score
nhists = []
nhists2d = []
total_vids = min(len(oldmp4s), len(newmp4s))
xcorr = np.zeros((total_vids, total_vids))
xcorr2d = np.zeros((total_vids, total_vids))
for i in range(total_vids):
datao = np.asarray(get_frames(open_data(oldmp4s[i])))
histo, _, _ = plt.hist(datao.flatten(), bins=255)
histo2d, _, _ = plt.hist(datao[-1, :, :].flatten(), bins=255)
for j in range(total_vids):
if i == 0:
# Only calculate the histograms once; it takes time
datan = np.asarray(get_frames(open_data(newmp4s[j])))
histn, _, _ = plt.hist(datan.flatten(), bins=255)
histn2d, _, _ = plt.hist(datan[-1, :, :].flatten(), bins=255)
histn = nhists[j]
histn2d = nhists2d[j]
rho, _ = spearmanr(histn, histo)
rho2d, _ = spearmanr(histn2d, histo2d)
xcorr[i, j] = rho
xcorr2d[i, j] = rho2d
similarity = np.mean(xcorr)
similarity2d = np.mean(xcorr2d)
log.info("Average 3D similarity: %s" % str(np.round(similarity, 5)))
log.info("Average 2D similarity: %s" % str(np.round(similarity2d, 5)))
if similarity < 0.5:
# For really low correlations, output the worst video pairing
# so that we can visually see what the issue was
minind = np.unravel_index(np.argmin(xcorr, axis=None), xcorr.shape)
oldvid = oldmp4s[minind[0]]
shutil.copyfile(oldvid, str(pathlib.Path(output, "old_video.mp4")))
newvid = newmp4s[minind[1]]
shutil.copyfile(newvid, str(pathlib.Path(output, "new_video.mp4")))
return np.round(similarity, 5), np.round(similarity2d, 5)
...@@ -75,6 +75,10 @@ def run_visual_metrics(config, jobs): ...@@ -75,6 +75,10 @@ def run_visual_metrics(config, jobs):
symbol=treeherder_info['symbol'] symbol=treeherder_info['symbol']
) )
# Store the platform name so we can use it to calculate
# the similarity metric against other tasks
job['worker'].setdefault('env', {})['TC_PLATFORM'] = platform
# run-on-projects needs to be set based on the dependent task # run-on-projects needs to be set based on the dependent task
attributes = dict(dep_job.attributes) attributes = dict(dep_job.attributes)
job['run-on-projects'] = attributes['run_on_projects'] job['run-on-projects'] = attributes['run_on_projects']
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment