Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Gaba
fenix
Commits
58fc0e53
Unverified
Commit
58fc0e53
authored
Apr 09, 2020
by
Johan Lorenzo
Committed by
GitHub
Apr 09, 2020
Browse files
Bug 1568466 - part 3: Enable shipit graph (#7314)
parent
ed466d34
Changes
7
Hide whitespace changes
Inline
Side-by-side
taskcluster/ci/config.yml
View file @
58fc0e53
...
...
@@ -26,7 +26,7 @@ taskgraph:
mobile
:
name
:
"
Fenix"
cached-task-prefix
:
project.mobile.fenix
decision-parameters
:
'
fenix_taskgraph:get_decision_parameters'
decision-parameters
:
'
fenix_taskgraph
.parameters
:get_decision_parameters'
workers
:
aliases
:
...
...
@@ -71,3 +71,9 @@ workers:
scriptworker
:
scope-prefix
:
project:mobile:fenix:releng
release-promotion
:
flavors
:
ship
:
target-tasks-method
:
release
taskcluster/fenix_taskgraph/__init__.py
View file @
58fc0e53
...
...
@@ -4,17 +4,7 @@
from
__future__
import
absolute_import
,
print_function
,
unicode_literals
import
os
import
re
from
importlib
import
import_module
from
six
import
text_type
from
voluptuous
import
All
,
Any
,
Range
,
Required
from
taskgraph.parameters
import
extend_parameters_schema
BETA_SEMVER
=
re
.
compile
(
r
'^v\d+\.\d+\.\d+-beta\.\d+$'
)
PRODUCTION_SEMVER
=
re
.
compile
(
r
'^v\d+\.\d+\.\d+(-rc\.\d+)?$'
)
def
register
(
graph_config
):
...
...
@@ -22,52 +12,16 @@ def register(graph_config):
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules
([
"job"
,
"worker_types"
,
"routes"
,
"target_tasks"
])
extend_parameters_schema
({
Required
(
"pull_request_number"
):
Any
(
All
(
int
,
Range
(
min
=
1
)),
None
),
Required
(
"release_type"
):
text_type
,
Required
(
"release_version"
):
text_type
,
})
_import_modules
([
"job"
,
"parameters"
,
"release_promotion"
,
"routes"
,
"target_tasks"
,
"worker_types"
,
])
def
_import_modules
(
modules
):
for
module
in
modules
:
import_module
(
".{}"
.
format
(
module
),
package
=
__name__
)
def
get_decision_parameters
(
graph_config
,
parameters
):
head_tag
=
parameters
[
"head_tag"
].
decode
(
"utf-8"
)
parameters
[
"release_type"
]
=
_resolve_release_type
(
head_tag
)
parameters
[
"release_version"
]
=
read_version_file
()
if
head_tag
:
parameters
[
"release_version"
]
=
head_tag
[
1
:]
pr_number
=
os
.
environ
.
get
(
"MOBILE_PULL_REQUEST_NUMBER"
,
None
)
parameters
[
"pull_request_number"
]
=
None
if
pr_number
is
None
else
int
(
pr_number
)
if
parameters
[
"tasks_for"
]
==
"github-release"
:
for
param_name
in
(
"release_type"
,
"release_version"
):
if
not
parameters
[
param_name
]:
raise
ValueError
(
'Cannot run github-release if "{}" is not defined. Got: {}'
.
format
(
param_name
,
parameters
[
param_name
]
)
)
parameters
[
"target_tasks_method"
]
=
"release"
def
read_version_file
():
with
open
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'..'
,
'..'
,
'version.txt'
))
as
f
:
return
f
.
read
().
strip
().
decode
(
'utf-8'
)
def
_resolve_release_type
(
head_tag
):
if
not
head_tag
:
return
""
elif
BETA_SEMVER
.
match
(
head_tag
):
return
"beta"
elif
PRODUCTION_SEMVER
.
match
(
head_tag
):
return
"production"
else
:
raise
ValueError
(
'Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)'
)
taskcluster/fenix_taskgraph/parameters.py
0 → 100644
View file @
58fc0e53
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from
__future__
import
absolute_import
,
print_function
,
unicode_literals
import
os
import
re
from
six
import
text_type
from
taskgraph.parameters
import
extend_parameters_schema
from
voluptuous
import
All
,
Any
,
Optional
,
Range
,
Required
BETA_SEMVER
=
re
.
compile
(
r
'^v\d+\.\d+\.\d+-beta\.\d+$'
)
PRODUCTION_SEMVER
=
re
.
compile
(
r
'^v\d+\.\d+\.\d+(-rc\.\d+)?$'
)
extend_parameters_schema
({
Required
(
"pull_request_number"
):
Any
(
All
(
int
,
Range
(
min
=
1
)),
None
),
Required
(
"release_type"
):
text_type
,
Optional
(
"shipping_phase"
):
Any
(
'build'
,
'ship'
,
None
),
Required
(
"version"
):
text_type
,
})
def
get_decision_parameters
(
graph_config
,
parameters
):
head_tag
=
parameters
[
"head_tag"
].
decode
(
"utf-8"
)
parameters
[
"release_type"
]
=
resolve_release_type
(
head_tag
)
parameters
[
"version"
]
=
head_tag
[
1
:]
if
head_tag
else
""
pr_number
=
os
.
environ
.
get
(
"MOBILE_PULL_REQUEST_NUMBER"
,
None
)
parameters
[
"pull_request_number"
]
=
None
if
pr_number
is
None
else
int
(
pr_number
)
if
parameters
[
"tasks_for"
]
==
"github-release"
:
for
param_name
in
(
"release_type"
,
"version"
):
if
not
parameters
[
param_name
]:
raise
ValueError
(
'Cannot run github-release if "{}" is not defined. Got: {}'
.
format
(
param_name
,
parameters
[
param_name
]
)
)
parameters
[
"target_tasks_method"
]
=
"release"
def
resolve_release_type
(
head_tag
):
if
not
head_tag
:
return
""
elif
BETA_SEMVER
.
match
(
head_tag
):
return
"beta"
elif
PRODUCTION_SEMVER
.
match
(
head_tag
):
return
"production"
else
:
raise
ValueError
(
'Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)'
)
taskcluster/fenix_taskgraph/release_promotion.py
0 → 100644
View file @
58fc0e53
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from
__future__
import
absolute_import
,
print_function
,
unicode_literals
from
taskgraph.actions.registry
import
register_callback_action
from
taskgraph.util.taskcluster
import
get_artifact
from
taskgraph.taskgraph
import
TaskGraph
from
taskgraph.decision
import
taskgraph_decision
from
taskgraph.parameters
import
Parameters
from
taskgraph.util.taskgraph
import
find_decision_task
,
find_existing_tasks_from_previous_kinds
from
.parameters
import
resolve_release_type
RELEASE_PROMOTION_PROJECTS
=
(
"https://github.com/JohanLorenzo/fenix"
,
)
def
is_release_promotion_available
(
parameters
):
return
parameters
[
'head_repository'
]
in
RELEASE_PROMOTION_PROJECTS
@
register_callback_action
(
name
=
'release-promotion'
,
title
=
'Ship Fenix'
,
symbol
=
'${input.release_promotion_flavor}'
,
description
=
"Ship Fenix"
,
generic
=
False
,
order
=
500
,
context
=
[],
available
=
is_release_promotion_available
,
schema
=
lambda
graph_config
:
{
'type'
:
'object'
,
'properties'
:
{
'build_number'
:
{
'type'
:
'integer'
,
'default'
:
1
,
'minimum'
:
1
,
'title'
:
'The release build number'
,
'description'
:
(
'The release build number. Starts at 1 per '
'release version, and increments on rebuild.'
),
},
'do_not_optimize'
:
{
'type'
:
'array'
,
'description'
:
(
'Optional: a list of labels to avoid optimizing out '
'of the graph (to force a rerun of, say, '
'funsize docker-image tasks).'
),
'items'
:
{
'type'
:
'string'
,
},
},
'revision'
:
{
'type'
:
'string'
,
'title'
:
'Optional: revision to ship'
,
'description'
:
(
'Optional: the revision to ship.'
),
},
'release_promotion_flavor'
:
{
'type'
:
'string'
,
'description'
:
'The flavor of release promotion to perform.'
,
'default'
:
'build'
,
'enum'
:
sorted
(
graph_config
[
'release-promotion'
][
'flavors'
].
keys
()),
},
'rebuild_kinds'
:
{
'type'
:
'array'
,
'description'
:
(
'Optional: an array of kinds to ignore from the previous '
'graph(s).'
),
'items'
:
{
'type'
:
'string'
,
},
},
'previous_graph_ids'
:
{
'type'
:
'array'
,
'description'
:
(
'Optional: an array of taskIds of decision or action '
'tasks from the previous graph(s) to use to populate '
'our `previous_graph_kinds`.'
),
'items'
:
{
'type'
:
'string'
,
},
},
'version'
:
{
'type'
:
'string'
,
'description'
:
(
'Optional: override the version for release promotion. '
"Occasionally we'll land a taskgraph fix in a later "
'commit, but want to act on a build from a previous '
'commit. If a version bump has landed in the meantime, '
'relying on the in-tree version will break things.'
),
'default'
:
''
,
},
},
"required"
:
[
'release_promotion_flavor'
,
'version'
,
'build_number'
],
}
)
def
release_promotion_action
(
parameters
,
graph_config
,
input
,
task_group_id
,
task_id
):
release_promotion_flavor
=
input
[
'release_promotion_flavor'
]
promotion_config
=
graph_config
[
'release-promotion'
][
'flavors'
][
release_promotion_flavor
]
target_tasks_method
=
promotion_config
[
'target-tasks-method'
].
format
(
project
=
parameters
[
'project'
]
)
rebuild_kinds
=
input
.
get
(
'rebuild_kinds'
)
or
promotion_config
.
get
(
'rebuild-kinds'
,
[])
do_not_optimize
=
input
.
get
(
'do_not_optimize'
)
or
promotion_config
.
get
(
'do-not-optimize'
,
[])
# make parameters read-write
parameters
=
dict
(
parameters
)
# Build previous_graph_ids from ``previous_graph_ids`` or ``revision``.
previous_graph_ids
=
input
.
get
(
'previous_graph_ids'
)
if
not
previous_graph_ids
:
previous_graph_ids
=
[
find_decision_task
(
parameters
,
graph_config
)]
# Download parameters from the first decision task
parameters
=
get_artifact
(
previous_graph_ids
[
0
],
"public/parameters.yml"
)
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph
=
{}
for
graph_id
in
previous_graph_ids
:
full_task_graph
=
get_artifact
(
graph_id
,
"public/full-task-graph.json"
)
combined_full_task_graph
.
update
(
full_task_graph
)
_
,
combined_full_task_graph
=
TaskGraph
.
from_json
(
combined_full_task_graph
)
parameters
[
'existing_tasks'
]
=
find_existing_tasks_from_previous_kinds
(
combined_full_task_graph
,
previous_graph_ids
,
rebuild_kinds
)
parameters
[
'do_not_optimize'
]
=
do_not_optimize
parameters
[
'target_tasks_method'
]
=
target_tasks_method
parameters
[
'build_number'
]
=
int
(
input
[
'build_number'
])
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters
[
'optimize_target_tasks'
]
=
True
parameters
[
'shipping_phase'
]
=
input
[
'release_promotion_flavor'
]
parameters
[
'version'
]
=
input
[
'version'
]
if
input
.
get
(
'version'
)
else
read_version_file
()
parameters
[
'head_tag'
]
=
'v{}'
.
format
(
parameters
[
'version'
])
parameters
[
'release_type'
]
=
resolve_release_type
(
parameters
[
'head_tag'
])
parameters
[
'pull_request_number'
]
=
None
# make parameters read-only
parameters
=
Parameters
(
**
parameters
)
taskgraph_decision
({
'root'
:
graph_config
.
root_dir
},
parameters
=
parameters
)
def
read_version_file
():
with
open
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'..'
,
'..'
,
'version.txt'
))
as
f
:
return
f
.
read
().
strip
().
decode
(
'utf-8'
)
taskcluster/fenix_taskgraph/target_tasks.py
View file @
58fc0e53
...
...
@@ -19,6 +19,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
@
_target_task
(
'release'
)
def
target_tasks_default
(
full_task_graph
,
parameters
,
graph_config
):
# TODO Use shipping-phase once we retire github-releases
def
filter
(
task
,
parameters
):
return
task
.
attributes
.
get
(
"release-type"
,
""
)
==
parameters
[
"release_type"
]
...
...
taskcluster/fenix_taskgraph/transforms/build.py
View file @
58fc0e53
...
...
@@ -97,7 +97,7 @@ def add_release_version(config, tasks):
for
task
in
tasks
:
if
task
.
pop
(
"include-release-version"
,
False
):
task
[
"run"
][
"gradlew"
].
append
(
'-PversionName={}'
.
format
(
config
.
params
[
"
release_
version"
])
'-PversionName={}'
.
format
(
config
.
params
[
"version"
])
)
yield
task
...
...
taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py
View file @
58fc0e53
...
...
@@ -19,10 +19,10 @@ transforms = TransformSequence()
def
make_task_description
(
config
,
jobs
):
for
job
in
jobs
:
product
=
"Fenix"
version
=
config
.
params
[
'
release_
version'
]
or
"{ver}"
version
=
config
.
params
[
'version'
]
or
"{ver}"
job
[
'worker'
][
'release-name'
]
=
'{product}-{version}-build{build_number}'
.
format
(
product
=
product
,
version
=
version
,
build_number
=
config
.
params
.
get
(
'build_number'
,
1
)
)
yield
job
\ No newline at end of file
yield
job
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment