job.py 5.72 KB
Newer Older
Johan Lorenzo's avatar
Johan Lorenzo committed
1
2
3
4
5
6
7
8
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util import path
Johan Lorenzo's avatar
Johan Lorenzo committed
9
from taskgraph.util.schema import Schema, taskref_or_string
Johan Lorenzo's avatar
Johan Lorenzo committed
10
11
12
13
14
from voluptuous import Required, Optional
from six import text_type

from pipes import quote as shell_quote

Johan Lorenzo's avatar
Johan Lorenzo committed
15
16
17
18
19
20
21
secret_schema = {
    Required("name"): text_type,
    Required("path"): text_type,
    Required("key"): text_type,
    Optional("json"): bool,
}

22
23
24
25
26
27
dummy_secret_schema = {
    Required("content"): text_type,
    Required("path"): text_type,
    Optional("json"): bool,
}

Johan Lorenzo's avatar
Johan Lorenzo committed
28
29
30
31
32
33
34
35
36
gradlew_schema = Schema({
    Required("using"): "gradlew",
    Optional("pre-gradlew"): [[text_type]],
    Required("gradlew"): [text_type],
    Optional("post-gradlew"): [[text_type]],
    # Base work directory used to set up the task.
    Required("workdir"): text_type,
    Optional("use-caches"): bool,
    Optional("secrets"): [secret_schema],
37
    Optional("dummy-secrets"): [dummy_secret_schema],
Johan Lorenzo's avatar
Johan Lorenzo committed
38
39
40
41
})

run_commands_schema = Schema({
    Required("using"): "run-commands",
42
    Optional("pre-commands"): [[text_type]],
Johan Lorenzo's avatar
Johan Lorenzo committed
43
44
45
46
    Required("commands"): [[taskref_or_string]],
    Required("workdir"): text_type,
    Optional("use-caches"): bool,
    Optional("secrets"): [secret_schema],
47
    Optional("dummy-secrets"): [dummy_secret_schema],
Johan Lorenzo's avatar
Johan Lorenzo committed
48
49
50
51
52
53
})


@run_job_using("docker-worker", "run-commands", schema=run_commands_schema)
def configure_run_commands_schema(config, job, taskdesc):
    run = job["run"]
54
    pre_commands = run.pop("pre-commands", [])
55
56
57
    pre_commands += [
        _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", [])
    ]
58
    pre_commands += [
Johan Lorenzo's avatar
Johan Lorenzo committed
59
60
        _generate_secret_command(secret) for secret in run.get("secrets", [])
    ]
61

Johan Lorenzo's avatar
Johan Lorenzo committed
62
63
64
65
66
67
    all_commands = pre_commands + run.pop("commands", [])

    run["command"] = _convert_commands_to_string(all_commands)
    _inject_secrets_scopes(run, taskdesc)
    _set_run_task_attributes(job)
    configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
Johan Lorenzo's avatar
Johan Lorenzo committed
68
69
70
71
72
73
74


@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
def configure_gradlew(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]

75
76
77
78
79
    worker.setdefault("env", {}).update({
        "ANDROID_SDK_ROOT": path.join(
            run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
        )
    })
Johan Lorenzo's avatar
Johan Lorenzo committed
80

Johan Lorenzo's avatar
Johan Lorenzo committed
81
82
83
    run["command"] = _extract_gradlew_command(run)
    _inject_secrets_scopes(run, taskdesc)
    _set_run_task_attributes(job)
Johan Lorenzo's avatar
Johan Lorenzo committed
84
85
86
    configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])


Johan Lorenzo's avatar
Johan Lorenzo committed
87
def _extract_gradlew_command(run):
88
    pre_gradle_commands = run.pop("pre-gradlew", [])
89
90
91
    pre_gradle_commands += [
        _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", [])
    ]
Johan Lorenzo's avatar
Johan Lorenzo committed
92
93
94
95
96
97
98
99
    pre_gradle_commands += [
        _generate_secret_command(secret) for secret in run.get("secrets", [])
    ]

    gradle_command = ["./gradlew"] + run.pop("gradlew")
    post_gradle_commands = run.pop("post-gradlew", [])

    commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
Johan Lorenzo's avatar
Johan Lorenzo committed
100
    return _convert_commands_to_string(commands)
Johan Lorenzo's avatar
Johan Lorenzo committed
101
102
103
104


def _generate_secret_command(secret):
    secret_command = [
105
        "taskcluster/scripts/get-secret.py",
Johan Lorenzo's avatar
Johan Lorenzo committed
106
107
108
109
110
111
112
113
        "-s", secret["name"],
        "-k", secret["key"],
        "-f", secret["path"],
    ]
    if secret.get("json"):
        secret_command.append("--json")

    return secret_command
Johan Lorenzo's avatar
Johan Lorenzo committed
114
115


116
117
118
119
120
121
122
123
124
125
126
127
def _generate_dummy_secret_command(secret):
    secret_command = [
        "taskcluster/scripts/write-dummy-secret.py",
        "-f", secret["path"],
        "-c", secret["content"],
    ]
    if secret.get("json"):
        secret_command.append("--json")

    return secret_command


Johan Lorenzo's avatar
Johan Lorenzo committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
def _convert_commands_to_string(commands):
    should_artifact_reference = False
    should_task_reference = False

    sanitized_commands = []
    for command in commands:
        sanitized_parts = []
        for part in command:
            if isinstance(part, dict):
                if "artifact-reference" in part:
                    part_string = part["artifact-reference"]
                    should_artifact_reference = True
                elif "task-reference" in part:
                    part_string = part["task-reference"]
                    should_task_reference = True
                else:
                    raise ValueError('Unsupported dict: {}'.format(part))
            else:
                part_string = part

            sanitized_parts.append(part_string)
        sanitized_commands.append(sanitized_parts)

    shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in sanitized_commands]
    full_string_command = " && ".join(shell_quoted_commands)

    if should_artifact_reference and should_task_reference:
        raise NotImplementedError('"arifact-reference" and "task-reference" cannot be both used')
    elif should_artifact_reference:
        return {"artifact-reference": full_string_command}
    elif should_task_reference:
        return {"task-reference": full_string_command}
    else:
        return full_string_command


def _inject_secrets_scopes(run, taskdesc):
    secrets = run.pop("secrets", [])
    scopes = taskdesc.setdefault("scopes", [])
    new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets]
    new_secret_scopes = list(set(new_secret_scopes))  # Scopes must not have any duplicates
    scopes.extend(new_secret_scopes)


def _set_run_task_attributes(job):
    run = job["run"]
    run["cwd"] = "{checkout}"
    run["using"] = "run-task"