305 lines
6.9 KiB
JavaScript
305 lines
6.9 KiB
JavaScript
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
import {clone} from "merge";
|
|
import merge from "./merge";
|
|
import slugid from "slugid";
|
|
import taskcluster from "taskcluster-client";
|
|
import * as image_builder from "./image_builder";
|
|
|
|
let maps = [];
|
|
let filters = [];
|
|
|
|
let tasks = new Map();
|
|
let tags = new Map();
|
|
let image_tasks = new Map();
|
|
let parameters = {};
|
|
|
|
let queue = new taskcluster.Queue({
|
|
rootUrl: process.env.TASKCLUSTER_PROXY_URL,
|
|
});
|
|
|
|
function fromNow(hours) {
|
|
let d = new Date();
|
|
d.setHours(d.getHours() + (hours|0));
|
|
return d.toJSON();
|
|
}
|
|
|
|
function parseRoutes(routes) {
|
|
let rv = [
|
|
`tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`,
|
|
...routes
|
|
];
|
|
|
|
// Notify about failures (except on try).
|
|
// Turned off, too noisy.
|
|
/*if (process.env.TC_PROJECT != "nss-try") {
|
|
rv.push(`notify.email.${process.env.TC_OWNER}.on-failed`,
|
|
`notify.email.${process.env.TC_OWNER}.on-exception`);
|
|
}*/
|
|
|
|
return rv;
|
|
}
|
|
|
|
function parseFeatures(list) {
|
|
return list.reduce((map, feature) => {
|
|
map[feature] = true;
|
|
return map;
|
|
}, {});
|
|
}
|
|
|
|
function parseArtifacts(artifacts) {
|
|
let copy = clone(artifacts);
|
|
Object.keys(copy).forEach(key => {
|
|
copy[key].expires = fromNow(copy[key].expires);
|
|
});
|
|
return copy;
|
|
}
|
|
|
|
function parseCollection(name) {
|
|
let collection = {};
|
|
collection[name] = true;
|
|
return collection;
|
|
}
|
|
|
|
function parseTreeherder(def) {
|
|
let treeherder = {
|
|
build: {
|
|
platform: def.platform
|
|
},
|
|
machine: {
|
|
platform: def.platform
|
|
},
|
|
symbol: def.symbol,
|
|
jobKind: def.kind
|
|
};
|
|
|
|
if (def.group) {
|
|
treeherder.groupSymbol = def.group;
|
|
}
|
|
|
|
if (def.collection) {
|
|
treeherder.collection = parseCollection(def.collection);
|
|
}
|
|
|
|
if (def.tier) {
|
|
treeherder.tier = def.tier;
|
|
}
|
|
|
|
return treeherder;
|
|
}
|
|
|
|
function convertTask(def) {
|
|
let scopes = [];
|
|
let dependencies = [];
|
|
|
|
let env = merge({
|
|
NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
|
|
NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION,
|
|
NSS_MAX_MP_PBE_ITERATION_COUNT: "100",
|
|
}, def.env || {});
|
|
|
|
if (def.parent) {
|
|
dependencies.push(def.parent);
|
|
env.TC_PARENT_TASK_ID = def.parent;
|
|
}
|
|
if (def.parents) {
|
|
dependencies = dependencies.concat(def.parents);
|
|
}
|
|
|
|
if (def.tests) {
|
|
env.NSS_TESTS = def.tests;
|
|
}
|
|
|
|
if (def.cycle) {
|
|
env.NSS_CYCLES = def.cycle;
|
|
}
|
|
if (def.kind === "build") {
|
|
// Disable leak checking during builds (bug 1579290).
|
|
if (env.ASAN_OPTIONS) {
|
|
env.ASAN_OPTIONS += ":detect_leaks=0";
|
|
} else {
|
|
env.ASAN_OPTIONS = "detect_leaks=0";
|
|
}
|
|
}
|
|
|
|
let payload = {
|
|
env,
|
|
command: def.command,
|
|
maxRunTime: def.maxRunTime || 3600
|
|
};
|
|
|
|
if (def.image) {
|
|
payload.image = def.image;
|
|
}
|
|
|
|
if (def.artifacts) {
|
|
payload.artifacts = parseArtifacts(def.artifacts);
|
|
}
|
|
|
|
if (def.features) {
|
|
payload.features = parseFeatures(def.features);
|
|
|
|
if (payload.features.allowPtrace) {
|
|
scopes.push("docker-worker:feature:allowPtrace");
|
|
}
|
|
}
|
|
|
|
if (def.scopes) {
|
|
// Need to add existing scopes in the task definition
|
|
scopes.push.apply(scopes, def.scopes)
|
|
}
|
|
|
|
let extra = Object.assign({
|
|
treeherder: parseTreeherder(def)
|
|
}, parameters);
|
|
|
|
return {
|
|
provisionerId: def.provisioner || `nss-${process.env.MOZ_SCM_LEVEL}`,
|
|
workerType: def.workerType || "linux",
|
|
schedulerId: process.env.TC_SCHEDULER_ID,
|
|
taskGroupId: process.env.TASK_ID,
|
|
|
|
scopes,
|
|
created: fromNow(0),
|
|
deadline: fromNow(24),
|
|
|
|
dependencies,
|
|
requires: def.requires || "all-completed",
|
|
routes: parseRoutes(def.routes || []),
|
|
|
|
metadata: {
|
|
name: def.name,
|
|
description: def.name,
|
|
owner: process.env.TC_OWNER,
|
|
source: process.env.TC_SOURCE
|
|
},
|
|
|
|
payload,
|
|
extra,
|
|
};
|
|
}
|
|
|
|
export function map(fun) {
|
|
maps.push(fun);
|
|
}
|
|
|
|
export function filter(fun) {
|
|
filters.push(fun);
|
|
}
|
|
|
|
export function addParameters(params) {
|
|
parameters = Object.assign(parameters, params);
|
|
}
|
|
|
|
export function clearFilters(fun) {
|
|
filters = [];
|
|
}
|
|
|
|
export function taggedTasks(tag) {
|
|
return tags[tag];
|
|
}
|
|
|
|
export function scheduleTask(def) {
|
|
let taskId = slugid.v4();
|
|
tasks.set(taskId, merge({}, def));
|
|
return taskId;
|
|
}
|
|
|
|
export async function submit() {
|
|
let promises = new Map();
|
|
|
|
for (let [taskId, task] of tasks) {
|
|
// Allow filtering tasks before we schedule them.
|
|
if (!filters.every(filter => filter(task))) {
|
|
continue;
|
|
}
|
|
|
|
// Allow changing tasks before we schedule them.
|
|
maps.forEach(map => { task = map(merge({}, task)) });
|
|
|
|
let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`;
|
|
if (task.group) {
|
|
log_id = `${task.group}::${log_id}`;
|
|
}
|
|
console.log(`+ Submitting ${log_id}.`);
|
|
|
|
// Index that task for each tag specified
|
|
if(task.tags) {
|
|
task.tags.map(tag => {
|
|
if(!tags[tag]) {
|
|
tags[tag] = [];
|
|
}
|
|
tags[tag].push(taskId);
|
|
});
|
|
}
|
|
|
|
let parent = task.parent;
|
|
|
|
// Convert the task definition.
|
|
task = await convertTask(task);
|
|
|
|
// Convert the docker image definition.
|
|
let image_def = task.payload.image;
|
|
if (image_def && image_def.hasOwnProperty("path")) {
|
|
let key = `${image_def.name}:${image_def.path}`;
|
|
let data = {};
|
|
|
|
// Check the cache first.
|
|
if (image_tasks.has(key)) {
|
|
data = image_tasks.get(key);
|
|
} else {
|
|
data.taskId = await image_builder.findTask(image_def);
|
|
data.isPending = !data.taskId;
|
|
|
|
// No task found.
|
|
if (data.isPending) {
|
|
let image_task = await image_builder.buildTask(image_def);
|
|
|
|
// Schedule a new image builder task immediately.
|
|
data.taskId = slugid.v4();
|
|
|
|
try {
|
|
await queue.createTask(data.taskId, convertTask(image_task));
|
|
} catch (e) {
|
|
console.error("! FAIL: Scheduling image builder task failed.");
|
|
continue; /* Skip this task on failure. */
|
|
}
|
|
}
|
|
|
|
// Store in cache.
|
|
image_tasks.set(key, data);
|
|
}
|
|
|
|
if (data.isPending) {
|
|
task.dependencies.push(data.taskId);
|
|
}
|
|
|
|
task.payload.image = {
|
|
path: "public/image.tar",
|
|
taskId: data.taskId,
|
|
type: "task-image"
|
|
};
|
|
}
|
|
|
|
// Wait for the parent task to be created before scheduling dependants.
|
|
let predecessor = parent ? promises.get(parent) : Promise.resolve();
|
|
|
|
promises.set(taskId, predecessor.then(() => {
|
|
// Schedule the task.
|
|
return queue.createTask(taskId, task).catch(err => {
|
|
console.error(`! FAIL: Scheduling ${log_id} failed.`, err);
|
|
});
|
|
}));
|
|
}
|
|
|
|
// Wait for all requests to finish.
|
|
if (promises.length) {
|
|
await Promise.all([...promises.values()]);
|
|
console.log("=== Total:", promises.length, "tasks. ===");
|
|
}
|
|
|
|
tasks.clear();
|
|
}
|