Metadata editing, releasing via the API

Left to do:
- test out including commit hash, when ruben
  finishes implementing this in the API.
- screenshot reconciliation.
master
Aaron Suen 2021-02-27 11:20:52 -05:00
parent 8f06601e86
commit 8362e41b0e
14 changed files with 618 additions and 272 deletions

66
README
View File

@ -39,8 +39,10 @@ been loaded. Settings become available in the order:
Recognized settings:
- conf: path to local config file, default' ~/.cdbrelease.json'.
- repo: REQUIRED path or URL to a git repository from which to create
the release; must have access to clone.
- srcrepo: REQUIRED path or URL to a git repository from which to create
the release; must have access to clone. Note that this may be
different from the "repo" setting, which is used for the published
repo in the package metadata.
- branch: git ref/branch/tag from the source repo to release, defaults
to master.
- execgit: path to execute git command, default 'git' (using $PATH)
@ -60,49 +62,21 @@ Recognized settings:
will be executed.
- cdbluapath: relative path to .cdbrelease.lua within the git
checkout/export, default '.cdbrelease.lua'.
- root: root URL of CDB API, default 'https://content.minetest.net/api'
- token: REQUIRED CDB API authentication token.
- user: username of package owner, defaults to username associated with
the API token.
- pkg: name of the package to release/edit.
- version: version of package to release, default none; if not set, then
no release will happen (but edits/screenshots can still apply).
- edit: enable automatic modification of package metadata (default
disabled)
- timeout: timeout (in seconds) for all CDB HTTP requests, default 60.
- force: if enabled, always create a new CDB release, even if one with
a matching title exists (assume the existing one may be corrupt).
========================================================================
In addition, all keys supported by the package metadata PUT request
(https://github.com/minetest/contentdb/blob/master/app/flatpages/help/api.md#packages)
are supported, and used in package metadata editing.
To create a release manually:
- In this project's dir, run:
node . --user="yourname" --token="yourapitoken" --pkg="yourpackage" \
--version="new-version-title"
- You can also set optional parameters:
--branch="..." to set the VCS source branch, if not master.
--min="..." to set the minimum Minetest version.
--max="..." to set the maximum Minetest version.
--fromgit="..." (see below)
- If a release with a matching name is detected then the release will
be aborted. Don't rely on this too much, as it may be fragile, e.g.
not work if a release exists but is pending approval.
- No news is good news. If you don't see anything then it probably
worked. Check on content.minetest.net to be certain.
........................................................................
To create a release automatically from a git repo:
- Create a .cdbrelease.lua file in the root of your project's git repo.
This should return a lua table containing key/value pairs to merge
into config (e.g. {version = "123"} maps to --version=123). You can
use .gitattributes and export tags.
- Commit (and if necessary push).
- In this project's dir, run:
node . --user="yourname" --token="yourapitoken" --repo="repourl"
- "repourl" should be anything that git can clone, including a local
path, a web URL, ssh URL, etc.
- You can also optionally set any parameters listed above in the
"release manually" section. Anything set locally will override the
settings from the repo. Technically you can store your username and
API Token in the repo, but this is bad practice.
N.B. code in .cdbrelease.lua is NOT sandboxed in any way!
There are significant security implications to using --repo with a
repo that you do not fully control. Use at your own risk.
========================================================================
========================================================================

View File

@ -1,77 +1,33 @@
'use strict';
const util = require('util');
const mylog = require('./mylog');
const config = require('./config');
const cdblib = require('./cdblib');
const cdbfetch = require('./cdbfetch');
const keepers = {};
`type title short_desc tags license media_license provides_str harddep_str
softdep_str repo website issueTracker forums desc`.match(/\S+/g)
.forEach(k => keepers[k] = true);
const sortser = obj => JSON.stringify(Array.isArray(obj) ? obj.sort() : obj);
module.exports = async () => {
const meta = {};
Object.keys(config)
.filter(k => keepers[k] && (config[k] || config[k] === ''))
.forEach(k => meta[k] = config[k]);
if(!Object.keys(meta)
.length)
return;
meta.name = config.pkg;
const pkgurl = `/packages/${config.user}/${config.pkg}/`;
mylog('checking existing package metadata...');
const resp = await cdbfetch(pkgurl);
const uri = config.root + '/packages/' + config.user + '/' + config.pkg + '/edit/';
await cdblib.login();
console.log('fetching package edit page...');
const $ = await cdblib.fetch(uri);
const fields = cdblib.getfields($);
let tagseen = {};
let tags = [];
(meta.tags || '').split(',')
.filter(x => /\S/.test(x))
.map(x => x.trim())
.forEach(x => {
const id = cdblib.findopt($, 'select#tags option', x);
if(!id || tagseen[id]) return;
tagseen[id] = true;
tags.push(id);
});
delete meta.tags;
'type license media_license'.split(' ')
const edits = {};
Object.keys(resp.body)
.filter(k => k !== 'screenshots' && config[k] !== undefined)
.forEach(k => {
if(meta[k])
meta[k] = cdblib.findopt($, `select#${k} option`, meta[k]);
if(sortser(resp.body[k]) !== sortser(k === 'long_description' ?
config[k].replace(/\r/g, '')
.replace(/\n/g, '\r\n') : config[k]))
edits[k] = config[k];
});
if(!Object.keys(edits)
.length)
return mylog('package metadata already up to date');
meta.media_license = meta.media_license || meta.license;
meta.license = meta.license || meta.media_license;
if(config.dryrun)
return mylog(`DRY RUN: package metadata edits: ${JSON.stringify(edits)}`);
if(!meta.repo && meta.repo !== '' && config.fromgit &&
config.fromgit.toLowerCase()
.startsWith('http'))
meta.repo = config.fromgit;
const forumid = (meta.forums || '').match(/[&\?]t=(\d+)/);
if(forumid) meta.forums = forumid[1];
Object.assign(fields, meta);
if(config.dryrun) {
console.log('dry run; not saving details');
fields.tags = tags;
console.log(util.inspect(fields));
return true;
}
let body = Object.keys(fields)
.sort()
.map(k => encodeURIComponent(k) + '=' + encodeURIComponent(fields[k]));
tags.forEach(k => body.push('tags=' + k));
body = body.join('&');
console.log('applying package detail changes...');
await cdblib.fetch(uri, 'post', body);
console.log('package details updated');
return true;
mylog('updating package metadata...');
await cdbfetch(pkgurl, 'post', edits, { json: true });
mylog('package metadata updated');
};

45
cdbfetch.js Normal file
View File

@ -0,0 +1,45 @@
'use strict';
const needle = require('needle');
const mylog = require('./mylog');
const config = require('./config');
const delay = () => new Promise(r => setTimeout(r, 2000 + Math.random() * 2000));
const maxattempts = 10;
async function fetch(uri, meth, data, opts, ...rest) {
if(uri.startsWith('/'))
uri = config.root.trimEnd('/') + uri;
meth = meth || 'get';
opts = opts || {};
opts.headers = opts.headers || {};
opts.headers.Authorization = `Bearer ${config.token}`;
opts.headers.Referer = uri;
opts.open_timeout = config.timeout * 1000;
opts.read_timeout = config.timeout * 1000;
opts.response_timeout = config.timeout * 1000;
if(!opts.hasOwnProperty('follow_max'))
opts.follow_max = 10;
for(let attempt = 1; attempt <= maxattempts; attempt++)
try {
mylog(`# ${meth} ${uri}`);
const resp = await needle(meth, uri, data, opts, ...rest);
if(!/^[23]\d\d/.test(resp.statusCode.toString())) {
const failure = {
statusCode: resp.statusCode,
statusMessage: resp.statusMessage
};
throw (Object.assign(new Error(JSON.stringify(failure)), failure));
}
return resp;
}
catch (err) {
if(attempt === maxattempts ||
err.statusCode && !/^(?:429|5\d\d)$/.test(err.statusCode.toString()) ||
!err.statusCode && !/remote end closed socket/i.test(err.message))
throw err;
await delay();
}
}
module.exports = fetch;

View File

@ -1,41 +0,0 @@
'use strict';
const needle = require('needle');
const config = require('./config');
const maxattempts = 10;
const cookiejar = {};
async function fetch(uri, meth, data, opts, ...rest) {
meth = meth || 'get';
opts = opts || {};
opts.headers = opts.headers || {};
opts.headers.Authorization = `Bearer ${config.token}`;
opts.headers.Referer = uri;
opts.cookies = cookiejar;
if (!opts.hasOwnProperty('follow_max'))
opts.follow_max = 10;
for (let attempt = 1; attempt <= maxattempts; attempt++)
try {
const resp = await needle(meth, uri, data, opts, ...rest);
if (resp.cookies)
Object.keys(resp.cookies)
.forEach(k => cookiejar[k] = resp.cookies[k]);
if (!/^[23]\d\d/.test(resp.statusCode.toString())) {
const failure = {
statusCode: resp.statusCode,
statusMessage: resp.statusMessage
};
throw (Object.assign(new Error(JSON.stringify(failure)), failure));
}
return resp;
}
catch (err) {
if (attempt === maxattempts
|| err.statusCode && !/^(?:429|5\d\d)$/.test(err.statusCode.toString())
|| !err.statusCode && !/remote end closed socket/i.test(err.message))
throw err;
await new Promise(r => setTimeout(r, 2000 + Math.random() * 2000));
}
}
module.exports = fetch;

View File

@ -1,73 +1,69 @@
'use strict';
const util = require('util');
const fs = require('fs');
const path = require('path');
const glob = require('glob');
const archiver = require('archiver');
const mylog = require('./mylog');
const config = require('./config');
const cdblib = require('./cdblib');
const cdbfetch = require('./cdbfetch');
const withtemp = require('./withtemp');
const spawn = require('./spawn');
async function checkalready() {
const uri = config.root + '/packages/' + config.user + '/' + config.pkg + '/';
console.log('fetching release list...');
let $ = await cdblib.fetch(uri);
let already;
$('a')
.each(function() {
if(already) return;
const a = $(this);
const h = a.attr('href');
if(!h || !/\/releases\/\d+\/download\//.test(h)) return;
if(a.text()
.trim() === config.version.trim())
already = true;
module.exports = async gitexport => {
const relurl = `/packages/${config.user}/${config.pkg}/releases/`;
if(!config.force) {
mylog('checking if new version is already released...');
const resp = await cdbfetch(relurl);
if(Array.isArray(resp.body) && resp.body.find(x => x.title === config.version))
return mylog(`version ${config.version} already released`);
}
await withtemp(async ziptmp => {
mylog('determining commit...');
const proc = spawn('git', [`--git-dir=${gitexport.path}`,
'show-ref',
config.branch
], { stdio: ['ignore', 'pipe', 'inherit'] });
let buff = '';
proc.stdout.on('data', x => buff += x.toString());
await proc.promise;
const comm = buff.match(/^[0-9a-f]{40}\s/);
if(!comm) throw `no commit hash found for ${config.branch}`;
const commit = comm[0].trim();
const zipfile = path.join(ziptmp, 'release.zip');
const zipstr = fs.createWriteStream(zipfile);
await gitexport(false, async gittmp => {
mylog('archiving release...');
const arch = archiver('zip', { zlib: { level: 9 } });
await new Promise((res, rej) => {
arch.on('error', rej);
arch.on('end', res);
arch.pipe(zipstr);
const g = glob('**', {
cwd: gittmp,
dot: false,
nodir: true,
follow: true
});
g.on('stat', f => arch.file(path.join(gittmp, f), { name: f }));
g.on('end', () => arch.finalize());
});
});
return already;
}
zipstr.end();
zipstr.close();
const params = {
title: config.version,
commit: commit,
method: 'zip',
file: { file: zipfile, content_type: 'application/zip' }
};
if(config.dryrun)
return mylog(`DRY RUN: package release: ${JSON.stringify(params)}`);
async function makerelease() {
const uri = config.root + '/packages/' + config.user + '/' + config.pkg + '/releases/new/';
console.log('fetching new release page...');
let $ = await cdblib.fetch(uri);
const fields = cdblib.getfields($);
fields.min_rel = cdblib.findopt($, 'select#min_rel option', config.min);
fields.max_rel = cdblib.findopt($, 'select#max_rel option', config.max);
fields.vcsLabel = config.branch;
fields.uploadOpt = 'vcs';
fields.title = config.version;
if(config.dryrun) {
console.log('dry run; not submitting');
console.log(util.inspect(fields));
return;
}
console.log('submitting new release...');
$ = await cdblib.fetch(uri, 'post', fields);
}
const delay = n => new Promise(r => setTimeout(r, n * 1000));
module.exports = async () => {
console.log('checking if new version is already released...');
if(await checkalready()) {
console.log('version ' + config.version + ' already released');
return;
}
await cdblib.login();
console.log('rechecking if new version is already pending...');
if(await checkalready()) {
console.log('version ' + config.version + ' already released');
return;
}
await makerelease();
if(config.dryrun) {
console.log('not waiting for release due to dry run');
return true;
}
console.log('waiting for new submission to be listed...');
for(let i = 0; i < config.timeout; i += config.poll) {
if(await checkalready()) {
console.log('version ' + config.version + ' release successful');
return true;
}
await delay(config.poll);
}
throw Error('timed out awaiting new release listing');
mylog('uploading release...');
await cdbfetch(`${relurl}new/`, 'post', params, { multipart: true });
mylog('release uploaded');
});
};

View File

@ -3,7 +3,8 @@
const path = require('path');
const minimist = require('minimist');
const config = {};
const proto = {};
const config = Object.create(proto);
module.exports = config;
const layers = {};
@ -17,9 +18,9 @@ function set(name, obj) {
Object.keys(config)
.forEach(k => delete config[k]);
layord.forEach(k => Object.assign(config, layers[k] || {}));
config.set = set;
return config;
}
proto.set = set;
set('cmdline', minimist(process.argv.slice(2)));
@ -29,12 +30,9 @@ set('default', {
execlua: 'lua5.1',
cdbjsonpath: '.cdb.json',
cdbluapath: '.cdbrelease.lua',
root: 'https://content.minetest.net/',
root: 'https://content.minetest.net/api',
token: null,
user: null,
pkg: null,
min: 'none',
max: 'none',
timeout: 60,
poll: 2
});

View File

@ -3,6 +3,7 @@
const fs = require('fs');
const util = require('util');
const config = require('./config');
const mylog = require('./mylog');
const spawn = require('./spawn');
const readFile = util.promisify(fs.readFile);
@ -32,7 +33,7 @@ module.exports = async cwd => {
(await readFile('luahook.lua'))
.toString();
console.log('loading metadata...');
mylog('loading lua metadata...');
let buff = '';
const hook = spawn('lua', ['-'], {
stdio: ['pipe', 'pipe', 'inherit'],

View File

@ -1,23 +1,9 @@
'use strict';
const fsx = require('fs-extra');
const tmp = require('tmp-promise');
const mylog = require('./mylog');
const config = require('./config');
const spawn = require('./spawn');
async function withtemp(func) {
const tmpdir = await tmp.dir({
mode: parseInt('700', 8),
prefix: 'cdbrelease-'
});
try {
await fsx.emptyDir(tmpdir.path);
return await func(tmpdir.path);
} finally {
await fsx.emptyDir(tmpdir.path);
await tmpdir.cleanup();
}
}
const withtemp = require('./withtemp');
async function rawexport(repodir, outdir) {
const garch = spawn('git', [
@ -26,7 +12,7 @@ async function rawexport(repodir, outdir) {
'--format=tar',
config.branch
], {
stdio: ['inherit', 'pipe', 'inherit']
stdio: ['ignore', 'pipe', 'inherit']
});
const untar = spawn('tar', [
'-C', outdir,
@ -41,26 +27,30 @@ async function rawexport(repodir, outdir) {
async function gitmirror(func) {
return await withtemp(async tmp => {
mylog('cloning source repository...');
await spawn('git', [
'clone',
'--bare',
'--depth=1',
'-b', config.branch,
config.repo,
config.srcrepo,
tmp
], {
stdio: 'inherit'
stdio: ['ignore', 'inherit', 'inherit']
})
.promise;
async function gitexport(full, f2) {
return withtemp(async t2 => {
mylog(`creating ${full?'full':'lean'} git export...`);
if(full)
await spawn('git', [
'clone',
'-b', config.branch,
tmp,
t2
])
], {
stdio: ['ignore', 'inherit', 'inherit']
})
.promise;
await rawexport(tmp, t2);
return await f2(t2);
@ -71,4 +61,4 @@ async function gitmirror(func) {
});
}
module.exports = gitmirror;
module.exports = gitmirror;

View File

@ -6,11 +6,12 @@ const fs = require('fs');
const util = require('util');
const path = require('path');
const config = require('./config');
// const cdbrelease = require('./cdbrelease');
// const cdbedit = require('./cdbedit');
// const cdbscreens = require('./cdbscreens');
const gitmirror = require('./gitmirror');
const gitlua = require('./gitlua');
const cdbfetch = require('./cdbfetch');
const cdbedit = require('./cdbedit');
const cdbrelease = require('./cdbrelease');
// const cdbscreens = require('./cdbscreens');
const readFile = util.promisify(fs.readFile);
const loadJsonConfig = async (fn, key, req) => {
@ -21,13 +22,18 @@ const loadJsonConfig = async (fn, key, req) => {
if(req) throw e;
console.warn(e.message || e);
}
};
function missingConfig(...opts) {
return new Error(`missing/invalid option${opts.length == 1 ? '' : 's'}: ${
opts.map(x => `--${x}=...`).join(', ')}`);
}
(async () => {
await loadJsonConfig(config.conf, 'conffile');
if(!config.repo)
throw 'missing/invalid option: --repo=...';
if(!config.srcrepo)
throw missingConfig('srcrepo');
await gitmirror(async gitexport => {
if(config.cdbjson || config.cdblua)
await gitexport(true, async full => {
@ -38,27 +44,27 @@ const loadJsonConfig = async (fn, key, req) => {
if(config.cdblua)
await gitlua(full);
});
if(!config.token) throw missingConfig('token');
if(!config.user) {
if(!config.root) throw missingConfig('root');
const resp = await cdbfetch('/whoami/');
if(resp.body && resp.body.username)
config.user = resp.body.username;
}
const missing = Object.keys(config)
.filter(k => k !== '_')
.filter(k => config[k] === null)
.sort();
if(missing.length)
throw missingConfig(...missing);
await Promise.all([
config.edit && cdbedit(),
config.version && cdbrelease(gitexport),
]);
});
const missing = [];
Object.keys(config)
.sort()
.forEach(k => {
if(k === '_') return;
const v = config[k];
if(v === null)
missing.push(`--${k}=...`);
});
if(missing.length)
throw 'missing/invalid options: ' + missing.join(' ');
config.root = config.root.replace(/\/+$/, '');
// if(config.version) {
// config.version = '' + config.version;
// config.edit = await cdbrelease() || config.edit;
// }
// if(config.edit)
// config.uploadimages = await cdbedit() || config.uploadimages;
// if(config.uploadimages && config.screenshots)
// await cdbscreens();
})();

3
mylog.js Normal file
View File

@ -0,0 +1,3 @@
'use strict';
module.exports = msg => console.warn(`cdbrelease: ${msg}`);

395
package-lock.json generated
View File

@ -9,6 +9,120 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.23.tgz",
"integrity": "sha512-EKhb5NveQ3NlW5EV7B0VRtDKwUfVey8LuJRl9pp5iW0se87/ZqLjG0PMf2MCzPXAJYWZN5Ltg7pHIAf9/Dm1tQ=="
},
"archiver": {
"version": "5.2.0",
"resolved": "http://127.0.0.1:4873/archiver/-/archiver-5.2.0.tgz",
"integrity": "sha512-QEAKlgQuAtUxKeZB9w5/ggKXh21bZS+dzzuQ0RPBC20qtDCbTyzqmisoeJP46MP39fg4B4IcyvR+yeyEBdblsQ==",
"requires": {
"archiver-utils": "^2.1.0",
"async": "^3.2.0",
"buffer-crc32": "^0.2.1",
"readable-stream": "^3.6.0",
"readdir-glob": "^1.0.0",
"tar-stream": "^2.1.4",
"zip-stream": "^4.0.4"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"archiver-utils": {
"version": "2.1.0",
"resolved": "http://127.0.0.1:4873/archiver-utils/-/archiver-utils-2.1.0.tgz",
"integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==",
"requires": {
"glob": "^7.1.4",
"graceful-fs": "^4.2.0",
"lazystream": "^1.0.0",
"lodash.defaults": "^4.2.0",
"lodash.difference": "^4.5.0",
"lodash.flatten": "^4.4.0",
"lodash.isplainobject": "^4.0.6",
"lodash.union": "^4.6.0",
"normalize-path": "^3.0.0",
"readable-stream": "^2.0.0"
},
"dependencies": {
"graceful-fs": {
"version": "4.2.6",
"resolved": "http://127.0.0.1:4873/graceful-fs/-/graceful-fs-4.2.6.tgz",
"integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ=="
},
"readable-stream": {
"version": "2.3.7",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "http://127.0.0.1:4873/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"async": {
"version": "3.2.0",
"resolved": "http://127.0.0.1:4873/async/-/async-3.2.0.tgz",
"integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw=="
},
"balanced-match": {
"version": "1.0.0",
"resolved": "http://127.0.0.1:4873/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"base64-js": {
"version": "1.5.1",
"resolved": "http://127.0.0.1:4873/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"bl": {
"version": "4.1.0",
"resolved": "http://127.0.0.1:4873/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
},
"dependencies": {
"inherits": {
"version": "2.0.4",
"resolved": "http://127.0.0.1:4873/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"readable-stream": {
"version": "3.6.0",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"bluebird": {
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
@ -19,6 +133,29 @@
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "http://127.0.0.1:4873/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "http://127.0.0.1:4873/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "http://127.0.0.1:4873/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"cheerio": {
"version": "1.0.0-rc.2",
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
@ -32,6 +169,69 @@
"parse5": "^3.0.1"
}
},
"compress-commons": {
"version": "4.0.2",
"resolved": "http://127.0.0.1:4873/compress-commons/-/compress-commons-4.0.2.tgz",
"integrity": "sha512-qhd32a9xgzmpfoga1VQEiLEwdKZ6Plnpx5UCgIsf89FSolyJ7WnifY4Gtjgv5WR6hWAyRaHxC5MiEhU/38U70A==",
"requires": {
"buffer-crc32": "^0.2.13",
"crc32-stream": "^4.0.1",
"normalize-path": "^3.0.0",
"readable-stream": "^3.6.0"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "http://127.0.0.1:4873/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "http://127.0.0.1:4873/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"crc-32": {
"version": "1.2.0",
"resolved": "http://127.0.0.1:4873/crc-32/-/crc-32-1.2.0.tgz",
"integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==",
"requires": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
}
},
"crc32-stream": {
"version": "4.0.2",
"resolved": "http://127.0.0.1:4873/crc32-stream/-/crc32-stream-4.0.2.tgz",
"integrity": "sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==",
"requires": {
"crc-32": "^1.2.0",
"readable-stream": "^3.4.0"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@ -94,11 +294,29 @@
"domelementtype": "1"
}
},
"end-of-stream": {
"version": "1.4.4",
"resolved": "http://127.0.0.1:4873/end-of-stream/-/end-of-stream-1.4.4.tgz",
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"requires": {
"once": "^1.4.0"
}
},
"entities": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
},
"exit-on-epipe": {
"version": "1.0.1",
"resolved": "http://127.0.0.1:4873/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw=="
},
"fs-constants": {
"version": "1.0.0",
"resolved": "http://127.0.0.1:4873/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs-extra": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
@ -109,6 +327,24 @@
"universalify": "^0.1.0"
}
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "http://127.0.0.1:4873/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"glob": {
"version": "7.1.6",
"resolved": "http://127.0.0.1:4873/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"graceful-fs": {
"version": "4.1.15",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
@ -135,11 +371,30 @@
"safer-buffer": ">= 2.1.2 < 3"
}
},
"ieee754": {
"version": "1.2.1",
"resolved": "http://127.0.0.1:4873/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
},
"inflight": {
"version": "1.0.6",
"resolved": "http://127.0.0.1:4873/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"isarray": {
"version": "1.0.0",
"resolved": "http://127.0.0.1:4873/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"jsonfile": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
@ -148,11 +403,76 @@
"graceful-fs": "^4.1.6"
}
},
"lazystream": {
"version": "1.0.0",
"resolved": "http://127.0.0.1:4873/lazystream/-/lazystream-1.0.0.tgz",
"integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=",
"requires": {
"readable-stream": "^2.0.5"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "http://127.0.0.1:4873/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"lodash": {
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
},
"lodash.defaults": {
"version": "4.2.0",
"resolved": "http://127.0.0.1:4873/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
},
"lodash.difference": {
"version": "4.5.0",
"resolved": "http://127.0.0.1:4873/lodash.difference/-/lodash.difference-4.5.0.tgz",
"integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw="
},
"lodash.flatten": {
"version": "4.4.0",
"resolved": "http://127.0.0.1:4873/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
},
"lodash.isplainobject": {
"version": "4.0.6",
"resolved": "http://127.0.0.1:4873/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
"integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
},
"lodash.union": {
"version": "4.6.0",
"resolved": "http://127.0.0.1:4873/lodash.union/-/lodash.union-4.6.0.tgz",
"integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg="
},
"minimatch": {
"version": "3.0.4",
"resolved": "http://127.0.0.1:4873/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
@ -173,6 +493,11 @@
"sax": "^1.2.4"
}
},
"normalize-path": {
"version": "3.0.0",
"resolved": "http://127.0.0.1:4873/normalize-path/-/normalize-path-3.0.0.tgz",
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
},
"nth-check": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
@ -181,6 +506,14 @@
"boolbase": "~1.0.0"
}
},
"once": {
"version": "1.4.0",
"resolved": "http://127.0.0.1:4873/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"os-tmpdir": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
@ -194,6 +527,21 @@
"@types/node": "*"
}
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "http://127.0.0.1:4873/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"printj": {
"version": "1.1.2",
"resolved": "http://127.0.0.1:4873/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ=="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "http://127.0.0.1:4873/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"readable-stream": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz",
@ -204,6 +552,14 @@
"util-deprecate": "^1.0.1"
}
},
"readdir-glob": {
"version": "1.1.1",
"resolved": "http://127.0.0.1:4873/readdir-glob/-/readdir-glob-1.1.1.tgz",
"integrity": "sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==",
"requires": {
"minimatch": "^3.0.4"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -227,6 +583,18 @@
"safe-buffer": "~5.1.0"
}
},
"tar-stream": {
"version": "2.2.0",
"resolved": "http://127.0.0.1:4873/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
},
"tmp": {
"version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@ -253,6 +621,33 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"wrappy": {
"version": "1.0.2",
"resolved": "http://127.0.0.1:4873/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"zip-stream": {
"version": "4.0.4",
"resolved": "http://127.0.0.1:4873/zip-stream/-/zip-stream-4.0.4.tgz",
"integrity": "sha512-a65wQ3h5gcQ/nQGWV1mSZCEzCML6EK/vyVPcrPNynySP1j3VBbQKh3nhC8CbORb+jfl2vXvh56Ul5odP1bAHqw==",
"requires": {
"archiver-utils": "^2.1.0",
"compress-commons": "^4.0.2",
"readable-stream": "^3.6.0"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "http://127.0.0.1:4873/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
}
}
}

View File

@ -6,8 +6,10 @@
"author": "Aaron Suen <warr1024@gmail.com>",
"license": "ISC",
"dependencies": {
"archiver": "^5.2.0",
"cheerio": "^1.0.0-rc.2",
"fs-extra": "^7.0.1",
"glob": "^7.1.6",
"minimist": "^1.2.0",
"needle": "^2.2.4",
"tmp-promise": "^1.0.5"

View File

@ -1,11 +1,12 @@
'use strict';
const childproc = require('child_process');
const mylog = require('./mylog');
const config = require('./config');
function spawn(path, params, ...args) {
path = config[`exec${path}`] || path;
console.log(`> ${path} ${params.join(' ')}`);
mylog(`> ${path} ${params.join(' ')}`);
const proc = childproc.spawn(path, params, ...args);
proc.promise = new Promise((res, rejraw) => {
function rej(e) {

20
withtemp.js Normal file
View File

@ -0,0 +1,20 @@
'use strict';
const fsx = require('fs-extra');
const tmp = require('tmp-promise');
async function withtemp(func) {
const tmpdir = await tmp.dir({
mode: parseInt('700', 8),
prefix: 'cdbrelease-'
});
try {
await fsx.emptyDir(tmpdir.path);
return await func(tmpdir.path);
} finally {
await fsx.emptyDir(tmpdir.path);
await tmpdir.cleanup();
}
}
module.exports = withtemp;