mirror of https://github.com/microsoft/vscode.git
eng - add `checksum` validation support to `nodejs` and `extensions` (#184877)
* eng - add `checksum` validation support to `remote` * eng - wire in more `checksum` support * eng - renames for remote fetching * eng - renames for remote fetching * eng - disable verbose * eng - always fetch verbose in CI * eng - 💄 * eng - add checksums for node * eng - report checksum matches * eng - fix build * eng - warn when not being able to check sum * eng - support checksums for built in extensions * eng - clear todo * eng - add nodejs metadata to product.json * 🆙 version * 🆙 distro * update distro * eng - switch to checksum file * cleanup alpine * fix alpine * fix bug * eng - fetch all from remote * eng - 💄 * eng - print checksums * eng - fix missing import * 🆙 distro * undo version change
This commit is contained in:
Benjamin Pasero
GitHub
parent
13b1b90a83
commit
8afb685c9b
|
|
@ -0,0 +1,34 @@
|
|||
dfb37570ef34ac04f34c26d0ec558df60a9665df5961c01c1657c0ca495f2f01 node-v16.17.1-aix-ppc64.tar.gz
|
||||
f9f02f7872e2e8ee54320fce13deb9d56904f32bb0615b6e21aa3371d8899150 node-v16.17.1-darwin-arm64.tar.gz
|
||||
09a45f60bfb9dfbea4f69044dc733ef983945acd92ca89ccccac267f3d71bd44 node-v16.17.1-darwin-arm64.tar.xz
|
||||
3db26761ad8493b894d42260d7e65094b7af9bc473588739e61bc1c32d6ff955 node-v16.17.1-darwin-x64.tar.gz
|
||||
8e7089956fa01cf7d0045945c0863d282dc6818fb0476237c1396497e29a4254 node-v16.17.1-darwin-x64.tar.xz
|
||||
35ccb95caf02cda3bd680da4350a8ae5d666a7a9eae3afe5c2a1b3ef29aef108 node-v16.17.1-headers.tar.gz
|
||||
554c8d1b4b16e0f4c073b9df7c49c893716a3a533f25ac646f23619f5ccee7df node-v16.17.1-headers.tar.xz
|
||||
adc7032888d4e672a4aac886baede8c04fccdd1a2e7ab4bcf325e3f336f44a3d node-v16.17.1-linux-arm64.tar.gz
|
||||
3dfb8fd8f6b97df69cdc56524abc906c50ef1d0bf091188616802e6c7c731389 node-v16.17.1-linux-arm64.tar.xz
|
||||
aeab05e35f1d2824ecfb88ca321f1408b44d292b2775f2890972c828e00216d0 node-v16.17.1-linux-armv7l.tar.gz
|
||||
a035ceefb5e16f5fce98c8ddfdf721b96eec20542c72fb8781bcbb6ef20c5550 node-v16.17.1-linux-armv7l.tar.xz
|
||||
1f48de7bed99e973c4c50f1b7fc99fc9af5144d093fd6d2b50a1e43b5818bf05 node-v16.17.1-linux-ppc64le.tar.gz
|
||||
70305934661f89fca64053b85317a75f233d5e3fdb2caa6546a19262a519cf20 node-v16.17.1-linux-ppc64le.tar.xz
|
||||
029dad48018bda07b481213816549b632059fc673c30fdc7a353e04619128344 node-v16.17.1-linux-s390x.tar.gz
|
||||
1a47f604944c6aff37cb7483503155671cdb34bda9bfb8962007bc440fa04d77 node-v16.17.1-linux-s390x.tar.xz
|
||||
da5658693243b3ecf6a4cba6751a71df1eb9e9703ca93b42a9404aed85f58ad0 node-v16.17.1-linux-x64.tar.gz
|
||||
06ba2eb34aa385967f5f58c87a44753f83212f6cccea892b33f80a2e7fda8384 node-v16.17.1-linux-x64.tar.xz
|
||||
12d10476ea7483298364c810c037b9316d1a73dc8c81cfeff7d794aecadde498 node-v16.17.1.pkg
|
||||
e423985f6019b2026f9a191adb56a96ae83ecd56cdf839cf94aa980168b7a90f node-v16.17.1.tar.gz
|
||||
6721feb4152d56d2c6b358ce397abd5a7f1daf09ee2e25c5021b9b4d3f86a330 node-v16.17.1.tar.xz
|
||||
9777e8c4b2864c5b54a0e4e9400f14887db68560a09b94b4113b560a64d1e680 node-v16.17.1-win-x64.7z
|
||||
ed290151efb417262b9808a70738d4ab79e9d53653a6a9f4b8dd97912e279dce node-v16.17.1-win-x64.zip
|
||||
0f8101648d5c9e49e89fee541da9e574f899716c32b7c51a732b1766b9fc4526 node-v16.17.1-win-x86.7z
|
||||
189b5e8b23226403e7b07a46614de19b444d369e694901e3668e2f549799cbcd node-v16.17.1-win-x86.zip
|
||||
1bdff65fb7642425c0d6826084d63c4be43520316f0ea0b46e6a51999a0ed7fc node-v16.17.1-x64.msi
|
||||
b737eb23a2c67c253b9364b5284123faf5220d567615bebd4ec4b81070e4d177 node-v16.17.1-x86.msi
|
||||
f518a70dcab7c3fac5b2e1ef100b4f628edfb160f4fafa9a94ef222da8a6e9ab win-x64/node.exe
|
||||
2f459a64647db493da63c790ce368ad54f59f086d9f22f59c5018680420197b3 win-x64/node.lib
|
||||
23215ce7d1e9de9777c3407239e7cf18d29d60f757b772219421ab361ac67c74 win-x64/node_pdb.7z
|
||||
8e32ec12028fd3e3147435be79a858ed9c870aaafa1fcb291362307ef3c47547 win-x64/node_pdb.zip
|
||||
2393aff88be19dbe0205cbde4ff0c1d89911b15de5c99c80f6e5e29604eecd12 win-x86/node.exe
|
||||
5018c3d42f3fbacbd06cb943b3f2696c8e67ca9bdf6864d0e263d6d6911dffd2 win-x86/node.lib
|
||||
05a4db56444a60ee70b0d2642d7f2d82a33339894d2d73bd07b1a41d6c869e04 win-x86/node_pdb.7z
|
||||
8f86eacb7f13a1bf6738cb0819d7854a2abca40fc2e9e1f91421e44ba52cad7e win-x86/node_pdb.zip
|
||||
|
|
@ -17,7 +17,6 @@ const rename = require('gulp-rename');
|
|||
const replace = require('gulp-replace');
|
||||
const filter = require('gulp-filter');
|
||||
const { getProductionDependencies } = require('./lib/dependencies');
|
||||
const { assetFromGithub } = require('./lib/github');
|
||||
const vfs = require('vinyl-fs');
|
||||
const packageJson = require('../package.json');
|
||||
const flatmap = require('gulp-flatmap');
|
||||
|
|
@ -43,7 +42,6 @@ const BUILD_TARGETS = [
|
|||
{ platform: 'win32', arch: 'x64' },
|
||||
{ platform: 'darwin', arch: 'x64' },
|
||||
{ platform: 'darwin', arch: 'arm64' },
|
||||
{ platform: 'linux', arch: 'ia32' },
|
||||
{ platform: 'linux', arch: 'x64' },
|
||||
{ platform: 'linux', arch: 'armhf' },
|
||||
{ platform: 'linux', arch: 'arm64' },
|
||||
|
|
@ -131,6 +129,33 @@ function getNodeVersion() {
|
|||
return target;
|
||||
}
|
||||
|
||||
function getNodeChecksum(nodeVersion, platform, arch) {
|
||||
let expectedName;
|
||||
switch (platform) {
|
||||
case 'win32':
|
||||
expectedName = `win-${arch}/node.exe`;
|
||||
break;
|
||||
|
||||
case 'darwin':
|
||||
case 'linux':
|
||||
expectedName = `node-v${nodeVersion}-${platform}-${arch}.tar.gz`;
|
||||
break;
|
||||
|
||||
case 'alpine':
|
||||
expectedName = `${platform}-${arch}/node`;
|
||||
break;
|
||||
}
|
||||
|
||||
const nodeJsChecksums = fs.readFileSync(path.join(REPO_ROOT, 'build', 'checksums', 'nodejs.txt'), 'utf8');
|
||||
for (const line of nodeJsChecksums.split('\n')) {
|
||||
const [checksum, name] = line.split(/\s+/);
|
||||
if (name === expectedName) {
|
||||
return checksum;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const nodeVersion = getNodeVersion();
|
||||
|
||||
BUILD_TARGETS.forEach(({ platform, arch }) => {
|
||||
|
|
@ -155,40 +180,57 @@ if (defaultNodeTask) {
|
|||
}
|
||||
|
||||
function nodejs(platform, arch) {
|
||||
const { remote } = require('./lib/gulpRemoteSource');
|
||||
const { fetchUrls, fetchGithub } = require('./lib/fetch');
|
||||
const untar = require('gulp-untar');
|
||||
const crypto = require('crypto');
|
||||
|
||||
if (arch === 'ia32') {
|
||||
arch = 'x86';
|
||||
}
|
||||
|
||||
if (platform === 'win32') {
|
||||
if (product.nodejsRepository) {
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from ${product.nodejsRepository}...`);
|
||||
return assetFromGithub(product.nodejsRepository, nodeVersion, name => name === `win-${arch}-node-patched.exe`)
|
||||
.pipe(rename('node.exe'));
|
||||
}
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from https://nodejs.org`);
|
||||
return remote(`/dist/v${nodeVersion}/win-${arch}/node.exe`, { base: 'https://nodejs.org', verbose: true })
|
||||
.pipe(rename('node.exe'));
|
||||
}
|
||||
|
||||
if (arch === 'alpine' || platform === 'alpine') {
|
||||
const imageName = arch === 'arm64' ? 'arm64v8/node' : 'node';
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from docker image ${imageName}`);
|
||||
const contents = cp.execSync(`docker run --rm ${imageName}:${nodeVersion}-alpine /bin/sh -c 'cat \`which node\`'`, { maxBuffer: 100 * 1024 * 1024, encoding: 'buffer' });
|
||||
return es.readArray([new File({ path: 'node', contents, stat: { mode: parseInt('755', 8) } })]);
|
||||
}
|
||||
|
||||
if (arch === 'armhf') {
|
||||
} else if (arch === 'armhf') {
|
||||
arch = 'armv7l';
|
||||
} else if (arch === 'alpine') {
|
||||
platform = 'alpine';
|
||||
arch = 'x64';
|
||||
}
|
||||
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from ${product.nodejs.repository}...`);
|
||||
|
||||
const checksumSha256 = getNodeChecksum(nodeVersion, platform, arch);
|
||||
|
||||
if (checksumSha256) {
|
||||
log(`Using SHA256 checksum for checking integrity: ${checksumSha256}`);
|
||||
} else {
|
||||
log.warn(`Unable to verify integrity of downloaded node.js binary because no SHA256 checksum was found!`);
|
||||
}
|
||||
|
||||
switch (platform) {
|
||||
case 'win32':
|
||||
return (product.nodejs.repository !== 'https://nodejs.org' ?
|
||||
fetchGithub(product.nodejs.repository, { version: product.nodejs.version, name: `win-${arch}-node.exe`, checksumSha256 }) :
|
||||
fetchUrls(`/dist/v${nodeVersion}/win-${arch}/node.exe`, { base: 'https://nodejs.org', checksumSha256 }))
|
||||
.pipe(rename('node.exe'));
|
||||
case 'darwin':
|
||||
case 'linux':
|
||||
return (product.nodejs.repository !== 'https://nodejs.org' ?
|
||||
fetchGithub(product.nodejs.repository, { version: product.nodejs.version, name: `node-v${nodeVersion}-${platform}-${arch}.tar.gz`, checksumSha256 }) :
|
||||
fetchUrls(`/dist/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: 'https://nodejs.org', checksumSha256 })
|
||||
).pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
|
||||
.pipe(filter('**/node'))
|
||||
.pipe(util.setExecutableBit('**'))
|
||||
.pipe(rename('node'));
|
||||
case 'alpine': {
|
||||
const imageName = arch === 'arm64' ? 'arm64v8/node' : 'node';
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from docker image ${imageName}`);
|
||||
const contents = cp.execSync(`docker run --rm ${imageName}:${nodeVersion}-alpine /bin/sh -c 'cat \`which node\`'`, { maxBuffer: 100 * 1024 * 1024, encoding: 'buffer' });
|
||||
if (checksumSha256) {
|
||||
const actualSHA256Checksum = crypto.createHash('sha256').update(contents).digest('hex');
|
||||
if (actualSHA256Checksum !== checksumSha256) {
|
||||
throw new Error(`Checksum mismatch for node.js from docker image (expected ${options.checksumSha256}, actual ${actualSHA256Checksum}))`);
|
||||
}
|
||||
}
|
||||
return es.readArray([new File({ path: 'node', contents, stat: { mode: parseInt('755', 8) } })]);
|
||||
}
|
||||
}
|
||||
log(`Downloading node.js ${nodeVersion} ${platform} ${arch} from https://nodejs.org`);
|
||||
return remote(`/dist/v${nodeVersion}/node-v${nodeVersion}-${platform}-${arch}.tar.gz`, { base: 'https://nodejs.org', verbose: true })
|
||||
.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
|
||||
.pipe(filter('**/node'))
|
||||
.pipe(util.setExecutableBit('**'))
|
||||
.pipe(rename('node'));
|
||||
}
|
||||
|
||||
function packageTask(type, platform, arch, sourceFolderName, destinationFolderName) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -20,6 +20,7 @@ const mkdirp = require('mkdirp');
|
|||
export interface IExtensionDefinition {
|
||||
name: string;
|
||||
version: string;
|
||||
sha256: string;
|
||||
repo: string;
|
||||
platforms?: string[];
|
||||
metadata: {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -22,10 +22,9 @@ const buffer = require('gulp-buffer');
|
|||
import * as jsoncParser from 'jsonc-parser';
|
||||
import webpack = require('webpack');
|
||||
import { getProductionDependencies } from './dependencies';
|
||||
import { getExtensionStream } from './builtInExtensions';
|
||||
import { IExtensionDefinition, getExtensionStream } from './builtInExtensions';
|
||||
import { getVersion } from './getVersion';
|
||||
import { remote, IOptions as IRemoteSrcOptions } from './gulpRemoteSource';
|
||||
import { assetFromGithub } from './github';
|
||||
import { fetchUrls, fetchGithub } from './fetch';
|
||||
|
||||
const root = path.dirname(path.dirname(__dirname));
|
||||
const commit = getVersion(root);
|
||||
|
|
@ -222,7 +221,7 @@ const baseHeaders = {
|
|||
'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
|
||||
};
|
||||
|
||||
export function fromMarketplace(serviceUrl: string, { name: extensionName, version, metadata }: IBuiltInExtension): Stream {
|
||||
export function fromMarketplace(serviceUrl: string, { name: extensionName, version, sha256, metadata }: IExtensionDefinition): Stream {
|
||||
const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
|
||||
|
||||
const [publisher, name] = extensionName.split('.');
|
||||
|
|
@ -230,16 +229,15 @@ export function fromMarketplace(serviceUrl: string, { name: extensionName, versi
|
|||
|
||||
fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
|
||||
|
||||
const options: IRemoteSrcOptions = {
|
||||
base: url,
|
||||
fetchOptions: {
|
||||
headers: baseHeaders
|
||||
}
|
||||
};
|
||||
|
||||
const packageJsonFilter = filter('package.json', { restore: true });
|
||||
|
||||
return remote('', options)
|
||||
return fetchUrls('', {
|
||||
base: url,
|
||||
nodeFetchOptions: {
|
||||
headers: baseHeaders
|
||||
},
|
||||
checksumSha256: sha256
|
||||
})
|
||||
.pipe(vzip.src())
|
||||
.pipe(filter('extension/**'))
|
||||
.pipe(rename(p => p.dirname = p.dirname!.replace(/^extension\/?/, '')))
|
||||
|
|
@ -250,14 +248,18 @@ export function fromMarketplace(serviceUrl: string, { name: extensionName, versi
|
|||
}
|
||||
|
||||
|
||||
export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream {
|
||||
export function fromGithub({ name, version, repo, sha256, metadata }: IExtensionDefinition): Stream {
|
||||
const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
|
||||
|
||||
fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
|
||||
|
||||
const packageJsonFilter = filter('package.json', { restore: true });
|
||||
|
||||
return assetFromGithub(new URL(repo).pathname, version, name => name.endsWith('.vsix'))
|
||||
return fetchGithub(new URL(repo).pathname, {
|
||||
version,
|
||||
name: name => name.endsWith('.vsix'),
|
||||
checksumSha256: sha256
|
||||
})
|
||||
.pipe(buffer())
|
||||
.pipe(vzip.src())
|
||||
.pipe(filter('extension/**'))
|
||||
|
|
@ -284,16 +286,9 @@ const marketplaceWebExtensionsExclude = new Set([
|
|||
'ms-vscode.vscode-js-profile-table'
|
||||
]);
|
||||
|
||||
interface IBuiltInExtension {
|
||||
name: string;
|
||||
version: string;
|
||||
repo: string;
|
||||
metadata: any;
|
||||
}
|
||||
|
||||
const productJson = JSON.parse(fs.readFileSync(path.join(__dirname, '../../product.json'), 'utf8'));
|
||||
const builtInExtensions: IBuiltInExtension[] = productJson.builtInExtensions || [];
|
||||
const webBuiltInExtensions: IBuiltInExtension[] = productJson.webBuiltInExtensions || [];
|
||||
const builtInExtensions: IExtensionDefinition[] = productJson.builtInExtensions || [];
|
||||
const webBuiltInExtensions: IExtensionDefinition[] = productJson.webBuiltInExtensions || [];
|
||||
|
||||
type ExtensionKind = 'ui' | 'workspace' | 'web';
|
||||
interface IExtensionManifest {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,147 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as es from 'event-stream';
|
||||
import fetch, { RequestInit } from 'node-fetch';
|
||||
import * as VinylFile from 'vinyl';
|
||||
import * as log from 'fancy-log';
|
||||
import * as ansiColors from 'ansi-colors';
|
||||
import * as crypto from 'crypto';
|
||||
import * as through2 from 'through2';
|
||||
import { Stream } from 'stream';
|
||||
|
||||
export interface IFetchOptions {
|
||||
base?: string;
|
||||
nodeFetchOptions?: RequestInit;
|
||||
verbose?: boolean;
|
||||
checksumSha256?: string;
|
||||
}
|
||||
|
||||
export function fetchUrls(urls: string[] | string, options: IFetchOptions): es.ThroughStream {
|
||||
if (options === undefined) {
|
||||
options = {};
|
||||
}
|
||||
|
||||
if (typeof options.base !== 'string' && options.base !== null) {
|
||||
options.base = '/';
|
||||
}
|
||||
|
||||
if (!Array.isArray(urls)) {
|
||||
urls = [urls];
|
||||
}
|
||||
|
||||
return es.readArray(urls).pipe(es.map<string, VinylFile | void>((data: string, cb) => {
|
||||
const url = [options.base, data].join('');
|
||||
fetchUrl(url, options).then(file => {
|
||||
cb(undefined, file);
|
||||
}, error => {
|
||||
cb(error);
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
export async function fetchUrl(url: string, options: IFetchOptions, retries = 10, retryDelay = 1000): Promise<VinylFile> {
|
||||
const verbose = !!options.verbose ?? (!!process.env['CI'] || !!process.env['BUILD_ARTIFACTSTAGINGDIRECTORY']);
|
||||
try {
|
||||
let startTime = 0;
|
||||
if (verbose) {
|
||||
log(`Start fetching ${ansiColors.magenta(url)}${retries !== 10 ? `(${10 - retries} retry}` : ''}`);
|
||||
startTime = new Date().getTime();
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options.nodeFetchOptions,
|
||||
signal: controller.signal as any /* Typings issue with lib.dom.d.ts */
|
||||
});
|
||||
if (verbose) {
|
||||
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
|
||||
}
|
||||
if (response.ok && (response.status >= 200 && response.status < 300)) {
|
||||
const contents = await response.buffer();
|
||||
if (options.checksumSha256) {
|
||||
const actualSHA256Checksum = crypto.createHash('sha256').update(contents).digest('hex');
|
||||
if (actualSHA256Checksum !== options.checksumSha256) {
|
||||
throw new Error(`Checksum mismatch for ${ansiColors.cyan(url)} (expected ${options.checksumSha256}, actual ${actualSHA256Checksum}))`);
|
||||
} else if (verbose) {
|
||||
log(`Verified SHA256 checksums match for ${ansiColors.cyan(url)}`);
|
||||
}
|
||||
} else if (verbose) {
|
||||
log(`Skipping checksum verification for ${ansiColors.cyan(url)} because no expected checksum was provided`);
|
||||
}
|
||||
if (verbose) {
|
||||
log(`Fetched response body buffer: ${ansiColors.magenta(`${(contents as Buffer).byteLength} bytes`)}`);
|
||||
}
|
||||
return new VinylFile({
|
||||
cwd: '/',
|
||||
base: options.base,
|
||||
path: url,
|
||||
contents
|
||||
});
|
||||
}
|
||||
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
} catch (e) {
|
||||
if (verbose) {
|
||||
log(`Fetching ${ansiColors.cyan(url)} failed: ${e}`);
|
||||
}
|
||||
if (retries > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
||||
return fetchUrl(url, options, retries - 1, retryDelay);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
const ghApiHeaders: Record<string, string> = {
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'VSCode Build',
|
||||
};
|
||||
if (process.env.GITHUB_TOKEN) {
|
||||
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
|
||||
}
|
||||
const ghDownloadHeaders = {
|
||||
...ghApiHeaders,
|
||||
Accept: 'application/octet-stream',
|
||||
};
|
||||
|
||||
export interface IGitHubAssetOptions {
|
||||
version: string;
|
||||
name: string | ((name: string) => boolean);
|
||||
checksumSha256?: string;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param repo for example `Microsoft/vscode`
|
||||
* @param version for example `16.17.1` - must be a valid releases tag
|
||||
* @param assetName for example (name) => name === `win-x64-node.exe` - must be an asset that exists
|
||||
* @returns a stream with the asset as file
|
||||
*/
|
||||
export function fetchGithub(repo: string, options: IGitHubAssetOptions): Stream {
|
||||
return fetchUrls(`/repos/${repo.replace(/^\/|\/$/g, '')}/releases/tags/v${options.version}`, {
|
||||
base: 'https://api.github.com',
|
||||
verbose: options.verbose,
|
||||
nodeFetchOptions: { headers: ghApiHeaders }
|
||||
}).pipe(through2.obj(async function (file, _enc, callback) {
|
||||
const assetFilter = typeof options.name === 'string' ? (name: string) => name === options.name : options.name;
|
||||
const asset = JSON.parse(file.contents.toString()).assets.find((a: { name: string }) => assetFilter(a.name));
|
||||
if (!asset) {
|
||||
return callback(new Error(`Could not find asset in release of ${repo} @ ${options.version}`));
|
||||
}
|
||||
try {
|
||||
callback(null, await fetchUrl(asset.url, {
|
||||
nodeFetchOptions: { headers: ghDownloadHeaders },
|
||||
verbose: options.verbose,
|
||||
checksumSha256: options.checksumSha256
|
||||
}));
|
||||
} catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
"use strict";
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.assetFromGithub = void 0;
|
||||
const node_fetch_1 = require("node-fetch");
|
||||
const gulpRemoteSource_1 = require("./gulpRemoteSource");
|
||||
const through2 = require("through2");
|
||||
const ghApiHeaders = {
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'VSCode Build',
|
||||
};
|
||||
if (process.env.GITHUB_TOKEN) {
|
||||
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
|
||||
}
|
||||
const ghDownloadHeaders = {
|
||||
...ghApiHeaders,
|
||||
Accept: 'application/octet-stream',
|
||||
};
|
||||
/**
|
||||
* @param repo for example `Microsoft/vscode`
|
||||
* @param version for example `16.17.1` - must be a valid releases tag
|
||||
* @param assetName for example (name) => name === `win-x64-node.exe` - must be an asset that exists
|
||||
* @returns a stream with the asset as file
|
||||
*/
|
||||
function assetFromGithub(repo, version, assetFilter) {
|
||||
return (0, gulpRemoteSource_1.remote)(`/repos/${repo.replace(/^\/|\/$/g, '')}/releases/tags/v${version}`, {
|
||||
base: 'https://api.github.com',
|
||||
fetchOptions: { headers: ghApiHeaders }
|
||||
}).pipe(through2.obj(async function (file, _enc, callback) {
|
||||
const asset = JSON.parse(file.contents.toString()).assets.find((a) => assetFilter(a.name));
|
||||
if (!asset) {
|
||||
return callback(new Error(`Could not find asset in release of ${repo} @ ${version}`));
|
||||
}
|
||||
const response = await (0, node_fetch_1.default)(asset.url, { headers: ghDownloadHeaders });
|
||||
if (response.ok) {
|
||||
file.contents = response.body.pipe(through2());
|
||||
callback(null, file);
|
||||
}
|
||||
else {
|
||||
return callback(new Error(`Request ${response.url} failed with status code: ${response.status}`));
|
||||
}
|
||||
}));
|
||||
}
|
||||
exports.assetFromGithub = assetFromGithub;
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2l0aHViLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZ2l0aHViLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7O2dHQUdnRzs7O0FBR2hHLDJDQUErQjtBQUMvQix5REFBNEM7QUFDNUMscUNBQXFDO0FBRXJDLE1BQU0sWUFBWSxHQUEyQjtJQUM1QyxNQUFNLEVBQUUsZ0NBQWdDO0lBQ3hDLFlBQVksRUFBRSxjQUFjO0NBQzVCLENBQUM7QUFDRixJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsWUFBWSxFQUFFO0lBQzdCLFlBQVksQ0FBQyxhQUFhLEdBQUcsUUFBUSxHQUFHLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLENBQUM7Q0FDakc7QUFDRCxNQUFNLGlCQUFpQixHQUFHO0lBQ3pCLEdBQUcsWUFBWTtJQUNmLE1BQU0sRUFBRSwwQkFBMEI7Q0FDbEMsQ0FBQztBQUVGOzs7OztHQUtHO0FBQ0gsU0FBZ0IsZUFBZSxDQUFDLElBQVksRUFBRSxPQUFlLEVBQUUsV0FBc0M7SUFDcEcsT0FBTyxJQUFBLHlCQUFNLEVBQUMsVUFBVSxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsRUFBRSxFQUFFLENBQUMsbUJBQW1CLE9BQU8sRUFBRSxFQUFFO1FBQ2pGLElBQUksRUFBRSx3QkFBd0I7UUFDOUIsWUFBWSxFQUFFLEVBQUUsT0FBTyxFQUFFLFlBQVksRUFBRTtLQUN2QyxDQUFDLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsS0FBSyxXQUFXLElBQUksRUFBRSxJQUFJLEVBQUUsUUFBUTtRQUN4RCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBbUIsRUFBRSxFQUFFLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO1FBQzdHLElBQUksQ0FBQyxLQUFLLEVBQUU7WUFDWCxPQUFPLFFBQVEsQ0FBQyxJQUFJLEtBQUssQ0FBQyxzQ0FBc0MsSUFBSSxNQUFNLE9BQU8sRUFBRSxDQUFDLENBQUMsQ0FBQztTQUN0RjtRQUNELE1BQU0sUUFBUSxHQUFHLE1BQU0sSUFBQSxvQkFBSyxFQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUUsRUFBRSxPQUFPLEVBQUUsaUJBQWlCLEVBQUUsQ0FBQyxDQUFDO1FBQ3hFLElBQUksUUFBUSxDQUFDLEVBQUUsRUFBRTtZQUNoQixJQUFJLENBQUMsUUFBUSxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7WUFDL0MsUUFBUSxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQztTQUNyQjthQUFNO1lBQ04sT0FBTyxRQUFRLENBQUMsSUFBSSxLQUFLLENBQUMsV0FBVyxRQUFRLENBQUMsR0FBRyw2QkFBNkIsUUFBUSxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUMsQ0FBQztTQUNsRztJQUVGLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDO0FBbEJELDBDQWtCQyJ9
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Stream } from 'stream';
|
||||
import fetch from 'node-fetch';
|
||||
import { remote } from './gulpRemoteSource';
|
||||
import * as through2 from 'through2';
|
||||
|
||||
const ghApiHeaders: Record<string, string> = {
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'VSCode Build',
|
||||
};
|
||||
if (process.env.GITHUB_TOKEN) {
|
||||
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
|
||||
}
|
||||
const ghDownloadHeaders = {
|
||||
...ghApiHeaders,
|
||||
Accept: 'application/octet-stream',
|
||||
};
|
||||
|
||||
/**
|
||||
* @param repo for example `Microsoft/vscode`
|
||||
* @param version for example `16.17.1` - must be a valid releases tag
|
||||
* @param assetName for example (name) => name === `win-x64-node.exe` - must be an asset that exists
|
||||
* @returns a stream with the asset as file
|
||||
*/
|
||||
export function assetFromGithub(repo: string, version: string, assetFilter: (name: string) => boolean): Stream {
|
||||
return remote(`/repos/${repo.replace(/^\/|\/$/g, '')}/releases/tags/v${version}`, {
|
||||
base: 'https://api.github.com',
|
||||
fetchOptions: { headers: ghApiHeaders }
|
||||
}).pipe(through2.obj(async function (file, _enc, callback) {
|
||||
const asset = JSON.parse(file.contents.toString()).assets.find((a: { name: string }) => assetFilter(a.name));
|
||||
if (!asset) {
|
||||
return callback(new Error(`Could not find asset in release of ${repo} @ ${version}`));
|
||||
}
|
||||
const response = await fetch(asset.url, { headers: ghDownloadHeaders });
|
||||
if (response.ok) {
|
||||
file.contents = response.body.pipe(through2());
|
||||
callback(null, file);
|
||||
} else {
|
||||
return callback(new Error(`Request ${response.url} failed with status code: ${response.status}`));
|
||||
}
|
||||
|
||||
}));
|
||||
}
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
"use strict";
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.remote = void 0;
|
||||
const es = require("event-stream");
|
||||
const node_fetch_1 = require("node-fetch");
|
||||
const VinylFile = require("vinyl");
|
||||
const through2 = require("through2");
|
||||
const log = require("fancy-log");
|
||||
const ansiColors = require("ansi-colors");
|
||||
function remote(urls, options) {
|
||||
if (options === undefined) {
|
||||
options = {};
|
||||
}
|
||||
if (typeof options.base !== 'string' && options.base !== null) {
|
||||
options.base = '/';
|
||||
}
|
||||
if (typeof options.buffer !== 'boolean') {
|
||||
options.buffer = true;
|
||||
}
|
||||
if (!Array.isArray(urls)) {
|
||||
urls = [urls];
|
||||
}
|
||||
return es.readArray(urls).pipe(es.map((data, cb) => {
|
||||
const url = [options.base, data].join('');
|
||||
fetchWithRetry(url, options).then(file => {
|
||||
cb(undefined, file);
|
||||
}, error => {
|
||||
cb(error);
|
||||
});
|
||||
}));
|
||||
}
|
||||
exports.remote = remote;
|
||||
async function fetchWithRetry(url, options, retries = 10, retryDelay = 1000) {
|
||||
try {
|
||||
let startTime = 0;
|
||||
if (options.verbose) {
|
||||
log(`Start fetching ${ansiColors.magenta(url)}${retries !== 10 ? `(${10 - retries} retry}` : ''}`);
|
||||
startTime = new Date().getTime();
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
|
||||
try {
|
||||
const response = await (0, node_fetch_1.default)(url, {
|
||||
...options.fetchOptions,
|
||||
signal: controller.signal /* Typings issue with lib.dom.d.ts */
|
||||
});
|
||||
if (options.verbose) {
|
||||
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
|
||||
}
|
||||
if (response.ok && (response.status >= 200 && response.status < 300)) {
|
||||
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
|
||||
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
|
||||
if (options.buffer && options.verbose) {
|
||||
log(`Fetched response body buffer: ${ansiColors.magenta(`${contents.byteLength} bytes`)}`);
|
||||
}
|
||||
return new VinylFile({
|
||||
cwd: '/',
|
||||
base: options.base,
|
||||
path: url,
|
||||
contents
|
||||
});
|
||||
}
|
||||
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
|
||||
}
|
||||
finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
if (options.verbose) {
|
||||
log(`Fetching ${ansiColors.cyan(url)} failed: ${e}`);
|
||||
}
|
||||
if (retries > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
||||
return fetchWithRetry(url, options, retries - 1, retryDelay);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ3VscFJlbW90ZVNvdXJjZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImd1bHBSZW1vdGVTb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7Z0dBR2dHOzs7QUFFaEcsbUNBQW1DO0FBQ25DLDJDQUFnRDtBQUNoRCxtQ0FBbUM7QUFDbkMscUNBQXFDO0FBQ3JDLGlDQUFpQztBQUNqQywwQ0FBMEM7QUFTMUMsU0FBZ0IsTUFBTSxDQUFDLElBQXVCLEVBQUUsT0FBaUI7SUFDaEUsSUFBSSxPQUFPLEtBQUssU0FBUyxFQUFFO1FBQzFCLE9BQU8sR0FBRyxFQUFFLENBQUM7S0FDYjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsSUFBSSxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLElBQUksRUFBRTtRQUM5RCxPQUFPLENBQUMsSUFBSSxHQUFHLEdBQUcsQ0FBQztLQUNuQjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsTUFBTSxLQUFLLFNBQVMsRUFBRTtRQUN4QyxPQUFPLENBQUMsTUFBTSxHQUFHLElBQUksQ0FBQztLQUN0QjtJQUVELElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxFQUFFO1FBQ3pCLElBQUksR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO0tBQ2Q7SUFFRCxPQUFPLEVBQUUsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQTJCLENBQUMsSUFBWSxFQUFFLEVBQUUsRUFBRSxFQUFFO1FBQ3BGLE1BQU0sR0FBRyxHQUFHLENBQUMsT0FBTyxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7UUFDMUMsY0FBYyxDQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUU7WUFDeEMsRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUNyQixDQUFDLEVBQUUsS0FBSyxDQUFDLEVBQUU7WUFDVixFQUFFLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDWCxDQUFDLENBQUMsQ0FBQztJQUNKLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDO0FBekJELHdCQXlCQztBQUVELEtBQUssVUFBVSxjQUFjLENBQUMsR0FBVyxFQUFFLE9BQWlCLEVBQUUsT0FBTyxHQUFHLEVBQUUsRUFBRSxVQUFVLEdBQUcsSUFBSTtJQUM1RixJQUFJO1FBQ0gsSUFBSSxTQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ2xCLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsa0JBQWtCLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsT0FBTyxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLEdBQUcsT0FBTyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7WUFDbkcsU0FBUyxHQUFHLElBQUksSUFBSSxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUM7U0FDakM7UUFDRCxNQUFNLFVBQVUsR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBQ3pDLE1BQU0sT0FBTyxHQUFHLFVBQVUsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxVQUFVLENBQUMsS0FBSyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FBQyxDQUFDO1FBQ2hFLElBQUk7WUFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUEsb0JBQUssRUFBQyxHQUFHLEVBQUU7Z0JBQ2pDLEdBQUcsT0FBTyxDQUFDLFlBQVk7Z0JBQ3ZCLE1BQU0sRUFBRSxVQUFVLENBQUMsTUFBYSxDQUFDLHFDQUFxQzthQUN0RSxDQUFDLENBQUM7WUFDSCxJQUFJLE9BQU8sQ0FBQyxPQUFPLEVBQUU7Z0JBQ3BCLEdBQUcsQ0FBQywyQkFBMkIsUUFBUSxDQUFDLE1BQU0sVUFBVSxVQUFVLENBQUMsT0FBTyxDQUFDLEdBQUcsSUFBSSxJQUFJLEVBQUUsQ0FBQyxPQUFPLEVBQUUsR0FBRyxTQUFTLEtBQUssQ0FBQyxFQUFFLENBQUMsQ0FBQzthQUN4SDtZQUNELElBQUksUUFBUSxDQUFDLEVBQUUsSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNLElBQUksR0FBRyxJQUFJLFFBQVEsQ0FBQyxNQUFNLEdBQUcsR0FBRyxDQUFDLEVBQUU7Z0JBQ3JFLG9JQUFvSTtnQkFDcEksTUFBTSxRQUFRLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsTUFBTSxRQUFRLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7Z0JBQzNGLElBQUksT0FBTyxDQUFDLE1BQU0sSUFBSSxPQUFPLENBQUMsT0FBTyxFQUFFO29CQUN0QyxHQUFHLENBQUMsaUNBQWlDLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBSSxRQUFtQixDQUFDLFVBQVUsUUFBUSxDQUFDLEVBQUUsQ0FBQyxDQUFDO2lCQUN2RztnQkFDRCxPQUFPLElBQUksU0FBUyxDQUFDO29CQUNwQixHQUFHLEVBQUUsR0FBRztvQkFDUixJQUFJLEVBQUUsT0FBTyxDQUFDLElBQUk7b0JBQ2xCLElBQUksRUFBRSxHQUFHO29CQUNULFFBQVE7aUJBQ1IsQ0FBQyxDQUFDO2FBQ0g7WUFDRCxNQUFNLElBQUksS0FBSyxDQUFDLFdBQVcsVUFBVSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsNkJBQTZCLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDO1NBQ2xHO2dCQUFTO1lBQ1QsWUFBWSxDQUFDLE9BQU8sQ0FBQyxDQUFDO1NBQ3RCO0tBQ0Q7SUFBQyxPQUFPLENBQUMsRUFBRTtRQUNYLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsWUFBWSxVQUFVLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDLENBQUM7U0FDckQ7UUFDRCxJQUFJLE9BQU8sR0FBRyxDQUFDLEVBQUU7WUFDaEIsTUFBTSxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxPQUFPLEVBQUUsVUFBVSxDQUFDLENBQUMsQ0FBQztZQUM5RCxPQUFPLGNBQWMsQ0FBQyxHQUFHLEVBQUUsT0FBTyxFQUFFLE9BQU8sR0FBRyxDQUFDLEVBQUUsVUFBVSxDQUFDLENBQUM7U0FDN0Q7UUFDRCxNQUFNLENBQUMsQ0FBQztLQUNSO0FBQ0YsQ0FBQyJ9
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as es from 'event-stream';
|
||||
import fetch, { RequestInit } from 'node-fetch';
|
||||
import * as VinylFile from 'vinyl';
|
||||
import * as through2 from 'through2';
|
||||
import * as log from 'fancy-log';
|
||||
import * as ansiColors from 'ansi-colors';
|
||||
|
||||
export interface IOptions {
|
||||
base?: string;
|
||||
buffer?: boolean;
|
||||
fetchOptions?: RequestInit;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
export function remote(urls: string[] | string, options: IOptions): es.ThroughStream {
|
||||
if (options === undefined) {
|
||||
options = {};
|
||||
}
|
||||
|
||||
if (typeof options.base !== 'string' && options.base !== null) {
|
||||
options.base = '/';
|
||||
}
|
||||
|
||||
if (typeof options.buffer !== 'boolean') {
|
||||
options.buffer = true;
|
||||
}
|
||||
|
||||
if (!Array.isArray(urls)) {
|
||||
urls = [urls];
|
||||
}
|
||||
|
||||
return es.readArray(urls).pipe(es.map<string, VinylFile | void>((data: string, cb) => {
|
||||
const url = [options.base, data].join('');
|
||||
fetchWithRetry(url, options).then(file => {
|
||||
cb(undefined, file);
|
||||
}, error => {
|
||||
cb(error);
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchWithRetry(url: string, options: IOptions, retries = 10, retryDelay = 1000): Promise<VinylFile> {
|
||||
try {
|
||||
let startTime = 0;
|
||||
if (options.verbose) {
|
||||
log(`Start fetching ${ansiColors.magenta(url)}${retries !== 10 ? `(${10 - retries} retry}` : ''}`);
|
||||
startTime = new Date().getTime();
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options.fetchOptions,
|
||||
signal: controller.signal as any /* Typings issue with lib.dom.d.ts */
|
||||
});
|
||||
if (options.verbose) {
|
||||
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
|
||||
}
|
||||
if (response.ok && (response.status >= 200 && response.status < 300)) {
|
||||
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
|
||||
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
|
||||
if (options.buffer && options.verbose) {
|
||||
log(`Fetched response body buffer: ${ansiColors.magenta(`${(contents as Buffer).byteLength} bytes`)}`);
|
||||
}
|
||||
return new VinylFile({
|
||||
cwd: '/',
|
||||
base: options.base,
|
||||
path: url,
|
||||
contents
|
||||
});
|
||||
}
|
||||
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
} catch (e) {
|
||||
if (options.verbose) {
|
||||
log(`Fetching ${ansiColors.cyan(url)} failed: ${e}`);
|
||||
}
|
||||
if (retries > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
||||
return fetchWithRetry(url, options, retries - 1, retryDelay);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -36,6 +36,7 @@
|
|||
{
|
||||
"name": "ms-vscode.js-debug-companion",
|
||||
"version": "1.0.18",
|
||||
"sha256": "b49ee134f452c88fe3de09ae62b7f77aa66d1d6dcd794c48e065bdc2c74d4a30",
|
||||
"repo": "https://github.com/microsoft/vscode-js-debug-companion",
|
||||
"metadata": {
|
||||
"id": "99cb0b7f-7354-4278-b8da-6cc79972169d",
|
||||
|
|
@ -51,6 +52,7 @@
|
|||
{
|
||||
"name": "ms-vscode.js-debug",
|
||||
"version": "1.78.0",
|
||||
"sha256": "9fbf0c15394fb436a4079b5f704d2718d305828be178b1219db85a9287e24870",
|
||||
"repo": "https://github.com/microsoft/vscode-js-debug",
|
||||
"metadata": {
|
||||
"id": "25629058-ddac-4e17-abba-74678e126c5d",
|
||||
|
|
@ -66,6 +68,7 @@
|
|||
{
|
||||
"name": "ms-vscode.vscode-js-profile-table",
|
||||
"version": "1.0.3",
|
||||
"sha256": "b9dab017506d9e6a469a0f82b392e4cb1d7a25a4843f1db8ba396cbee209cfc5",
|
||||
"repo": "https://github.com/microsoft/vscode-js-profile-visualizer",
|
||||
"metadata": {
|
||||
"id": "7e52b41b-71ad-457b-ab7e-0620f1fc4feb",
|
||||
|
|
@ -78,5 +81,9 @@
|
|||
"publisherDisplayName": "Microsoft"
|
||||
}
|
||||
}
|
||||
]
|
||||
],
|
||||
"nodejs": {
|
||||
"repository": "https://nodejs.org",
|
||||
"version": "16.17.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ else {
|
|||
// Running out of sources
|
||||
if (Object.keys(product).length === 0) {
|
||||
Object.assign(product, {
|
||||
version: '1.78.0-dev',
|
||||
version: '1.80.0-dev',
|
||||
nameShort: 'Code - OSS Dev',
|
||||
nameLong: 'Code - OSS Dev',
|
||||
applicationName: 'code-oss',
|
||||
|
|
|
|||
Loading…
Reference in New Issue