louis_lifu
/
vscode
mirror of https://github.com/microsoft/vscode.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build: cache built-in extensions to avoid rate limiting (#156918)

This commit is contained in:
Connor Peet 2022-08-02 15:27:16 -07:00 committed by GitHub
parent 8b27dcb1f8
commit ca48c64699
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 57 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
build/azure-pipelines/product-compile.yml
View File

@ -56,6 +56,13 @@ steps:
cacheHitVar: NODE_MODULES_RESTORED
displayName: Restore node_modules cache
# Cache built-in extensions to avoid GH rate limits.
- task: Cache@2
inputs:
key: '"$(Agent.OS)" | product.json'
path: .build/builtInExtensions
displayName: Restore built-in extensions
- script: |
set -e
tar -xzf .build/node_modules_cache/cache.tgz
@ -109,6 +116,11 @@ steps:
node build/azure-pipelines/mixin
displayName: Mix in quality
- script: |
set -e
node build/lib/builtInExtensions.js
displayName: Download missing built-in extensions
- script: |
set -e
yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check

2
build/gulpfile.extensions.js
View File

@ -236,8 +236,8 @@ exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;
const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions'));
const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
cleanExtensionsBuildTask,
task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))),
task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build'))),
));
gulp.task(compileExtensionsBuildTask);

2
build/gulpfile.vscode.web.js
View File

@ -229,7 +229,7 @@ function packageTask(sourceFolderName, destinationFolderName) {
const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series(
task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')),
task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true).pipe(gulp.dest('.build/web'))),
task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build/web'))),
task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))),
task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')),
));
gulp.task(compileWebExtensionsBuildTask);

20
build/lib/builtInExtensions.js
View File

@ -4,7 +4,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.getBuiltInExtensions = void 0;
exports.getBuiltInExtensions = exports.getExtensionStream = void 0;
const fs = require("fs");
const path = require("path");
const os = require("os");
@ -44,6 +44,21 @@ function isUpToDate(extension) {
return false;
}
}
function getExtensionDownloadStream(extension) {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
}
function getExtensionStream(extension) {
// if the extension exists on disk, use those files instead of downloading anew
if (isUpToDate(extension)) {
log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎'));
return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true })
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
}
return getExtensionDownloadStream(extension);
}
exports.getExtensionStream = getExtensionStream;
function syncMarketplaceExtension(extension) {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
@ -52,8 +67,7 @@ function syncMarketplaceExtension(extension) {
return es.readArray([]);
}
rimraf.sync(getExtensionPath(extension));
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
return getExtensionDownloadStream(extension)
.pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
}

21
build/lib/builtInExtensions.ts
View File

@ -68,10 +68,26 @@ function isUpToDate(extension: IExtensionDefinition): boolean {
}
}
function getExtensionDownloadStream(extension: IExtensionDefinition) {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
}
export function getExtensionStream(extension: IExtensionDefinition) {
// if the extension exists on disk, use those files instead of downloading anew
if (isUpToDate(extension)) {
log('[extensions]', `${extension.name}@${extension.version} up to date`, ansiColors.green('✔︎'));
return vfs.src(['**'], { cwd: getExtensionPath(extension), dot: true })
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`));
}
return getExtensionDownloadStream(extension);
}
function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
if (isUpToDate(extension)) {
log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
return es.readArray([]);
@ -79,8 +95,7 @@ function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
rimraf.sync(getExtensionPath(extension));
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
return getExtensionDownloadStream(extension)
.pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
}

8
build/lib/extensions.js
View File

@ -25,6 +25,7 @@ const buffer = require('gulp-buffer');
const jsoncParser = require("jsonc-parser");
const dependencies_1 = require("./dependencies");
const _ = require("underscore");
const builtInExtensions_1 = require("./builtInExtensions");
const util = require('./util');
const root = path.dirname(path.dirname(__dirname));
const commit = util.getVersion(root);
@ -312,16 +313,15 @@ function packageLocalExtensionsStream(forWeb) {
.pipe(util2.setExecutableBit(['**/*.sh'])));
}
exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) {
function packageMarketplaceExtensionsStream(forWeb) {
const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : [])
];
const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions
.map(extension => {
const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data) => {
const src = (0, builtInExtensions_1.getExtensionStream)(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`));
return updateExtensionPackageJSON(src, (data) => {
delete data.scripts;
delete data.dependencies;
delete data.devDependencies;

8
build/lib/extensions.ts
View File

@ -25,6 +25,7 @@ import * as jsoncParser from 'jsonc-parser';
import webpack = require('webpack');
import { getProductionDependencies } from './dependencies';
import _ = require('underscore');
import { getExtensionStream } from './builtInExtensions';
const util = require('./util');
const root = path.dirname(path.dirname(__dirname));
const commit = util.getVersion(root);
@ -381,7 +382,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
);
}
export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream {
export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : [])
@ -390,9 +391,8 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServi
es.merge(
...marketplaceExtensionsDescriptions
.map(extension => {
const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data: any) => {
const src = getExtensionStream(extension).pipe(rename(p => p.dirname = `extensions/${p.dirname}`));
return updateExtensionPackageJSON(src, (data: any) => {
delete data.scripts;
delete data.dependencies;
delete data.devDependencies;