use vsce to determine what to package when bundling extensions

fixes #29054

build/gulpfile.vscode.js

@@ -30,6 +30,7 @@ const product = require('../product.json');
 const shrinkwrap = require('../npm-shrinkwrap.json');
 const crypto = require('crypto');
 const i18n = require('./lib/i18n');
+const glob = require('glob');
 
 const productDependencies = Object.keys(product.dependencies || {});
 const dependencies = Object.keys(shrinkwrap.dependencies)
@@ -46,6 +47,11 @@ const builtInExtensions = [
 	{ name: 'ms-vscode.node-debug2', version: '1.14.0' }
 ];
 
+const excludedExtensions = [
+	'vscode-api-tests',
+	'vscode-colorize-tests'
+];
+
 const vscodeEntryPoints = _.flatten([
 	buildfile.entrypoint('vs/workbench/electron-browser/workbench.main'),
 	buildfile.base,
@@ -222,40 +228,41 @@ function packageTask(platform, arch, opts) {
 			.pipe(rename(function (path) { path.dirname = path.dirname.replace(new RegExp('^' + out), 'out'); }))
 			.pipe(util.setExecutableBit(['**/*.sh']));
 
-		const extensionsList = [
-			'extensions/*/**',
-			'!extensions/*/src/**',
-			'!extensions/*/out/**/test/**',
-			'!extensions/*/test/**',
-			'!extensions/*/build/**',
-			'!extensions/**/node_modules/@types/**',
-			'!extensions/*/{client,server}/src/**',
-			'!extensions/*/{client,server}/test/**',
-			'!extensions/*/{client,server}/out/**/test/**',
-			'!extensions/*/{client,server}/out/**/typings/**',
-			'!extensions/**/.vscode/**',
-			'!extensions/**/tsconfig.json',
-			'!extensions/typescript/bin/**',
-			'!extensions/vscode-api-tests/**',
-			'!extensions/vscode-colorize-tests/**',
-			...builtInExtensions.map(e => `!extensions/${e.name}/**`)
-		];
+		const root = path.resolve(path.join(__dirname, '..'));
+		const localExtensionDescriptions = glob.sync('extensions/*/package.json')
+			.map(manifestPath => {
+				const extensionPath = path.dirname(path.join(root, manifestPath));
+				const extensionName = path.basename(extensionPath);
+				return { name: extensionName, path: extensionPath };
+			})
+			.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
+			.filter(({ name }) => builtInExtensions.every(b => b.name !== name));
 
-		const nlsFilter = filter('**/*.nls.json', { restore: true });
-		const extensions = gulp.src(extensionsList, { base: '.' })
-			// TODO@Dirk: this filter / buffer is here to make sure the nls.json files are buffered
-			.pipe(nlsFilter)
-			.pipe(buffer())
-			.pipe(nlsDev.createAdditionalLanguageFiles(languages, path.join(__dirname, '..', 'i18n')))
-			.pipe(nlsFilter.restore);
+		const localExtensions = es.merge(...localExtensionDescriptions.map(extension => {
+			const nlsFilter = filter('**/*.nls.json', { restore: true });
+
+			return ext.fromLocal(extension.path)
+				.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`))
+				// 	// TODO@Dirk: this filter / buffer is here to make sure the nls.json files are buffered
+				.pipe(nlsFilter)
+				.pipe(buffer())
+				.pipe(nlsDev.createAdditionalLanguageFiles(languages, path.join(__dirname, '..', 'i18n')))
+				.pipe(nlsFilter.restore);
+		}));
+
+		const localExtensionDependencies = gulp.src('extensions/node_modules/**', { base: '.' });
 
 		const marketplaceExtensions = es.merge(...builtInExtensions.map(extension => {
-			return ext.src(extension.name, extension.version)
+			return ext.fromMarketplace(extension.name, extension.version)
 				.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
 		}));
 
-		const sources = es.merge(src, extensions, marketplaceExtensions)
-			.pipe(filter(['**', '!**/*.js.map']));
+		const sources = es.merge(
+			src,
+			localExtensions,
+			localExtensionDependencies,
+			marketplaceExtensions
+		).pipe(filter(['**', '!**/*.js.map']));
 
 		let version = packageJson.version;
 		const quality = product.quality;

build/lib/extensions.js

@@ -4,7 +4,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-var event_stream_1 = require("event-stream");
+var es = require("event-stream");
 var assign = require("object-assign");
 var remote = require("gulp-remote-src");
 var flatmap = require('gulp-flatmap');
@@ -14,8 +14,29 @@ var rename = require('gulp-rename');
 var util = require('gulp-util');
 var buffer = require('gulp-buffer');
 var json = require('gulp-json-editor');
+var fs = require("fs");
+var path = require("path");
+var vsce = require("vsce");
+var File = require("vinyl");
+function fromLocal(extensionPath) {
+    var result = es.through();
+    vsce.listFiles({ cwd: extensionPath })
+        .then(function (fileNames) {
+        var files = fileNames
+            .map(function (fileName) { return path.join(extensionPath, fileName); })
+            .map(function (filePath) { return new File({
+            path: filePath,
+            base: extensionPath,
+            contents: fs.createReadStream(filePath)
+        }); });
+        es.readArray(files).pipe(result);
+    })
+        .catch(function (err) { return result.emit('error', err); });
+    return result;
+}
+exports.fromLocal = fromLocal;
 function error(err) {
-    var result = event_stream_1.through();
+    var result = es.through();
     setTimeout(function () { return result.emit('error', err); });
     return result;
 }
@@ -23,7 +44,7 @@ var baseHeaders = {
     'X-Market-Client-Id': 'VSCode Build',
     'User-Agent': 'VSCode Build',
 };
-function src(extensionName, version) {
+function fromMarketplace(extensionName, version) {
     var filterType = 7;
     var value = extensionName;
     var criterium = { filterType: filterType, value: value };
@@ -93,4 +114,4 @@ function src(extensionName, version) {
         }));
     }));
 }
-exports.src = src;
+exports.fromMarketplace = fromMarketplace;

build/lib/extensions.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { through } from 'event-stream';
+import * as es from 'event-stream';
 import { Stream } from 'stream';
 import assign = require('object-assign');
 import remote = require('gulp-remote-src');
@@ -14,9 +14,33 @@ const rename = require('gulp-rename');
 const util = require('gulp-util');
 const buffer = require('gulp-buffer');
 const json = require('gulp-json-editor');
+import * as fs from 'fs';
+import * as path from 'path';
+import * as vsce from 'vsce';
+import * as File from 'vinyl';
+
+export function fromLocal(extensionPath: string): Stream {
+	const result = es.through();
+
+	vsce.listFiles({ cwd: extensionPath })
+		.then(fileNames => {
+			const files = fileNames
+				.map(fileName => path.join(extensionPath, fileName))
+				.map(filePath => new File({
+					path: filePath,
+					base: extensionPath,
+					contents: fs.createReadStream(filePath) as any
+				}));
+
+			es.readArray(files).pipe(result);
+		})
+		.catch(err => result.emit('error', err));
+
+	return result;
+}
 
 function error(err: any): Stream {
-	const result = through();
+	const result = es.through();
 	setTimeout(() => result.emit('error', err));
 	return result;
 }
@@ -26,7 +50,7 @@ const baseHeaders = {
 	'User-Agent': 'VSCode Build',
 };
 
-export function src(extensionName: string, version: string): Stream {
+export function fromMarketplace(extensionName: string, version: string): Stream {
 	const filterType = 7;
 	const value = extensionName;
 	const criterium = { filterType, value };

extensions/bat/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/clojure/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/coffeescript/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/configuration-editing/.vscodeignore

@@ -0,0 +1,4 @@
+test/**
+src/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/cpp/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/csharp/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/css/.vscodeignore

@@ -0,0 +1,6 @@
+test/**
+client/src/**
+client/tsconfig.json
+server/src/**
+server/tsconfig.json
+server/node_modules/@types/**

extensions/diff/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/docker/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/emmet/.vscodeignore

@@ -0,0 +1,4 @@
+test/**
+src/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/extension-editing/.vscodeignore

@@ -0,0 +1,4 @@
+test/**
+src/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/fsharp/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/git/.vscodeignore

@@ -0,0 +1,5 @@
+src/**
+test/**
+out/test/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/git/npm-shrinkwrap.json

@@ -13,9 +13,9 @@
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz"
     },
     "vscode-extension-telemetry": {
-      "version": "0.0.6",
-      "from": "vscode-extension-telemetry@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.6.tgz"
+      "version": "0.0.7",
+      "from": "vscode-extension-telemetry@>=0.0.7 <0.0.8",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.7.tgz"
     },
     "vscode-nls": {
       "version": "2.0.2",

extensions/git/package.json

@@ -633,8 +633,8 @@
   },
   "dependencies": {
     "iconv-lite": "0.4.15",
-    "vscode-extension-telemetry": "^0.0.7",
-    "vscode-nls": "^2.0.1"
+    "vscode-extension-telemetry": "0.0.7",
+    "vscode-nls": "2.0.2"
   },
   "devDependencies": {
     "@types/mocha": "^2.2.41",

extensions/git/vsc-extension-quickstart.md

@@ -1,33 +0,0 @@
-# Welcome to your first VS Code Extension
-
-## What's in the folder
-* This folder contains all of the files necessary for your extension
-* `package.json` - this is the manifest file in which you declare your extension and command.
-The sample plugin registers a command and defines its title and command name. With this information
-VS Code can show the command in the command palette. It doesn’t yet need to load the plugin.
-* `src/extension.ts` - this is the main file where you will provide the implementation of your command.
-The file exports one function, `activate`, which is called the very first time your extension is
-activated (in this case by executing the command). Inside the `activate` function we call `registerCommand`.
-We pass the function containing the implementation of the command as the second parameter to
-`registerCommand`.
-
-## Get up and running straight away
-* press `F5` to open a new window with your extension loaded
-* run your command from the command palette by pressing (`Ctrl+Shift+P` or `Cmd+Shift+P` on Mac) and typing `Hello World`
-* set breakpoints in your code inside `src/extension.ts` to debug your extension
-* find output from your extension in the debug console
-
-## Make changes
-* you can relaunch the extension from the debug toolbar after changing code in `src/extension.ts`
-* you can also reload (`Ctrl+R` or `Cmd+R` on Mac) the VS Code window with your extension to load your changes
-
-## Explore the API
-* you can open the full set of our API when you open the file `node_modules/vscode/vscode.d.ts`
-
-## Run tests
-* open the debug viewlet (`Ctrl+Shift+D` or `Cmd+Shift+D` on Mac) and from the launch configuration dropdown pick `Launch Tests`
-* press `F5` to run the tests in a new window with your extension loaded
-* see the output of the test result in the debug console
-* make changes to `test/extension.test.ts` or create new test files inside the `test` folder
-    * by convention, the test runner will only consider files matching the name pattern `**.test.ts`
-    * you can create folders inside the `test` folder to structure your tests any way you want

extensions/gitsyntax/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/go/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/groovy/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/grunt/.vscodeignore

@@ -0,0 +1,3 @@
+test/**
+src/**
+tsconfig.json

extensions/gulp/.vscodeignore

@@ -0,0 +1,2 @@
+src/**
+tsconfig.json

extensions/handlebars/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/hlsl/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/html/.vscodeignore

@@ -0,0 +1,9 @@
+test/**
+server/tsconfig.json
+server/node_modules/@types/**
+server/src/**
+server/test/**
+server/out/test/**
+client/tsconfig.json
+client/src/**
+npm-shrinkwrap.json

extensions/html/npm-shrinkwrap.json

@@ -8,9 +8,9 @@
       "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-0.18.0.tgz"
     },
     "vscode-extension-telemetry": {
-      "version": "0.0.6",
-      "from": "vscode-extension-telemetry@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.6.tgz"
+      "version": "0.0.7",
+      "from": "vscode-extension-telemetry@>=0.0.7 <0.0.8",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.7.tgz"
     },
     "vscode-jsonrpc": {
       "version": "3.1.0-alpha.1",
@@ -38,4 +38,4 @@
       "resolved": "https://registry.npmjs.org/winreg/-/winreg-1.2.3.tgz"
     }
   }
-}
+}

extensions/html/package.json

@@ -191,13 +191,13 @@
     }
   },
   "dependencies": {
-    "vscode-extension-telemetry": "^0.0.7",
-    "vscode-languageclient": "^3.1.0-alpha.1",
-    "vscode-languageserver-types": "^3.0.3",
-    "vscode-nls": "^2.0.2"
+    "vscode-extension-telemetry": "0.0.7",
+    "vscode-languageclient": "3.1.0-alpha.1",
+    "vscode-languageserver-types": "3.0.3",
+    "vscode-nls": "2.0.2"
   },
   "devDependencies": {
     "@types/node": "^6.0.51",
     "@types/mocha": "^2.2.33"
   }
-}
+}

extensions/ini/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/jake/.vscodeignore

@@ -0,0 +1,2 @@
+src/**
+tsconfig.json

extensions/java/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/javascript/.vscodeignore

@@ -0,0 +1,4 @@
+test/**
+src/**/*.ts
+tsconfig.json
+npm-shrinkwrap.json

extensions/json/.vscodeignore

@@ -0,0 +1,7 @@
+test/**
+client/tsconfig.json
+client/src/**
+server/tsconfig.json
+server/src/**
+server/node_modules/@types/**
+npm-shrinkwrap.json

extensions/json/npm-shrinkwrap.json

@@ -8,9 +8,9 @@
       "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-0.18.0.tgz"
     },
     "vscode-extension-telemetry": {
-      "version": "0.0.6",
-      "from": "vscode-extension-telemetry@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.6.tgz"
+      "version": "0.0.7",
+      "from": "vscode-extension-telemetry@>=0.0.7 <0.0.8",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.7.tgz"
     },
     "vscode-jsonrpc": {
       "version": "3.1.0-alpha.1",
@@ -38,4 +38,4 @@
       "resolved": "https://registry.npmjs.org/winreg/-/winreg-1.2.3.tgz"
     }
   }
-}
+}

extensions/json/package.json

@@ -117,16 +117,18 @@
     },
     "configurationDefaults": {
       "[json]": {
-        "editor.quickSuggestions": { "strings": true }
+        "editor.quickSuggestions": {
+          "strings": true
+        }
       }
     }
   },
   "dependencies": {
-    "vscode-extension-telemetry": "^0.0.7",
-    "vscode-languageclient": "^3.1.0-alpha.1",
-    "vscode-nls": "^2.0.2"
+    "vscode-extension-telemetry": "0.0.7",
+    "vscode-languageclient": "3.1.0-alpha.1",
+    "vscode-nls": "2.0.2"
   },
   "devDependencies": {
     "@types/node": "^6.0.51"
   }
-}
+}

extensions/less/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/lua/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/make/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/markdown/.vscodeignore

@@ -0,0 +1,4 @@
+test/**
+src/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/markdown/npm-shrinkwrap.json

@@ -58,13 +58,13 @@
       "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.3.tgz"
     },
     "vscode-extension-telemetry": {
-      "version": "0.0.6",
-      "from": "vscode-extension-telemetry@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.6.tgz"
+      "version": "0.0.7",
+      "from": "vscode-extension-telemetry@>=0.0.7 <0.0.8",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.7.tgz"
     },
     "vscode-nls": {
       "version": "2.0.2",
-      "from": "vscode-nls@latest",
+      "from": "vscode-nls@>=2.0.1 <3.0.0",
       "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-2.0.2.tgz"
     },
     "winreg": {
@@ -73,4 +73,4 @@
       "resolved": "https://registry.npmjs.org/winreg/-/winreg-1.2.3.tgz"
     }
   }
-}
+}

extensions/markdown/package.json

@@ -227,11 +227,11 @@
 		"update-grammar": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ./syntaxes/gulpfile.js"
 	},
 	"dependencies": {
-		"highlight.js": "^9.11.0",
-		"markdown-it": "^8.3.1",
+		"highlight.js": "9.5.0",
+		"markdown-it": "8.2.2",
 		"markdown-it-named-headers": "0.0.4",
-		"vscode-extension-telemetry": "^0.0.7",
-		"vscode-nls": "^2.0.2"
+		"vscode-extension-telemetry": "0.0.7",
+		"vscode-nls": "2.0.2"
 	},
 	"devDependencies": {
 		"@types/node": "^7.0.4",

extensions/merge-conflict/.vscodeignore

@@ -0,0 +1,2 @@
+src/**
+tsconfig.json

extensions/npm/.vscodeignore

@@ -0,0 +1,2 @@
+src/**
+tsconfig.json

extensions/objective-c/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/perl/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/php/.vscodeignore

@@ -1 +1,6 @@
 test/**
+build/**
+out/test/**
+src/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/php/package.nls.json

@@ -1,4 +1,4 @@
-{
+{
 	"configuration.suggest.basic": "Configures if the built-in PHP language suggestions are enabled. The support suggests PHP globals and variables.",
 	"configuration.validate.enable": "Enable/disable built-in PHP validation.",
 	"configuration.validate.executablePath": "Points to the PHP executable.",

extensions/powershell/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/pug/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/python/.vscodeignore

@@ -1 +1,3 @@
 test/**
+src/**
+tsconfig.json

extensions/r/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/razor/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/ruby/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/rust/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/scss/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/shaderlab/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/shellscript/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/sql/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/swift/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/theme-seti/.vscodeignore

@@ -0,0 +1 @@
+build/**

extensions/typescript/.vscodeignore

@@ -0,0 +1,5 @@
+build/**
+src/**
+test/**
+tsconfig.json
+npm-shrinkwrap.json

extensions/typescript/npm-shrinkwrap.json

@@ -13,9 +13,9 @@
       "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz"
     },
     "vscode-extension-telemetry": {
-      "version": "0.0.6",
-      "from": "vscode-extension-telemetry@>=0.0.6 <0.0.7",
-      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.6.tgz"
+      "version": "0.0.7",
+      "from": "vscode-extension-telemetry@>=0.0.7 <0.0.8",
+      "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.0.7.tgz"
     },
     "vscode-nls": {
       "version": "2.0.1",

extensions/typescript/package.json

@@ -13,8 +13,8 @@
   "enableProposedApi": true,
   "dependencies": {
     "semver": "4.3.6",
-    "vscode-extension-telemetry": "^0.0.7",
-    "vscode-nls": "^2.0.1"
+    "vscode-extension-telemetry": "0.0.7",
+    "vscode-nls": "2.0.1"
   },
   "devDependencies": {
     "@types/node": "^7.0.4",

extensions/vb/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/xml/.vscodeignore

@@ -0,0 +1 @@
+test/**

extensions/yaml/.vscodeignore

@@ -0,0 +1 @@
+test/**

package.json

@@ -112,6 +112,7 @@
     "underscore": "^1.8.2",
     "vinyl": "^0.4.5",
     "vinyl-fs": "^2.4.3",
+    "vsce": "^1.25.1",
     "vscode-nls-dev": "^2.0.1"
   },
   "repository": {