432 lines
12 KiB
JavaScript
432 lines
12 KiB
JavaScript
const fs = require("fs");
|
|
const path = require("path");
|
|
|
|
const ncp = require("ncp").ncp;
|
|
const { rimrafSync } = require("rimraf");
|
|
|
|
const {
|
|
validateFilesPresent,
|
|
execCmdSync,
|
|
autodetectPlatformAndArch,
|
|
} = require("../../../scripts/util/index");
|
|
|
|
const { generateAndCopyConfigYamlSchema } = require("./generate-copy-config");
|
|
const { installAndCopyNodeModules } = require("./install-copy-nodemodule");
|
|
const { copySqlite, copyEsbuild } = require("./install-copy-sqlite-esbuild");
|
|
const { installNodeModules } = require("./install-nodemodules");
|
|
const { writeBuildTimestamp, continueDir } = require("./utils");
|
|
|
|
// Clear folders that will be packaged to ensure clean slate
|
|
rimrafSync(path.join(__dirname, "..", "bin"));
|
|
rimrafSync(path.join(__dirname, "..", "out"));
|
|
fs.mkdirSync(path.join(__dirname, "..", "out", "node_modules"), {
|
|
recursive: true,
|
|
});
|
|
const guiDist = path.join(__dirname, "..", "..", "..", "gui", "dist");
|
|
if (!fs.existsSync(guiDist)) {
|
|
fs.mkdirSync(guiDist, { recursive: true });
|
|
}
|
|
|
|
// Get the target to package for
|
|
let target = undefined;
|
|
const args = process.argv;
|
|
if (args[2] === "--target") {
|
|
target = args[3];
|
|
}
|
|
|
|
let os;
|
|
let arch;
|
|
if (target) {
|
|
[os, arch] = target.split("-");
|
|
} else {
|
|
[os, arch] = autodetectPlatformAndArch();
|
|
}
|
|
|
|
if (os === "alpine") {
|
|
os = "linux";
|
|
}
|
|
if (arch === "armhf") {
|
|
arch = "arm64";
|
|
}
|
|
target = `${os}-${arch}`;
|
|
console.log("[info] Using target: ", target);
|
|
|
|
const exe = os === "win32" ? ".exe" : "";
|
|
|
|
const isInGitHubAction = !!process.env.GITHUB_ACTIONS;
|
|
|
|
const isArmTarget =
|
|
target === "darwin-arm64" ||
|
|
target === "linux-arm64" ||
|
|
target === "win32-arm64";
|
|
|
|
const isWinTarget = target?.startsWith("win");
|
|
const isLinuxTarget = target?.startsWith("linux");
|
|
const isMacTarget = target?.startsWith("darwin");
|
|
|
|
void (async () => {
|
|
console.log("[info] Packaging extension for target ", target);
|
|
|
|
// Make sure we have an initial timestamp file
|
|
writeBuildTimestamp();
|
|
|
|
await Promise.all([generateAndCopyConfigYamlSchema(), installNodeModules()]);
|
|
|
|
process.chdir(path.join(continueDir, "gui"));
|
|
|
|
if (isInGitHubAction) {
|
|
execCmdSync("npm run build");
|
|
}
|
|
|
|
// Copy over the dist folder to the JetBrains extension //
|
|
const intellijExtensionWebviewPath = path.join(
|
|
"..",
|
|
"extensions",
|
|
"intellij",
|
|
"src",
|
|
"main",
|
|
"resources",
|
|
"webview",
|
|
);
|
|
|
|
const indexHtmlPath = path.join(intellijExtensionWebviewPath, "index.html");
|
|
fs.copyFileSync(indexHtmlPath, "tmp_index.html");
|
|
rimrafSync(intellijExtensionWebviewPath);
|
|
fs.mkdirSync(intellijExtensionWebviewPath, { recursive: true });
|
|
|
|
await new Promise((resolve, reject) => {
|
|
ncp("dist", intellijExtensionWebviewPath, (error) => {
|
|
if (error) {
|
|
console.warn(
|
|
"[error] Error copying React app build to JetBrains extension: ",
|
|
error,
|
|
);
|
|
reject(error);
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
|
|
// Put back index.html
|
|
if (fs.existsSync(indexHtmlPath)) {
|
|
rimrafSync(indexHtmlPath);
|
|
}
|
|
fs.copyFileSync("tmp_index.html", indexHtmlPath);
|
|
fs.unlinkSync("tmp_index.html");
|
|
|
|
console.log("[info] Copied gui build to JetBrains extension");
|
|
|
|
// Then copy over the dist folder to the VSCode extension //
|
|
const vscodeGuiPath = path.join("../extensions/vscode/gui");
|
|
fs.mkdirSync(vscodeGuiPath, { recursive: true });
|
|
await new Promise((resolve, reject) => {
|
|
ncp("dist", vscodeGuiPath, (error) => {
|
|
if (error) {
|
|
console.log(
|
|
"Error copying React app build to VSCode extension: ",
|
|
error,
|
|
);
|
|
reject(error);
|
|
} else {
|
|
console.log("Copied gui build to VSCode extension");
|
|
resolve();
|
|
}
|
|
});
|
|
});
|
|
|
|
if (!fs.existsSync(path.join("dist", "assets", "index.js"))) {
|
|
throw new Error("gui build did not produce index.js");
|
|
}
|
|
if (!fs.existsSync(path.join("dist", "assets", "index.css"))) {
|
|
throw new Error("gui build did not produce index.css");
|
|
}
|
|
|
|
// Copy over native / wasm modules //
|
|
process.chdir("../extensions/vscode");
|
|
|
|
fs.mkdirSync("bin", { recursive: true });
|
|
|
|
// onnxruntime-node
|
|
await new Promise((resolve, reject) => {
|
|
ncp(
|
|
path.join(__dirname, "../../../core/node_modules/onnxruntime-node/bin"),
|
|
path.join(__dirname, "../bin"),
|
|
{
|
|
dereference: true,
|
|
},
|
|
(error) => {
|
|
if (error) {
|
|
console.warn("[info] Error copying onnxruntime-node files", error);
|
|
reject(error);
|
|
}
|
|
resolve();
|
|
},
|
|
);
|
|
});
|
|
if (target) {
|
|
// If building for production, only need the binaries for current platform
|
|
try {
|
|
if (!target.startsWith("darwin")) {
|
|
rimrafSync(path.join(__dirname, "../bin/napi-v3/darwin"));
|
|
}
|
|
if (!target.startsWith("linux")) {
|
|
rimrafSync(path.join(__dirname, "../bin/napi-v3/linux"));
|
|
}
|
|
if (!target.startsWith("win")) {
|
|
rimrafSync(path.join(__dirname, "../bin/napi-v3/win32"));
|
|
}
|
|
|
|
// Also don't want to include cuda/shared/tensorrt binaries, they are too large
|
|
if (target.startsWith("linux")) {
|
|
const filesToRemove = [
|
|
"libonnxruntime_providers_cuda.so",
|
|
"libonnxruntime_providers_shared.so",
|
|
"libonnxruntime_providers_tensorrt.so",
|
|
];
|
|
filesToRemove.forEach((file) => {
|
|
const filepath = path.join(
|
|
__dirname,
|
|
"../bin/napi-v3/linux/x64",
|
|
file,
|
|
);
|
|
if (fs.existsSync(filepath)) {
|
|
fs.rmSync(filepath);
|
|
}
|
|
});
|
|
}
|
|
} catch (e) {
|
|
console.warn("[info] Error removing unused binaries", e);
|
|
}
|
|
}
|
|
console.log("[info] Copied onnxruntime-node");
|
|
|
|
// tree-sitter-wasm
|
|
fs.mkdirSync("out", { recursive: true });
|
|
|
|
await new Promise((resolve, reject) => {
|
|
ncp(
|
|
path.join(__dirname, "../../../core/node_modules/tree-sitter-wasms/out"),
|
|
path.join(__dirname, "../out/tree-sitter-wasms"),
|
|
{ dereference: true },
|
|
(error) => {
|
|
if (error) {
|
|
console.warn("[error] Error copying tree-sitter-wasm files", error);
|
|
reject(error);
|
|
} else {
|
|
resolve();
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
const filesToCopy = [
|
|
"../../../core/vendor/tree-sitter.wasm",
|
|
"../../../core/llm/llamaTokenizerWorkerPool.mjs",
|
|
"../../../core/llm/llamaTokenizer.mjs",
|
|
"../../../core/llm/tiktokenWorkerPool.mjs",
|
|
"../../../core/util/start_ollama.sh",
|
|
];
|
|
|
|
for (const f of filesToCopy) {
|
|
fs.copyFileSync(
|
|
path.join(__dirname, f),
|
|
path.join(__dirname, "..", "out", path.basename(f)),
|
|
);
|
|
console.log(`[info] Copied ${path.basename(f)}`);
|
|
}
|
|
|
|
// tree-sitter tag query files
|
|
// ncp(
|
|
// path.join(
|
|
// __dirname,
|
|
// "../../../core/node_modules/llm-code-highlighter/dist/tag-qry",
|
|
// ),
|
|
// path.join(__dirname, "../out/tag-qry"),
|
|
// (error) => {
|
|
// if (error)
|
|
// console.warn("Error copying code-highlighter tag-qry files", error);
|
|
// },
|
|
// );
|
|
|
|
// textmate-syntaxes
|
|
await new Promise((resolve, reject) => {
|
|
ncp(
|
|
path.join(__dirname, "../textmate-syntaxes"),
|
|
path.join(__dirname, "../gui/textmate-syntaxes"),
|
|
(error) => {
|
|
if (error) {
|
|
console.warn("[error] Error copying textmate-syntaxes", error);
|
|
reject(error);
|
|
} else {
|
|
resolve();
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
// GitHub Actions doesn't support ARM, so we need to download pre-saved binaries
|
|
// 02/07/25 - the above comment is out of date, there is now support for ARM runners on GitHub Actions
|
|
if (isArmTarget) {
|
|
// lancedb binary
|
|
const packageToInstall = {
|
|
"darwin-arm64": "@lancedb/vectordb-darwin-arm64",
|
|
"linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
|
|
"win32-arm64": "@lancedb/vectordb-win32-arm64-msvc",
|
|
}[target];
|
|
console.log(
|
|
"[info] Downloading pre-built lancedb binary: " + packageToInstall,
|
|
);
|
|
|
|
await Promise.all([
|
|
copyEsbuild(target),
|
|
copySqlite(target),
|
|
installAndCopyNodeModules(packageToInstall, "@lancedb"),
|
|
]);
|
|
} else {
|
|
// Download esbuild from npm in tmp and copy over
|
|
console.log("[info] npm installing esbuild binary");
|
|
await installAndCopyNodeModules("esbuild@0.17.19", "@esbuild");
|
|
}
|
|
|
|
console.log("[info] Copying sqlite node binding from core");
|
|
await new Promise((resolve, reject) => {
|
|
ncp(
|
|
path.join(__dirname, "../../../core/node_modules/sqlite3/build"),
|
|
path.join(__dirname, "../out/build"),
|
|
{ dereference: true },
|
|
(error) => {
|
|
if (error) {
|
|
console.warn("[error] Error copying sqlite3 files", error);
|
|
reject(error);
|
|
} else {
|
|
resolve();
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
// Copied here as well for the VS Code test suite
|
|
await new Promise((resolve, reject) => {
|
|
ncp(
|
|
path.join(__dirname, "../../../core/node_modules/sqlite3/build"),
|
|
path.join(__dirname, "../out"),
|
|
{ dereference: true },
|
|
(error) => {
|
|
if (error) {
|
|
console.warn("[error] Error copying sqlite3 files", error);
|
|
reject(error);
|
|
} else {
|
|
resolve();
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
// Copy node_modules for pre-built binaries
|
|
const NODE_MODULES_TO_COPY = [
|
|
"esbuild",
|
|
"@esbuild",
|
|
"@lancedb",
|
|
"@vscode/ripgrep",
|
|
"workerpool",
|
|
];
|
|
|
|
fs.mkdirSync("out/node_modules", { recursive: true });
|
|
|
|
await Promise.all(
|
|
NODE_MODULES_TO_COPY.map(
|
|
(mod) =>
|
|
new Promise((resolve, reject) => {
|
|
fs.mkdirSync(`out/node_modules/${mod}`, { recursive: true });
|
|
ncp(
|
|
`node_modules/${mod}`,
|
|
`out/node_modules/${mod}`,
|
|
{ dereference: true },
|
|
function (error) {
|
|
if (error) {
|
|
console.error(`[error] Error copying ${mod}`, error);
|
|
reject(error);
|
|
} else {
|
|
console.log(`[info] Copied ${mod}`);
|
|
resolve();
|
|
}
|
|
},
|
|
);
|
|
}),
|
|
),
|
|
);
|
|
|
|
// delete esbuild/bin because platform-specific @esbuild is downloaded
|
|
fs.rmSync(`out/node_modules/esbuild/bin`, { recursive: true });
|
|
|
|
console.log(`[info] Copied ${NODE_MODULES_TO_COPY.join(", ")}`);
|
|
|
|
// Copy over any worker files
|
|
fs.cpSync(
|
|
"node_modules/jsdom/lib/jsdom/living/xhr/xhr-sync-worker.js",
|
|
"out/xhr-sync-worker.js",
|
|
);
|
|
|
|
// Validate the all of the necessary files are present
|
|
validateFilesPresent([
|
|
// Queries used to create the index for @code context provider
|
|
"tree-sitter/code-snippet-queries/c_sharp.scm",
|
|
|
|
// Queries used for @outline and @highlights context providers
|
|
"tag-qry/tree-sitter-c_sharp-tags.scm",
|
|
|
|
// onnx runtime bindngs
|
|
`bin/napi-v3/${os}/${arch}/onnxruntime_binding.node`,
|
|
`bin/napi-v3/${os}/${arch}/${
|
|
isMacTarget
|
|
? "libonnxruntime.1.14.0.dylib"
|
|
: isLinuxTarget
|
|
? "libonnxruntime.so.1.14.0"
|
|
: "onnxruntime.dll"
|
|
}`,
|
|
|
|
// Code/styling for the sidebar
|
|
"gui/assets/index.js",
|
|
"gui/assets/index.css",
|
|
|
|
// Tutorial
|
|
"media/move-chat-panel-right.md",
|
|
"continue_tutorial.py",
|
|
"config_schema.json",
|
|
|
|
// Embeddings model
|
|
"models/all-MiniLM-L6-v2/config.json",
|
|
"models/all-MiniLM-L6-v2/special_tokens_map.json",
|
|
"models/all-MiniLM-L6-v2/tokenizer_config.json",
|
|
"models/all-MiniLM-L6-v2/tokenizer.json",
|
|
"models/all-MiniLM-L6-v2/vocab.txt",
|
|
"models/all-MiniLM-L6-v2/onnx/model_quantized.onnx",
|
|
|
|
// node_modules (it's a bit confusing why this is necessary)
|
|
`node_modules/@vscode/ripgrep/bin/rg${exe}`,
|
|
|
|
// out directory (where the extension.js lives)
|
|
// "out/extension.js", This is generated afterward by vsce
|
|
// web-tree-sitter
|
|
"out/tree-sitter.wasm",
|
|
// Worker required by jsdom
|
|
"out/xhr-sync-worker.js",
|
|
// SQLite3 Node native module
|
|
"out/build/Release/node_sqlite3.node",
|
|
|
|
// out/node_modules (to be accessed by extension.js)
|
|
`out/node_modules/@vscode/ripgrep/bin/rg${exe}`,
|
|
`out/node_modules/@esbuild/${
|
|
target === "win32-arm64"
|
|
? "esbuild.exe"
|
|
: target === "win32-x64"
|
|
? "win32-x64/esbuild.exe"
|
|
: `${target}/bin/esbuild`
|
|
}`,
|
|
`out/node_modules/@lancedb/vectordb-${target}${isWinTarget ? "-msvc" : ""}${isLinuxTarget ? "-gnu" : ""}/index.node`,
|
|
`out/node_modules/esbuild/lib/main.js`,
|
|
]);
|
|
|
|
process.exit(0);
|
|
})();
|