uinstinct
parent
3c5332ac0a
commit
dd353c6571
174
binary/build.js
174
binary/build.js
|
|
@ -3,13 +3,10 @@ const fs = require("fs");
|
|||
const path = require("path");
|
||||
const ncp = require("ncp").ncp;
|
||||
const { rimrafSync } = require("rimraf");
|
||||
const {
|
||||
validateFilesPresent,
|
||||
execCmdSync,
|
||||
autodetectPlatformAndArch,
|
||||
} = require("../scripts/util");
|
||||
const { validateFilesPresent } = require("../scripts/util");
|
||||
const { downloadRipgrep } = require("./utils/ripgrep");
|
||||
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
|
||||
const { fork } = require("child_process");
|
||||
|
||||
const bin = path.join(__dirname, "bin");
|
||||
const out = path.join(__dirname, "out");
|
||||
|
|
@ -29,8 +26,6 @@ function cleanSlate() {
|
|||
const esbuildOutputFile = "out/index.js";
|
||||
let targets = [...ALL_TARGETS];
|
||||
|
||||
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
|
||||
|
||||
const assetBackups = [
|
||||
"node_modules/win-ca/lib/crypt32-ia32.node.bak",
|
||||
"node_modules/win-ca/lib/crypt32-x64.node.bak",
|
||||
|
|
@ -78,67 +73,6 @@ async function buildWithEsbuild() {
|
|||
});
|
||||
}
|
||||
|
||||
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
|
||||
console.log(`Copying ${packageName} to ${toCopy}`);
|
||||
// This is a way to install only one package without npm trying to install all the dependencies
|
||||
// Create a temporary directory for installing the package
|
||||
const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
|
||||
const tempDir = path.join(
|
||||
__dirname,
|
||||
"tmp",
|
||||
`continue-node_modules-${adjustedName}`,
|
||||
);
|
||||
const currentDir = process.cwd();
|
||||
|
||||
// // Remove the dir we will be copying to
|
||||
// rimrafSync(`node_modules/${toCopy}`);
|
||||
|
||||
// // Ensure the temporary directory exists
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// Move to the temporary directory
|
||||
process.chdir(tempDir);
|
||||
|
||||
// Initialize a new package.json and install the package
|
||||
execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
|
||||
|
||||
console.log(
|
||||
`Contents of: ${packageName}`,
|
||||
fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
|
||||
);
|
||||
|
||||
// Without this it seems the file isn't completely written to disk
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
// Copy the installed package back to the current directory
|
||||
await new Promise((resolve, reject) => {
|
||||
ncp(
|
||||
path.join(tempDir, "node_modules", toCopy),
|
||||
path.join(currentDir, "node_modules", toCopy),
|
||||
{ dereference: true },
|
||||
(error) => {
|
||||
if (error) {
|
||||
console.error(
|
||||
`[error] Error copying ${packageName} package`,
|
||||
error,
|
||||
);
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
// Clean up the temporary directory
|
||||
// rimrafSync(tempDir);
|
||||
|
||||
// Return to the original directory
|
||||
process.chdir(currentDir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and installs ripgrep binaries for the specified target
|
||||
*
|
||||
|
|
@ -181,16 +115,35 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
|||
),
|
||||
);
|
||||
|
||||
console.log("[info] Downloading prebuilt lancedb...");
|
||||
const copyLanceDBPromises = [];
|
||||
for (const target of targets) {
|
||||
if (TARGET_TO_LANCEDB[target]) {
|
||||
console.log(`[info] Downloading for ${target}...`);
|
||||
await installNodeModuleInTempDirAndCopyToCurrent(
|
||||
TARGET_TO_LANCEDB[target],
|
||||
"@lancedb",
|
||||
);
|
||||
if (!TARGET_TO_LANCEDB[target]) {
|
||||
continue;
|
||||
}
|
||||
console.log(`[info] Downloading for ${target}...`);
|
||||
const child = fork("./utils/copy-lancedb.js", { stdio: "inherit" });
|
||||
child.send({
|
||||
payload: {
|
||||
packageName: TARGET_TO_LANCEDB[target],
|
||||
toCopy: "@lancedb",
|
||||
},
|
||||
});
|
||||
copyLanceDBPromises.push(
|
||||
new Promise((resolve, reject) => {
|
||||
child.on("message", (msg) => {
|
||||
if (msg.error) {
|
||||
reject();
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}),
|
||||
);
|
||||
}
|
||||
await Promise.all(copyLanceDBPromises).catch(() => {
|
||||
console.error("[error] Failed to copy LanceDB");
|
||||
process.exit(1);
|
||||
});
|
||||
console.log("[info] Copied all LanceDB");
|
||||
|
||||
// tree-sitter-wasm
|
||||
const treeSitterWasmsDir = path.join(out, "tree-sitter-wasms");
|
||||
|
|
@ -252,59 +205,31 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
|||
"out/llamaTokenizerWorkerPool.mjs",
|
||||
);
|
||||
|
||||
const buildBinaryPromises = [];
|
||||
console.log("[info] Building binaries with pkg...");
|
||||
for (const target of targets) {
|
||||
const targetDir = `bin/${target}`;
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
console.log(`[info] Building ${target}...`);
|
||||
execCmdSync(
|
||||
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
|
||||
const child = fork("./utils/bundle-binary.js", { stdio: "inherit" });
|
||||
child.send({
|
||||
payload: {
|
||||
target,
|
||||
},
|
||||
});
|
||||
buildBinaryPromises.push(
|
||||
new Promise((resolve, reject) => {
|
||||
child.on("message", (msg) => {
|
||||
if (msg.error) {
|
||||
reject();
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
// Download and unzip prebuilt sqlite3 binary for the target
|
||||
console.log("[info] Downloading node-sqlite3");
|
||||
|
||||
const downloadUrl =
|
||||
// node-sqlite3 doesn't have a pre-built binary for win32-arm64
|
||||
target === "win32-arm64"
|
||||
? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
|
||||
: `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
|
||||
target
|
||||
}.tar.gz`;
|
||||
|
||||
execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
|
||||
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
|
||||
|
||||
// Copy to build directory for testing
|
||||
try {
|
||||
const [platform, arch] = target.split("-");
|
||||
if (platform === currentPlatform && arch === currentArch) {
|
||||
fs.copyFileSync(
|
||||
`${targetDir}/build/Release/node_sqlite3.node`,
|
||||
`build/node_sqlite3.node`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("[warn] Could not copy node_sqlite to build");
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
fs.unlinkSync(`${targetDir}/build.tar.gz`);
|
||||
|
||||
// copy @lancedb to bin folders
|
||||
console.log("[info] Copying @lancedb files to bin");
|
||||
fs.copyFileSync(
|
||||
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
|
||||
`${targetDir}/index.node`,
|
||||
);
|
||||
|
||||
// Download and install ripgrep for the target
|
||||
await downloadRipgrepForTarget(target, targetDir);
|
||||
|
||||
// Informs the `continue-binary` of where to look for node_sqlite3.node
|
||||
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
|
||||
fs.writeFileSync(`${targetDir}/package.json`, "");
|
||||
}
|
||||
await Promise.all(buildBinaryPromises).catch(() => {
|
||||
console.error("[error] Failed to build binaries");
|
||||
process.exit(1);
|
||||
});
|
||||
console.log("[info] All binaries built");
|
||||
|
||||
// Cleanup - this is needed when running locally
|
||||
fs.rmSync("out/package.json");
|
||||
|
|
@ -331,4 +256,5 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
|||
validateFilesPresent(pathsToVerify);
|
||||
|
||||
console.log("[info] Done!");
|
||||
process.exit(0);
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* @file Builds the binary for the specified target. It is intended to run as a child process.
|
||||
*/
|
||||
const {
|
||||
execCmdSync,
|
||||
autodetectPlatformAndArch,
|
||||
} = require("../../scripts/util");
|
||||
const { downloadRipgrep } = require("./ripgrep");
|
||||
const { TARGET_TO_LANCEDB } = require("../utils/targets");
|
||||
const fs = require("fs");
|
||||
|
||||
/**
|
||||
* @param {string} target the platform to build for
|
||||
*/
|
||||
async function bundleForBinary(target) {
|
||||
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
|
||||
|
||||
const targetDir = `bin/${target}`;
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
console.log(`[info] Building ${target}...`);
|
||||
execCmdSync(
|
||||
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
|
||||
);
|
||||
|
||||
// Download and unzip prebuilt sqlite3 binary for the target
|
||||
console.log("[info] Downloading node-sqlite3");
|
||||
|
||||
const downloadUrl =
|
||||
// node-sqlite3 doesn't have a pre-built binary for win32-arm64
|
||||
target === "win32-arm64"
|
||||
? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
|
||||
: `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
|
||||
target
|
||||
}.tar.gz`;
|
||||
|
||||
execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
|
||||
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
|
||||
|
||||
// Copy to build directory for testing
|
||||
try {
|
||||
const [platform, arch] = target.split("-");
|
||||
if (platform === currentPlatform && arch === currentArch) {
|
||||
fs.copyFileSync(
|
||||
`${targetDir}/build/Release/node_sqlite3.node`,
|
||||
`build/node_sqlite3.node`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("[warn] Could not copy node_sqlite to build");
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
fs.unlinkSync(`${targetDir}/build.tar.gz`);
|
||||
|
||||
// copy @lancedb to bin folders
|
||||
console.log("[info] Copying @lancedb files to bin");
|
||||
fs.copyFileSync(
|
||||
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
|
||||
`${targetDir}/index.node`,
|
||||
);
|
||||
|
||||
// Download and install ripgrep for the target
|
||||
await downloadRipgrep(target, targetDir);
|
||||
|
||||
// Informs the `continue-binary` of where to look for node_sqlite3.node
|
||||
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
|
||||
fs.writeFileSync(`${targetDir}/package.json`, "");
|
||||
}
|
||||
|
||||
process.on("message", (msg) => {
|
||||
bundleForBinary(msg.payload.target)
|
||||
.then(() => process.send({ done: true }))
|
||||
.catch((error) => {
|
||||
console.error(error); // show the error in the parent process
|
||||
process.send({ error: true });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* @file Copy lancedb to the current directory. It is intended to run as a child process.
|
||||
*/
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const ncp = require("ncp").ncp;
|
||||
const { execCmdSync } = require("../../scripts/util");
|
||||
|
||||
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
|
||||
console.log(`Copying ${packageName} to ${toCopy}`);
|
||||
// This is a way to install only one package without npm trying to install all the dependencies
|
||||
// Create a temporary directory for installing the package
|
||||
const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
|
||||
const tempDir = path.join(
|
||||
__dirname,
|
||||
"..",
|
||||
"tmp",
|
||||
`continue-node_modules-${adjustedName}`,
|
||||
);
|
||||
const currentDir = process.cwd();
|
||||
|
||||
// // Remove the dir we will be copying to
|
||||
// rimrafSync(`node_modules/${toCopy}`);
|
||||
|
||||
// // Ensure the temporary directory exists
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// Move to the temporary directory
|
||||
process.chdir(tempDir);
|
||||
|
||||
// Initialize a new package.json and install the package
|
||||
execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
|
||||
|
||||
console.log(
|
||||
`Contents of: ${packageName}`,
|
||||
fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
|
||||
);
|
||||
|
||||
// Without this it seems the file isn't completely written to disk
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
// Copy the installed package back to the current directory
|
||||
await new Promise((resolve, reject) => {
|
||||
ncp(
|
||||
path.join(tempDir, "node_modules", toCopy),
|
||||
path.join(currentDir, "node_modules", toCopy),
|
||||
{ dereference: true },
|
||||
(error) => {
|
||||
if (error) {
|
||||
console.error(
|
||||
`[error] Error copying ${packageName} package`,
|
||||
error,
|
||||
);
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
// Clean up the temporary directory
|
||||
// rimrafSync(tempDir);
|
||||
|
||||
// Return to the original directory
|
||||
process.chdir(currentDir);
|
||||
}
|
||||
}
|
||||
|
||||
process.on("message", (msg) => {
|
||||
installNodeModuleInTempDirAndCopyToCurrent(
|
||||
msg.payload.packageName,
|
||||
msg.payload.toCopy,
|
||||
)
|
||||
.then(() => process.send({ done: true }))
|
||||
.catch((error) => {
|
||||
console.error(error); // show the error in the parent process
|
||||
process.send({ error: true });
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue