louis_lifu
/
continue
mirror of https://github.com/continuedev/continue.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

⏩ Preview to main (#1299) 

* 🩹 for now, skip onboarding in jetbrains

* 🩹 temporarily don't show use codebase on jetbrains

* 🐛 use system certificates in binary

* 🔖 update jetbrains version

* 🩹 correctly contruct set of certs

* 🔖 bump intellij version to 0.0.45

* 🩹 update to support images for gpt-4-turbo

* 🐛 fix image support autodetection

* ⚡️ again, improve image support autodetection

* 🐛 set supportsCompletions based on useLegacyCompletionsEndpoint model setting

Closes #1132

* 📝 useLegacyCompletionsEndpoint within OpenAI docs

* 🔧 forceCompletionsEndpointType option

* Revert "🔧 forceCompletionsEndpointType option"

This reverts commit dd51fcbb7f.

* 🩹 set default useLegacyCompletionsEndpoint to undefined

* 🩹 look for bedrock credentials in homedir

* 🩹 use title for autodetect

* 🐛 set supportsCompletions based on useLegacyCompletionsEndpoint model setting

Closes #1132

* 📝 useLegacyCompletionsEndpoint within OpenAI docs

* 🩹 set default useLegacyCompletionsEndpoint to undefined

* 🩹 look for bedrock credentials in homedir

* 🩹 use title for autodetect

* Fix slash command params loading

Existing slash commands expect an object named
"params" so mapping to "options" here caused
params to be undefined within the run scope. I
renamed from 'm' to 's' just to avoid potential
confusion with the model property mapping above.

* Add outputDir param to /share

* Enable basic tilde expansion for /share outputDir

* Add ability to specify workspace for /share

* Add datetimestamp to exported session filename

* Use `.`, `./`, or `.\` for current workspace

* Add description of outputDir param for /share

* Ensure replacement only at start of string

* Create user-specified directory if necessary

* Change "Continue" to "Assistant" in export

* Consolidate to single replace regex

* Reformat markdown code blocks

Currently, user-selected code blocks are formatted
with range in file (rif) info on the same line as
the triple backticks, which means that when
exported to markdown they don't have the language
info needed on that line for syntax highlighting.
This update moves the rif info to the following
line as a comment in the language of the file and
with the language info in the correct place.

Before:
```example.ts (3-6)

function fib(n) {
    if (n <= 1) return n;
    return fib(n - 2) + fib(n - 1);
}
```

After:
```ts
// example.ts (3-6)

function fib(n) {
    if (n <= 1) return n;
    return fib(n - 2) + fib(n - 1);
}
```

* Tidy regex to capture filename

* Tidy regex to capture filename

* Ensure adjacent codeblocks separated by newline

* Aesthetic tweaks to output format

* ✨ disableInFiles option for autocomplete

* feat(httpContextProvider): load AC on fetch client (#1150)

Co-authored-by: Bertrand Pinel <bertrand.pinel@pole-emploi.fr>

* ✨ global filewatcher for config.json/ts changes

* 🐛 retry webview requests so that first cmd+L works

* ✨ Improved onboarding experience (#1155)

* 🚸 onboarding improvements

* 🧑‍💻 keyboard shortcuts to toggle autocomplete and open config.json

* ⚡️ improve detection of terminal code blocks

* 🚧 onboarding improvements

* 🚧 more onboarding improvements

* 💄 last session button

* 🚸 show more fallback options in dropdown

* 💄 add sectioning to models page

* 💄 clean up delete model button

* 💄 make tooltip look nicer

* 🚸 download Ollama button

* 💄 local LLM onboarding

* 🐛 select correct terminal on "runCommand" message

* 💄 polish onboarding

* 💚 fix gui build errors

* 📝 add /v1 to OpenAI examples in docs

* 🚑 hotfix for not iterable error

* ✨ add Cohere as Embeddings Provider

* 💄 add llama3 to UI

* 🔥 remove disable indexing

* 🍱 update continue logo

* 🐛 fix language undefined bug

* 🐛 fix merge mistake

* 📝 rename googlepalmapi.md to googlegeminiapi.md

* 📝 update mistral models

* Rename to geminiapi & change filename this time

* ✨ global request options (#1153)

* ✨ global request options

* 🐛 fix jira context provider by injecting fetch

* ✨ request options for embeddings providers

* ✨ add Cohere as Reranker (#1159)

* ♻️ use custom requestOptions with CohereEmbeddingsProvider

* Update preIndexedDocs.ts (#1154)

Add WordPress and WooCommerce as preIndexedDocs.

* 🩹 remove example "outputDir" from default config

* Fix slash command params loading (#1084)

Existing slash commands expect an object named
"params" so mapping to "options" here caused
params to be undefined within the run scope. I
renamed from 'm' to 's' just to avoid potential
confusion with the model property mapping above.

* 🐛 don't index if no open workspace folders

* 💄 improve onboarding language

* 🚸 improve onboarding

* 🐛 stop loading when error

* 💄 replace text in input box

* Respect Retry-After header when available from 429 responses (#1182)

* 🩹 remove dead code for exponential backoff

This has been replaced by the withExponentialBackoff helper

* 🩹 respect Retry-After header when available

* 🚸 update inline tips language

* ✨ input box history

* 📌 update package-locks

* 🔊 log errors in prepackage

* 🐛 err to string

* 📌 pin llama-tokenizer-js

* 📌 update lockfile

* 🚚 change /docs to docs.

* 📦 package win-ca dependencies in binary

* 🔥 remove unpopular models from UI

* 🍱 new logo in jetbrains

* 🎨 use node-fetch everywhere

* 🚸 immediately select newly added models

* 🚸 spell out Alt instead of using symbol

* 🔥 remove config shortcut

* 🐛 fix changing model bug

* 🩹 de-duplicate before adding models

* 🔧 add embeddingsProvider specific request options

* 🎨 refactor to always use node-fetch from LLM

* 🔥 remove duplicate tokens generated

* 🔊 add timestamp to JetBrains logs

* 🎨 maxStopWords for Groq

* 🐛 fix groq provider calling /completions

* 🐛 correctly adhere to LanceDB table name spec

* 🐛 fix sqlite NOT NULL constraint failed error with custom model

* Fix issue where Accept/Reject All only accepts/rejects a single diff hunk. (#1197)

* Fix issues parsing Ollama /api/show endpoint payloads. (#1199)

* ✨ model role for inlineEdit

* 🩹 various small updates

* 🐛 fix openai image support

* 🔖 update gradle version

* 🍱 update jetbrains icon

* 🐛 fix autocomplete in notebook cells

* 🔥 remove unused media

* 🔥 remove unused files

* Fix schema to allow 'AUTODETECT' sentinel for model when provider is 'groq'. (#1203)

* 🐛 small improvements

* Fix issue with @codebase provider when n becomes odd due to a divide by 2 during the full text search portion of the query. (#1204)

* 🐛 add skipLines

* ✨ URLContextProvider

* 🥅 improved error handling for codebase indexing

* 🏷️ use official Git extension types

* ➕ declare vscode.git extension dependency

* ⚡️ use reranker for docs context provider

* 🚸 Use templating in default customCommand

* 🎨 use U+23CE

* 🚸 disable autocomplete in commit message box

* 🩹 add gems to default ignored paths

* ⚡️ format markdown blocks as comments in .ipynb completions

* 🐛 don't strip port in URL

* 🐛 fix "gemini" provider spacing issues

* 📦 update posthog version

* CON-1067: Failed state seems to be toggling as intended

* 🏷️ update types.ts

* 🐛 fix copy/paste/cut behavior in VS Code notebooks

* ✨ llama3 prompt template

* Rework for proper initialization on start up

* CON-1067 Clean-up

* CON-1067 more clean-up

* Add indexingNotLoaded state

* CON-1067 communicate progress to frontend

* 🐛 fix undefined prefix, suffix and language for `/edit` (#1216)

* 🐛 add .bind to fix templating in systemMessage

* CON-1067 clean up

* 🐛 small improvements to autocomplete

* Update DocsContextProvider.ts (#1217)

I fixed a bug where you were sending the query variable (which holds the base URL of the doc) to the rerank method, and it made no sense to rerank the chunks based on a URL. So I changed it to extras.fullInput because it should rerank based on the user input, which should provide better results.

* 📝 select-provider.md update

* 🐛 fix merge errors

* Nate/autocomplete-metrics (#1230)

* ⚡️ use context.selectedCompletionInfo, deduplicate logs

* ⚡️ don't reject if user keeps typing same as completion

* ⚡️ vscode autocomplete edge cases

* 🚧 WIP on vscode autocomplete

* ⚡️ better bracket handlng

* ⚡️ improved multi-line detection

* Active file default context (#1231)

* 🚸 include currently active file by default

* 🚸 warn if non-autocomplete model being used

* ✨ try hole filling template for gpt

* 💄 ui for no context

* ⚡️ leave out bottom of excessively large files

* 🚧 experimenting with perplexity style streaming

* 🐛 fix #1237

* 💚 fix type error

* ⚡️ improve LSP usage in autocomplete

* 🐛 fix content parsing regression in /edit

* add PySide6 docs to preindexed docs (#1236)

* CON-232 bring custom docs to top, alphabetize doc results, make scrol… (#1239)

* CON-232 bring custom docs to top, alphabetize doc results, make scrollable

* CON-232 cleanup

---------

Co-authored-by: Justin Milner <jmilner@jmilner-lt2.deka.local>

* CON-1067 condense some things

* 🚚 [Auxiliary -> Continue] Sidebar

* 🔊 log completion options in ~/.continue/sessions

* CON-1067 wrong ret val fix

* CON-1067: fixes from testing

* ⚡️ filter out completions that are only punctuation/space

* ⚡️ inject intellisense docs, no multi-line on comments

* ⚡️ crawl type definitions for autocomplete

* ⚡️ truncate function text

* ⚡️ cache LSP calls

* ⚡️ find recently edited ranges with perfect prefix match

* 🐛 fix gif paths

* ⚡️ bring back double new line stop words

* 📌 add yarn lock files

* 🐛 allow language keywords to be generated

* 💄 toggle on help button

* 🎨 defaultContext option

* 🐛 fix lancedb bug by upgrading

* 🐛 fix groq stop tokens

* 🐛 preventDefault to avoid double paste

* 🚸 don't repeatedly override cmd+J

* 🧑‍💻 fix npm run test in core

* 📝 change description

* 🐛 silence Ollama invalid server state warning

* ⚡️ more accurate getTerminalContents

* ⚡️ make getTerminalContents more accurate

* 🧑‍💻 use yarn instead of npm

* 👷 fix yarn add --no-save in prepackge

* 🐛 correctly read entire notebook file contents

* ➕ import handlebars

* 🔥 remove unnecessary migrations

* ⚡️ improve /comment prompt

* Add debug terminal context menu (#1261)

* Add --no-dependencies to vsce package (#1255)

This is not needed because we bundle first with esbuild, and
vsce pack has issues with modern package managers.

see: microsoft/vscode-vsce#421 (comment)

* ui: change line decoration color to use vscode theme (#1253)

* ui: change line decoration color to use vscode theme

to give user a more consistent experience by letting the decoration color to user the color defined in the theme.

* fix: incorrect color item

should be line background not text background
because the decoration is for the whole line

* 🎨 refactor indexing state into a single object

* CON-223 Correct diff streaming abort (#1263)

Co-authored-by: Justin Milner <jmilner@jmilner-lt2.deka.local>

* 📦 switch to pnpm (#1265)

* 🎨 use pnpm instead of yarn

* ➕ make dependencies explicit for pnpm

* 🐛 add powershell to extension mapping, and default to ext

* 🎨 make llamatokenizer commonjs compatible

* ➕ add esbuild optional deps

* 🚚 rename vendor/node_modules -> modules

* 🔖 update core version

* 🐛 fixes for transformers.js compatibility

* 🔖 update core version

* 🎨 set modelPath in constructor

* 🎨 fix transformers.js import

* 🎨 eslint enforce import extensions

* 🎨 require -> import

* 🚸 notify user if not diff in /commit

* 💄 Improve colors of the IntelliJ tool window icon (#1273)

Without this fix, the continue icon sticks out from the other toolwindow icons,
resulting in an inconsistent appearance of the whole IDE and creates a feeling
that the continue plugin "doesn't fit, something must be broken".

According to
https://plugins.jetbrains.com/docs/intellij/icons.html#new-ui-icon-colors
specific colors are needed to work nicely with dark and light modes. Bonus is
that the active tool window icon color then changes automatically to white.

Co-authored-by: Lukas Baron <LukasBaron@gmail.com>

* ✨ send Bearer token to Ollama if apiKey set

* 🐛 fix slash command bug

* 🧑‍💻 add onnxruntime (--save-dev) for types

* 🐛 don't apply to file when running code block in terminal

* 🐛 avoid double paste

* ✨ gpt-4o

* 🎨 pass uniqueId to constructor

* 🚸 focus without scrolling into vie

* 🎨 refactoring for continue-proxy LLM (#1277)

* 🐛 continue server client fixes

* 🎨 refactor OpenAI _getHeaders

* 🎨 pass ideSettings to llmFromDescription

* ⚡️ improve add docstring command

* 💚 ci updates

* 🐛 fix repeated paste bug

* 🐛 fix build script

* 🩹 various small improvements

* 📌 pin esbuild 0.17.19

* 🧑‍💻 pnpm i gui in prepackage

* 🐛 show all diff changes in vscode

* 🩹 getMetaKeyName

* 🐛 fix reading of unopened ipynb files

* ⚡️ gpt-4o system prompt

* 💄 make font size configurable in config.json ui.fontSize

* 🩹 properly dispose of diff handler

* 🐛 fix indexing status display

* ⚡️ context pruning

* 🎨 update free trial models

* fix: remove some backup files generated by pkg if present (#1287)

---------

Co-authored-by: Roger Meier <r.meier@siemens.com>
Co-authored-by: Peter Zaback <pzaback@gmail.com>
Co-authored-by: Bertrand P <49525332+Berber31@users.noreply.github.com>
Co-authored-by: Bertrand Pinel <bertrand.pinel@pole-emploi.fr>
Co-authored-by: Maxime Brunet <max@brnt.mx>
Co-authored-by: Jose Vega <bloguea.y.gana@gmail.com>
Co-authored-by: Nejc Habjan <hab.nejc@gmail.com>
Co-authored-by: Chad Yates <cyates@dynfxdigital.com>
Co-authored-by: Justin Milner <jmilner@jmilner-lt2.deka.local>
Co-authored-by: 小颚虫 <123357481+5eqn@users.noreply.github.com>
Co-authored-by: 5eqn <491100866@qq.com>
Co-authored-by: Pixel <54180211+pixelsortr@users.noreply.github.com>
Co-authored-by: Justin Milner <42585006+justinmilner1@users.noreply.github.com>
Co-authored-by: Devin Gould <djgould0628@gmail.com>
Co-authored-by: Dipta Mahardhika <146386738+d-mahard@users.noreply.github.com>
Co-authored-by: tnglemongrass <113173292+tnglemongrass@users.noreply.github.com>
Co-authored-by: Lukas Baron <LukasBaron@gmail.com>
Co-authored-by: Fernando <fernando.sanchez.jr@gmail.com>
This commit is contained in:
Nate Sesti 2024-05-16 22:15:44 -07:00 committed by GitHub
parent b2c7b074c2
commit bcd2a8eea4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
260 changed files with 36739 additions and 36611 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/tasks.json vendored
View File

@ -110,8 +110,8 @@
{
"label": "install-all-dependencies",
"type": "shell",
"windows": { "command": "./install-dependencies.ps1" },
"command": "./install-dependencies.sh",
"windows": { "command": "./scripts/install-dependencies.ps1" },
"command": "./scripts/install-dependencies.sh",
"problemMatcher": [] // Empty so users are not prompted to select progress reporting
},
//

11
CONTRIBUTING.md
View File

@ -56,9 +56,12 @@ Continue is continuously improving, but a feature isn't complete until it is ref
### Environment Setup
#### VS Code
#### Pre-requisites
Pre-requisite: You should have Node.js version 20.11.0 (LTS) or higher installed. You can get it on [nodejs.org](https://nodejs.org/en/download) or, if you are using NVM (Node Version Manager), you can set the correct version of Node.js for this project by running the following command in the root of the project:
- You should have Node.js version 20.11.0 (LTS) or higher installed. You can get it on [nodejs.org](https://nodejs.org/en/download) or, if you are using NVM (Node Version Manager), you can set the correct version of Node.js for this project by running the following command in the root of the project:
- Continue uses `pnpm` to manage node_modules. You can install `pnpm` globally with `npm install -g pnpm`, or another method described in [their docs](https://pnpm.io/installation#using-npm).
#### VS Code
```bash
nvm use
@ -77,7 +80,7 @@ nvm use
1. The new VS Code window with the extension is referred to as the _Host VS Code_
2. The window you started debugging from is referred to as the _Main VS Code_
4. To package the extension, run `npm run package` in the `extensions/vscode` directory. This will generate `extensions/vscode/build/continue-patch.vsix`, which you can install by right-clicking and selecting "Install Extension VSIX".
4. To package the extension, run `pnpm package` in the `extensions/vscode` directory. This will generate `extensions/vscode/build/continue-patch.vsix`, which you can install by right-clicking and selecting "Install Extension VSIX".
> Note: Breakpoints can be used in both the `core` and `extensions/vscode` folders while debugging, but are not currently supported inside of `gui` code. Hot-reloading is enabled with Vite, so if you make any changes to the `gui`, they should be automatically reflected without rebuilding. Similarly, any changes to `core` or `extensions/vscode` will be automatically included by just reloading the _Host VS Code_ window with cmd/ctrl+shift+p "Reload Window".
@ -86,7 +89,7 @@ nvm use
Pre-requisite: You should use the Intellij IDE, which can be downloaded [here](https://www.jetbrains.com/idea/download). Either Ultimate or Community (free) will work. Continue is built with JDK version 19.
1. Clone the repository
2. Run `install-dependencies.sh` or `install-dependencies.ps1` on Windows
2. Run `scripts/install-dependencies.sh` or `scripts/install-dependencies.ps1` on Windows
3. Run `cd extensions/vscode && node scripts/prepackage.js` (this will copy over the built React application to the proper JetBrains directory)
4. Select the "Run Plugin" Gradle configuration and click the "Run" or "Debug" button
5. To package the extension, choose the "Build Plugin" Gradle configuration

95
binary/build.js
View File

@ -1,7 +1,18 @@
const esbuild = require("esbuild");
const { execSync } = require("child_process");
const fs = require("fs");
const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
function execCmdSync(cmd) {
try {
execSync(cmd);
} catch (err) {
console.error(`Error executing command '${cmd}': `, err.output.toString());
process.exit(1);
}
}
const esbuildOutputFile = "out/index.js";
const platforms = ["darwin", "linux", "win32"];
@ -9,6 +20,10 @@ const architectures = ["x64", "arm64"];
let targets = platforms.flatMap((platform) =>
architectures.map((arch) => `${platform}-${arch}`),
);
const assetBackups = [
"node_modules/win-ca/lib/crypt32-ia32.node.bak",
"node_modules/win-ca/lib/crypt32-x64.node.bak"
];
let esbuildOnly = false;
for (let i = 2; i < process.argv.length; i++) {
@ -28,9 +43,63 @@ const targetToLanceDb = {
"win32-x64": "@lancedb/vectordb-win32-x64-msvc",
};
async function installNodeModuleInTempDirAndCopyToCurrent(package, toCopy) {
console.log(`Copying ${package} to ${toCopy}`);
// This is a way to install only one package without npm trying to install all the dependencies
// Create a temporary directory for installing the package
const tempDir = path.join(
__dirname,
"tmp",
`continue-node_modules-${toCopy}`,
);
const currentDir = process.cwd();
// Remove the dir we will be copying to
rimrafSync(`node_modules/${toCopy}`);
// Ensure the temporary directory exists
fs.mkdirSync(tempDir, { recursive: true });
try {
// Move to the temporary directory
process.chdir(tempDir);
// Initialize a new package.json and install the package
execCmdSync(`npm init -y && npm i -f ${package} --no-save`);
console.log(
`Contents of: ${package}`,
fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
);
// Copy the installed package back to the current directory
await new Promise((resolve, reject) => {
ncp(
path.join(tempDir, "node_modules", toCopy),
path.join(currentDir, "node_modules", toCopy),
{ dereference: true },
(error) => {
if (error) {
console.error(`[error] Error copying ${package} package`, error);
reject(error);
} else {
resolve();
}
},
);
});
} finally {
// Clean up the temporary directory
rimrafSync(tempDir);
// Return to the original directory
process.chdir(currentDir);
}
}
(async () => {
// console.log("[info] Building with ncc...");
// execSync(`npx ncc build src/index.ts -o out`);
// execCmdSync(`npx ncc build src/index.ts -o out`);
// Copy node_modules for pre-built binaries
const DYNAMIC_IMPORTS = [
@ -80,10 +149,20 @@ const targetToLanceDb = {
for (const target of targets) {
if (targetToLanceDb[target]) {
console.log(`[info] Downloading ${target}...`);
execSync(`npm install -f ${targetToLanceDb[target]} --no-save`);
await installNodeModuleInTempDirAndCopyToCurrent(
targetToLanceDb[target],
"@lancedb",
);
}
}
console.log("[info] Cleaning up artifacts from previous builds...");
// delete asset backups generated by previous pkg invocations, if present
for (const assetPath of assetBackups) {
fs.rmSync(assetPath, {force: true});
}
console.log("[info] Building with esbuild...");
// Bundles the extension into one file
await esbuild.build({
@ -120,7 +199,7 @@ const targetToLanceDb = {
const targetDir = `bin/${target}`;
fs.mkdirSync(targetDir, { recursive: true });
console.log(`[info] Building ${target}...`);
execSync(
execCmdSync(
`npx pkg --no-bytecode --public-packages "*" --public pkgJson/${target} --out-path ${targetDir}`,
);
@ -128,8 +207,8 @@ const targetToLanceDb = {
const downloadUrl = `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
target === "win32-arm64" ? "win32-ia32" : target
}.tar.gz`;
execSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
execSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
fs.copyFileSync(
`${targetDir}/build/Release/node_sqlite3.node`,
`${targetDir}/node_sqlite3.node`,
@ -143,10 +222,10 @@ const targetToLanceDb = {
// Download and unzip prebuilt esbuild binary for the target
console.log(`[info] Downloading esbuild for ${target}...`);
// Version is pinned to 0.19.11 in package.json to make sure that they match
execSync(
execCmdSync(
`curl -o ${targetDir}/esbuild.tgz https://registry.npmjs.org/@esbuild/${target}/-/${target}-0.19.11.tgz`,
);
execSync(`tar -xzvf ${targetDir}/esbuild.tgz -C ${targetDir}`);
execCmdSync(`tar -xzvf ${targetDir}/esbuild.tgz -C ${targetDir}`);
if (target.startsWith("win32")) {
fs.cpSync(`${targetDir}/package/esbuild.exe`, `${targetDir}/esbuild.exe`);
} else {
@ -158,7 +237,7 @@ const targetToLanceDb = {
recursive: true,
});
}
// execSync(
// execCmdSync(
// `npx pkg out/index.js --target node18-darwin-arm64 --no-bytecode --public-packages "*" --public -o bin/pkg`
// );
console.log("[info] Done!");

2816
binary/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
binary/package.json
View File

@ -34,6 +34,7 @@
"@vercel/ncc": "^0.38.1",
"esbuild": "0.19.11",
"pkg": "^5.8.1",
"rimraf": "^5.0.7",
"typescript": "^5.3.3"
},
"dependencies": {

5916
binary/pnpm-lock.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

27
core/.eslintrc.json Normal file
View File

@ -0,0 +1,27 @@
{
"root": true,
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 6,
"sourceType": "module"
},
"extends": ["plugin:require-extensions/recommended"],
"plugins": ["@typescript-eslint", "import", "require-extensions"],
"rules": {
"quotes": ["warn", "double", {}],
"import/extensions": ["error", "always", {
"js": "never",
"jsx": "never",
"ts": "never",
"tsx": "never"
}],
"@typescript-eslint/naming-convention": "warn",
"@typescript-eslint/semi": "warn",
"curly": "warn",
"eqeqeq": "warn",
"no-throw-literal": "warn",
"semi": "off"
},
"ignorePatterns": ["out", "dist", "**/*.d.ts"]
}

4
core/autocomplete/ast.ts
View File

@ -1,6 +1,6 @@
import Parser from "web-tree-sitter";
import { RangeInFileWithContents } from "../commands/util";
import { getParserForFile } from "../util/treeSitter";
import { RangeInFileWithContents } from "../commands/util.js";
import { getParserForFile } from "../util/treeSitter.js";
export async function getAst(
filepath: string,

4
core/autocomplete/cache.ts
View File

@ -1,7 +1,7 @@
import { open } from "sqlite";
import sqlite3 from "sqlite3";
import { DatabaseConnection } from "../indexing/refreshIndex";
import { getTabAutocompleteCacheSqlitePath } from "../util/paths";
import { DatabaseConnection } from "../indexing/refreshIndex.js";
import { getTabAutocompleteCacheSqlitePath } from "../util/paths.js";
export class AutocompleteLruCache {
private static capacity: number = 1000;

4
core/autocomplete/charStream.ts
View File

@ -30,7 +30,9 @@ export async function* noFirstCharNewline(stream: AsyncGenerator<string>) {
for await (let char of stream) {
if (first) {
first = false;
if (char === "\n") return;
if (char === "\n") {
return;
}
}
yield char;
}

50
core/autocomplete/completionProvider.ts
View File

@ -2,31 +2,37 @@ import Handlebars from "handlebars";
import ignore from "ignore";
import path from "path";
import { v4 as uuidv4 } from "uuid";
import { IDE, ILLM, Position, Range, TabAutocompleteOptions } from "..";
import { RangeInFileWithContents } from "../commands/util";
import { ConfigHandler } from "../config/handler";
import { streamLines } from "../diff/util";
import OpenAI from "../llm/llms/OpenAI";
import { getBasename } from "../util";
import { logDevData } from "../util/devdata";
import { RangeInFileWithContents } from "../commands/util.js";
import { ConfigHandler } from "../config/handler.js";
import { streamLines } from "../diff/util.js";
import {
IDE,
ILLM,
Position,
Range,
TabAutocompleteOptions,
} from "../index.js";
import OpenAI from "../llm/llms/OpenAI.js";
import { logDevData } from "../util/devdata.js";
import { getBasename } from "../util/index.js";
import {
COUNT_COMPLETION_REJECTED_AFTER,
DEFAULT_AUTOCOMPLETE_OPTS,
} from "../util/parameters";
import { Telemetry } from "../util/posthog";
import { getRangeInString } from "../util/ranges";
import AutocompleteLruCache from "./cache";
} from "../util/parameters.js";
import { Telemetry } from "../util/posthog.js";
import { getRangeInString } from "../util/ranges.js";
import AutocompleteLruCache from "./cache.js";
import {
noFirstCharNewline,
onlyWhitespaceAfterEndOfLine,
stopOnUnmatchedClosingBracket,
} from "./charStream";
} from "./charStream.js";
import {
constructAutocompletePrompt,
languageForFilepath,
} from "./constructPrompt";
import { isOnlyPunctuationAndWhitespace } from "./filter";
import { AutocompleteLanguageInfo } from "./languages";
} from "./constructPrompt.js";
import { isOnlyPunctuationAndWhitespace } from "./filter.js";
import { AutocompleteLanguageInfo } from "./languages.js";
import {
avoidPathLine,
noTopLevelKeywordsMidline,
@ -34,11 +40,11 @@ import {
stopAtRepeatingLines,
stopAtSimilarLine,
streamWithNewLines,
} from "./lineStream";
import { AutocompleteSnippet } from "./ranking";
import { RecentlyEditedRange } from "./recentlyEdited";
import { getTemplateForModel } from "./templates";
import { GeneratorReuseManager } from "./util";
} from "./lineStream.js";
import { AutocompleteSnippet } from "./ranking.js";
import { RecentlyEditedRange } from "./recentlyEdited.js";
import { getTemplateForModel } from "./templates.js";
import { GeneratorReuseManager } from "./util.js";
export interface AutocompleteInput {
completionId: string;
@ -157,7 +163,9 @@ export async function getTabCompletion(
}
// Model
if (!llm) return;
if (!llm) {
return;
}
if (llm instanceof OpenAI) {
llm.useLegacyCompletionsEndpoint = true;
} else if (

18
core/autocomplete/constructPrompt.ts
View File

@ -1,21 +1,25 @@
import Parser from "web-tree-sitter";
import { TabAutocompleteOptions } from "..";
import { RangeInFileWithContents } from "../commands/util";
import { RangeInFileWithContents } from "../commands/util.js";
import { TabAutocompleteOptions } from "../index.js";
import {
countTokens,
pruneLinesFromBottom,
pruneLinesFromTop,
} from "../llm/countTokens";
import { getAst, getTreePathAtCursor } from "./ast";
import { AutocompleteLanguageInfo, LANGUAGES, Typescript } from "./languages";
} from "../llm/countTokens.js";
import { getAst, getTreePathAtCursor } from "./ast.js";
import {
AutocompleteLanguageInfo,
LANGUAGES,
Typescript,
} from "./languages.js";
import {
AutocompleteSnippet,
fillPromptWithSnippets,
rankSnippets,
removeRangeFromSnippets,
} from "./ranking";
import { RecentlyEditedRange, findMatchingRange } from "./recentlyEdited";
} from "./ranking.js";
import { RecentlyEditedRange, findMatchingRange } from "./recentlyEdited.js";
export function languageForFilepath(
filepath: string,

2
core/autocomplete/languages.ts
View File

@ -15,7 +15,7 @@ export const Typescript = {
export const Python = {
// """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks.
// This stops the model from trying to complete the start of a new markdown block
stopWords: ["def", "class", '"""#'],
stopWords: ["def", "class", "\"\"\"#"],
comment: "#",
endOfLine: [],
};

4
core/autocomplete/lineStream.ts
View File

@ -1,6 +1,6 @@
import { distance } from "fastest-levenshtein";
import { DiffLine } from "..";
import { LineStream } from "../diff/util";
import { DiffLine } from "../index.js";
import { LineStream } from "../diff/util.js";
export async function* noTopLevelKeywordsMidline(
lines: LineStream,

6
core/autocomplete/ranking.ts
View File

@ -1,6 +1,6 @@
import { Range } from "..";
import { RangeInFileWithContents } from "../commands/util";
import { countTokens } from "../llm/countTokens";
import { Range } from "../index.js";
import { RangeInFileWithContents } from "../commands/util.js";
import { countTokens } from "../llm/countTokens.js";
export type AutocompleteSnippet = RangeInFileWithContents & {
score?: number;

2
core/autocomplete/recentlyEdited.ts
View File

@ -1,4 +1,4 @@
import { RangeInFile } from "..";
import { RangeInFile } from "../index.js";
export type RecentlyEditedRange = RangeInFile & {
timestamp: number;

4
core/autocomplete/retrieval.ts
View File

@ -1,5 +1,5 @@
import { BranchAndDir, Chunk } from "..";
import { FullTextSearchCodebaseIndex } from "../indexing/FullTextSearch";
import { BranchAndDir, Chunk } from "../index.js";
import { FullTextSearchCodebaseIndex } from "../indexing/FullTextSearch.js";
export async function fullTextRetrieve(
prefix: string,

4
core/autocomplete/slidingWindow.ts
View File

@ -1,5 +1,5 @@
import { RangeInFileWithContents } from "../commands/util";
import { AutocompleteSnippet, jaccardSimilarity } from "./ranking";
import { RangeInFileWithContents } from "../commands/util.js";
import { AutocompleteSnippet, jaccardSimilarity } from "./ranking.js";
function* slidingWindow(
content: string,

4
core/autocomplete/templates.ts
View File

@ -1,7 +1,7 @@
// Fill in the middle prompts
import { CompletionOptions } from "..";
import { AutocompleteSnippet } from "./ranking";
import { CompletionOptions } from "../index.js";
import { AutocompleteSnippet } from "./ranking.js";
interface AutocompleteTemplate {
template:

23
core/commands/index.ts
View File

@ -1,7 +1,7 @@
import { CustomCommand, SlashCommand, SlashCommandDescription } from "..";
import { stripImages } from "../llm/countTokens";
import { renderTemplatedString } from "../llm/llms";
import SlashCommands from "./slash";
import { CustomCommand, SlashCommand, SlashCommandDescription } from "../index.js";
import { stripImages } from "../llm/countTokens.js";
import { renderTemplatedString } from "../llm/llms/index.js";
import SlashCommands from "./slash/index.js";
export function slashFromCustomCommand(
customCommand: CustomCommand,
@ -28,10 +28,17 @@ export function slashFromCustomCommand(
const messages = [...history];
// Find the last chat message with this slash command and replace it with the user input
for (let i = messages.length - 1; i >= 0; i--) {
if (
messages[i].role === "user" &&
stripImages(messages[i].content).startsWith(`/${customCommand.name}`)
) {
const {role, content} = messages[i];
if (role !== "user") {
continue;
}
if (Array.isArray(content) && content.some((part) => part.text?.startsWith(`/${customCommand.name}`))) {
messages[i] = { ...messages[i], content: content.map((part) => {
return part.text?.startsWith(`/${customCommand.name}`) ? {...part, text: promptUserInput } : part;
}) };
break;
} else if (typeof content === "string" && content.startsWith(`/${customCommand.name}`)) {
messages[i] = { ...messages[i], content: promptUserInput };
break;
}

6
core/commands/slash/cmd.ts
View File

@ -1,6 +1,6 @@
import { SlashCommand } from "../..";
import { streamLines } from "../../diff/util";
import { removeQuotesAndEscapes } from "../../util";
import { SlashCommand } from "../../index.js";
import { streamLines } from "../../diff/util.js";
import { removeQuotesAndEscapes } from "../../util/index.js";
const GenerateTerminalCommand: SlashCommand = {
name: "cmd",

7
core/commands/slash/comment.ts
View File

@ -1,5 +1,5 @@
import { SlashCommand } from "../..";
import EditSlashCommand from "./edit";
import { SlashCommand } from "../../index.js";
import EditSlashCommand from "./edit.js";
const CommentSlashCommand: SlashCommand = {
name: "comment",
@ -7,7 +7,8 @@ const CommentSlashCommand: SlashCommand = {
run: async function* (sdk) {
for await (const update of EditSlashCommand.run({
...sdk,
input: "Write comments for this code",
input:
"Write comments for this code. Do not change anything about the code itself.",
})) {
yield update;
}

10
core/commands/slash/commit.ts
View File

@ -1,11 +1,17 @@
import { SlashCommand } from "../..";
import { stripImages } from "../../llm/countTokens";
import { SlashCommand } from "../../index.js";
import { stripImages } from "../../llm/countTokens.js";
const CommitMessageCommand: SlashCommand = {
name: "commit",
description: "Generate a commit message for current changes",
run: async function* ({ ide, llm, input }) {
const diff = await ide.getDiff();
if (!diff || diff.trim() === "") {
yield "No changes detected. Make sure you are in a git repository with current changes."
return;
}
const prompt = `${diff}\n\nGenerate a commit message for the above set of changes. First, give a single sentence, no more than 80 characters. Then, after 2 line breaks, give a list of no more than 5 short bullet points, each no more than 40 characters. Output nothing except for the commit message, and don't surround it in quotes.`;
for await (const chunk of llm.streamChat([
{ role: "user", content: prompt },

6
core/commands/slash/draftIssue.ts
View File

@ -1,6 +1,6 @@
import { ChatMessage, SlashCommand } from "../..";
import { stripImages } from "../../llm/countTokens";
import { removeQuotesAndEscapes } from "../../util";
import { ChatMessage, SlashCommand } from "../../index.js";
import { stripImages } from "../../llm/countTokens.js";
import { removeQuotesAndEscapes } from "../../util/index.js";
const PROMPT = (
input: string,

12
core/commands/slash/edit.ts
View File

@ -1,4 +1,3 @@
import { ContextItemWithId, ILLM, SlashCommand } from "../..";
import {
filterCodeBlockLines,
filterEnglishLinesAtEnd,
@ -6,17 +5,18 @@ import {
fixCodeLlamaFirstLineIndentation,
stopAtLines,
streamWithNewLines,
} from "../../autocomplete/lineStream";
import { streamLines } from "../../diff/util";
import { stripImages } from "../../llm/countTokens";
} from "../../autocomplete/lineStream.js";
import { streamLines } from "../../diff/util.js";
import { ContextItemWithId, ILLM, SlashCommand } from "../../index.js";
import { stripImages } from "../../llm/countTokens.js";
import {
dedentAndGetCommonWhitespace,
getMarkdownLanguageTagForFile,
} from "../../util";
} from "../../util/index.js";
import {
RangeInFileWithContents,
contextItemToRangeInFileWithContents,
} from "../util";
} from "../util.js";
const PROMPT = `Take the file prefix and suffix into account, but only rewrite the code_to_edit as specified in the user_request. The code you write in modified_code_to_edit will replace the code between the code_to_edit tags. Do NOT preface your answer or write anything other than code. The </modified_code_to_edit> tag should be written to indicate the end of the modified code section. Do not ever use nested tags.

6
core/commands/slash/http.ts
View File

@ -1,5 +1,5 @@
import { SlashCommand } from "../..";
import { removeQuotesAndEscapes } from "../../util";
import { SlashCommand } from "../../index.js";
import { removeQuotesAndEscapes } from "../../util/index.js";
const HttpSlashCommand: SlashCommand = {
name: "http",
@ -26,7 +26,7 @@ const HttpSlashCommand: SlashCommand = {
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (done) {break;}
const decoded = new TextDecoder("utf-8").decode(value);
yield decoded;
}

18
core/commands/slash/index.ts
View File

@ -1,12 +1,12 @@
import GenerateTerminalCommand from "./cmd";
import CommentSlashCommand from "./comment";
import CommitMessageCommand from "./commit";
import DraftIssueCommand from "./draftIssue";
import EditSlashCommand from "./edit";
import HttpSlashCommand from "./http";
import ShareSlashCommand from "./share";
import StackOverflowSlashCommand from "./stackOverflow";
import ReviewMessageCommand from "./review";
import GenerateTerminalCommand from "./cmd.js";
import CommentSlashCommand from "./comment.js";
import CommitMessageCommand from "./commit.js";
import DraftIssueCommand from "./draftIssue.js";
import EditSlashCommand from "./edit.js";
import HttpSlashCommand from "./http.js";
import ShareSlashCommand from "./share.js";
import StackOverflowSlashCommand from "./stackOverflow.js";
import ReviewMessageCommand from "./review.js";
export default [
DraftIssueCommand,

8
core/commands/slash/review.ts
View File

@ -1,6 +1,6 @@
import { SlashCommand } from "../..";
import { stripImages } from "../../llm/countTokens";
import { ChatMessage } from "../..";
import { SlashCommand } from "../../index.js";
import { stripImages } from "../../llm/countTokens.js";
import { ChatMessage } from "../../index.js";
const prompt = `
Review the following code, focusing on Readability, Maintainability, Code Smells, Speed, and Memory Performance. Provide feedback with these guidelines:
@ -17,7 +17,7 @@ function getLastUserHistory(history: ChatMessage[]): string {
.reverse()
.find((message) => message.role === "user");
if (!lastUserHistory) return "";
if (!lastUserHistory) {return "";}
if (lastUserHistory.content instanceof Array) {
return lastUserHistory.content.reduce(

10
core/commands/slash/share.ts
View File

@ -1,9 +1,9 @@
import path from "path";
import * as fs from "fs";
import { homedir } from "os";
import { SlashCommand } from "../..";
import { languageForFilepath } from "../../autocomplete/constructPrompt";
import { stripImages } from "../../llm/countTokens";
import { SlashCommand } from "../../index.js";
import { languageForFilepath } from "../../autocomplete/constructPrompt.js";
import { stripImages } from "../../llm/countTokens.js";
// If useful elsewhere, helper funcs should move to core/util/index.ts or similar
function getOffsetDatetime(date: Date): Date {
@ -31,7 +31,7 @@ function reformatCodeBlocks(msgText: string): string {
},
);
// Appease the markdown linter
return msgText.replace(/```\n```/g, '```\n\n```');
return msgText.replace(/```\n```/g, "```\n\n```");
}
const ShareSlashCommand: SlashCommand = {
@ -69,7 +69,7 @@ const ShareSlashCommand: SlashCommand = {
outputDir = outputDir.replace(/^~/, homedir);
} else if (
outputDir.startsWith("./") ||
outputDir.startsWith(`.\\`) ||
outputDir.startsWith(".\\") ||
outputDir === "."
) {
const workspaceDirs = await ide.getWorkspaceDirs();

4
core/commands/slash/stackOverflow.ts
View File

@ -1,5 +1,5 @@
import { ChatMessageRole, FetchFunction, SlashCommand } from "../..";
import { pruneStringFromBottom, stripImages } from "../../llm/countTokens";
import { ChatMessageRole, FetchFunction, SlashCommand } from "../../index.js";
import { pruneStringFromBottom, stripImages } from "../../llm/countTokens.js";
const SERVER_URL = "https://proxy-server-l6vsfbzhba-uw.a.run.app";
const PROMPT = (

2
core/commands/util.ts
View File

@ -1,4 +1,4 @@
import { ContextItemWithId } from "..";
import { ContextItemWithId } from "../index.js";
export interface RangeInFileWithContents {
filepath: string;

87
core/config/default.ts
View File

@ -1,32 +1,36 @@
import { ContextProviderWithParams, SerializedContinueConfig } from "..";
import {
ContextProviderWithParams,
SerializedContinueConfig,
} from "../index.js";
export const defaultConfig: SerializedContinueConfig = {
models: [
{
title: "GPT-4-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Llama3 70b (Free Trial)",
provider: "free-trial",
model: "llama3-70b",
systemMessage:
"You are an expert software developer. You give helpful and concise responses. Whenever you write a code block you include the language after the opening ticks.",
},
{
title: "GPT-4o (Free Trial)",
provider: "free-trial",
model: "gpt-4o",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Claude 3 Sonnet (Free Trial)",
provider: "free-trial",
model: "claude-3-sonnet-20240229",
},
{
title: "GPT-4 Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
},
{
title: "GPT-3.5-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-3.5-turbo",
},
{
title: "Gemini Pro (Free Trial)",
provider: "free-trial",
model: "gemini-pro",
},
{
title: "Mixtral (Free Trial)",
provider: "free-trial",
model: "mistral-8x7b",
},
],
customCommands: [
{
@ -45,31 +49,32 @@ export const defaultConfig: SerializedContinueConfig = {
export const defaultConfigJetBrains: SerializedContinueConfig = {
models: [
{
title: "GPT-4-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Llama3 70b (Free Trial)",
provider: "free-trial",
model: "llama3-70b",
systemMessage:
"You are an expert software developer. You give helpful and concise responses. Whenever you write a code block you include the language after the opening ticks.",
},
{
title: "GPT-4o (Free Trial)",
provider: "free-trial",
model: "gpt-4o",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Claude 3 Sonnet (Free Trial)",
provider: "free-trial",
model: "claude-3-sonnet-20240229",
},
{
title: "GPT-4 Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
},
{
title: "GPT-3.5-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-3.5-turbo",
},
{
title: "Gemini Pro (Free Trial)",
provider: "free-trial",
model: "gemini-pro",
},
{
title: "Mixtral (Free Trial)",
provider: "free-trial",
model: "mistral-8x7b",
},
],
customCommands: [
{

27
core/config/handler.ts
View File

@ -1,11 +1,11 @@
import { ContinueConfig, ContinueRcJson, IDE, ILLM } from "..";
import { IdeSettings } from "../protocol";
import { Telemetry } from "../util/posthog";
import { ContinueConfig, ContinueRcJson, IDE, ILLM } from "../index.js";
import { IdeSettings } from "../protocol.js";
import { Telemetry } from "../util/posthog.js";
import {
BrowserSerializedContinueConfig,
finalToBrowserConfig,
loadFullConfigNode,
} from "./load";
} from "./load.js";
export class ConfigHandler {
private savedConfig: ContinueConfig | undefined;
@ -13,12 +13,12 @@ export class ConfigHandler {
constructor(
private readonly ide: IDE,
private ideSettingsPromise: Promise<IdeSettings>,
private ideSettings: IdeSettings,
private readonly writeLog: (text: string) => Promise<void>,
private readonly onConfigUpdate: () => void,
) {
this.ide = ide;
this.ideSettingsPromise = ideSettingsPromise;
this.ideSettings = ideSettings;
this.writeLog = writeLog;
this.onConfigUpdate = onConfigUpdate;
try {
@ -29,7 +29,7 @@ export class ConfigHandler {
}
updateIdeSettings(ideSettings: IdeSettings) {
this.ideSettingsPromise = Promise.resolve(ideSettings);
this.ideSettings = ideSettings;
this.reloadConfig();
}
@ -61,21 +61,14 @@ export class ConfigHandler {
}
const ideInfo = await this.ide.getIdeInfo();
const ideSettings = await this.ideSettingsPromise;
let remoteConfigServerUrl = undefined;
try {
remoteConfigServerUrl =
typeof ideSettings.remoteConfigServerUrl !== "string" ||
ideSettings.remoteConfigServerUrl === ""
? undefined
: new URL(ideSettings.remoteConfigServerUrl);
} catch (e) {}
const uniqueId = await this.ide.getUniqueId();
this.savedConfig = await loadFullConfigNode(
this.ide.readFile.bind(this.ide),
workspaceConfigs,
remoteConfigServerUrl,
this.ideSettings,
ideInfo.ideType,
uniqueId,
this.writeLog,
);
this.savedConfig.allowAnonymousTelemetry =

103
core/config/load.ts
View File

@ -1,5 +1,14 @@
import * as fs from "fs";
import path from "path";
import {
slashCommandFromDescription,
slashFromCustomCommand,
} from "../commands/index.js";
import CustomContextProviderClass from "../context/providers/CustomContextProvider.js";
import FileContextProvider from "../context/providers/FileContextProvider.js";
import { contextProviderClassFromName } from "../context/providers/index.js";
import { AllRerankers } from "../context/rerankers/index.js";
import { LLMReranker } from "../context/rerankers/llm.js";
import {
BrowserSerializedContinueConfig,
Config,
@ -16,24 +25,16 @@ import {
RerankerDescription,
SerializedContinueConfig,
SlashCommand,
} from "..";
import {
slashCommandFromDescription,
slashFromCustomCommand,
} from "../commands";
import { contextProviderClassFromName } from "../context/providers";
import CustomContextProviderClass from "../context/providers/CustomContextProvider";
import FileContextProvider from "../context/providers/FileContextProvider";
import { AllRerankers } from "../context/rerankers";
import { LLMReranker } from "../context/rerankers/llm";
import { AllEmbeddingsProviders } from "../indexing/embeddings";
import TransformersJsEmbeddingsProvider from "../indexing/embeddings/TransformersJsEmbeddingsProvider";
import { BaseLLM } from "../llm";
import { llmFromDescription } from "../llm/llms";
import CustomLLMClass from "../llm/llms/CustomLLM";
import { copyOf } from "../util";
import { fetchwithRequestOptions } from "../util/fetchWithOptions";
import mergeJson from "../util/merge";
} from "../index.js";
import TransformersJsEmbeddingsProvider from "../indexing/embeddings/TransformersJsEmbeddingsProvider.js";
import { AllEmbeddingsProviders } from "../indexing/embeddings/index.js";
import { BaseLLM } from "../llm/index.js";
import CustomLLMClass from "../llm/llms/CustomLLM.js";
import { llmFromDescription } from "../llm/llms/index.js";
import { IdeSettings } from "../protocol.js";
import { fetchwithRequestOptions } from "../util/fetchWithOptions.js";
import { copyOf } from "../util/index.js";
import mergeJson from "../util/merge.js";
import {
getConfigJsPath,
getConfigJsPathForRemote,
@ -41,14 +42,13 @@ import {
getConfigJsonPathForRemote,
getConfigTsPath,
getContinueDotEnv,
migrate,
} from "../util/paths";
} from "../util/paths.js";
import {
defaultContextProvidersJetBrains,
defaultContextProvidersVsCode,
defaultSlashCommandsJetBrains,
defaultSlashCommandsVscode,
} from "./default";
} from "./default.js";
const { execSync } = require("child_process");
function resolveSerializedConfig(filepath: string): SerializedContinueConfig {
@ -82,7 +82,7 @@ const configMergeKeys = {
function loadSerializedConfig(
workspaceConfigs: ContinueRcJson[],
remoteConfigServerUrl: URL | undefined,
ideSettings: IdeSettings,
ideType: IdeType,
): SerializedContinueConfig {
const configPath = getConfigJsonPath(ideType);
@ -97,38 +97,10 @@ function loadSerializedConfig(
config.allowAnonymousTelemetry = true;
}
migrate("codeContextProvider", () => {
if (!config.contextProviders?.filter((cp) => cp.name === "code")?.length) {
config.contextProviders = [
...(config.contextProviders || []),
{
name: "code",
params: {},
},
];
}
fs.writeFileSync(configPath, JSON.stringify(config, undefined, 2), "utf8");
});
migrate("docsContextProvider1", () => {
if (!config.contextProviders?.filter((cp) => cp.name === "docs")?.length) {
config.contextProviders = [
...(config.contextProviders || []),
{
name: "docs",
params: {},
},
];
}
fs.writeFileSync(configPath, JSON.stringify(config, undefined, 2), "utf8");
});
if (remoteConfigServerUrl) {
if (ideSettings.remoteConfigServerUrl) {
try {
const remoteConfigJson = resolveSerializedConfig(
getConfigJsonPathForRemote(remoteConfigServerUrl),
getConfigJsonPathForRemote(ideSettings.remoteConfigServerUrl),
);
config = mergeJson(config, remoteConfigJson, "merge", configMergeKeys);
} catch (e) {
@ -197,6 +169,8 @@ function isContextProviderWithParams(
async function intermediateToFinalConfig(
config: Config,
readFile: (filepath: string) => Promise<string>,
ideSettings: IdeSettings,
uniqueId: string,
writeLog: (log: string) => Promise<void>,
): Promise<ContinueConfig> {
// Auto-detect models
@ -206,11 +180,15 @@ async function intermediateToFinalConfig(
const llm = await llmFromDescription(
desc,
readFile,
uniqueId,
ideSettings,
writeLog,
config.completionOptions,
config.systemMessage,
);
if (!llm) continue;
if (!llm) {
continue;
}
if (llm.model === "AUTODETECT") {
try {
@ -224,6 +202,8 @@ async function intermediateToFinalConfig(
title: llm.title + " - " + modelName,
},
readFile,
uniqueId,
ideSettings,
writeLog,
copyOf(config.completionOptions),
config.systemMessage,
@ -282,6 +262,8 @@ async function intermediateToFinalConfig(
autocompleteLlm = await llmFromDescription(
config.tabAutocompleteModel,
readFile,
uniqueId,
ideSettings,
writeLog,
config.completionOptions,
config.systemMessage,
@ -471,15 +453,12 @@ async function buildConfigTs() {
async function loadFullConfigNode(
readFile: (filepath: string) => Promise<string>,
workspaceConfigs: ContinueRcJson[],
remoteConfigServerUrl: URL | undefined,
ideSettings: IdeSettings,
ideType: IdeType,
uniqueId: string,
writeLog: (log: string) => Promise<void>,
): Promise<ContinueConfig> {
let serialized = loadSerializedConfig(
workspaceConfigs,
remoteConfigServerUrl,
ideType,
);
let serialized = loadSerializedConfig(workspaceConfigs, ideSettings, ideType);
let intermediate = serializedToIntermediateConfig(serialized);
const configJsContents = await buildConfigTs();
@ -499,10 +478,10 @@ async function loadFullConfigNode(
}
// Remote config.js
if (remoteConfigServerUrl) {
if (ideSettings.remoteConfigServerUrl) {
try {
const configJsPathForRemote = getConfigJsPathForRemote(
remoteConfigServerUrl,
ideSettings.remoteConfigServerUrl,
);
const module = await require(configJsPathForRemote);
delete require.cache[require.resolve(configJsPathForRemote)];
@ -518,6 +497,8 @@ async function loadFullConfigNode(
const finalConfig = await intermediateToFinalConfig(
intermediate,
readFile,
ideSettings,
uniqueId,
writeLog,
);
return finalConfig;

43
core/config/onboarding.ts
View File

@ -1,4 +1,4 @@
import { SerializedContinueConfig } from "..";
import { SerializedContinueConfig } from "../index.js";
export function setupOptimizedMode(
config: SerializedContinueConfig,
@ -6,31 +6,32 @@ export function setupOptimizedMode(
return {
...config,
models: [
{
title: "GPT-4-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Llama3 70b (Free Trial)",
provider: "free-trial",
model: "llama3-70b",
systemMessage:
"You are an expert software developer. You give helpful and concise responses. Whenever you write a code block you include the language after the opening ticks.",
},
{
title: "GPT-4o (Free Trial)",
provider: "free-trial",
model: "gpt-4o",
systemMessage:
"You are an expert software developer. You give helpful and concise responses.",
},
{
title: "Claude 3 Sonnet (Free Trial)",
provider: "free-trial",
model: "claude-3-sonnet-20240229",
},
{
title: "GPT-4 Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-4-turbo",
},
{
title: "GPT-3.5-Turbo (Free Trial)",
provider: "free-trial",
model: "gpt-3.5-turbo",
},
{
title: "Gemini Pro (Free Trial)",
provider: "free-trial",
model: "gemini-pro",
},
{
title: "Mixtral (Free Trial)",
provider: "free-trial",
model: "mistral-8x7b",
},
],
tabAutocompleteModel: {
title: "Tab Autocomplete",

5
core/config/types.ts
View File

@ -438,10 +438,6 @@ declare global {
description: string;
params?: { [key: string]: any };
run: (sdk: ContinueSDK) => AsyncGenerator<string | undefined>;
// If true, this command will be run in NodeJs and have access to the filesystem and other Node-only APIs
// You must make sure to dynamically import any Node-only dependencies in your command so that it doesn't break in the browser
runInNodeJs?: boolean;
}
// Config
@ -527,6 +523,7 @@ declare global {
| "gpt-3.5-turbo-0613"
| "gpt-4-32k"
| "gpt-4-turbo"
| "gpt-4o"
| "gpt-4-turbo-preview"
| "gpt-4-vision-preview"
// Mistral

4
core/config/util.ts
View File

@ -1,6 +1,6 @@
import { readFileSync, writeFileSync } from "fs";
import { ModelDescription } from "..";
import { editConfigJson, getConfigJsonPath } from "../util/paths";
import { ModelDescription } from "../index.js";
import { editConfigJson, getConfigJsonPath } from "../util/paths.js";
function stringify(obj: any, indentation?: number): string {
return JSON.stringify(

2
core/context/index.ts
View File

@ -5,7 +5,7 @@ import {
ContextSubmenuItem,
IContextProvider,
LoadSubmenuItemsArgs,
} from "..";
} from "../index.js";
export abstract class BaseContextProvider implements IContextProvider {
options: { [key: string]: any };

6
core/context/providers/CodeContextProvider.ts
View File

@ -1,12 +1,12 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
import { CodeSnippetsCodebaseIndex } from "../../indexing/CodeSnippetsIndex";
} from "../../index.js";
import { CodeSnippetsCodebaseIndex } from "../../indexing/CodeSnippetsIndex.js";
class CodeContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

6
core/context/providers/CodeHighlightsContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { getBasename } from "../../util";
} from "../../index.js";
import { getBasename } from "../../util/index.js";
// import { getHighlightsThatFit, ILLMContextSizer } from "llm-code-highlighter/dist/index.continue";

6
core/context/providers/CodeOutlineContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { getBasename } from "../../util";
} from "../../index.js";
import { getBasename } from "../../util/index.js";
// import { getOutlines } from "llm-code-highlighter/dist/index.continue";

6
core/context/providers/CodebaseContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { retrieveContextItemsFromEmbeddings } from "../retrieval/retrieval";
} from "../../index.js";
import { retrieveContextItemsFromEmbeddings } from "../retrieval/retrieval.js";
class CodebaseContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

6
core/context/providers/CurrentFileContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { getBasename } from "../../util";
} from "../../index.js";
import { getBasename } from "../../util/index.js";
import { BaseContextProvider } from "../index.js";
class CurrentFileContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

2
core/context/providers/CustomContextProvider.ts
View File

@ -5,7 +5,7 @@ import {
CustomContextProvider,
IContextProvider,
LoadSubmenuItemsArgs,
} from "../..";
} from "../../index.js";
class CustomContextProviderClass implements IContextProvider {
custom: CustomContextProvider;

10
core/context/providers/DatabaseContextProvider.ts
View File

@ -1,11 +1,11 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
} from "../../index.js";
class DatabaseContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {
@ -47,7 +47,7 @@ class DatabaseContextProvider extends BaseContextProvider {
let contextItem = {
name: `${connectionName}-all-tables-schemas`,
description: `Schema for all tables.`,
description: "Schema for all tables.",
content: prompt,
};
@ -101,7 +101,7 @@ class DatabaseContextProvider extends BaseContextProvider {
let contextItem = {
id: `${connection.name}.all`,
title: `${connection.name} all table schemas`,
description: ``,
description: "",
};
contextItems.push(contextItem);
@ -110,7 +110,7 @@ class DatabaseContextProvider extends BaseContextProvider {
let contextItem = {
id: `${connection.name}.${tableName}`,
title: `${connection.name}.${tableName} schema`,
description: ``,
description: "",
};
contextItems.push(contextItem);

4
core/context/providers/DiffContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class DiffContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

8
core/context/providers/DocsContextProvider.ts
View File

@ -1,13 +1,13 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
import configs from "../../indexing/docs/preIndexedDocs";
import TransformersJsEmbeddingsProvider from "../../indexing/embeddings/TransformersJsEmbeddingsProvider";
} from "../../index.js";
import configs from "../../indexing/docs/preIndexedDocs.js";
import TransformersJsEmbeddingsProvider from "../../indexing/embeddings/TransformersJsEmbeddingsProvider.js";
class DocsContextProvider extends BaseContextProvider {
static DEFAULT_N_RETRIEVE = 30;

6
core/context/providers/FileContextProvider.ts
View File

@ -1,12 +1,12 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
import { getBasename, getLastNPathParts } from "../../util";
} from "../../index.js";
import { getBasename, getLastNPathParts } from "../../util/index.js";
class FileContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/FileTreeContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
interface Directory {
name: string;

8
core/context/providers/FolderContextProvider.ts
View File

@ -1,17 +1,17 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
import { getBasename, getLastNPathParts } from "../../util";
} from "../../index.js";
import { getBasename, getLastNPathParts } from "../../util/index.js";
class FolderContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {
title: "folder",
displayTitle: "Folders",
displayTitle: "Folder",
description: "Type to search",
type: "submenu",
};

4
core/context/providers/GitHubIssuesContextProvider.ts
View File

@ -1,11 +1,11 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
} from "../../index.js";
class GitHubIssuesContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

34
core/context/providers/GitLabMergeRequestContextProvider.ts
View File

@ -1,6 +1,6 @@
import { AxiosInstance, AxiosError } from "axios";
import {BaseContextProvider} from "..";
import { ContextProviderExtras, ContextItem, ContextProviderDescription } from "../..";
import {BaseContextProvider} from "../index.js";
import { ContextProviderExtras, ContextItem, ContextProviderDescription } from "../../index.js";
interface RemoteBranchInfo {
branch: string | null;
@ -70,10 +70,10 @@ const getSubprocess = async (extras: ContextProviderExtras) => {
class GitLabMergeRequestContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {
title: 'gitlab-mr',
displayTitle: 'GitLab Merge Request',
description: 'Reference comments in a GitLab Merge Request',
type: 'normal'
title: "gitlab-mr",
displayTitle: "GitLab Merge Request",
description: "Reference comments in a GitLab Merge Request",
type: "normal"
};
private async getApi(): Promise<AxiosInstance> {
@ -83,7 +83,7 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
const token = this.options.token;
if(!token) {
throw new Error(`GitLab Private Token is required!`);
throw new Error("GitLab Private Token is required!");
}
return Axios.create({
@ -99,13 +99,13 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
const subprocess = await getSubprocess(extras);
const branchName = await subprocess(`git branch --show-current`);
const branchName = await subprocess("git branch --show-current");
const branchRemote = await subprocess(
`git config branch.${branchName}.remote`
);
const branchInfo = await subprocess(`git branch -vv`);
const branchInfo = await subprocess("git branch -vv");
const currentBranchInfo = branchInfo
.split("\n")
@ -121,7 +121,7 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
const remoteUrl = await subprocess(`git remote get-url ${branchRemote}`);
const urlMatches = RegExp(`:(?<project>.*).git`).exec(remoteUrl);
const urlMatches = RegExp(":(?<project>.*).git").exec(remoteUrl);
const project = urlMatches?.groups?.["project"] ?? null;
@ -157,8 +157,8 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
for (const mergeRequest of mergeRequests) {
const parts = [
`# GitLab Merge Request\ntitle: "${mergeRequest.title}"\ndescription: "${mergeRequest.description ?? 'None'}"`,
`## Comments`,
`# GitLab Merge Request\ntitle: "${mergeRequest.title}"\ndescription: "${mergeRequest.description ?? "None"}"`,
"## Comments",
];
const comments = await api.get<Array<GitLabComment>>(
@ -195,7 +195,7 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
}
const commentFormatter = async (comment: GitLabComment) => {
const commentLabel = comment.body.includes("```suggestion") ? 'Code Suggestion' : 'Comment';
const commentLabel = comment.body.includes("```suggestion") ? "Code Suggestion" : "Comment";
let result = `#### ${commentLabel}\nauthor: "${comment.author.name}"\ndate: "${comment.created_at}"\nresolved: ${
comment.resolved ? "Yes" : "No"
}`;
@ -240,13 +240,13 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
{
name: mergeRequest.title,
content,
description: `Comments from the Merge Request for this branch.`,
description: "Comments from the Merge Request for this branch.",
},
);
}
} catch (ex) {
let content = `# GitLab Merge Request\n\nError getting merge request. `;
let content = "# GitLab Merge Request\n\nError getting merge request. ";
if (ex instanceof AxiosError) {
if (ex.response) {
const errorMessage = ex.response?.data ? ex.response.data.message ?? JSON.stringify(ex.response?.data) : `${ex.response.status}: ${ex.response.statusText}`;
@ -262,9 +262,9 @@ class GitLabMergeRequestContextProvider extends BaseContextProvider {
result.push(
{
name: `GitLab Merge Request`,
name: "GitLab Merge Request",
content,
description: `Error getting the Merge Request for this branch.`,
description: "Error getting the Merge Request for this branch.",
},
);
}

4
core/context/providers/GoogleContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class GoogleContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/HttpContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class HttpContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

6
core/context/providers/JiraIssuesContextProvider/JiraClient.ts
View File

@ -1,4 +1,4 @@
import { RequestOptions } from "../../..";
import { RequestOptions } from "../../../index.js";
const { convert: adf2md } = require("adf-to-md");
interface JiraClientOptions {
@ -67,7 +67,7 @@ export class JiraClient {
private authHeader;
constructor(options: JiraClientOptions) {
this.options = {
issueQuery: `assignee = currentUser() AND resolution = Unresolved order by updated DESC`,
issueQuery: "assignee = currentUser() AND resolution = Unresolved order by updated DESC",
apiVersion: "3",
requestOptions: {},
...options,
@ -142,7 +142,7 @@ export class JiraClient {
this.baseUrl +
`/search?fields=summary&jql=${
this.options.issueQuery ??
`assignee = currentUser() AND resolution = Unresolved order by updated DESC`
"assignee = currentUser() AND resolution = Unresolved order by updated DESC"
}`,
),
{

6
core/context/providers/JiraIssuesContextProvider/index.ts
View File

@ -1,12 +1,12 @@
import { BaseContextProvider } from "../..";
import { BaseContextProvider } from "../../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../../..";
import { JiraClient } from "./JiraClient";
} from "../../../index.js";
import { JiraClient } from "./JiraClient.js";
class JiraIssuesContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/LocalsProvider.ts
View File

@ -1,11 +1,11 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
} from "../../index.js";
class LocalsProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/OSContextProvider.ts
View File

@ -1,12 +1,12 @@
//os.platform()
//os.arch()
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
import os from "os";
class OSContextProvider extends BaseContextProvider {

6
core/context/providers/OpenFilesContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { getBasename } from "../../util";
} from "../../index.js";
import { getBasename } from "../../util/index.js";
class OpenFilesContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/PostgresContextProvider.ts
View File

@ -1,11 +1,11 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
ContextSubmenuItem,
LoadSubmenuItemsArgs,
} from "../..";
} from "../../index.js";
class PostgresContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

8
core/context/providers/ProblemsContextProvider.ts
View File

@ -1,10 +1,10 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
import { getBasename } from "../../util";
} from "../../index.js";
import { getBasename } from "../../util/index.js";
class ProblemsContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {
@ -46,7 +46,7 @@ class ProblemsContextProvider extends BaseContextProvider {
? [
{
description: "Problems in current file",
content: `There are no problems found in the open file.`,
content: "There are no problems found in the open file.",
name: "No problems found",
},
]

4
core/context/providers/SearchContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class SearchContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/TerminalContextProvider.ts
View File

@ -1,9 +1,9 @@
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class TerminalContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

4
core/context/providers/URLContextProvider.ts
View File

@ -1,12 +1,12 @@
import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import { NodeHtmlMarkdown } from "node-html-markdown";
import { BaseContextProvider } from "..";
import { BaseContextProvider } from "../index.js";
import {
ContextItem,
ContextProviderDescription,
ContextProviderExtras,
} from "../..";
} from "../../index.js";
class URLContextProvider extends BaseContextProvider {
static description: ContextProviderDescription = {

46
core/context/providers/index.ts
View File

@ -1,26 +1,26 @@
import { BaseContextProvider } from "..";
import { ContextProviderName } from "../..";
import CodeContextProvider from "./CodeContextProvider";
import CodebaseContextProvider from "./CodebaseContextProvider";
import CurrentFileContextProvider from "./CurrentFileContextProvider";
import DatabaseContextProvider from "./DatabaseContextProvider";
import DiffContextProvider from "./DiffContextProvider";
import DocsContextProvider from "./DocsContextProvider";
import FileTreeContextProvider from "./FileTreeContextProvider";
import FolderContextProvider from "./FolderContextProvider";
import GitHubIssuesContextProvider from "./GitHubIssuesContextProvider";
import GitLabMergeRequestContextProvider from "./GitLabMergeRequestContextProvider";
import GoogleContextProvider from "./GoogleContextProvider";
import HttpContextProvider from "./HttpContextProvider";
import JiraIssuesContextProvider from "./JiraIssuesContextProvider";
import LocalsProvider from "./LocalsProvider";
import OSContextProvider from "./OSContextProvider";
import OpenFilesContextProvider from "./OpenFilesContextProvider";
import PostgresContextProvider from "./PostgresContextProvider";
import ProblemsContextProvider from "./ProblemsContextProvider";
import SearchContextProvider from "./SearchContextProvider";
import TerminalContextProvider from "./TerminalContextProvider";
import URLContextProvider from "./URLContextProvider";
import { ContextProviderName } from "../../index.js";
import { BaseContextProvider } from "../index.js";
import CodeContextProvider from "./CodeContextProvider.js";
import CodebaseContextProvider from "./CodebaseContextProvider.js";
import CurrentFileContextProvider from "./CurrentFileContextProvider.js";
import DatabaseContextProvider from "./DatabaseContextProvider.js";
import DiffContextProvider from "./DiffContextProvider.js";
import DocsContextProvider from "./DocsContextProvider.js";
import FileTreeContextProvider from "./FileTreeContextProvider.js";
import FolderContextProvider from "./FolderContextProvider.js";
import GitHubIssuesContextProvider from "./GitHubIssuesContextProvider.js";
import GitLabMergeRequestContextProvider from "./GitLabMergeRequestContextProvider.js";
import GoogleContextProvider from "./GoogleContextProvider.js";
import HttpContextProvider from "./HttpContextProvider.js";
import JiraIssuesContextProvider from "./JiraIssuesContextProvider/index.js";
import LocalsProvider from "./LocalsProvider.js";
import OSContextProvider from "./OSContextProvider.js";
import OpenFilesContextProvider from "./OpenFilesContextProvider.js";
import PostgresContextProvider from "./PostgresContextProvider.js";
import ProblemsContextProvider from "./ProblemsContextProvider.js";
import SearchContextProvider from "./SearchContextProvider.js";
import TerminalContextProvider from "./TerminalContextProvider.js";
import URLContextProvider from "./URLContextProvider.js";
const Providers: (typeof BaseContextProvider)[] = [
DiffContextProvider,

4
core/context/rerankers/cohere.ts
View File

@ -1,5 +1,5 @@
import fetch from "node-fetch";
import { Chunk, Reranker } from "../..";
import { Chunk, Reranker } from "../../index.js";
export class CohereReranker implements Reranker {
name = "cohere";
@ -18,7 +18,7 @@ export class CohereReranker implements Reranker {
) {}
async rerank(query: string, chunks: Chunk[]): Promise<number[]> {
let apiBase = this.params.apiBase ?? CohereReranker.defaultOptions.apiBase
let apiBase = this.params.apiBase ?? CohereReranker.defaultOptions.apiBase;
if (!apiBase.endsWith("/")) {
apiBase += "/";
}

6
core/context/rerankers/freeTrial.ts
View File

@ -1,7 +1,7 @@
import fetch from "node-fetch";
import { Chunk, Reranker } from "../..";
import { getHeaders } from "../../continueServer/stubs/headers";
import { SERVER_URL } from "../../util/parameters";
import { Chunk, Reranker } from "../../index.js";
import { getHeaders } from "../../continueServer/stubs/headers.js";
import { SERVER_URL } from "../../util/parameters.js";
export class FreeTrialReranker implements Reranker {
name = "free-trial";

10
core/context/rerankers/index.ts
View File

@ -1,8 +1,8 @@
import { RerankerName } from "../..";
import { CohereReranker } from "./cohere";
import { FreeTrialReranker } from "./freeTrial";
import { LLMReranker } from "./llm";
import { VoyageReranker } from "./voyage";
import { RerankerName } from "../../index.js";
import { CohereReranker } from "./cohere.js";
import { FreeTrialReranker } from "./freeTrial.js";
import { LLMReranker } from "./llm.js";
import { VoyageReranker } from "./voyage.js";
export const AllRerankers: { [key in RerankerName]: any } = {
cohere: CohereReranker,

4
core/context/rerankers/llm.ts
View File

@ -1,5 +1,5 @@
import { Chunk, ILLM, Reranker } from "../..";
import { getBasename } from "../../util";
import { Chunk, ILLM, Reranker } from "../../index.js";
import { getBasename } from "../../util/index.js";
const RERANK_PROMPT = (
query: string,

2
core/context/rerankers/voyage.ts
View File

@ -1,5 +1,5 @@
import fetch from "node-fetch";
import { Chunk, Reranker } from "../..";
import { Chunk, Reranker } from "../../index.js";
export class VoyageReranker implements Reranker {
name = "voyage";

4
core/context/retrieval/fullTextSearch.ts
View File

@ -1,5 +1,5 @@
import { BranchAndDir, Chunk } from "../..";
import { FullTextSearchCodebaseIndex } from "../../indexing/FullTextSearch";
import { BranchAndDir, Chunk } from "../../index.js";
import { FullTextSearchCodebaseIndex } from "../../indexing/FullTextSearch.js";
export async function retrieveFts(
query: string,
n: number,

19
core/context/retrieval/retrieval.ts
View File

@ -1,9 +1,14 @@
import { BranchAndDir, Chunk, ContextItem, ContextProviderExtras } from "../..";
import { LanceDbIndex } from "../../indexing/LanceDbIndex";
import {
BranchAndDir,
Chunk,
ContextItem,
ContextProviderExtras,
} from "../../index.js";
import { LanceDbIndex } from "../../indexing/LanceDbIndex.js";
import { deduplicateArray, getBasename } from "../../util";
import { RETRIEVAL_PARAMS } from "../../util/parameters";
import { retrieveFts } from "./fullTextSearch";
import { deduplicateArray, getBasename } from "../../util/index.js";
import { RETRIEVAL_PARAMS } from "../../util/parameters.js";
import { retrieveFts } from "./fullTextSearch.js";
function deduplicateChunks(chunks: Chunk[]): Chunk[] {
return deduplicateArray(chunks, (a, b) => {
@ -121,9 +126,9 @@ export async function retrieveContextItemsFromEmbeddings(
);
results.sort(
(a, b) => scores[results.indexOf(b)] - scores[results.indexOf(a)],
(a, b) => scores[results.indexOf(a)] - scores[results.indexOf(b)],
);
results = results.slice(0, nFinal);
results = results.slice(-nFinal);
}
if (results.length === 0) {

5
core/continueServer/interface.ts
View File

@ -1,4 +1,4 @@
import { Chunk } from "..";
import { Chunk } from "../index.js";
export interface EmbeddingsCacheChunk {
vector: number[];
@ -19,6 +19,9 @@ export interface EmbeddingsCacheResponse<T extends ArtifactType> {
}
export interface IContinueServerClient {
connected: boolean;
url: URL | undefined;
getUserToken(): Promise<string | undefined>;
getConfig(): Promise<{ configJson: string; configJs: string }>;
getFromIndexCache<T extends ArtifactType>(
keys: string[],

24
core/continueServer/stubs/client.ts
View File

@ -2,13 +2,31 @@ import {
ArtifactType,
EmbeddingsCacheResponse,
IContinueServerClient,
} from "../interface";
} from "../interface.js";
export class ContinueServerClient implements IContinueServerClient {
url: URL | undefined;
constructor(
private readonly serverUrl: string,
serverUrl: string | undefined,
private readonly userToken: Promise<string | undefined>,
) {}
) {
try {
this.url =
typeof serverUrl !== "string" || serverUrl === ""
? undefined
: new URL(serverUrl);
} catch (e) {
console.warn("Invalid Continue server url", e);
this.url = undefined;
}
}
getUserToken(): Promise<string | undefined> {
return this.userToken;
}
connected: boolean = false;
public async getConfig(): Promise<{ configJson: string; configJs: string }> {
throw new Error("Not Implemented");

4
core/diff/streamDiff.ts
View File

@ -1,5 +1,5 @@
import { DiffLine } from "..";
import { LineStream, matchLine } from "./util";
import { DiffLine } from "../index.js";
import { LineStream, matchLine } from "./util.js";
/**
* https://blog.jcoglan.com/2017/02/12/the-myers-diff-algorithm-part-1/

4
core/diff/util.ts
View File

@ -1,6 +1,6 @@
import { distance } from "fastest-levenshtein";
import { ChatMessage } from "..";
import { stripImages } from "../llm/countTokens";
import { ChatMessage } from "../index.js";
import { stripImages } from "../llm/countTokens.js";
export type LineStream = AsyncGenerator<string>;

8
core/index.d.ts vendored
View File

@ -37,6 +37,7 @@ export interface Chunk extends ChunkWithoutID {
export interface IndexingProgressUpdate {
progress: number;
desc: string;
status: "starting" | "indexing" | "done" | "failed" | "paused" | "disabled";
}
export type PromptTemplate =
@ -439,10 +440,6 @@ export interface SlashCommand {
description: string;
params?: { [key: string]: any };
run: (sdk: ContinueSDK) => AsyncGenerator<string | undefined>;
// If true, this command will be run in NodeJs and have access to the filesystem and other Node-only APIs
// You must make sure to dynamically import any Node-only dependencies in your command so that it doesn't break in the browser
runInNodeJs?: boolean;
}
// Config
@ -519,6 +516,7 @@ type ModelProvider =
| "deepinfra"
| "flowise"
| "groq"
| "continue-proxy"
| "custom";
export type ModelName =
@ -529,6 +527,7 @@ export type ModelName =
| "gpt-4"
| "gpt-3.5-turbo-0613"
| "gpt-4-32k"
| "gpt-4o"
| "gpt-4-turbo"
| "gpt-4-turbo-preview"
| "gpt-4-vision-preview"
@ -700,6 +699,7 @@ export interface TabAutocompleteOptions {
export interface ContinueUIConfig {
codeBlockToolbarPosition?: "top" | "bottom";
fontSize?: number;
}
interface ContextMenuConfig {

25
core/indexing/CodeSnippetsIndex.ts
View File

@ -7,20 +7,20 @@ import {
IDE,
IndexTag,
IndexingProgressUpdate,
} from "..";
import { getBasename } from "../util";
} from "../index.js";
import { getBasename } from "../util/index.js";
import {
getLanguageForFile,
getParserForFile,
supportedLanguages,
} from "../util/treeSitter";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
} from "../util/treeSitter.js";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex.js";
import {
CodebaseIndex,
IndexResultType,
MarkCompleteCallback,
RefreshIndexResults,
} from "./types";
} from "./types.js";
export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
artifactId = "codeSnippets";
@ -118,7 +118,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
// Add snippets to sqlite
for (const snippet of snippets) {
const { lastID } = await db.run(
`INSERT INTO code_snippets (path, cacheKey, content, title, startLine, endLine) VALUES (?, ?, ?, ?, ?, ?)`,
"INSERT INTO code_snippets (path, cacheKey, content, title, startLine, endLine) VALUES (?, ?, ?, ?, ?, ?)",
[
compute.path,
compute.cacheKey,
@ -130,7 +130,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
);
await db.run(
`INSERT INTO code_snippets_tags (snippetId, tag) VALUES (?, ?)`,
"INSERT INTO code_snippets_tags (snippetId, tag) VALUES (?, ?)",
[lastID, tagString],
);
}
@ -138,6 +138,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
yield {
desc: `Indexing ${compute.path}`,
progress: i / results.compute.length,
status: "indexing",
};
markComplete([compute], IndexResultType.Compute);
}
@ -145,10 +146,10 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
for (let i = 0; i < results.del.length; i++) {
const del = results.del[i];
const deleted = await db.run(
`DELETE FROM code_snippets WHERE path = ? AND cacheKey = ?`,
"DELETE FROM code_snippets WHERE path = ? AND cacheKey = ?",
[del.path, del.cacheKey],
);
await db.run(`DELETE FROM code_snippets_tags WHERE snippetId = ?`, [
await db.run("DELETE FROM code_snippets_tags WHERE snippetId = ?", [
deleted.lastID,
]);
markComplete([del], IndexResultType.Delete);
@ -156,13 +157,13 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
for (let i = 0; i < results.addTag.length; i++) {
const snippetsWithPath = await db.all(
`SELECT * FROM code_snippets WHERE cacheKey = ?`,
"SELECT * FROM code_snippets WHERE cacheKey = ?",
[results.addTag[i].cacheKey],
);
for (const snippet of snippetsWithPath) {
await db.run(
`INSERT INTO code_snippets_tags (snippetId, tag) VALUES (?, ?)`,
"INSERT INTO code_snippets_tags (snippetId, tag) VALUES (?, ?)",
[snippet.id, tagString],
);
}
@ -189,7 +190,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
static async getForId(id: number): Promise<ContextItem> {
const db = await SqliteDb.get();
const row = await db.get(`SELECT * FROM code_snippets WHERE id = ?`, [id]);
const row = await db.get("SELECT * FROM code_snippets WHERE id = ?", [id]);
return {
name: row.title,

21
core/indexing/FullTextSearch.ts
View File

@ -1,13 +1,13 @@
import { BranchAndDir, Chunk, IndexTag, IndexingProgressUpdate } from "..";
import { RETRIEVAL_PARAMS } from "../util/parameters";
import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
import { BranchAndDir, Chunk, IndexTag, IndexingProgressUpdate } from "../index.js";
import { RETRIEVAL_PARAMS } from "../util/parameters.js";
import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex.js";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex.js";
import {
CodebaseIndex,
IndexResultType,
MarkCompleteCallback,
RefreshIndexResults,
} from "./types";
} from "./types.js";
export class FullTextSearchCodebaseIndex implements CodebaseIndex {
artifactId: string = "sqliteFts";
@ -43,17 +43,17 @@ export class FullTextSearchCodebaseIndex implements CodebaseIndex {
// Insert chunks
const chunks = await db.all(
`SELECT * FROM chunks WHERE path = ? AND cacheKey = ?`,
"SELECT * FROM chunks WHERE path = ? AND cacheKey = ?",
[item.path, item.cacheKey],
);
for (let chunk of chunks) {
const { lastID } = await db.run(
`INSERT INTO fts (path, content) VALUES (?, ?)`,
"INSERT INTO fts (path, content) VALUES (?, ?)",
[item.path, chunk.content],
);
await db.run(
`INSERT INTO fts_metadata (id, path, cacheKey, chunkId) VALUES (?, ?, ?, ?)`,
"INSERT INTO fts_metadata (id, path, cacheKey, chunkId) VALUES (?, ?, ?, ?)",
[lastID, item.path, item.cacheKey, chunk.id],
);
}
@ -61,6 +61,7 @@ export class FullTextSearchCodebaseIndex implements CodebaseIndex {
yield {
progress: i / results.compute.length,
desc: `Indexing ${item.path}`,
status: "indexing",
};
markComplete([item], IndexResultType.Compute);
}
@ -78,10 +79,10 @@ export class FullTextSearchCodebaseIndex implements CodebaseIndex {
// Delete
for (const item of results.del) {
const { lastID } = await db.run(
`DELETE FROM fts_metadata WHERE path = ? AND cacheKey = ?`,
"DELETE FROM fts_metadata WHERE path = ? AND cacheKey = ?",
[item.path, item.cacheKey],
);
await db.run(`DELETE FROM fts WHERE rowid = ?`, [lastID]);
await db.run("DELETE FROM fts WHERE rowid = ?", [lastID]);
markComplete([item], IndexResultType.Delete);
}

28
core/indexing/LanceDbIndex.ts
View File

@ -1,25 +1,25 @@
// NOTE: vectordb requirement must be listed in extensions/vscode to avoid error
import { v4 as uuidv4 } from "uuid";
import { Table } from "vectordb";
import { IContinueServerClient } from "../continueServer/interface.js";
import {
BranchAndDir,
Chunk,
EmbeddingsProvider,
IndexTag,
IndexingProgressUpdate,
} from "..";
import { ContinueServerClient } from "../continueServer/stubs/client";
import { MAX_CHUNK_SIZE } from "../llm/constants";
import { getBasename } from "../util";
import { getLanceDbPath } from "../util/paths";
import { chunkDocument } from "./chunk/chunk";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
} from "../index.js";
import { MAX_CHUNK_SIZE } from "../llm/constants.js";
import { getBasename } from "../util/index.js";
import { getLanceDbPath } from "../util/paths.js";
import { chunkDocument } from "./chunk/chunk.js";
import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex.js";
import {
CodebaseIndex,
IndexResultType,
PathAndCacheKey,
RefreshIndexResults,
} from "./types";
} from "./types.js";
// LanceDB converts to lowercase, so names must all be lowercase
interface LanceDbRow {
@ -40,7 +40,7 @@ export class LanceDbIndex implements CodebaseIndex {
constructor(
private readonly embeddingsProvider: EmbeddingsProvider,
private readonly readFile: (filepath: string) => Promise<string>,
private readonly continueServerClient?: ContinueServerClient,
private readonly continueServerClient?: IContinueServerClient,
) {}
private tableNameForTag(tag: IndexTag) {
@ -176,7 +176,7 @@ export class LanceDbIndex implements CodebaseIndex {
};
// Check remote cache
if (this.continueServerClient !== undefined) {
if (this.continueServerClient?.connected) {
try {
const keys = results.compute.map(({ cacheKey }) => cacheKey);
const resp = await this.continueServerClient.getFromIndexCache(
@ -250,7 +250,7 @@ export class LanceDbIndex implements CodebaseIndex {
data.contents,
);
yield { progress, desc };
yield { progress, desc, status: "indexing" };
} else {
await addComputedLanceDbRows(update, computedRows);
computedRows = [];
@ -304,7 +304,11 @@ export class LanceDbIndex implements CodebaseIndex {
}
markComplete(results.del, IndexResultType.Delete);
yield { progress: 1, desc: "Completed Calculating Embeddings" };
yield {
progress: 1,
desc: "Completed Calculating Embeddings",
status: "done",
};
}
private async _retrieveForTag(

31
core/indexing/chunk/ChunkCodebaseIndex.ts
View File

@ -1,15 +1,15 @@
import { Chunk, IndexTag, IndexingProgressUpdate } from "../..";
import { ContinueServerClient } from "../../continueServer/stubs/client";
import { MAX_CHUNK_SIZE } from "../../llm/constants";
import { getBasename } from "../../util";
import { DatabaseConnection, SqliteDb, tagToString } from "../refreshIndex";
import { IContinueServerClient } from "../../continueServer/interface.js";
import { Chunk, IndexTag, IndexingProgressUpdate } from "../../index.js";
import { MAX_CHUNK_SIZE } from "../../llm/constants.js";
import { getBasename } from "../../util/index.js";
import { DatabaseConnection, SqliteDb, tagToString } from "../refreshIndex.js";
import {
CodebaseIndex,
IndexResultType,
MarkCompleteCallback,
RefreshIndexResults,
} from "../types";
import { chunkDocument } from "./chunk";
} from "../types.js";
import { chunkDocument } from "./chunk.js";
export class ChunkCodebaseIndex implements CodebaseIndex {
static artifactId: string = "chunks";
@ -17,7 +17,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
constructor(
private readonly readFile: (filepath: string) => Promise<string>,
private readonly continueServerClient?: ContinueServerClient,
private readonly continueServerClient: IContinueServerClient,
) {
this.readFile = readFile;
}
@ -53,7 +53,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
async function handleChunk(chunk: Chunk) {
const { lastID } = await db.run(
`INSERT INTO chunks (cacheKey, path, idx, startLine, endLine, content) VALUES (?, ?, ?, ?, ?, ?)`,
"INSERT INTO chunks (cacheKey, path, idx, startLine, endLine, content) VALUES (?, ?, ?, ?, ?, ?)",
[
chunk.digest,
chunk.filepath,
@ -64,14 +64,14 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
],
);
await db.run(`INSERT INTO chunk_tags (chunkId, tag) VALUES (?, ?)`, [
await db.run("INSERT INTO chunk_tags (chunkId, tag) VALUES (?, ?)", [
lastID,
tagString,
]);
}
// Check the remote cache
if (this.continueServerClient !== undefined) {
if (this.continueServerClient.connected) {
try {
const keys = results.compute.map(({ cacheKey }) => cacheKey);
const resp = await this.continueServerClient.getFromIndexCache(
@ -113,6 +113,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
yield {
progress: i / results.compute.length,
desc: `Chunking ${getBasename(item.path)}`,
status: "indexing",
};
markComplete([item], IndexResultType.Compute);
}
@ -120,12 +121,12 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
// Add tag
for (const item of results.addTag) {
const chunksWithPath = await db.all(
`SELECT * FROM chunks WHERE cacheKey = ?`,
"SELECT * FROM chunks WHERE cacheKey = ?",
[item.cacheKey],
);
for (const chunk of chunksWithPath) {
await db.run(`INSERT INTO chunk_tags (chunkId, tag) VALUES (?, ?)`, [
await db.run("INSERT INTO chunk_tags (chunkId, tag) VALUES (?, ?)", [
chunk.id,
tagString,
]);
@ -152,12 +153,12 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
// Delete
for (const item of results.del) {
const deleted = await db.run(`DELETE FROM chunks WHERE cacheKey = ?`, [
const deleted = await db.run("DELETE FROM chunks WHERE cacheKey = ?", [
item.cacheKey,
]);
// Delete from chunk_tags
await db.run(`DELETE FROM chunk_tags WHERE chunkId = ?`, [
await db.run("DELETE FROM chunk_tags WHERE chunkId = ?", [
deleted.lastID,
]);

4
core/indexing/chunk/basic.ts
View File

@ -1,5 +1,5 @@
import { ChunkWithoutID } from "../..";
import { countTokens } from "../../llm/countTokens";
import { ChunkWithoutID } from "../../index.js";
import { countTokens } from "../../llm/countTokens.js";
export function* basicChunker(
contents: string,

12
core/indexing/chunk/chunk.ts
View File

@ -1,9 +1,9 @@
import { Chunk, ChunkWithoutID } from "../..";
import { MAX_CHUNK_SIZE } from "../../llm/constants";
import { countTokens } from "../../llm/countTokens";
import { supportedLanguages } from "../../util/treeSitter";
import { basicChunker } from "./basic";
import { codeChunker } from "./code";
import { Chunk, ChunkWithoutID } from "../../index.js";
import { MAX_CHUNK_SIZE } from "../../llm/constants.js";
import { countTokens } from "../../llm/countTokens.js";
import { supportedLanguages } from "../../util/treeSitter.js";
import { basicChunker } from "./basic.js";
import { codeChunker } from "./code.js";
async function* chunkDocumentWithoutId(
filepath: string,

6
core/indexing/chunk/code.ts
View File

@ -1,7 +1,7 @@
import { SyntaxNode } from "web-tree-sitter";
import { ChunkWithoutID } from "../..";
import { countTokens } from "../../llm/countTokens";
import { getParserForFile } from "../../util/treeSitter";
import { ChunkWithoutID } from "../../index.js";
import { countTokens } from "../../llm/countTokens.js";
import { getParserForFile } from "../../util/treeSitter.js";
function collapsedReplacement(node: SyntaxNode): string {
if (node.type === "statement_block") {

6
core/indexing/chunk/markdown.ts
View File

@ -1,6 +1,6 @@
import { ChunkWithoutID } from "../..";
import { countTokens } from "../../llm/countTokens";
import { basicChunker } from "./basic";
import { ChunkWithoutID } from "../../index.js";
import { countTokens } from "../../llm/countTokens.js";
import { basicChunker } from "./basic.js";
export function cleanFragment(
fragment: string | undefined,

8
core/indexing/docs/article.ts
View File

@ -1,9 +1,9 @@
import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import { Chunk } from "../..";
import { MAX_CHUNK_SIZE } from "../../llm/constants";
import { cleanFragment, cleanHeader } from "../chunk/markdown";
import { PageData } from "./crawl";
import { Chunk } from "../../index.js";
import { MAX_CHUNK_SIZE } from "../../llm/constants.js";
import { cleanFragment, cleanHeader } from "../chunk/markdown.js";
import { PageData } from "./crawl.js";
export type ArticleComponent = {
title: string;

18
core/indexing/docs/db.ts
View File

@ -1,10 +1,10 @@
import { Database, open } from "sqlite";
import sqlite3 from "sqlite3";
import { Chunk } from "../..";
import { getDocsSqlitePath, getLanceDbPath } from "../../util/paths";
import { Chunk } from "../../index.js";
import { getDocsSqlitePath, getLanceDbPath } from "../../util/paths.js";
import { downloadPreIndexedDocs } from "./preIndexed";
import { default as configs } from "./preIndexedDocs";
import { downloadPreIndexedDocs } from "./preIndexed.js";
import { default as configs } from "./preIndexedDocs.js";
const DOCS_TABLE_NAME = "docs";
@ -69,7 +69,7 @@ export async function retrieveDocs(
const tableNames = await lance.tableNames();
if (!tableNames.includes(DOCS_TABLE_NAME)) {
const downloaded = await downloadDocs();
if (downloaded) return downloaded;
if (downloaded) {return downloaded;}
}
const table = await lance.openTable(DOCS_TABLE_NAME);
@ -83,7 +83,7 @@ export async function retrieveDocs(
if ((!docs || docs.length === 0) && !nested) {
const downloaded = await downloadDocs();
if (downloaded) return downloaded;
if (downloaded) {return downloaded;}
}
return docs.map((doc) => ({
@ -128,7 +128,7 @@ export async function addDocs(
// Only after add it to SQLite
const db = await getDBDocs();
await db.run(
`INSERT INTO docs (title, baseUrl) VALUES (?, ?)`,
"INSERT INTO docs (title, baseUrl) VALUES (?, ?)",
title,
baseUrl.toString(),
);
@ -138,12 +138,12 @@ export async function listDocs(): Promise<
{ title: string; baseUrl: string }[]
> {
const db = await getDBDocs();
const docs = db.all(`SELECT title, baseUrl FROM docs`);
const docs = db.all("SELECT title, baseUrl FROM docs");
return docs;
}
export async function hasDoc(baseUrl: string) {
const db = await getDBDocs();
const doc = await db.get(`SELECT title FROM docs WHERE baseUrl =?`, baseUrl);
const doc = await db.get("SELECT title FROM docs WHERE baseUrl =?", baseUrl);
return !!doc;
}

23
core/indexing/docs/index.ts
View File

@ -1,12 +1,8 @@
import {
Chunk,
EmbeddingsProvider,
IndexingProgressUpdate,
} from "../..";
import { Chunk, EmbeddingsProvider, IndexingProgressUpdate } from "../../index.js";
import { crawlPage } from "./crawl";
import { addDocs, hasDoc } from "./db";
import { pageToArticle, chunkArticle, Article } from "./article";
import { Article, chunkArticle, pageToArticle } from "./article.js";
import { crawlPage } from "./crawl.js";
import { addDocs, hasDoc } from "./db.js";
export async function* indexDocs(
title: string,
@ -17,6 +13,7 @@ export async function* indexDocs(
yield {
progress: 1,
desc: "Already indexed",
status: "done",
};
return;
}
@ -24,19 +21,21 @@ export async function* indexDocs(
yield {
progress: 0,
desc: "Finding subpages",
status: "indexing",
};
const articles: Article[] = [];
for await (const page of crawlPage(baseUrl)) {
const article = pageToArticle(page);
if (!article) continue;
if (!article) {continue;}
articles.push(article);
yield {
progress: 0,
desc: `Finding subpages (${page.path})`,
status: "indexing",
};
}
@ -47,14 +46,15 @@ export async function* indexDocs(
yield {
progress: Math.max(1, Math.floor(100 / (articles.length + 1))),
desc: `${article.subpath}`,
status: "indexing",
};
const subpathEmbeddings = await embeddingsProvider.embed(
chunkArticle(article).map(chunk => {
chunkArticle(article).map((chunk) => {
chunks.push(chunk);
return chunk.content;
})
}),
);
embeddings.push(...subpathEmbeddings);
@ -65,5 +65,6 @@ export async function* indexDocs(
yield {
progress: 1,
desc: "Done",
status: "done",
};
}

4
core/indexing/docs/preIndexed.ts
View File

@ -1,5 +1,5 @@
import { Chunk } from "../..";
import { addDocs } from "./db";
import { Chunk } from "../../index.js";
import { addDocs } from "./db.js";
const request = require("request");

2
core/indexing/embeddings/BaseEmbeddingsProvider.ts
View File

@ -1,4 +1,4 @@
import { EmbedOptions, EmbeddingsProvider, FetchFunction } from "../..";
import { EmbedOptions, EmbeddingsProvider, FetchFunction } from "../../index.js";
class BaseEmbeddingsProvider implements EmbeddingsProvider {
options: EmbedOptions;

6
core/indexing/embeddings/CohereEmbeddingsProvider.ts
View File

@ -1,7 +1,7 @@
import { Response } from "node-fetch";
import { EmbedOptions } from "../..";
import { withExponentialBackoff } from "../../util/withExponentialBackoff";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import { EmbedOptions } from "../../index.js";
import { withExponentialBackoff } from "../../util/withExponentialBackoff.js";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
class CohereEmbeddingsProvider extends BaseEmbeddingsProvider {
static maxBatchSize = 96;

6
core/indexing/embeddings/DeepInfraEmbeddingsProvider.ts
View File

@ -1,6 +1,6 @@
import { EmbedOptions } from "../..";
import { withExponentialBackoff } from "../../util/withExponentialBackoff";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import { EmbedOptions } from "../../index.js";
import { withExponentialBackoff } from "../../util/withExponentialBackoff.js";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
class DeepInfraEmbeddingsProvider extends BaseEmbeddingsProvider {
static defaultOptions: Partial<EmbedOptions> | undefined = {

10
core/indexing/embeddings/FreeTrialEmbeddingsProvider.ts
View File

@ -1,9 +1,9 @@
import { Response } from "node-fetch";
import { EmbedOptions } from "../..";
import { getHeaders } from "../../continueServer/stubs/headers";
import { SERVER_URL } from "../../util/parameters";
import { withExponentialBackoff } from "../../util/withExponentialBackoff";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import { EmbedOptions } from "../../index.js";
import { getHeaders } from "../../continueServer/stubs/headers.js";
import { SERVER_URL } from "../../util/parameters.js";
import { withExponentialBackoff } from "../../util/withExponentialBackoff.js";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
class FreeTrialEmbeddingsProvider extends BaseEmbeddingsProvider {
static maxBatchSize = 128;

6
core/indexing/embeddings/OllamaEmbeddingsProvider.ts
View File

@ -1,6 +1,6 @@
import { EmbedOptions, FetchFunction } from "../..";
import { withExponentialBackoff } from "../../util/withExponentialBackoff";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import { EmbedOptions, FetchFunction } from "../../index.js";
import { withExponentialBackoff } from "../../util/withExponentialBackoff.js";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
async function embedOne(
chunk: string,

6
core/indexing/embeddings/OpenAIEmbeddingsProvider.ts
View File

@ -1,7 +1,7 @@
import { Response } from "node-fetch";
import { EmbedOptions } from "../..";
import { withExponentialBackoff } from "../../util/withExponentialBackoff";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import { EmbedOptions } from "../../index.js";
import { withExponentialBackoff } from "../../util/withExponentialBackoff.js";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
class OpenAIEmbeddingsProvider extends BaseEmbeddingsProvider {
// https://platform.openai.com/docs/api-reference/embeddings/create is 2048

14
core/indexing/embeddings/TransformersJsEmbeddingsProvider.ts
View File

@ -1,11 +1,8 @@
import {
PipelineType,
env,
pipeline,
} from "../../vendor/node_modules/@xenova/transformers";
// @ts-ignore
import { PipelineType, env, pipeline } from "../../vendor/modules/@xenova/transformers/src/transformers.js";
import path from "path";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider";
import BaseEmbeddingsProvider from "./BaseEmbeddingsProvider.js";
env.allowLocalModels = true;
env.allowRemoteModels = false;
@ -31,8 +28,11 @@ class EmbeddingsPipeline {
export class TransformersJsEmbeddingsProvider extends BaseEmbeddingsProvider {
static MaxGroupSize: number = 4;
constructor() {
constructor(modelPath?: string) {
super({ model: "all-MiniLM-L2-v6" }, () => Promise.resolve(null));
if (modelPath) {
env.localModelPath = modelPath;
}
}
get id(): string {

12
core/indexing/embeddings/index.ts
View File

@ -1,9 +1,9 @@
import { EmbeddingsProviderName } from "../..";
import CohereEmbeddingsProvider from "./CohereEmbeddingsProvider";
import FreeTrialEmbeddingsProvider from "./FreeTrialEmbeddingsProvider";
import OllamaEmbeddingsProvider from "./OllamaEmbeddingsProvider";
import OpenAIEmbeddingsProvider from "./OpenAIEmbeddingsProvider";
import TransformersJsEmbeddingsProvider from "./TransformersJsEmbeddingsProvider";
import { EmbeddingsProviderName } from "../../index.js";
import CohereEmbeddingsProvider from "./CohereEmbeddingsProvider.js";
import FreeTrialEmbeddingsProvider from "./FreeTrialEmbeddingsProvider.js";
import OllamaEmbeddingsProvider from "./OllamaEmbeddingsProvider.js";
import OpenAIEmbeddingsProvider from "./OpenAIEmbeddingsProvider.js";
import TransformersJsEmbeddingsProvider from "./TransformersJsEmbeddingsProvider.js";
export const AllEmbeddingsProviders: {
[key in EmbeddingsProviderName]: any;

64
core/indexing/indexCodebase.ts
View File

@ -1,12 +1,12 @@
import { IDE, IndexTag, IndexingProgressUpdate } from "..";
import { ConfigHandler } from "../config/handler";
import { ContinueServerClient } from "../continueServer/stubs/client";
import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex";
import { FullTextSearchCodebaseIndex } from "./FullTextSearch";
import { LanceDbIndex } from "./LanceDbIndex";
import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex";
import { getComputeDeleteAddRemove } from "./refreshIndex";
import { CodebaseIndex } from "./types";
import { ConfigHandler } from "../config/handler.js";
import { IContinueServerClient } from "../continueServer/interface.js";
import { IDE, IndexTag, IndexingProgressUpdate } from "../index.js";
import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex.js";
import { FullTextSearchCodebaseIndex } from "./FullTextSearch.js";
import { LanceDbIndex } from "./LanceDbIndex.js";
import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex.js";
import { getComputeDeleteAddRemove } from "./refreshIndex.js";
import { CodebaseIndex } from "./types.js";
export class PauseToken {
constructor(private _paused: boolean) {}
@ -21,21 +21,12 @@ export class PauseToken {
}
export class CodebaseIndexer {
private continueServerClient?: ContinueServerClient;
constructor(
private readonly configHandler: ConfigHandler,
private readonly ide: IDE,
private readonly pauseToken: PauseToken,
private readonly continueServerUrl: string | undefined,
private readonly userToken: Promise<string | undefined>,
) {
if (continueServerUrl) {
this.continueServerClient = new ContinueServerClient(
continueServerUrl,
userToken,
);
}
}
private readonly continueServerClient: IContinueServerClient,
) {}
private async getIndexesToBuild(): Promise<CodebaseIndex[]> {
const config = await this.configHandler.loadConfig();
@ -62,12 +53,28 @@ export class CodebaseIndexer {
abortSignal: AbortSignal,
): AsyncGenerator<IndexingProgressUpdate> {
if (workspaceDirs.length === 0) {
yield {
progress: 0,
desc: "Nothing to index",
status: "disabled",
};
return;
}
const config = await this.configHandler.loadConfig();
if (config.disableIndexing) {
yield {
progress: 0,
desc: "Indexing is disabled in the config.json",
status: "disabled",
};
return;
} else {
yield {
progress: 0,
desc: "Starting indexing",
status: "starting",
};
}
const indexesToBuild = await this.getIndexesToBuild();
@ -81,6 +88,7 @@ export class CodebaseIndexer {
yield {
progress: 0,
desc: "Starting indexing...",
status: "starting",
};
for (let directory of workspaceDirs) {
@ -115,11 +123,20 @@ export class CodebaseIndexer {
yield {
progress: 1,
desc: "Indexing cancelled",
status: "disabled",
};
return;
}
while (this.pauseToken.paused) {
await new Promise((resolve) => setTimeout(resolve, 100));
if (this.pauseToken.paused) {
yield {
progress: completedDirs / workspaceDirs.length,
desc: "Paused",
status: "paused",
};
while (this.pauseToken.paused) {
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
yield {
@ -128,6 +145,7 @@ export class CodebaseIndexer {
(completedIndexes + progress) / indexesToBuild.length) /
workspaceDirs.length,
desc,
status: "indexing",
};
}
completedIndexes++;
@ -136,6 +154,7 @@ export class CodebaseIndexer {
(completedDirs + completedIndexes / indexesToBuild.length) /
workspaceDirs.length,
desc: "Completed indexing " + codebaseIndex.artifactId,
status: "indexing",
};
} catch (e) {
console.warn(
@ -148,6 +167,7 @@ export class CodebaseIndexer {
yield {
progress: completedDirs / workspaceDirs.length,
desc: "Indexing Complete",
status: "done",
};
}
}

12
core/indexing/refreshIndex.ts
View File

@ -2,8 +2,8 @@ import crypto from "crypto";
import * as fs from "fs";
import { Database, open } from "sqlite";
import sqlite3 from "sqlite3";
import { IndexTag, IndexingProgressUpdate } from "..";
import { getIndexSqlitePath } from "../util/paths";
import { IndexTag, IndexingProgressUpdate } from "../index.js";
import { getIndexSqlitePath } from "../util/paths.js";
import {
CodebaseIndex,
IndexResultType,
@ -11,7 +11,7 @@ import {
MarkCompleteCallback,
PathAndCacheKey,
RefreshIndexResults,
} from "./types";
} from "./types.js";
export type DatabaseConnection = Database<sqlite3.Database>;
@ -157,7 +157,7 @@ async function getAddRemoveForTag(
switch (resultType) {
case AddRemoveResultType.Add:
await db.run(
`INSERT INTO tag_catalog (path, cacheKey, lastUpdated, dir, branch, artifactId) VALUES (?, ?, ?, ?, ?, ?)`,
"INSERT INTO tag_catalog (path, cacheKey, lastUpdated, dir, branch, artifactId) VALUES (?, ?, ?, ?, ?, ?)",
path,
cacheKey,
newLastUpdatedTimestamp,
@ -243,7 +243,7 @@ async function getTagsFromGlobalCache(
): Promise<IndexTag[]> {
const db = await SqliteDb.get();
const stmt = await db.prepare(
`SELECT dir, branch, artifactId FROM global_cache WHERE cacheKey = ? AND artifactId = ?`,
"SELECT dir, branch, artifactId FROM global_cache WHERE cacheKey = ? AND artifactId = ?",
);
const rows = await stmt.all(cacheKey, artifactId);
return rows;
@ -356,7 +356,7 @@ export class GlobalCacheCodeBaseIndex implements CodebaseIndex {
return this.deleteOrRemoveTag(cacheKey, tag);
}),
]);
yield { progress: 1, desc: "Done updating global cache" };
yield { progress: 1, desc: "Done updating global cache", status: "done" };
}
private async computeOrAddTag(

2
core/indexing/types.ts
View File

@ -1,4 +1,4 @@
import { IndexTag, IndexingProgressUpdate } from "..";
import { IndexTag, IndexingProgressUpdate } from "../index.js";
export enum IndexResultType {
Compute = "compute",

Some files were not shown because too many files have changed in this diff Show More