convert to use interface, fix type-related bug, more

binary/build.js

@@ -47,10 +47,11 @@ async function installNodeModuleInTempDirAndCopyToCurrent(package, toCopy) {
   console.log(`Copying ${package} to ${toCopy}`);
   // This is a way to install only one package without npm trying to install all the dependencies
   // Create a temporary directory for installing the package
+  const adjustedName = toCopy.replace(/^@/, "").replace("/", "-");
   const tempDir = path.join(
     __dirname,
     "tmp",
-    `continue-node_modules-${toCopy}`,
+    `continue-node_modules-${adjustedName}`,
   );
   const currentDir = process.cwd();
 

binary/src/core.ts

@@ -128,12 +128,9 @@ export class Core {
 
     // Context providers
     on("context/addDocs", async (msg) => {
-      const SiteIndexingConfig = msg.data;
       for await (const _ of indexDocs(
-        SiteIndexingConfig.title,
-        new URL(SiteIndexingConfig.rootUrl),
+        msg.data,
         new TransformersJsEmbeddingsProvider(),
-        SiteIndexingConfig.maxDepth
       )) {
       }
     });

core/context/providers/DocsContextProvider.ts

@@ -114,7 +114,7 @@ class DocsContextProvider extends BaseContextProvider {
         return 1;
       } else {
         // Secondary criterion: Alphabetical order when both items are in the same category
-        return a.title.localeCompare(b.title);
+        return a.title.toString().localeCompare(b.title.toString());
       }
     });
 

core/index.d.ts

@@ -154,6 +154,13 @@ export interface ContextSubmenuItem {
   description: string;
 }
 
+export interface SiteIndexingConfig {
+  startUrl: string;
+  rootUrl: string;
+  title: string;
+  maxDepth?: number;
+}
+
 export interface IContextProvider {
   get description(): ContextProviderDescription;
 

core/indexing/docs/crawl.ts

@@ -130,6 +130,7 @@ export type PageData = {
 };
 
 export async function* crawlPage(url: URL, maxDepth: number = 3): AsyncGenerator<PageData> {
+  console.log("starting crawl")
   const { baseUrl, basePath } = splitUrl(url);
   let paths: { path: string; depth: number }[] = [{ path: basePath, depth: 0 }];
   
@@ -141,9 +142,10 @@ export async function* crawlPage(url: URL, maxDepth: number = 3): AsyncGenerator
     const promises = batch.map(({ path, depth }) => getLinksFromUrl(baseUrl, path).then(links => ({ links, path, depth }))); // Adjust for depth tracking
 
     const results = await Promise.all(promises);
-
+    console.log("results length: ", results.length)
     for (const { links: { html, links: linksArray }, path, depth } of results) {
       if (html !== "" && depth <= maxDepth) { // Check depth
+        console.log("Depth: ", depth)
         yield {
           url: url.toString(),
           path,
@@ -153,6 +155,7 @@ export async function* crawlPage(url: URL, maxDepth: number = 3): AsyncGenerator
       
       // Ensure we only add links if within depth limit
       if (depth < maxDepth) {
+        console.log("Depth: ", depth)
         for (let link of linksArray) {
           if (!paths.some(p => p.path === link)) {
             paths.push({ path: link, depth: depth + 1 }); // Increment depth for new paths
@@ -163,4 +166,5 @@ export async function* crawlPage(url: URL, maxDepth: number = 3): AsyncGenerator
 
     index += batch.length; // Proceed to next batch
   }
+  console.log("crawl completed")
 }

core/indexing/docs/index.ts

@@ -3,14 +3,16 @@ import { Chunk, EmbeddingsProvider, IndexingProgressUpdate } from "../../index.j
 import { Article, chunkArticle, pageToArticle } from "./article.js";
 import { crawlPage } from "./crawl.js";
 import { addDocs, hasDoc } from "./db.js";
+import {SiteIndexingConfig } from "../../index.js"
 
 export async function* indexDocs(
-  title: string,
-  baseUrl: URL,
+  siteIndexingConfig: SiteIndexingConfig,
   embeddingsProvider: EmbeddingsProvider,
-  maxDepth?: number
 ): AsyncGenerator<IndexingProgressUpdate> {
-  if (await hasDoc(baseUrl.toString())) {
+  console.log("In core indexDocs. maxDepth -> ", siteIndexingConfig.maxDepth, " Base Url -> ", siteIndexingConfig.startUrl)
+  const startUrl = new URL(siteIndexingConfig.startUrl)
+  
+  if (await hasDoc(siteIndexingConfig.startUrl.toString())) {
     yield {
       progress: 1,
       desc: "Already indexed",
@@ -27,10 +29,12 @@ export async function* indexDocs(
 
   const articles: Article[] = [];
 
-  for await (const page of crawlPage(baseUrl, maxDepth)) {
+  console.log("starting crawl loop")
+  for await (const page of crawlPage(startUrl, siteIndexingConfig.maxDepth)) {
     const article = pageToArticle(page);
-    if (!article) {continue;}
+    if (!article) { continue; }
 
+    console.log("pushing article")
     articles.push(article);
 
     yield {
@@ -61,7 +65,7 @@ export async function* indexDocs(
     embeddings.push(...subpathEmbeddings);
   }
 
-  await addDocs(title, baseUrl, chunks, embeddings);
+  await addDocs(siteIndexingConfig.title, startUrl, chunks, embeddings);
 
   yield {
     progress: 1,

core/indexing/docs/preIndexedDocs.ts

@@ -1,9 +1,4 @@
-export interface SiteIndexingConfig {
-  startUrl: string;
-  rootUrl: string;
-  title: string;
-  maxDepth?: number;
-}
+import {SiteIndexingConfig} from "../../index.js";
 
 const configs: SiteIndexingConfig[] = [
   {

core/protocol.ts

@@ -10,9 +10,9 @@ import {
   RangeInFile,
   SerializedContinueConfig,
   SessionInfo,
+  SiteIndexingConfig
 } from ".";
 import { AutocompleteInput } from "./autocomplete/completionProvider";
-import { SiteIndexingConfig } from "./indexing/docs/preIndexedDocs";
 import { IdeProtocol } from "./web/webviewProtocol";
 
 export type ProtocolGeneratorType<T> = AsyncGenerator<{
@@ -63,7 +63,7 @@ export type Protocol = {
     { title: string },
     Promise<ContextSubmenuItem[]>,
   ];
-  "context/addDocs": [SiteIndexingConfig, void];
+  "context/addDocs": [siteIndexingConfig: SiteIndexingConfig, Promise<void>];
   "autocomplete/complete": [AutocompleteInput, Promise<string[]>];
   "autocomplete/cancel": [undefined, void];
   "autocomplete/accept": [{ completionId: string }, void];

core/web/webviewProtocol.ts

@@ -105,6 +105,7 @@ export type WebviewProtocol = Protocol &
   };
 
 export type ReverseWebviewProtocol = {
+  addDocs: [{  startUrl: string, rootUrl: string; title: string; maxDepth?: number;}]
   setInactive: [undefined, void];
   configUpdate: [undefined, void];
   submitMessage: [{ message: any }, void]; // any -> JSONContent from TipTap

extensions/vscode/scripts/prepackage.js

@@ -309,7 +309,9 @@ const exe = os === "win32" ? ".exe" : "";
     console.log(`Copying ${package} to ${toCopy}`);
     // This is a way to install only one package without npm trying to install all the dependencies
     // Create a temporary directory for installing the package
-    const tempDir = `/tmp/continue-node_modules-${toCopy}`;
+    const adjustedName = toCopy.replace(/^@/, "").replace("/", "-");
+
+    const tempDir = `/tmp/continue-node_modules-${adjustedName}`;
     const currentDir = process.cwd();
 
     // Remove the dir we will be copying to

extensions/vscode/src/webviewProtocol.ts

@@ -25,6 +25,7 @@ import { v4 as uuidv4 } from "uuid";
 import * as vscode from "vscode";
 import { VerticalPerLineDiffManager } from "./diff/verticalPerLine/manager";
 import { getExtensionUri } from "./util/vscode";
+import { SiteIndexingConfig } from "core/index";
 
 export async function showTutorial() {
   const tutorialPath = path.join(
@@ -44,7 +45,7 @@ export async function showTutorial() {
   await vscode.window.showTextDocument(doc, { preview: false });
 }
 
-export class VsCodeWebviewProtocol {
+ export class VsCodeWebviewProtocol {
   listeners = new Map<keyof WebviewProtocol, ((message: Message) => any)[]>();
   abortedMessageIds: Set<string> = new Set();
 
@@ -555,18 +556,25 @@ export class VsCodeWebviewProtocol {
       }
     });
     this.on("context/addDocs", (msg) => {
-      const SiteIndexingConfig = msg.data
+      const siteIndexingConfig: SiteIndexingConfig = {
+        startUrl: msg.data.url,
+        rootUrl: msg.data.url,
+        title: msg.data.title,
+        maxDepth: 4
+      };    
+      // const siteIndexingConfig = msg.data
+      
       const embeddingsProvider = new TransformersJsEmbeddingsProvider();
+      console.log("In vscode addDocs")
       vscode.window.withProgress(
         {
           location: vscode.ProgressLocation.Notification,
-          title: `Indexing ${SiteIndexingConfig.title}`,
+          title: `Indexing ${siteIndexingConfig.title}`,
           cancellable: false,
         },
         async (progress) => {
           for await (const update of indexDocs(
-            SiteIndexingConfig.title,
-            new URL(SiteIndexingConfig.url),
+            siteIndexingConfig,
             embeddingsProvider,
           )) {
             progress.report({
@@ -576,7 +584,7 @@ export class VsCodeWebviewProtocol {
           }
 
           vscode.window.showInformationMessage(
-            `🎉 Successfully indexed ${SiteIndexingConfig.title}`,
+            `🎉 Successfully indexed ${siteIndexingConfig.title}`,
           );
 
           this.request("refreshSubmenuItems", undefined);

gui/src/components/dialogs/AddDocsDialog.tsx

@@ -6,7 +6,14 @@ import { Button, Input } from "..";
 import { SubmenuContextProvidersContext } from "../../App";
 import { setShowDialog } from "../../redux/slices/uiStateSlice";
 import { postToIde } from "../../util/ide";
-import { SiteIndexingConfig } from "core/indexing/docs/preIndexedDocs";
+import { SiteIndexingConfig } from "core/index";
+
+// export interface SiteIndexingConfig {
+//   startUrl: string;
+//   rootUrl: string;
+//   title: string;
+//   maxDepth?: number;
+// }
 
 const GridDiv = styled.div`
   display: grid;
@@ -75,8 +82,14 @@ function AddDocsDialog() {
         disabled={!docsUrl || !urlValid}
         className="ml-auto"
         onClick={() => {
-          const siteindexingConfig = { url: docsUrl, title: docsTitle, maxDepth:maxDepth }
-          postToIde("context/addDocs", siteindexingConfig);
+          const siteIndexingConfig: SiteIndexingConfig = {
+            startUrl: docsUrl,
+            rootUrl: docsUrl,
+            title: docsTitle,
+            maxDepth: maxDepth
+          };          
+          console.log("triggering addDocs. maxDepth: ", maxDepth)
+          postToIde("context/addDocs", siteIndexingConfig);
           setDocsTitle("");
           setDocsUrl("");
           setMaxDepth(defaultMaxDepth)