🚧 full draft of code context provider

core/autocomplete/retrieval.ts

@@ -1,6 +1,5 @@
-import { Chunk } from "..";
+import { Chunk, IndexTag } from "..";
 import { FullTextSearchCodebaseIndex } from "../indexing/FullTextSearch";
-import { IndexTag } from "../indexing/types";
 
 export async function fullTextRetrieve(
   prefix: string,

core/context/providers/CodeContextProvider.ts

@@ -0,0 +1,44 @@
+import { BaseContextProvider } from "..";
+import {
+  ContextItem,
+  ContextProviderDescription,
+  ContextProviderExtras,
+  ContextSubmenuItem,
+  LoadSubmenuItemsArgs,
+} from "../..";
+import { CodeSnippetsCodebaseIndex } from "../../indexing/CodeSnippetsIndex";
+
+class CodeContextProvider extends BaseContextProvider {
+  static description: ContextProviderDescription = {
+    title: "code",
+    displayTitle: "Code",
+    description: "Type to search",
+    type: "submenu",
+  };
+
+  async getContextItems(
+    query: string,
+    extras: ContextProviderExtras,
+  ): Promise<ContextItem[]> {
+    // Assume the query is the id as returned by loadSubmenuItems
+    return [await CodeSnippetsCodebaseIndex.getForId(parseInt(query, 10))];
+  }
+
+  async loadSubmenuItems(
+    args: LoadSubmenuItemsArgs,
+  ): Promise<ContextSubmenuItem[]> {
+    const tags = await args.ide.getTags("codeSnippets");
+    const snippets = await Promise.all(
+      tags.map((tag) => CodeSnippetsCodebaseIndex.getAll(tag)),
+    );
+
+    const submenuItems: ContextSubmenuItem[] = [];
+    for (const snippetList of snippets) {
+      submenuItems.push(...snippetList);
+    }
+
+    return submenuItems;
+  }
+}
+
+export default CodeContextProvider;

core/context/providers/index.ts

@@ -1,5 +1,6 @@
 import { BaseContextProvider } from "..";
 import { ContextProviderName } from "../..";
+import CodeContextProvider from "./CodeContextProvider";
 import CodebaseContextProvider from "./CodebaseContextProvider";
 import DatabaseContextProvider from "./DatabaseContextProvider";
 import DiffContextProvider from "./DiffContextProvider";
@@ -36,6 +37,7 @@ const Providers: (typeof BaseContextProvider)[] = [
   JiraIssuesContextProvider,
   PostgresContextProvider,
   DatabaseContextProvider,
+  CodeContextProvider,
 ];
 
 export function contextProviderClassFromName(

core/context/retrieval.ts

@@ -1,7 +1,7 @@
-import { Chunk, ContextItem, ContextProviderExtras, ILLM } from "..";
+import { Chunk, ContextItem, ContextProviderExtras, ILLM, IndexTag } from "..";
 import { FullTextSearchCodebaseIndex } from "../indexing/FullTextSearch";
 import { LanceDbIndex } from "../indexing/LanceDbIndex";
-import { IndexTag } from "../indexing/types";
+
 import { llmCanGenerateInParallel } from "../llm/autodetect";
 import { getBasename } from "../util";
 

core/index.d.ts

@@ -16,7 +16,7 @@ declare global {
     postIntellijMessage?: (
       messageType: string,
       data: any,
-      messageIde: string
+      messageIde: string,
     ) => void;
   }
 }
@@ -73,17 +73,17 @@ export interface ILLM extends LLMOptions {
 
   streamComplete(
     prompt: string,
-    options?: LLMFullCompletionOptions
+    options?: LLMFullCompletionOptions,
   ): AsyncGenerator<string, LLMReturnValue>;
 
   streamChat(
     messages: ChatMessage[],
-    options?: LLMFullCompletionOptions
+    options?: LLMFullCompletionOptions,
   ): AsyncGenerator<ChatMessage, LLMReturnValue>;
 
   chat(
     messages: ChatMessage[],
-    options?: LLMFullCompletionOptions
+    options?: LLMFullCompletionOptions,
   ): Promise<ChatMessage>;
 
   countTokens(text: string): number;
@@ -123,10 +123,10 @@ export interface CustomContextProvider {
   type?: ContextProviderType;
   getContextItems(
     query: string,
-    extras: ContextProviderExtras
+    extras: ContextProviderExtras,
   ): Promise<ContextItem[]>;
   loadSubmenuItems?: (
-    args: LoadSubmenuItemsArgs
+    args: LoadSubmenuItemsArgs,
   ) => Promise<ContextSubmenuItem[]>;
 }
 
@@ -141,7 +141,7 @@ export interface IContextProvider {
 
   getContextItems(
     query: string,
-    extras: ContextProviderExtras
+    extras: ContextProviderExtras,
   ): Promise<ContextItem[]>;
 
   loadSubmenuItems(args: LoadSubmenuItemsArgs): Promise<ContextSubmenuItem[]>;
@@ -287,15 +287,15 @@ export interface CustomLLMWithOptionals {
   streamCompletion?: (
     prompt: string,
     options: CompletionOptions,
-    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>
+    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
   ) => AsyncGenerator<string>;
   streamChat?: (
     messages: ChatMessage[],
     options: CompletionOptions,
-    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>
+    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
   ) => AsyncGenerator<string>;
   listModels?: (
-    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>
+    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
   ) => Promise<string[]>;
 }
 
@@ -328,6 +328,12 @@ export interface IdeInfo {
   remoteName: string;
 }
 
+export interface IndexTag {
+  directory: string;
+  branch: string;
+  artifactId: string;
+}
+
 export interface IDE {
   getIdeInfo(): Promise<IdeInfo>;
   getDiff(): Promise<string>;
@@ -349,12 +355,12 @@ export interface IDE {
   showLines(
     filepath: string,
     startLine: number,
-    endLine: number
+    endLine: number,
   ): Promise<void>;
   showDiff(
     filepath: string,
     newContents: string,
-    stepIndex: number
+    stepIndex: number,
   ): Promise<void>;
   getOpenFiles(): Promise<string[]>;
   getPinnedFiles(): Promise<string[]>;
@@ -363,6 +369,7 @@ export interface IDE {
   getProblems(filepath?: string | undefined): Promise<Problem[]>;
   getBranch(dir: string): Promise<string>;
   getStats(directory: string): Promise<{ [path: string]: number }>;
+  getTags(artifactId: string): Promise<IndexTag[]>;
 }
 
 // Slash Commands

core/indexing/CodeSnippetsIndex.ts

@@ -1,16 +1,23 @@
 import fs from "fs";
 import path from "path";
-import { ChunkWithoutID, IDE, IndexingProgressUpdate } from "..";
+import {
+  ChunkWithoutID,
+  ContextItem,
+  ContextSubmenuItem,
+  IDE,
+  IndexTag,
+  IndexingProgressUpdate,
+} from "..";
+import { getBasename } from "../util";
 import {
   getLanguageForFile,
   getParserForFile,
   supportedLanguages,
 } from "../util/treeSitter";
-import { DatabaseConnection, SqliteDb } from "./refreshIndex";
+import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
 import {
   CodebaseIndex,
   IndexResultType,
-  IndexTag,
   MarkCompleteCallback,
   RefreshIndexResults,
 } from "./types";
@@ -29,6 +36,13 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
         startLine INTEGER NOT NULL,
         endLine INTEGER NOT NULL
     )`);
+
+    await db.exec(`CREATE TABLE IF NOT EXISTS code_snippets_tags (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      tag TEXT NOT NULL,
+      snippetId INTEGER NOT NULL,
+      FOREIGN KEY (snippetId) REFERENCES code_snippets (id)
+  )`);
   }
 
   private getQuerySource(filepath: string) {
@@ -39,6 +53,9 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       "tag-qry",
       `tree-sitter-${fullLangName}-tags.scm`,
     );
+    if (!fs.existsSync(sourcePath)) {
+      return "";
+    }
     return fs.readFileSync(sourcePath).toString();
   }
 
@@ -72,6 +89,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
   ): AsyncGenerator<IndexingProgressUpdate, any, unknown> {
     const db = await SqliteDb.get();
     await this._createTables(db);
+    const tagString = tagToString(tag);
 
     for (let i = 0; i < results.compute.length; i++) {
       const compute = results.compute[i];
@@ -82,7 +100,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
 
       // Add snippets to sqlite
       for (const snippet of snippets) {
-        await db.run(
+        const { lastID } = await db.run(
           `INSERT INTO code_snippets (path, cacheKey, content, startLine, endLine) VALUES (?, ?, ?, ?, ?)`,
           [
             compute.path,
@@ -92,6 +110,11 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
             snippet.endLine,
           ],
         );
+
+        await db.run(
+          `INSERT INTO code_snippets_tags (snippetId, tag) VALUES (?, ?)`,
+          [lastID, tagString],
+        );
       }
 
       yield {
@@ -100,5 +123,78 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex {
       };
       markComplete([compute], IndexResultType.Compute);
     }
+
+    for (let i = 0; i < results.del.length; i++) {
+      const del = results.del[i];
+      const deleted = await db.run(
+        `DELETE FROM code_snippets WHERE path = ? AND cacheKey = ?`,
+        [del.path, del.cacheKey],
+      );
+      await db.run(`DELETE FROM code_snippets_tags WHERE snippetId = ?`, [
+        deleted.lastID,
+      ]);
+      markComplete([del], IndexResultType.Delete);
+    }
+
+    for (let i = 0; i < results.addTag.length; i++) {
+      const snippetsWithPath = await db.all(
+        `SELECT * FROM code_snippets WHERE cacheKey = ?`,
+        [results.addTag[i].cacheKey],
+      );
+
+      for (const snippet of snippetsWithPath) {
+        await db.run(
+          `INSERT INTO code_snippet_tags (snippetId, tag) VALUES (?, ?)`,
+          [snippet.id, tagString],
+        );
+      }
+
+      markComplete([results.addTag[i]], IndexResultType.AddTag);
+    }
+
+    for (let i = 0; i < results.removeTag.length; i++) {
+      const item = results.removeTag[i];
+      await db.run(
+        `
+        DELETE FROM code_snippets_tags
+        WHERE tag = ?
+          AND snippetId IN (
+            SELECT id FROM code_snippets
+            WHERE cacheKey = ? AND path = ?
+          )
+      `,
+        [tagString, item.cacheKey, item.path],
+      );
+      markComplete([results.removeTag[i]], IndexResultType.RemoveTag);
+    }
+  }
+
+  static async getForId(id: number): Promise<ContextItem> {
+    const db = await SqliteDb.get();
+    const row = await db.get(`SELECT * FROM code_snippets WHERE id = ?`, [id]);
+
+    return {
+      name: getBasename(row.path),
+      description: getBasename(row.path, 2),
+      content: `\`\`\`${getBasename(row.path)}\n${row.content}\n\`\`\``,
+    };
+  }
+
+  static async getAll(tag: IndexTag): Promise<ContextSubmenuItem[]> {
+    const db = await SqliteDb.get();
+    const rows = await db.all(
+      `SELECT *
+      FROM code_snippets cs
+      JOIN code_snippets_tags cst ON cs.id = cst.snippetId
+      WHERE cst.tag = ?;
+      `,
+      [tagToString(tag)],
+    );
+
+    return rows.map((row) => ({
+      title: getBasename(row.path),
+      description: getBasename(row.path, 2),
+      id: row.id.toString(),
+    }));
   }
 }

core/indexing/FullTextSearch.ts

@@ -1,9 +1,8 @@
-import { Chunk, IndexingProgressUpdate } from "..";
+import { Chunk, IndexTag, IndexingProgressUpdate } from "..";
 import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
 import {
   CodebaseIndex,
   IndexResultType,
-  IndexTag,
   MarkCompleteCallback,
   RefreshIndexResults,
 } from "./types";

core/indexing/LanceDbIndex.ts

@@ -1,6 +1,11 @@
 // NOTE: vectordb requirement must be listed in extensions/vscode to avoid error
 import { v4 as uuidv4 } from "uuid";
-import { Chunk, EmbeddingsProvider, IndexingProgressUpdate } from "..";
+import {
+  Chunk,
+  EmbeddingsProvider,
+  IndexTag,
+  IndexingProgressUpdate,
+} from "..";
 import { MAX_CHUNK_SIZE } from "../llm/constants";
 import { getBasename } from "../util";
 import { getLanceDbPath } from "../util/paths";
@@ -9,7 +14,6 @@ import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex";
 import {
   CodebaseIndex,
   IndexResultType,
-  IndexTag,
   PathAndCacheKey,
   RefreshIndexResults,
 } from "./types";

core/indexing/README.md

@@ -0,0 +1,30 @@
+# Indexing
+
+Continue uses a tagging system along with content addressing to ensure that nothing needs to be indexed twice. When you change branches, Continue will only re-index the files that are newly modified and that we don't already have a copy of. This system can be used across many different "artifacts" just by implementing the `CodebaseIndex` class.
+
+_artifact_: something that is generated by indexing and then saved to be used later (e.g. emeddings, full-text search index, or a table of top-level code snippets in each file)
+
+_cacheKey_: a key that determines whether two files can be considered the same to avoid re-indexing (always hash of file contents at this point)
+
+_`CodebaseIndex`_: a class that makes it easy to use the indexing system to help you generate a new artifact
+
+The indexing process does the following:
+
+1. Check the modified timestamps of all files in the repo (this may seem extreme, but checking timestamps is significantly faster than actually reading a file. Git does the same thing.)
+2. Compare these to a "catalog" (stored in SQLite) of the last time that we indexed each of these files to get a list of files to "add" or "remove". If the file exists in the repo but not in the catalog, then we must "add" the file. If it exists in the catalog but not the repo, we must "remove" the file. If it exists in both and was modified after last indexed, then we must update the file. In this case we also add it to the "add" list.
+3. For each file to "add", check whether it was indexed on another branch. Here we use a SQLite table that acts as a cache for indexed files. If we find an entry in this table for a file with the same cacheKey, then we only need to add a tag to this entry for the current branch ("addTag"). Otherwise, we must "compute" the artifact.
+4. For each file in "remove", check whether it was indexed on another branch. If we find only one entry with the same cacheKey (presumably this should be the entry for the current branch, or something has gone wrong), then this entry should be removed and there will be no more branches that need the artifact, so we want to "delete" it. If there is more than one tag on this artifact, then we should just remove the tag for this branch ("removeTag").
+5. After having calculated these four lists of files ("compute", "delete", "addTag", "removeTag"), we pass them to the `CodebaseIndex` so that it can update whatever index-specific storage it might have. Many of them use SQLite and/or LanceDB. The `CodebaseIndex` implements a method called "update" that accepts the four lists and yields progress updates as it iterates over the lists. These progress updates are used to officially mark a file as having been indexed, so that if the extension is closed mid-indexing we don't falsely record progress.
+
+## Existing `CodebaseIndex`es
+
+All indexes must be returned by `getIndexesToBuild` in [`indexCodebase.ts`](./indexCodebase.ts) if they are to be used.
+
+`CodeSnippetsCodebaseIndex`: uses tree-sitter queries to get a list of functions, classes, and other top-level code objects in each file
+`FullTextSearchCodebaseIndex`: creates a full-text search index using SQLite FTS5
+`ChunkCodebaseIndex`: chunks files recursively by code structure, for use in other embeddings providers like `LanceDbCodebaseIndex`
+`LanceDbCodebaseIndex`: calculates embeddings for each chunk and adds them to the LanceDB vector database, with metadata going into SQLite
+
+## Known problems:
+
+- `FullTextSearchCodebaseIndex` doesn't differentiate between tags (branch, repo), so results may come from any branch/repo. LanceDB does this by creating separate tables for each tag (see `tableNameForTag`). The chunk index does this with a second table

core/indexing/chunk/ChunkCodebaseIndex.ts

@@ -1,11 +1,10 @@
-import { IndexingProgressUpdate } from "../..";
+import { IndexTag, IndexingProgressUpdate } from "../..";
 import { MAX_CHUNK_SIZE } from "../../llm/constants";
 import { getBasename } from "../../util";
 import { DatabaseConnection, SqliteDb, tagToString } from "../refreshIndex";
 import {
   CodebaseIndex,
   IndexResultType,
-  IndexTag,
   MarkCompleteCallback,
   RefreshIndexResults,
 } from "../types";
@@ -106,7 +105,17 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
 
     // Remove tag
     for (const item of results.removeTag) {
-      await db.run(`DELETE FROM chunk_tags WHERE tag = ?`, [tagString]);
+      await db.run(
+        `
+        DELETE FROM chunk_tags
+        WHERE tag = ?
+          AND chunkId IN (
+            SELECT id FROM chunks
+            WHERE cacheKey = ? AND path = ?
+          )
+      `,
+        [tagString, item.cacheKey, item.path],
+      );
       markComplete([item], IndexResultType.RemoveTag);
     }
 

core/indexing/indexCodebase.ts

@@ -1,10 +1,11 @@
-import { IDE, IndexingProgressUpdate } from "..";
+import { IDE, IndexTag, IndexingProgressUpdate } from "..";
 import { ConfigHandler } from "../config/handler";
+import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex";
 import { FullTextSearchCodebaseIndex } from "./FullTextSearch";
 import { LanceDbIndex } from "./LanceDbIndex";
 import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex";
 import { getComputeDeleteAddRemove } from "./refreshIndex";
-import { CodebaseIndex, IndexTag } from "./types";
+import { CodebaseIndex } from "./types";
 
 export class PauseToken {
   constructor(private _paused: boolean) {}
@@ -39,6 +40,7 @@ export class CodebaseIndexer {
         this.ide.readFile.bind(this.ide),
       ),
       new FullTextSearchCodebaseIndex(),
+      new CodeSnippetsCodebaseIndex(this.ide),
     ];
 
     return indexes;

core/indexing/refreshIndex.ts

@@ -2,12 +2,11 @@ import crypto from "crypto";
 import * as fs from "fs";
 import { Database, open } from "sqlite";
 import sqlite3 from "sqlite3";
-import { IndexingProgressUpdate } from "..";
+import { IndexTag, IndexingProgressUpdate } from "..";
 import { getIndexSqlitePath } from "../util/paths";
 import {
   CodebaseIndex,
   IndexResultType,
-  IndexTag,
   LastModifiedMap,
   MarkCompleteCallback,
   PathAndCacheKey,

core/indexing/types.ts

@@ -1,4 +1,4 @@
-import { IndexingProgressUpdate } from "..";
+import { IndexTag, IndexingProgressUpdate } from "..";
 
 export enum IndexResultType {
   Compute = "compute",
@@ -26,12 +26,6 @@ export type PathAndCacheKey = {
   cacheKey: string;
 };
 
-export interface IndexTag {
-  directory: string;
-  branch: string;
-  artifactId: string;
-}
-
 export type RefreshIndexResults = {
   compute: PathAndCacheKey[];
   del: PathAndCacheKey[];

core/util/filesystem.ts

@@ -1,8 +1,12 @@
 import * as fs from "fs";
-import { ContinueRcJson, IDE, IdeInfo, Problem, Range } from "..";
+import { ContinueRcJson, IDE, IdeInfo, IndexTag, Problem, Range } from "..";
+
 import { getContinueGlobalPath } from "./paths";
 
 class FileSystemIde implements IDE {
+  getTags(artifactId: string): Promise<IndexTag[]> {
+    return Promise.resolve([]);
+  }
   getIdeInfo(): Promise<IdeInfo> {
     return Promise.resolve({
       ideType: "vscode",

core/util/messageIde.ts

@@ -1,9 +1,12 @@
-import { ContinueRcJson, IDE, IdeInfo, Problem, Range } from "..";
+import { ContinueRcJson, IDE, IdeInfo, IndexTag, Problem, Range } from "..";
 
 export class MessageIde implements IDE {
   constructor(
     private readonly request: (messageType: string, data: any) => Promise<any>,
   ) {}
+  getTags(artifactId: string): Promise<IndexTag[]> {
+    return this.request("getTags", artifactId);
+  }
   getIdeInfo(): Promise<IdeInfo> {
     return this.request("getIdeInfo", undefined);
   }

core/web/webviewProtocol.ts

@@ -2,10 +2,12 @@ import {
   ContextItemWithId,
   ContinueRcJson,
   DiffLine,
+  IndexTag,
   Problem,
   Range,
 } from "..";
 import { RangeInFileWithContents } from "../commands/util";
+
 import { Protocol } from "../protocol";
 
 export type IdeProtocol = {
@@ -45,6 +47,7 @@ export type IdeProtocol = {
   getTerminalContents: [undefined, string];
   isTelemetryEnabled: [undefined, boolean];
   getUniqueId: [undefined, string];
+  getTags: [string, IndexTag[]];
 };
 
 export type WebviewProtocol = Protocol &

extensions/vscode/src/ideProtocol.ts

@@ -1,10 +1,9 @@
+import * as child_process from "child_process";
 import { exec } from "child_process";
+import { ContinueRcJson, IDE, IdeInfo, IndexTag, Problem, Range } from "core";
 import { getContinueGlobalPath } from "core/util/paths";
 import * as path from "path";
 import * as vscode from "vscode";
-
-import * as child_process from "child_process";
-import { ContinueRcJson, IDE, IdeInfo, Problem, Range } from "core";
 import { DiffManager } from "./diff/horizontal";
 import { VsCodeIdeUtils } from "./util/ideUtils";
 import { traverseDirectory } from "./util/traverseDirectory";
@@ -20,6 +19,26 @@ class VsCodeIde implements IDE {
   constructor(private readonly diffManager: DiffManager) {
     this.ideUtils = new VsCodeIdeUtils();
   }
+  async getTags(artifactId: string): Promise<IndexTag[]> {
+    const workspaceDirs = await this.getWorkspaceDirs();
+
+    const branches = (await Promise.race([
+      Promise.all(workspaceDirs.map((dir) => this.getBranch(dir))),
+      new Promise((resolve) => {
+        setTimeout(() => {
+          resolve(["NONE"]);
+        }, 500);
+      }),
+    ])) as string[];
+
+    const tags: IndexTag[] = workspaceDirs.map((directory, i) => ({
+      directory,
+      branch: branches[i],
+      artifactId,
+    }));
+
+    return tags;
+  }
   getIdeInfo(): Promise<IdeInfo> {
     return Promise.resolve({
       ideType: "vscode",

extensions/vscode/tag-qry/tree-sitter-c-tags.scm

@@ -0,0 +1,9 @@
+(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class
+
+(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class
+
+(function_declarator declarator: (identifier) @name.definition.function) @definition.function
+
+(type_definition declarator: (type_identifier) @name.definition.type) @definition.type
+
+(enum_specifier name: (type_identifier) @name.definition.type) @definition.type

extensions/vscode/tag-qry/tree-sitter-c_sharp-tags.scm

@@ -0,0 +1,46 @@
+(class_declaration
+ name: (identifier) @name.definition.class
+ ) @definition.class
+
+(class_declaration
+   bases: (base_list (_) @name.reference.class)
+ ) @reference.class
+
+(interface_declaration
+ name: (identifier) @name.definition.interface
+ ) @definition.interface
+
+(interface_declaration
+ bases: (base_list (_) @name.reference.interface)
+ ) @reference.interface
+
+(method_declaration
+ name: (identifier) @name.definition.method
+ ) @definition.method
+
+(object_creation_expression
+ type: (identifier) @name.reference.class
+ ) @reference.class
+
+(type_parameter_constraints_clause
+ target: (identifier) @name.reference.class
+ ) @reference.class
+
+(type_constraint
+ type: (identifier) @name.reference.class
+ ) @reference.class
+
+(variable_declaration
+ type: (identifier) @name.reference.class
+ ) @reference.class
+
+(invocation_expression
+ function:
+  (member_access_expression
+    name: (identifier) @name.reference.send
+ )
+) @reference.send
+
+(namespace_declaration
+ name: (identifier) @name.definition.module
+) @definition.module

extensions/vscode/tag-qry/tree-sitter-cpp-tags.scm

@@ -0,0 +1,15 @@
+(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class
+
+(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class
+
+(function_declarator declarator: (identifier) @name.definition.function) @definition.function
+
+(function_declarator declarator: (field_identifier) @name.definition.function) @definition.function
+
+(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @scope name: (identifier) @name.definition.method)) @definition.method
+
+(type_definition declarator: (type_identifier) @name.definition.type) @definition.type
+
+(enum_specifier name: (type_identifier) @name.definition.type) @definition.type
+
+(class_specifier name: (type_identifier) @name.definition.class) @definition.class

extensions/vscode/tag-qry/tree-sitter-elisp-tags.scm

@@ -0,0 +1,8 @@
+;; defun/defsubst
+(function_definition name: (symbol) @name.definition.function) @definition.function
+
+;; Treat macros as function definitions for the sake of TAGS.
+(macro_definition name: (symbol) @name.definition.function) @definition.function
+
+;; Match function calls
+(list (symbol) @name.reference.function) @reference.function

extensions/vscode/tag-qry/tree-sitter-elixir-tags.scm

@@ -0,0 +1,54 @@
+; Definitions
+
+; * modules and protocols
+(call
+  target: (identifier) @ignore
+  (arguments (alias) @name.definition.module)
+  (#match? @ignore "^(defmodule|defprotocol)$")) @definition.module
+
+; * functions/macros
+(call
+  target: (identifier) @ignore
+  (arguments
+    [
+      ; zero-arity functions with no parentheses
+      (identifier) @name.definition.function
+      ; regular function clause
+      (call target: (identifier) @name.definition.function)
+      ; function clause with a guard clause
+      (binary_operator
+        left: (call target: (identifier) @name.definition.function)
+        operator: "when")
+    ])
+  (#match? @ignore "^(def|defp|defdelegate|defguard|defguardp|defmacro|defmacrop|defn|defnp)$")) @definition.function
+
+; References
+
+; ignore calls to kernel/special-forms keywords
+(call
+  target: (identifier) @ignore
+  (#match? @ignore "^(def|defp|defdelegate|defguard|defguardp|defmacro|defmacrop|defn|defnp|defmodule|defprotocol|defimpl|defstruct|defexception|defoverridable|alias|case|cond|else|for|if|import|quote|raise|receive|require|reraise|super|throw|try|unless|unquote|unquote_splicing|use|with)$"))
+
+; ignore module attributes
+(unary_operator
+  operator: "@"
+  operand: (call
+    target: (identifier) @ignore))
+
+; * function call
+(call
+  target: [
+   ; local
+   (identifier) @name.reference.call
+   ; remote
+   (dot
+     right: (identifier) @name.reference.call)
+  ]) @reference.call
+
+; * pipe into function call
+(binary_operator
+  operator: "|>"
+  right: (identifier) @name.reference.call) @reference.call
+
+; * modules
+(alias) @name.reference.module @reference.module

extensions/vscode/tag-qry/tree-sitter-elm-tags.scm

@@ -0,0 +1,19 @@
+(value_declaration (function_declaration_left (lower_case_identifier) @name.definition.function)) @definition.function
+
+(function_call_expr (value_expr (value_qid) @name.reference.function)) @reference.function
+(exposed_value (lower_case_identifier) @name.reference.function)) @reference.function
+(type_annotation ((lower_case_identifier) @name.reference.function) (colon)) @reference.function
+
+(type_declaration ((upper_case_identifier) @name.definition.type) ) @definition.type
+
+(type_ref (upper_case_qid (upper_case_identifier) @name.reference.type)) @reference.type
+(exposed_type (upper_case_identifier) @name.reference.type)) @reference.type
+
+(type_declaration (union_variant (upper_case_identifier) @name.definition.union)) @definition.union
+
+(value_expr (upper_case_qid (upper_case_identifier) @name.reference.union)) @reference.union
+
+
+(module_declaration 
+    (upper_case_qid (upper_case_identifier)) @name.definition.module
+) @definition.module

extensions/vscode/tag-qry/tree-sitter-go-tags.scm

@@ -0,0 +1,30 @@
+(
+  (comment)* @doc
+  .
+  (function_declaration
+    name: (identifier) @name.definition.function) @definition.function
+  (#strip! @doc "^//\\s*")
+  (#set-adjacent! @doc @definition.function)
+)
+
+(
+  (comment)* @doc
+  .
+  (method_declaration
+    name: (field_identifier) @name.definition.method) @definition.method
+  (#strip! @doc "^//\\s*")
+  (#set-adjacent! @doc @definition.method)
+)
+
+(call_expression
+  function: [
+    (identifier) @name.reference.call
+    (parenthesized_expression (identifier) @name.reference.call)
+    (selector_expression field: (field_identifier) @name.reference.call)
+    (parenthesized_expression (selector_expression field: (field_identifier) @name.reference.call))
+  ]) @reference.call
+
+(type_spec
+  name: (type_identifier) @name.definition.type) @definition.type
+
+(type_identifier) @name.reference.type @reference.type

extensions/vscode/tag-qry/tree-sitter-java-tags.scm

@@ -0,0 +1,20 @@
+(class_declaration
+  name: (identifier) @name.definition.class) @definition.class
+
+(method_declaration
+  name: (identifier) @name.definition.method) @definition.method
+
+(method_invocation
+  name: (identifier) @name.reference.call
+  arguments: (argument_list) @reference.call)
+
+(interface_declaration
+  name: (identifier) @name.definition.interface) @definition.interface
+
+(type_list
+  (type_identifier) @name.reference.implementation) @reference.implementation
+
+(object_creation_expression
+  type: (type_identifier) @name.reference.class) @reference.class
+
+(superclass (type_identifier) @name.reference.class) @reference.class

extensions/vscode/tag-qry/tree-sitter-javascript-tags.scm

@@ -0,0 +1,88 @@
+(
+  (comment)* @doc
+  .
+  (method_definition
+    name: (property_identifier) @name.definition.method) @definition.method
+  (#not-eq? @name.definition.method "constructor")
+  (#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
+  (#select-adjacent! @doc @definition.method)
+)
+
+(
+  (comment)* @doc
+  .
+  [
+    (class
+      name: (_) @name.definition.class)
+    (class_declaration
+      name: (_) @name.definition.class)
+  ] @definition.class
+  (#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
+  (#select-adjacent! @doc @definition.class)
+)
+
+(
+  (comment)* @doc
+  .
+  [
+    (function
+      name: (identifier) @name.definition.function)
+    (function_declaration
+      name: (identifier) @name.definition.function)
+    (generator_function
+      name: (identifier) @name.definition.function)
+    (generator_function_declaration
+      name: (identifier) @name.definition.function)
+  ] @definition.function
+  (#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
+  (#select-adjacent! @doc @definition.function)
+)
+
+(
+  (comment)* @doc
+  .
+  (lexical_declaration
+    (variable_declarator
+      name: (identifier) @name.definition.function
+      value: [(arrow_function) (function)]) @definition.function)
+  (#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
+  (#select-adjacent! @doc @definition.function)
+)
+
+(
+  (comment)* @doc
+  .
+  (variable_declaration
+    (variable_declarator
+      name: (identifier) @name.definition.function
+      value: [(arrow_function) (function)]) @definition.function)
+  (#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
+  (#select-adjacent! @doc @definition.function)
+)
+
+(assignment_expression
+  left: [
+    (identifier) @name.definition.function
+    (member_expression
+      property: (property_identifier) @name.definition.function)
+  ]
+  right: [(arrow_function) (function)]
+) @definition.function
+
+(pair
+  key: (property_identifier) @name.definition.function
+  value: [(arrow_function) (function)]) @definition.function
+
+(
+  (call_expression
+    function: (identifier) @name.reference.call) @reference.call
+  (#not-match? @name.reference.call "^(require)$")
+)
+
+(call_expression
+  function: (member_expression
+    property: (property_identifier) @name.reference.call)
+  arguments: (_) @reference.call)
+
+(new_expression
+  constructor: (_) @name.reference.class) @reference.class

extensions/vscode/tag-qry/tree-sitter-ocaml-tags.scm

@@ -0,0 +1,116 @@
+; Modules
+;--------
+
+(
+  (comment)? @doc .
+  (module_definition (module_binding (module_name) @name.definition.module) @definition.module)
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+(module_path (module_name) @name.reference.module) @reference.module
+
+; Modules types
+;--------------
+
+(
+  (comment)? @doc .
+  (module_type_definition (module_type_name) @name.definition.interface) @definition.interface
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+(module_type_path (module_type_name) @name.reference.implementation) @reference.implementation
+
+; Functions
+;----------
+
+(
+  (comment)? @doc .
+  (value_definition
+    [
+      (let_binding
+        pattern: (value_name) @name.definition.function
+        (parameter))
+      (let_binding
+        pattern: (value_name) @name.definition.function
+        body: [(fun_expression) (function_expression)])
+    ] @definition.function
+  )
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+(
+  (comment)? @doc .
+  (external (value_name) @name.definition.function) @definition.function
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+(application_expression
+  function: (value_path (value_name) @name.reference.call)) @reference.call
+
+(infix_expression
+  left: (value_path (value_name) @name.reference.call)
+  (infix_operator) @reference.call
+  (#eq? @reference.call "@@"))
+
+(infix_expression
+  (infix_operator) @reference.call
+  right: (value_path (value_name) @name.reference.call)
+  (#eq? @reference.call "|>"))
+
+; Operator
+;---------
+
+(
+  (comment)? @doc .
+  (value_definition
+    (let_binding
+      pattern: (parenthesized_operator [
+        (prefix_operator)
+        (infix_operator)
+        (hash_operator)
+        (indexing_operator)
+        (let_operator)
+        (and_operator)
+        (match_operator)
+      ] @name.definition.function)) @definition.function)
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+[
+  (prefix_operator)
+  (sign_operator)
+  (infix_operator)
+  (hash_operator)
+  (indexing_operator)
+  (let_operator)
+  (and_operator)
+  (match_operator)
+] @name.reference.call @reference.call
+
+; Classes
+;--------
+
+(
+  (comment)? @doc .
+  [
+    (class_definition (class_binding (class_name) @name.definition.class) @definition.class)
+    (class_type_definition (class_type_binding (class_type_name) @name.definition.class) @definition.class)
+  ]
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+[
+  (class_path (class_name) @name.reference.class)
+  (class_type_path (class_type_name) @name.reference.class)
+] @reference.class
+
+; Methods
+;--------
+
+(
+  (comment)? @doc .
+  (method_definition (method_name) @name.definition.method) @definition.method
+  (#strip! @doc "^\\(\\*\\*?\\s*|\\s\\*\\)$")
+)
+
+(method_invocation (method_name) @name.reference.call) @reference.call

extensions/vscode/tag-qry/tree-sitter-php-tags.scm

@@ -0,0 +1,26 @@
+(class_declaration
+  name: (name) @name.definition.class) @definition.class
+
+(function_definition
+  name: (name) @name.definition.function) @definition.function
+
+(method_declaration
+  name: (name) @name.definition.function) @definition.function
+
+(object_creation_expression
+  [
+    (qualified_name (name) @name.reference.class)
+    (variable_name (name) @name.reference.class)
+  ]) @reference.class
+
+(function_call_expression
+  function: [
+    (qualified_name (name) @name.reference.call)
+    (variable_name (name)) @name.reference.call
+  ]) @reference.call
+
+(scoped_call_expression
+  name: (name) @name.reference.call) @reference.call
+
+(member_call_expression
+  name: (name) @name.reference.call) @reference.call

extensions/vscode/tag-qry/tree-sitter-python-tags.scm

@@ -0,0 +1,12 @@
+(class_definition
+  name: (identifier) @name.definition.class) @definition.class
+
+(function_definition
+  name: (identifier) @name.definition.function) @definition.function
+
+(call
+  function: [
+      (identifier) @name.reference.call
+      (attribute
+        attribute: (identifier) @name.reference.call)
+  ]) @reference.call

extensions/vscode/tag-qry/tree-sitter-ql-tags.scm

@@ -0,0 +1,26 @@
+(classlessPredicate
+  name: (predicateName) @name.definition.function) @definition.function
+
+(memberPredicate
+  name: (predicateName) @name.definition.method) @definition.method
+
+(aritylessPredicateExpr
+  name: (literalId) @name.reference.call) @reference.call
+
+(module
+  name: (moduleName) @name.definition.module) @definition.module
+
+(dataclass
+  name: (className) @name.definition.class) @definition.class
+
+(datatype
+  name: (className) @name.definition.class) @definition.class
+
+(datatypeBranch
+  name: (className) @name.definition.class) @definition.class
+
+(qualifiedRhs
+  name: (predicateName) @name.reference.call) @reference.call
+
+(typeExpr
+  name: (className) @name.reference.type) @reference.type

extensions/vscode/tag-qry/tree-sitter-ruby-tags.scm

@@ -0,0 +1,64 @@
+; Method definitions
+
+(
+  (comment)* @doc
+  .
+  [
+    (method
+      name: (_) @name.definition.method) @definition.method
+    (singleton_method
+      name: (_) @name.definition.method) @definition.method
+  ]
+  (#strip! @doc "^#\\s*")
+  (#select-adjacent! @doc @definition.method)
+)
+
+(alias
+  name: (_) @name.definition.method) @definition.method
+
+(setter
+  (identifier) @ignore)
+
+; Class definitions
+
+(
+  (comment)* @doc
+  .
+  [
+    (class
+      name: [
+        (constant) @name.definition.class
+        (scope_resolution
+          name: (_) @name.definition.class)
+      ]) @definition.class
+    (singleton_class
+      value: [
+        (constant) @name.definition.class
+        (scope_resolution
+          name: (_) @name.definition.class)
+      ]) @definition.class
+  ]
+  (#strip! @doc "^#\\s*")
+  (#select-adjacent! @doc @definition.class)
+)
+
+; Module definitions
+
+(
+  (module
+    name: [
+      (constant) @name.definition.module
+      (scope_resolution
+        name: (_) @name.definition.module)
+    ]) @definition.module
+)
+
+; Calls
+
+(call method: (identifier) @name.reference.call) @reference.call
+
+(
+  [(identifier) (constant)] @name.reference.call @reference.call
+  (#is-not? local)
+  (#not-match? @name.reference.call "^(lambda|load|require|require_relative|__FILE__|__LINE__)$")
+)

extensions/vscode/tag-qry/tree-sitter-rust-tags.scm

@@ -0,0 +1,60 @@
+; ADT definitions
+
+(struct_item
+    name: (type_identifier) @name.definition.class) @definition.class
+
+(enum_item
+    name: (type_identifier) @name.definition.class) @definition.class
+
+(union_item
+    name: (type_identifier) @name.definition.class) @definition.class
+
+; type aliases
+
+(type_item
+    name: (type_identifier) @name.definition.class) @definition.class
+
+; method definitions
+
+(declaration_list
+    (function_item
+        name: (identifier) @name.definition.method)) @definition.method
+
+; function definitions
+
+(function_item
+    name: (identifier) @name.definition.function) @definition.function
+
+; trait definitions
+(trait_item
+    name: (type_identifier) @name.definition.interface) @definition.interface
+
+; module definitions
+(mod_item
+    name: (identifier) @name.definition.module) @definition.module
+
+; macro definitions
+
+(macro_definition
+    name: (identifier) @name.definition.macro) @definition.macro
+
+; references
+
+(call_expression
+    function: (identifier) @name.reference.call) @reference.call
+
+(call_expression
+    function: (field_expression
+        field: (field_identifier) @name.reference.call)) @reference.call
+
+(macro_invocation
+    macro: (identifier) @name.reference.call) @reference.call
+
+; implementations
+
+(impl_item
+    trait: (type_identifier) @name.reference.implementation) @reference.implementation
+
+(impl_item
+    type: (type_identifier) @name.reference.implementation
+    !trait) @reference.implementation

extensions/vscode/tag-qry/tree-sitter-typescript-tags.scm

@@ -0,0 +1,23 @@
+(function_signature
+  name: (identifier) @name.definition.function) @definition.function
+
+(method_signature
+  name: (property_identifier) @name.definition.method) @definition.method
+
+(abstract_method_signature
+  name: (property_identifier) @name.definition.method) @definition.method
+
+(abstract_class_declaration
+  name: (type_identifier) @name.definition.class) @definition.class
+
+(module
+  name: (identifier) @name.definition.module) @definition.module
+
+(interface_declaration
+  name: (type_identifier) @name.definition.interface) @definition.interface
+
+(type_annotation
+  (type_identifier) @name.reference.type) @reference.type
+
+(new_expression
+  constructor: (identifier) @name.reference.class) @reference.class

extensions/vscode/yarn.lock

@@ -696,6 +696,11 @@
   resolved "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz"
   integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==
 
+"@lancedb/vectordb-darwin-arm64@*":
+  version "0.4.12"
+  resolved "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.12.tgz"
+  integrity sha512-38/rkJRlWXkPWXuj9onzvbrhnIWcIUQjgEp5G9v5ixPosBowm7A4j8e2Q8CJMsVSNcVX2JLqwWVldiWegZFuYw==
+
 "@lancedb/vectordb-darwin-arm64@0.4.10":
   version "0.4.10"
   resolved "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.10.tgz"