fix(search-contexts): Fix issue where a repository would not appear in a search context if it was created after the search context was created (#354)

## Problem

If a repository is added **after** a search context (e.g., a new repository is synced from the code host), then it will never be added to the context even if it should be included. The workaround is to restart the instance.

## Solution

This PR adds a call to re-sync all search contexts whenever a connection is successfully synced. This PR adds the `@sourcebot/shared` package that contains `syncSearchContexts.ts` (previously in web) and it's dependencies (namely the entitlements system).

## Why another package?

Because the `syncSearchContexts` call is now called from:
1. `initialize.ts` in **web** - handles syncing search contexts on startup and whenever the config is modified in watch mode. This is the same as before.
2. `connectionManager.ts` in **backend** - syncs the search contexts whenever a connection is successfully synced.

## Follow-up devex work
Two things:
1. We have several very thin shared packages (i.e., `crypto`, `error`, and `logger`) that we can probably fold into this "general" shared package. `schemas` and `db` _feels_ like they should remain separate (mostly because they are "code-gen" packages).
2. When running `yarn dev`, any changes made to the shared package will only get picked if you `ctrl+c` and restart the instance. Would be nice if we have watch mode work across package dependencies in the monorepo.

Author: brendan-kellam

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Delete account join request when redeeming an invite. [#352](https://github.com/sourcebot-dev/sourcebot/pull/352)
+- Fix issue where a repository would not be included in a search context if the context was created before the repository. [#354](https://github.com/sourcebot-dev/sourcebot/pull/354)
 
 ## [4.3.0] - 2025-06-11
 

Dockerfile

@@ -43,12 +43,14 @@ COPY ./packages/schemas ./packages/schemas
 COPY ./packages/crypto ./packages/crypto
 COPY ./packages/error ./packages/error
 COPY ./packages/logger ./packages/logger
+COPY ./packages/shared ./packages/shared
 
 RUN yarn workspace @sourcebot/db install
 RUN yarn workspace @sourcebot/schemas install
 RUN yarn workspace @sourcebot/crypto install
 RUN yarn workspace @sourcebot/error install
 RUN yarn workspace @sourcebot/logger install
+RUN yarn workspace @sourcebot/shared install
 # ------------------------------------
 
 # ------ Build Web ------
@@ -92,6 +94,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Fixes arm64 timeouts
 RUN yarn workspace @sourcebot/web install
@@ -132,6 +135,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 RUN yarn workspace @sourcebot/backend install
 RUN yarn workspace @sourcebot/backend build
 
@@ -215,6 +219,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Configure dependencies
 RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl util-linux unzip

LICENSE

@@ -2,7 +2,7 @@ Copyright (c) 2025 Taqla Inc.
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and "packages/web/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All content that resides under the "ee/", "packages/web/src/ee/", and "packages/shared/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
 - All third party components incorporated into the Sourcebot Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 

Makefile

@@ -34,6 +34,8 @@ clean:
 		packages/error/dist \
 		packages/mcp/node_modules \
 		packages/mcp/dist \
+		packages/shared/node_modules \
+		packages/shared/dist \
 		.sourcebot
 
 soft-reset:

package.json

@@ -16,7 +16,7 @@
         "dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
         "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
         "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset",
-        "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db}' run build"
+        "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build"
     },
     "devDependencies": {
         "cross-env": "^7.0.3",

packages/backend/package.json

@@ -33,9 +33,9 @@
         "@sourcebot/error": "workspace:*",
         "@sourcebot/logger": "workspace:*",
         "@sourcebot/schemas": "workspace:*",
+        "@sourcebot/shared": "workspace:*",
         "@t3-oss/env-core": "^0.12.0",
         "@types/express": "^5.0.0",
-        "ajv": "^8.17.1",
         "argparse": "^2.0.1",
         "bullmq": "^5.34.10",
         "cross-fetch": "^4.0.0",
@@ -50,7 +50,6 @@
         "posthog-node": "^4.2.1",
         "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
-        "strip-json-comments": "^5.0.1",
         "zod": "^3.24.3"
     }
 }

packages/backend/src/connectionManager.ts

@@ -9,6 +9,7 @@ import { BackendError, BackendException } from "@sourcebot/error";
 import { captureEvent } from "./posthog.js";
 import { env } from "./env.js";
 import * as Sentry from "@sentry/node";
+import { loadConfig, syncSearchContexts } from "@sourcebot/shared";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -264,7 +265,7 @@ export class ConnectionManager implements IConnectionManager {
 
     private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
         this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
-        const { connectionId } = job.data;
+        const { connectionId, orgId } = job.data;
 
         let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
             where: { id: connectionId },
@@ -289,7 +290,25 @@ export class ConnectionManager implements IConnectionManager {
                         notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
                 syncedAt: new Date()
             }
-        })
+        });
+
+        // After a connection has synced, we need to re-sync the org's search contexts as
+        // there may be new repos that match the search context's include/exclude patterns.
+        if (env.CONFIG_PATH) {
+            try {
+                const config = await loadConfig(env.CONFIG_PATH);
+
+                await syncSearchContexts({
+                    db: this.db,
+                    orgId,
+                    contexts: config.contexts,
+                });
+            } catch (err) {
+                this.logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`);
+                Sentry.captureException(err);
+            }
+        }
+
 
         captureEvent('backend_connection_sync_job_completed', {
             connectionId: connectionId,

packages/backend/src/constants.ts

@@ -16,4 +16,4 @@ export const DEFAULT_SETTINGS: Settings = {
     repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
     repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
     enablePublicAccess: false,
-}
+}

packages/backend/src/index.ts

@@ -10,7 +10,8 @@ import { PrismaClient } from "@sourcebot/db";
 import { env } from "./env.js";
 import { createLogger } from "@sourcebot/logger";
 
-const logger = createLogger('index');
+const logger = createLogger('backend-entrypoint');
+
 
 // Register handler for normal exit
 process.on('exit', (code) => {
@@ -72,3 +73,4 @@ main(prisma, context)
     .finally(() => {
         logger.info("Shutting down...");
     });
+

packages/backend/src/main.ts

@@ -7,40 +7,16 @@ import { ConnectionManager } from './connectionManager.js';
 import { RepoManager } from './repoManager.js';
 import { env } from './env.js';
 import { PromClient } from './promClient.js';
-import { isRemotePath } from './utils.js';
-import { readFile } from 'fs/promises';
-import stripJsonComments from 'strip-json-comments';
-import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
-import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
-import { Ajv } from "ajv";
+import { loadConfig } from '@sourcebot/shared';
 
 const logger = createLogger('backend-main');
-const ajv = new Ajv({
-    validateFormats: false,
-});
 
 const getSettings = async (configPath?: string) => {
     if (!configPath) {
         return DEFAULT_SETTINGS;
     }
 
-    const configContent = await (async () => {
-        if (isRemotePath(configPath)) {
-            const response = await fetch(configPath);
-            if (!response.ok) {
-                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
-            }
-            return response.text();
-        } else {
-            return readFile(configPath, { encoding: 'utf-8' });
-        }
-    })();
-
-    const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
-    const isValidConfig = ajv.validate(indexSchema, config);
-    if (!isValidConfig) {
-        throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
-    }
+    const config = await loadConfig(configPath);
 
     return {
         ...DEFAULT_SETTINGS,