chore(api): Changed the search api to return raw source (instead of base64 encoding) (#356)

This PR alters the behaviour of the search api (and all apis that depend on it) to return raw source code instead of a base64 encoding. Reasoning: we are decoding it on the client in multiple different places, so it would be beneficial to decode it in a single spot.

**Note**: This is a **breaking change** to the API surface. However, since the API surface is still unofficial/unsupported, I will roll this as a patch version change. See #101

Author: brendan-kellam

CHANGELOG.md

@@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+<!-- @NOTE: this release includes a API change that affects the MCP package (@sourcebot/mcp). On release, bump the MCP package's version and delete this message. -->
+
 ### Fixed
 - Delete account join request when redeeming an invite. [#352](https://github.com/sourcebot-dev/sourcebot/pull/352)
 - Fix issue where a repository would not be included in a search context if the context was created before the repository. [#354](https://github.com/sourcebot-dev/sourcebot/pull/354)
 
+### Changed
+- Changed search api (and all apis that depend on it) to return raw source code instead of base64 encoded string. ([356](https://github.com/sourcebot-dev/sourcebot/pull/356)).
+
+
 ## [4.3.0] - 2025-06-11
 
 ### Added

packages/mcp/CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Updated API client to match the latest Sourcebot release. [#356](https://github.com/sourcebot-dev/sourcebot/pull/356)
+
 ## [1.0.2] - 2025-05-28
 
 ### Changed

packages/mcp/src/index.ts

@@ -8,7 +8,7 @@ import { z } from 'zod';
 import { listRepos, search, getFileSource } from './client.js';
 import { env, numberSchema } from './env.js';
 import { TextContent } from './types.js';
-import { base64Decode, isServiceError } from './utils.js';
+import { isServiceError } from './utils.js';
 
 // Create MCP server
 const server = new McpServer({
@@ -114,8 +114,7 @@ server.tool(
 
             if (includeCodeSnippets) {
                 const snippets = file.chunks.map(chunk => {
-                    const content = base64Decode(chunk.content);
-                    return `\`\`\`\n${content}\n\`\`\``
+                    return `\`\`\`\n${chunk.content}\n\`\`\``
                 }).join('\n');
                 text += `\n\n${snippets}`;
             }
@@ -201,7 +200,7 @@ server.tool(
 
         const content: TextContent[] = [{
             type: "text",
-            text: `file: ${fileName}\nrepository: ${repoId}\nlanguage: ${response.language}\nsource:\n${base64Decode(response.source)}`,
+            text: `file: ${fileName}\nrepository: ${repoId}\nlanguage: ${response.language}\nsource:\n${response.source}`,
         }]
 
         return {

packages/mcp/src/utils.ts

@@ -1,10 +1,5 @@
 import { ServiceError } from "./types.js";
 
-// From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem
-export const base64Decode = (base64: string): string => {
-    const binString = atob(base64);
-    return Buffer.from(Uint8Array.from(binString, (m) => m.codePointAt(0)!).buffer).toString();
-}
 
 export const isServiceError = (data: unknown): data is ServiceError => {
     return typeof data === 'object' &&

packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { base64Decode, getCodeHostInfoForRepo, unwrapServiceError } from "@/lib/utils";
+import { getCodeHostInfoForRepo, unwrapServiceError } from "@/lib/utils";
 import { useBrowseParams } from "@/app/[domain]/browse/hooks/useBrowseParams";
 import { useQuery } from "@tanstack/react-query";
 import { getFileSource } from "@/features/search/fileSourceApi";
@@ -88,7 +88,7 @@ export const CodePreviewPanel = () => {
             </div>
             <Separator />
             <PureCodePreviewPanel
-                source={base64Decode(fileSourceResponse.source)}
+                source={fileSourceResponse.source}
                 language={fileSourceResponse.language}
                 repoName={repoName}
                 path={path}

packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx

@@ -7,7 +7,6 @@ import { useDomain } from "@/hooks/useDomain";
 import { SymbolIcon } from "@radix-ui/react-icons";
 import { SetStateAction, Dispatch, useMemo } from "react";
 import { getFileSource } from "@/features/search/fileSourceApi";
-import { base64Decode } from "@/lib/utils";
 import { unwrapServiceError } from "@/lib/utils";
 
 interface CodePreviewPanelProps {
@@ -41,10 +40,8 @@ export const CodePreviewPanel = ({
             }, domain)
         ),
         select: (data) => {
-            const decodedSource = base64Decode(data.source);
-
             return {
-                content: decodedSource,
+                content: data.source,
                 filepath: previewedFile.fileName.text,
                 matches: previewedFile.chunks,
                 link: previewedFile.webUrl,

packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx

@@ -1,8 +1,7 @@
 'use client';
 
-import { useCallback, useMemo } from "react";
+import { useCallback } from "react";
 import { SearchResultFile, SearchResultChunk } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 
 
@@ -17,17 +16,12 @@ export const FileMatch = ({
     file,
     onOpen: _onOpen,
 }: FileMatchProps) => {
-
-    const content = useMemo(() => {
-        return base64Decode(match.content);
-    }, [match.content]);
-
     const onOpen = useCallback((isCtrlKeyPressed: boolean) => {
         const startLineNumber = match.contentStart.lineNumber;
-        const endLineNumber = content.trimEnd().split('\n').length + startLineNumber - 1;
+        const endLineNumber = match.content.trimEnd().split('\n').length + startLineNumber - 1;
 
         _onOpen(startLineNumber, endLineNumber, isCtrlKeyPressed);
-    }, [content, match.contentStart.lineNumber, _onOpen]);
+    }, [match.content, match.contentStart.lineNumber, _onOpen]);
 
     // If it's just the title, don't show a code preview
     if (match.matchRanges.length === 0) {
@@ -57,7 +51,7 @@ export const FileMatch = ({
                 lineNumbersOffset={match.contentStart.lineNumber}
                 renderWhitespace={true}
             >
-                {content}
+                {match.content}
             </LightweightCodeHighlighter>
         </div>
     );

packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx

@@ -5,7 +5,6 @@ import { PathHeader } from "@/app/[domain]/components/pathHeader";
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 import { FindRelatedSymbolsResponse } from "@/features/codeNav/types";
 import { RepositoryInfo, SourceRange } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 import { useMemo, useRef } from "react";
 import useCaptureEvent from "@/hooks/useCaptureEvent";
 import { useVirtualizer } from "@tanstack/react-virtual";
@@ -155,10 +154,6 @@ const ReferenceListItem = ({
     onClick,
     onMouseEnter,
 }: ReferenceListItemProps) => {
-    const decodedLineContent = useMemo(() => {
-        return base64Decode(lineContent);
-    }, [lineContent]);
-
     const highlightRanges = useMemo(() => [range], [range]);
 
     return (
@@ -174,7 +169,7 @@ const ReferenceListItem = ({
                 lineNumbersOffset={range.start.lineNumber}
                 renderWhitespace={false}
             >
-                {decodedLineContent}
+                {lineContent}
             </LightweightCodeHighlighter>
         </div>
     )

packages/web/src/ee/features/codeNav/components/symbolHoverPopup/symbolDefinitionPreview.tsx

@@ -3,7 +3,6 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 import { useMemo } from "react";
 import { SourceRange } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 
 interface SymbolDefinitionPreviewProps {
     symbolDefinition: {
@@ -21,10 +20,6 @@ export const SymbolDefinitionPreview = ({
     const { lineContent, language, range } = symbolDefinition;
     const highlightRanges = useMemo(() => [range], [range]);
 
-    const decodedLineContent = useMemo(() => {
-        return base64Decode(lineContent);
-    }, [lineContent]);
-
     return (
         <div className="flex flex-col gap-2 mb-2">
             <Tooltip
@@ -55,7 +50,7 @@ export const SymbolDefinitionPreview = ({
                 lineNumbersOffset={range.start.lineNumber}
                 renderWhitespace={false}
             >
-                {decodedLineContent}
+                {lineContent}
             </LightweightCodeHighlighter>
         </div>
     )

packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts

@@ -1,7 +1,6 @@
 import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/review-agent/types";
 import { getFileSource } from "@/features/search/fileSourceApi";
 import { fileSourceResponseSchema } from "@/features/search/schemas";
-import { base64Decode } from "@/lib/utils";
 import { isServiceError } from "@/lib/utils";
 import { env } from "@/env.mjs";
 import { createLogger } from "@sourcebot/logger";
@@ -24,7 +23,7 @@ export const fetchFileContent = async (pr_payload: sourcebot_pr_payload, filenam
     }
 
     const fileSourceResponse = fileSourceResponseSchema.parse(response);
-    const fileContent = base64Decode(fileSourceResponse.source);
+    const fileContent = fileSourceResponse.source;
 
     const fileContentContext: sourcebot_context = {
         type: "file_content",