Multimodality: Upload PDFs and Images (#135)

Author: bracesproul

next.config.mjs

@@ -1,4 +1,10 @@
 /** @type {import('next').NextConfig} */
-const nextConfig = {};
+const nextConfig = {
+  experimental: {
+    serverActions: {
+      bodySizeLimit: "10mb",
+    },
+  },
+};
 
 export default nextConfig;

package.json

@@ -53,7 +53,7 @@
     "tailwind-merge": "^3.0.2",
     "tailwindcss-animate": "^1.0.7",
     "use-stick-to-bottom": "^1.0.46",
-    "uuid": "^11.0.5",
+    "uuid": "^11.1.0",
     "zod": "^3.24.2"
   },
   "devDependencies": {
@@ -64,6 +64,7 @@
     "@types/react": "^19.0.8",
     "@types/react-dom": "^19.0.3",
     "@types/react-syntax-highlighter": "^15.5.13",
+    "@types/uuid": "^10.0.0",
     "autoprefixer": "^10.4.20",
     "dotenv": "^16.4.7",
     "eslint": "^9.19.0",
@@ -81,8 +82,7 @@
     "typescript-eslint": "^8.22.0"
   },
   "overrides": {
-    "react-is": "^19.0.0-rc-69d4b800-20241021",
-    "@langchain/langgraph-checkpoint": "^0.0.16"
+    "react-is": "^19.0.0-rc-69d4b800-20241021"
   },
   "packageManager": "[email protected]"
 }

pnpm-lock.yaml

@@ -0,0 +1,37 @@
+import React from "react";
+import type { Base64ContentBlock } from "@langchain/core/messages";
+import { MultimodalPreview } from "../ui/MultimodalPreview";
+import { cn } from "@/lib/utils";
+
+interface ContentBlocksPreviewProps {
+  blocks: Base64ContentBlock[];
+  onRemove: (idx: number) => void;
+  size?: "sm" | "md" | "lg";
+  className?: string;
+}
+
+/**
+ * Renders a preview of content blocks with optional remove functionality.
+ * Uses cn utility for robust class merging.
+ */
+export const ContentBlocksPreview: React.FC<ContentBlocksPreviewProps> = ({
+  blocks,
+  onRemove,
+  size = "md",
+  className,
+}) => {
+  if (!blocks.length) return null;
+  return (
+    <div className={cn("flex flex-wrap gap-2 p-3.5 pb-0", className)}>
+      {blocks.map((block, idx) => (
+        <MultimodalPreview
+          key={idx}
+          block={block}
+          removable
+          onRemove={() => onRemove(idx)}
+          size={size}
+        />
+      ))}
+    </div>
+  );
+};

src/components/thread/ContentBlocksPreview.tsx

@@ -21,6 +21,8 @@ import {
   PanelRightClose,
   SquarePen,
   XIcon,
+  Plus,
+  CircleX,
 } from "lucide-react";
 import { useQueryState, parseAsBoolean } from "nuqs";
 import { StickToBottom, useStickToBottomContext } from "use-stick-to-bottom";
@@ -36,6 +38,8 @@ import {
   TooltipProvider,
   TooltipTrigger,
 } from "../ui/tooltip";
+import { useFileUpload } from "@/hooks/use-file-upload";
+import { ContentBlocksPreview } from "./ContentBlocksPreview";
 import {
   useArtifactOpen,
   ArtifactContent,
@@ -122,6 +126,14 @@ export function Thread() {
     parseAsBoolean.withDefault(false),
   );
   const [input, setInput] = useState("");
+  const {
+    contentBlocks,
+    setContentBlocks,
+    handleFileUpload,
+    dropRef,
+    removeBlock,
+    resetBlocks,
+  } = useFileUpload();
   const [firstTokenReceived, setFirstTokenReceived] = useState(false);
   const isLargeScreen = useMediaQuery("(min-width: 1024px)");
 
@@ -183,13 +195,17 @@ export function Thread() {
 
   const handleSubmit = (e: FormEvent) => {
     e.preventDefault();
-    if (!input.trim() || isLoading) return;
+    if ((input.trim().length === 0 && contentBlocks.length === 0) || isLoading)
+      return;
     setFirstTokenReceived(false);
 
     const newHumanMessage: Message = {
       id: uuidv4(),
       type: "human",
-      content: input,
+      content: [
+        ...(input.trim().length > 0 ? [{ type: "text", text: input }] : []),
+        ...contentBlocks,
+      ] as Message["content"],
     };
 
     const toolMessages = ensureToolCallsHaveResponses(stream.messages);
@@ -214,6 +230,7 @@ export function Thread() {
     );
 
     setInput("");
+    setContentBlocks([]);
   };
 
   const handleRegenerate = (
@@ -423,11 +440,18 @@ export function Thread() {
 
                   <ScrollToBottom className="animate-in fade-in-0 zoom-in-95 absolute bottom-full left-1/2 mb-4 -translate-x-1/2" />
 
-                  <div className="bg-muted relative z-10 mx-auto mb-8 w-full max-w-3xl rounded-2xl border shadow-xs">
+                  <div
+                    ref={dropRef}
+                    className="bg-muted relative z-10 mx-auto mb-8 w-full max-w-3xl rounded-2xl border shadow-xs"
+                  >
                     <form
                       onSubmit={handleSubmit}
                       className="mx-auto grid max-w-3xl grid-rows-[1fr_auto] gap-2"
                     >
+                      <ContentBlocksPreview
+                        blocks={contentBlocks}
+                        onRemove={removeBlock}
+                      />
                       <textarea
                         value={input}
                         onChange={(e) => setInput(e.target.value)}
@@ -448,7 +472,7 @@ export function Thread() {
                         className="field-sizing-content resize-none border-none bg-transparent p-3.5 pb-0 shadow-none ring-0 outline-none focus:ring-0 focus:outline-none"
                       />
 
-                      <div className="flex items-center justify-between p-2 pt-4">
+                      <div className="flex items-center gap-6 p-2 pt-4">
                         <div>
                           <div className="flex items-center space-x-2">
                             <Switch
@@ -464,19 +488,40 @@ export function Thread() {
                             </Label>
                           </div>
                         </div>
+                        <Label
+                          htmlFor="file-input"
+                          className="flex cursor-pointer items-center gap-2"
+                        >
+                          <Plus className="size-5 text-gray-600" />
+                          <span className="text-sm text-gray-600">
+                            Upload PDF or Image
+                          </span>
+                        </Label>
+                        <input
+                          id="file-input"
+                          type="file"
+                          onChange={handleFileUpload}
+                          multiple
+                          accept="image/jpeg,image/png,image/gif,image/webp,application/pdf"
+                          className="hidden"
+                        />
                         {stream.isLoading ? (
                           <Button
                             key="stop"
                             onClick={() => stream.stop()}
+                            className="ml-auto"
                           >
                             <LoaderCircle className="h-4 w-4 animate-spin" />
                             Cancel
                           </Button>
                         ) : (
                           <Button
                             type="submit"
-                            className="shadow-md transition-all"
-                            disabled={isLoading || !input.trim()}
+                            className="ml-auto shadow-md transition-all"
+                            disabled={
+                              isLoading ||
+                              (!input.trim() && contentBlocks.length === 0)
+                            }
                           >
                             Send
                           </Button>

src/components/thread/index.tsx

@@ -5,6 +5,8 @@ import { getContentString } from "../utils";
 import { cn } from "@/lib/utils";
 import { Textarea } from "@/components/ui/textarea";
 import { BranchSwitcher, CommandBar } from "./shared";
+import { MultimodalPreview } from "@/components/ui/MultimodalPreview";
+import type { Base64ContentBlock } from "@langchain/core/messages";
 
 function EditableContent({
   value,
@@ -32,6 +34,36 @@ function EditableContent({
   );
 }
 
+// Type guard for Base64ContentBlock
+function isBase64ContentBlock(block: unknown): block is Base64ContentBlock {
+  if (typeof block !== "object" || block === null || !("type" in block))
+    return false;
+  // file type (legacy)
+  if (
+    (block as { type: unknown }).type === "file" &&
+    "source_type" in block &&
+    (block as { source_type: unknown }).source_type === "base64" &&
+    "mime_type" in block &&
+    typeof (block as { mime_type?: unknown }).mime_type === "string" &&
+    ((block as { mime_type: string }).mime_type.startsWith("image/") ||
+      (block as { mime_type: string }).mime_type === "application/pdf")
+  ) {
+    return true;
+  }
+  // image type (new)
+  if (
+    (block as { type: unknown }).type === "image" &&
+    "source_type" in block &&
+    (block as { source_type: unknown }).source_type === "base64" &&
+    "mime_type" in block &&
+    typeof (block as { mime_type?: unknown }).mime_type === "string" &&
+    (block as { mime_type: string }).mime_type.startsWith("image/")
+  ) {
+    return true;
+  }
+  return false;
+}
+
 export function HumanMessage({
   message,
   isLoading,
@@ -84,9 +116,34 @@ export function HumanMessage({
             onSubmit={handleSubmitEdit}
           />
         ) : (
-          <p className="bg-muted ml-auto w-fit rounded-3xl px-4 py-2 whitespace-pre-wrap">
-            {contentString}
-          </p>
+          <div className="flex flex-col gap-2">
+            {/* Render images and files if no text */}
+            {Array.isArray(message.content) && message.content.length > 0 && (
+              <div className="flex flex-col items-end gap-2">
+                {message.content.reduce<React.ReactNode[]>(
+                  (acc, block, idx) => {
+                    if (isBase64ContentBlock(block)) {
+                      acc.push(
+                        <MultimodalPreview
+                          key={idx}
+                          block={block}
+                          size="md"
+                        />,
+                      );
+                    }
+                    return acc;
+                  },
+                  [],
+                )}
+              </div>
+            )}
+            {/* Render text if present, otherwise fallback to file/image name */}
+            {contentString ? (
+              <p className="bg-muted ml-auto w-fit rounded-3xl px-4 py-2 text-right whitespace-pre-wrap">
+                {contentString}
+              </p>
+            ) : null}
+          </div>
         )}
 
         <div

src/components/thread/messages/human.tsx

@@ -1,5 +1,11 @@
 import type { Message } from "@langchain/langgraph-sdk";
 
+/**
+ * Extracts a string summary from a message's content, supporting multimodal (text, image, file, etc.).
+ * - If text is present, returns the joined text.
+ * - If not, returns a label for the first non-text modality (e.g., 'Image', 'Other').
+ * - If unknown, returns 'Multimodal message'.
+ */
 export function getContentString(content: Message["content"]): string {
   if (typeof content === "string") return content;
   const texts = content