pass multiple feedback URLs (#353)

Author: vbarda

.github/workflows/eval.yml

@@ -26,14 +26,14 @@ jobs:
         with:
           python-version: "3.11"
           poetry-version: "1.7.1"
-          cache-key: lint
+          cache-key: eval
 
       - name: Install dependencies
         run: poetry install --with dev
 
       - name: Evaluate
         env:
-          LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
+          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }}

backend/graph.py

@@ -1,4 +1,5 @@
 import os
+from collections import defaultdict
 from typing import Annotated, Literal, Sequence, TypedDict
 
 import weaviate
@@ -25,6 +26,7 @@
 from langchain_openai import ChatOpenAI
 from langchain_weaviate import WeaviateVectorStore
 from langgraph.graph import END, StateGraph, add_messages
+from langsmith import Client as LangsmithClient
 
 from backend.constants import WEAVIATE_DOCS_INDEX_NAME
 from backend.ingest import get_embeddings_model
@@ -105,6 +107,8 @@
 COHERE_MODEL_KEY = "cohere_command"
 GROQ_LLAMA_3_MODEL_KEY = "groq_llama_3"
 
+FEEDBACK_KEYS = ["user_score", "user_click"]
+
 
 def update_documents(
     _: list[Document], right: list[Document] | list[dict]
@@ -127,6 +131,7 @@ class AgentState(TypedDict):
     messages: Annotated[list[BaseMessage], add_messages]
     # for convenience in evaluations
     answer: str
+    feedback_urls: dict[str, list[str]]
 
 
 gpt_3_5 = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0, streaming=True)
@@ -260,8 +265,27 @@ def get_chat_history(messages: Sequence[BaseMessage]) -> Sequence[BaseMessage]:
     return chat_history
 
 
+def get_feedback_urls(config: RunnableConfig) -> dict[str, list[str]]:
+    ls_client = LangsmithClient()
+    run_id = config["configurable"].get("run_id")
+    if run_id is None:
+        return {}
+
+    tokens = ls_client.create_presigned_feedback_tokens(run_id, FEEDBACK_KEYS)
+    key_to_token_urls = defaultdict(list)
+
+    for token_idx, token in enumerate(tokens):
+        key_idx = token_idx % len(FEEDBACK_KEYS)
+        key = FEEDBACK_KEYS[key_idx]
+        key_to_token_urls[key].append(token.url)
+    return key_to_token_urls
+
+
 def synthesize_response(
-    state: AgentState, model: LanguageModelLike, prompt_template: str
+    state: AgentState,
+    config: RunnableConfig,
+    model: LanguageModelLike,
+    prompt_template: str,
 ) -> AgentState:
     prompt = ChatPromptTemplate.from_messages(
         [
@@ -282,16 +306,24 @@ def synthesize_response(
             ),
         }
     )
-    return {"messages": [synthesized_response], "answer": synthesized_response.content}
+    # finally, add feedback URLs so that users can leave feedback
+    feedback_urls = get_feedback_urls(config)
+    return {
+        "messages": [synthesized_response],
+        "answer": synthesized_response.content,
+        "feedback_urls": feedback_urls,
+    }
 
 
-def synthesize_response_default(state: AgentState) -> AgentState:
-    return synthesize_response(state, llm, RESPONSE_TEMPLATE)
+def synthesize_response_default(
+    state: AgentState, config: RunnableConfig
+) -> AgentState:
+    return synthesize_response(state, config, llm, RESPONSE_TEMPLATE)
 
 
-def synthesize_response_cohere(state: AgentState) -> AgentState:
+def synthesize_response_cohere(state: AgentState, config: RunnableConfig) -> AgentState:
     model = llm.bind(documents=state["documents"])
-    return synthesize_response(state, model, COHERE_RESPONSE_TEMPLATE)
+    return synthesize_response(state, config, model, COHERE_RESPONSE_TEMPLATE)
 
 
 def route_to_response_synthesizer(

frontend/app/components/ChatMessageBubble.tsx

@@ -130,14 +130,15 @@ const createAnswerElements = (
 
 export function ChatMessageBubble(props: {
   message: Message;
-  feedbackUrls?: Record<string, string>;
+  feedbackUrls?: Record<string, string[]>;
   aiEmoji?: string;
   isMostRecent: boolean;
   messageCompleted: boolean;
 }) {
   const { type, content } = props.message;
-  const responseFeedbackUrl = props.feedbackUrls?.[RESPONSE_FEEDBACK_KEY];
-  const sourceFeedbackUrl = props.feedbackUrls?.[SOURCE_CLICK_KEY];
+  const responseFeedbackUrls =
+    props.feedbackUrls?.[RESPONSE_FEEDBACK_KEY] ?? [];
+  const sourceFeedbackUrls = props.feedbackUrls?.[SOURCE_CLICK_KEY] ?? [];
   const isUser = type === "human";
   const [isLoading, setIsLoading] = useState(false);
   const [feedback, setFeedback] = useState<Feedback | null>(null);
@@ -161,23 +162,27 @@ export function ChatMessageBubble(props: {
   };
 
   const sendUserFeedback = async (score: number) => {
-    if (responseFeedbackUrl === undefined) {
+    if (responseFeedbackUrls.length === 0) {
       return;
     }
     if (isLoading) {
       return;
     }
     setIsLoading(true);
     try {
-      const data = await sendFeedback({
-        feedbackUrl: responseFeedbackUrl,
-        score,
-        feedbackId: feedback?.feedback_id,
-        comment,
-        isExplicit: true,
-      });
-      if (data.code === 200) {
-        setFeedback({ score, feedback_id: data.feedbackId });
+      const feedbackResponses = [];
+      for (const feedbackUrl of responseFeedbackUrls) {
+        const data = await sendFeedback({
+          feedbackUrl,
+          score,
+          feedbackId: feedback?.feedback_id,
+          comment,
+          isExplicit: true,
+        });
+        feedbackResponses.push(data);
+      }
+      if (feedbackResponses.every((response) => response.code === 200)) {
+        setFeedback({ score, feedback_id: feedbackResponses[0].feedbackId });
         score == 1 ? animateButton("upButton") : animateButton("downButton");
         if (comment) {
           setComment("");
@@ -272,7 +277,7 @@ export function ChatMessageBubble(props: {
                           filteredSources.map(() => false),
                         )
                       }
-                      feedbackUrl={sourceFeedbackUrl}
+                      feedbackUrls={sourceFeedbackUrls}
                     />
                   </Box>
                 ))}
@@ -306,7 +311,7 @@ export function ChatMessageBubble(props: {
               variant="outline"
               colorScheme={feedback === null ? "green" : "gray"}
               onClick={() => {
-                if (feedback === null && responseFeedbackUrl) {
+                if (feedback === null && responseFeedbackUrls) {
                   sendUserFeedback(1);
                   animateButton("upButton");
                 } else {
@@ -322,7 +327,7 @@ export function ChatMessageBubble(props: {
               variant="outline"
               colorScheme={feedback === null ? "red" : "gray"}
               onClick={() => {
-                if (feedback === null && responseFeedbackUrl) {
+                if (feedback === null && responseFeedbackUrls) {
                   sendUserFeedback(0);
                   animateButton("downButton");
                 } else {

frontend/app/components/SourceBubble.tsx

@@ -8,19 +8,19 @@ export function SourceBubble({
   highlighted,
   onMouseEnter,
   onMouseLeave,
-  feedbackUrl,
+  feedbackUrls,
 }: {
   source: Source;
   highlighted: boolean;
   onMouseEnter: () => any;
   onMouseLeave: () => any;
-  feedbackUrl?: string;
+  feedbackUrls: string[];
 }) {
   return (
     <Card
       onClick={async () => {
         window.open(source.url, "_blank");
-        if (feedbackUrl) {
+        for (const feedbackUrl of feedbackUrls) {
           await sendFeedback({
             feedbackUrl,
             value: source.url,

frontend/app/hooks/useStreamState.ts

@@ -10,7 +10,7 @@ export interface StreamState {
   status: "inflight" | "error" | "done";
   messages?: Message[];
   documents?: Document[];
-  feedbackUrls?: Record<string, string>;
+  feedbackUrls?: Record<string, string[]>;
 }
 
 export interface StreamStateProps {
@@ -68,8 +68,7 @@ export function useStreamState(): StreamStateProps {
         input: messages == null ? null : { messages },
         config,
         streamMode: ["messages", "values"],
-        signal: controller.signal,
-        feedbackKeys: [RESPONSE_FEEDBACK_KEY, SOURCE_CLICK_KEY],
+        signal: controller.signal
       });
 
       for await (const chunk of stream) {
@@ -94,6 +93,7 @@ export function useStreamState(): StreamStateProps {
               ...streamStates[threadId],
               status: "inflight",
               documents: data["documents"],
+              feedbackUrls: data["feedback_urls"]
             },
           }));
         } else if (chunk.event === "error") {
@@ -104,15 +104,6 @@ export function useStreamState(): StreamStateProps {
               status: "error",
             },
           }));
-        } else if (chunk.event === "feedback") {
-          setStreamStates((streamStates) => ({
-            ...streamStates,
-            [threadId]: {
-              ...streamStates[threadId],
-              feedbackUrls: chunk.data,
-              status: "inflight",
-            },
-          }));
         } else if (chunk.event === "end") {
           setStreamStates((streamStates) => ({
             ...streamStates,

frontend/app/hooks/useThreadList.ts

@@ -76,19 +76,25 @@ export function useThreadList(userId: string): ThreadListProps {
     setOffset((prevOffset) => prevOffset + PAGE_SIZE);
   }, [areThreadsLoading]);
 
-  const createThread = useCallback(async (name: string) => {
-    const saved = await client.threads.create({ metadata: { name, userId } });
-    dispatch({ type: "add", threads: [saved] });
-    return saved;
-  }, [userId]);
+  const createThread = useCallback(
+    async (name: string) => {
+      const saved = await client.threads.create({ metadata: { name, userId } });
+      dispatch({ type: "add", threads: [saved] });
+      return saved;
+    },
+    [userId],
+  );
 
-  const updateThread = useCallback(async (thread_id: string, name: string) => {
-    const saved = await client.threads.update(thread_id, {
-      metadata: { name, userId },
-    });
-    dispatch({ type: "add", threads: [saved] });
-    return saved;
-  }, [userId]);
+  const updateThread = useCallback(
+    async (thread_id: string, name: string) => {
+      const saved = await client.threads.update(thread_id, {
+        metadata: { name, userId },
+      });
+      dispatch({ type: "add", threads: [saved] });
+      return saved;
+    },
+    [userId],
+  );
 
   const deleteThread = useCallback(
     async (thread_id: string) => {