From a5afb7894b50f68e563eaa738a587b4ba7bed255 Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Tue, 19 Mar 2024 17:06:19 +0100 Subject: [PATCH 01/36] Add MistralAI provider Add RunnableWithKwargs Add first optimization --- libs/core/langchain_core/output_parsers/pydantic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/core/langchain_core/output_parsers/pydantic.py b/libs/core/langchain_core/output_parsers/pydantic.py index 73444d45af283..4fa525f8360bd 100644 --- a/libs/core/langchain_core/output_parsers/pydantic.py +++ b/libs/core/langchain_core/output_parsers/pydantic.py @@ -60,6 +60,10 @@ def parse_result( json_object = super().parse_result(result) return self._parse_obj(json_object) + def parse_result( + self, result: List[Generation], *, partial: bool = False + ) -> TBaseModel: + pass def parse(self, text: str) -> TBaseModel: return super().parse(text) From 1909eb312adabcc3e897921d40a6732202ee4183 Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Mon, 22 Apr 2024 12:09:35 +0200 Subject: [PATCH 02/36] Fix _amake_session() --- .../langchain_community/indexes/_sql_record_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/indexes/_sql_record_manager.py b/libs/community/langchain_community/indexes/_sql_record_manager.py index 544e828df2a82..c326998f4cbd4 100644 --- a/libs/community/langchain_community/indexes/_sql_record_manager.py +++ b/libs/community/langchain_community/indexes/_sql_record_manager.py @@ -175,7 +175,7 @@ def _make_session(self) -> Generator[Session, None, None]: async def _amake_session(self) -> AsyncGenerator[AsyncSession, None]: """Create a session and close it after use.""" - if not isinstance(self.session_factory, async_sessionmaker): + if not isinstance(self.engine, AsyncEngine): raise AssertionError("This method is not supported for sync engines.") async with self.session_factory() as session: From b329fe7bd5edfd10644bf06d718cb9fa62667c35 Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:40:17 -0700 Subject: [PATCH 03/36] core[patch]: Release 0.1.46 (#20891) --- libs/core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/pyproject.toml b/libs/core/pyproject.toml index 860ed2d26540d..ddc44a952e404 100644 --- a/libs/core/pyproject.toml +++ b/libs/core/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain-core" -version = "0.1.45" +version = "0.1.46" description = "Building applications with LLMs through composability" authors = [] license = "MIT" From 842d84f23955753d0caea97eb3d60bc05b93c513 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 25 Apr 2024 09:36:54 -0700 Subject: [PATCH 04/36] partner: Upstage quick documentation update (#20869) * Updating the provider docs page. The RAG example was meant to be moved to cookbook, but was merged by mistake. * Fix bug in Groundedness Check --------- Co-authored-by: JuHyung-Son Co-authored-by: Erick Friis --- .../docs/integrations/providers/upstage.ipynb | 81 +------------------ .../tools/groundedness_check.py | 18 +++-- ...ss_check.py => test_groundedness_check.py} | 18 +++++ .../unit_tests/test_groundedness_check.py | 2 + 4 files changed, 35 insertions(+), 84 deletions(-) rename libs/partners/upstage/tests/integration_tests/{test_groundness_check.py => test_groundedness_check.py} (53%) diff --git a/docs/docs/integrations/providers/upstage.ipynb b/docs/docs/integrations/providers/upstage.ipynb index e3571e63ae31d..c6885d532f628 100644 --- a/docs/docs/integrations/providers/upstage.ipynb +++ b/docs/docs/integrations/providers/upstage.ipynb @@ -53,7 +53,7 @@ "| Chat | Build assistants using Solar Mini Chat | `from langchain_upstage import ChatUpstage` | [Go](../../chat/upstage) |\n", "| Text Embedding | Embed strings to vectors | `from langchain_upstage import UpstageEmbeddings` | [Go](../../text_embedding/upstage) |\n", "| Groundedness Check | Verify groundedness of assistant's response | `from langchain_upstage import GroundednessCheck` | [Go](../../tools/upstage_groundedness_check) |\n", - "| Layout Analysis | Serialize documents with tables and figures | `from langchain_upstage import UpstageLayoutAnalysis` | [Go](../../document_loaders/upstage) |\n", + "| Layout Analysis | Serialize documents with tables and figures | `from langchain_upstage import UpstageLayoutAnalysisLoader` | [Go](../../document_loaders/upstage) |\n", "\n", "See [documentations](https://developers.upstage.ai/) for more details about the features." ] @@ -172,10 +172,10 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_upstage import UpstageLayoutAnalysis\n", + "from langchain_upstage import UpstageLayoutAnalysisLoader\n", "\n", "file_path = \"/PATH/TO/YOUR/FILE.pdf\"\n", - "layzer = UpstageLayoutAnalysis(file_path, split=\"page\")\n", + "layzer = UpstageLayoutAnalysisLoader(file_path, split=\"page\")\n", "\n", "# For improved memory efficiency, consider using the lazy_load method to load documents page by page.\n", "docs = layzer.load() # or layzer.lazy_load()\n", @@ -183,81 +183,6 @@ "for doc in docs[:3]:\n", " print(doc)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieval-Augmented Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import List\n", - "\n", - "from langchain_community.vectorstores import DocArrayInMemorySearch\n", - "from langchain_core.documents.base import Document\n", - "from langchain_core.output_parsers import StrOutputParser\n", - "from langchain_core.prompts import ChatPromptTemplate\n", - "from langchain_core.runnables import RunnablePassthrough\n", - "from langchain_core.runnables.base import RunnableSerializable\n", - "from langchain_upstage import (\n", - " ChatUpstage,\n", - " GroundednessCheck,\n", - " UpstageEmbeddings,\n", - " UpstageLayoutAnalysis,\n", - ")\n", - "\n", - "model = ChatUpstage()\n", - "\n", - "file_path = (\n", - " \"/PATH/TO/YOUR/FILE.pdf\" # e.g. \"libs/partners/upstage/tests/examples/solar.pdf\"\n", - ")\n", - "loader = UpstageLayoutAnalysis(file_path=file_path, split=\"element\")\n", - "docs = loader.load()\n", - "\n", - "vectorstore = DocArrayInMemorySearch.from_documents(docs, embedding=UpstageEmbeddings())\n", - "retriever = vectorstore.as_retriever()\n", - "\n", - "template = \"\"\"Answer the question based only on the following context:\n", - "{context}\n", - "\n", - "Question: {question}\n", - "\"\"\"\n", - "prompt = ChatPromptTemplate.from_template(template)\n", - "output_parser = StrOutputParser()\n", - "\n", - "question = \"ASK ANY QUESTION HERE.\" # e.g. \"How many parameters in SOLAR model?\"\n", - "\n", - "retrieved_docs = retriever.get_relevant_documents(question)\n", - "\n", - "groundedness_check = GroundednessCheck()\n", - "groundedness = \"\"\n", - "while groundedness != \"grounded\":\n", - " chain: RunnableSerializable = RunnablePassthrough() | prompt | model | output_parser\n", - "\n", - " result = chain.invoke(\n", - " {\n", - " \"context\": retrieved_docs,\n", - " \"question\": question,\n", - " }\n", - " )\n", - "\n", - " # convert all Documents to string\n", - " def formatDocumentsAsString(docs: List[Document]) -> str:\n", - " return \"\\n\".join([doc.page_content for doc in docs])\n", - "\n", - " groundedness = groundedness_check.run(\n", - " {\n", - " \"context\": formatDocumentsAsString(retrieved_docs),\n", - " \"assistant_message\": result,\n", - " }\n", - " ).content" - ] } ], "metadata": { diff --git a/libs/partners/upstage/langchain_upstage/tools/groundedness_check.py b/libs/partners/upstage/langchain_upstage/tools/groundedness_check.py index d423e7aa43d3e..68d9453b40ec6 100644 --- a/libs/partners/upstage/langchain_upstage/tools/groundedness_check.py +++ b/libs/partners/upstage/langchain_upstage/tools/groundedness_check.py @@ -1,5 +1,5 @@ import os -from typing import Literal, Optional, Type, Union +from typing import Any, Literal, Optional, Type, Union from langchain_core.callbacks import ( AsyncCallbackManagerForToolRun, @@ -8,6 +8,7 @@ from langchain_core.messages import AIMessage, HumanMessage from langchain_core.pydantic_v1 import BaseModel, Field, SecretStr from langchain_core.tools import BaseTool +from langchain_core.utils import convert_to_secret_str from langchain_upstage import ChatUpstage @@ -51,11 +52,14 @@ class GroundednessCheck(BaseTool): args_schema: Type[BaseModel] = GroundednessCheckInput - def __init__(self, upstage_api_key: Optional[SecretStr] = None): + def __init__(self, **kwargs: Any) -> None: + upstage_api_key = kwargs.get("upstage_api_key", None) + if not upstage_api_key: + upstage_api_key = kwargs.get("api_key", None) if not upstage_api_key: upstage_api_key = SecretStr(os.getenv("UPSTAGE_API_KEY", "")) - else: - upstage_api_key = upstage_api_key + upstage_api_key = convert_to_secret_str(upstage_api_key) + if ( not upstage_api_key or not upstage_api_key.get_secret_value() @@ -76,7 +80,9 @@ def _run( run_manager: Optional[CallbackManagerForToolRun] = None, ) -> Union[str, Literal["grounded", "notGrounded", "notSure"]]: """Use the tool.""" - response = self.api_wrapper.invoke([HumanMessage(context), AIMessage(query)]) + response = self.api_wrapper.invoke( + [HumanMessage(context), AIMessage(query)], stream=False + ) return str(response.content) async def _arun( @@ -86,6 +92,6 @@ async def _arun( run_manager: Optional[AsyncCallbackManagerForToolRun] = None, ) -> Union[str, Literal["grounded", "notGrounded", "notSure"]]: response = await self.api_wrapper.ainvoke( - [HumanMessage(context), AIMessage(query)] + [HumanMessage(context), AIMessage(query)], stream=False ) return str(response.content) diff --git a/libs/partners/upstage/tests/integration_tests/test_groundness_check.py b/libs/partners/upstage/tests/integration_tests/test_groundedness_check.py similarity index 53% rename from libs/partners/upstage/tests/integration_tests/test_groundness_check.py rename to libs/partners/upstage/tests/integration_tests/test_groundedness_check.py index ccb0a1c94bc35..29059f47e132d 100644 --- a/libs/partners/upstage/tests/integration_tests/test_groundness_check.py +++ b/libs/partners/upstage/tests/integration_tests/test_groundedness_check.py @@ -1,3 +1,8 @@ +import os + +import openai +import pytest + from langchain_upstage import GroundednessCheck @@ -8,6 +13,19 @@ def test_langchain_upstage_groundedness_check() -> None: assert output in ["grounded", "notGrounded", "notSure"] + api_key = os.environ.get("UPSTAGE_API_KEY", None) + + tool = GroundednessCheck(upstage_api_key=api_key) + output = tool.run({"context": "foo bar", "query": "bar foo"}) + + assert output in ["grounded", "notGrounded", "notSure"] + + +def test_langchain_upstage_groundedness_check_fail_with_wrong_api_key() -> None: + tool = GroundednessCheck(api_key="wrong-key") + with pytest.raises(openai.AuthenticationError): + tool.run({"context": "foo bar", "query": "bar foo"}) + async def test_langchain_upstage_groundedness_check_async() -> None: """Test Upstage Groundedness Check asynchronous.""" diff --git a/libs/partners/upstage/tests/unit_tests/test_groundedness_check.py b/libs/partners/upstage/tests/unit_tests/test_groundedness_check.py index cba5f1bd9e6d9..4891dec01b0dd 100644 --- a/libs/partners/upstage/tests/unit_tests/test_groundedness_check.py +++ b/libs/partners/upstage/tests/unit_tests/test_groundedness_check.py @@ -8,3 +8,5 @@ def test_initialization() -> None: """Test embedding model initialization.""" GroundednessCheck() + GroundednessCheck(upstage_api_key="key") + GroundednessCheck(api_key="key") From a5643858962779504363aff9240ddf87e8cbeb09 Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:40:26 -0700 Subject: [PATCH 05/36] core[minor], langchain[patch], community[patch]: mv StructuredQuery (#20849) mv StructuredQuery to core --- .../vectorstores/tencentvectordb.py | 2 +- libs/core/langchain_core/structured_query.py | 143 +++++++++++++++ .../chains/query_constructor/base.py | 6 +- .../langchain/chains/query_constructor/ir.py | 163 +++--------------- .../chains/query_constructor/parser.py | 2 +- .../retrievers/self_query/astradb.py | 2 +- .../langchain/retrievers/self_query/base.py | 2 +- .../langchain/retrievers/self_query/chroma.py | 2 +- .../retrievers/self_query/dashvector.py | 2 +- .../self_query/databricks_vector_search.py | 2 +- .../retrievers/self_query/deeplake.py | 2 +- .../langchain/retrievers/self_query/dingo.py | 2 +- .../retrievers/self_query/elasticsearch.py | 2 +- .../langchain/retrievers/self_query/milvus.py | 2 +- .../retrievers/self_query/mongodb_atlas.py | 2 +- .../retrievers/self_query/myscale.py | 2 +- .../retrievers/self_query/opensearch.py | 2 +- .../retrievers/self_query/pgvector.py | 2 +- .../retrievers/self_query/pinecone.py | 2 +- .../langchain/retrievers/self_query/qdrant.py | 2 +- .../langchain/retrievers/self_query/redis.py | 3 +- .../retrievers/self_query/supabase.py | 2 +- .../retrievers/self_query/tencentvectordb.py | 2 +- .../retrievers/self_query/timescalevector.py | 2 +- .../retrievers/self_query/vectara.py | 2 +- .../retrievers/self_query/weaviate.py | 2 +- .../chains/query_constructor/test_parser.py | 4 +- .../retrievers/self_query/test_astradb.py | 3 +- .../retrievers/self_query/test_base.py | 4 +- .../retrievers/self_query/test_chroma.py | 3 +- .../retrievers/self_query/test_dashvector.py | 4 +- .../test_databricks_vector_search.py | 4 +- .../retrievers/self_query/test_deeplake.py | 3 +- .../retrievers/self_query/test_dingo.py | 3 +- .../self_query/test_elasticsearch.py | 3 +- .../retrievers/self_query/test_milvus.py | 4 +- .../self_query/test_mongodb_atlas.py | 3 +- .../retrievers/self_query/test_myscale.py | 4 +- .../retrievers/self_query/test_opensearch.py | 3 +- .../retrievers/self_query/test_pgvector.py | 4 +- .../retrievers/self_query/test_pinecone.py | 3 +- .../retrievers/self_query/test_redis.py | 4 +- .../retrievers/self_query/test_supabase.py | 3 +- .../self_query/test_tencentvectordb.py | 3 +- .../self_query/test_timescalevector.py | 4 +- .../retrievers/self_query/test_vectara.py | 3 +- .../retrievers/self_query/test_weaviate.py | 3 +- 47 files changed, 232 insertions(+), 199 deletions(-) create mode 100644 libs/core/langchain_core/structured_query.py diff --git a/libs/community/langchain_community/vectorstores/tencentvectordb.py b/libs/community/langchain_community/vectorstores/tencentvectordb.py index 53036b4802e02..d9c070f177db2 100644 --- a/libs/community/langchain_community/vectorstores/tencentvectordb.py +++ b/libs/community/langchain_community/vectorstores/tencentvectordb.py @@ -110,11 +110,11 @@ def translate_filter( lc_filter: str, allowed_fields: Optional[Sequence[str]] = None ) -> str: from langchain.chains.query_constructor.base import fix_filter_directive - from langchain.chains.query_constructor.ir import FilterDirective from langchain.chains.query_constructor.parser import get_parser from langchain.retrievers.self_query.tencentvectordb import ( TencentVectorDBTranslator, ) + from langchain_core.structured_query import FilterDirective tvdb_visitor = TencentVectorDBTranslator(allowed_fields) flt = cast( diff --git a/libs/core/langchain_core/structured_query.py b/libs/core/langchain_core/structured_query.py new file mode 100644 index 0000000000000..eb61cea6b507f --- /dev/null +++ b/libs/core/langchain_core/structured_query.py @@ -0,0 +1,143 @@ +"""Internal representation of a structured query language.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import Enum +from typing import Any, List, Optional, Sequence, Union + +from langchain_core.pydantic_v1 import BaseModel + + +class Visitor(ABC): + """Defines interface for IR translation using visitor pattern.""" + + allowed_comparators: Optional[Sequence[Comparator]] = None + allowed_operators: Optional[Sequence[Operator]] = None + + def _validate_func(self, func: Union[Operator, Comparator]) -> None: + if isinstance(func, Operator) and self.allowed_operators is not None: + if func not in self.allowed_operators: + raise ValueError( + f"Received disallowed operator {func}. Allowed " + f"comparators are {self.allowed_operators}" + ) + if isinstance(func, Comparator) and self.allowed_comparators is not None: + if func not in self.allowed_comparators: + raise ValueError( + f"Received disallowed comparator {func}. Allowed " + f"comparators are {self.allowed_comparators}" + ) + + @abstractmethod + def visit_operation(self, operation: Operation) -> Any: + """Translate an Operation.""" + + @abstractmethod + def visit_comparison(self, comparison: Comparison) -> Any: + """Translate a Comparison.""" + + @abstractmethod + def visit_structured_query(self, structured_query: StructuredQuery) -> Any: + """Translate a StructuredQuery.""" + + +def _to_snake_case(name: str) -> str: + """Convert a name into snake_case.""" + snake_case = "" + for i, char in enumerate(name): + if char.isupper() and i != 0: + snake_case += "_" + char.lower() + else: + snake_case += char.lower() + return snake_case + + +class Expr(BaseModel): + """Base class for all expressions.""" + + def accept(self, visitor: Visitor) -> Any: + """Accept a visitor. + + Args: + visitor: visitor to accept + + Returns: + result of visiting + """ + return getattr(visitor, f"visit_{_to_snake_case(self.__class__.__name__)}")( + self + ) + + +class Operator(str, Enum): + """Enumerator of the operations.""" + + AND = "and" + OR = "or" + NOT = "not" + + +class Comparator(str, Enum): + """Enumerator of the comparison operators.""" + + EQ = "eq" + NE = "ne" + GT = "gt" + GTE = "gte" + LT = "lt" + LTE = "lte" + CONTAIN = "contain" + LIKE = "like" + IN = "in" + NIN = "nin" + + +class FilterDirective(Expr, ABC): + """A filtering expression.""" + + +class Comparison(FilterDirective): + """A comparison to a value.""" + + comparator: Comparator + attribute: str + value: Any + + def __init__( + self, comparator: Comparator, attribute: str, value: Any, **kwargs: Any + ) -> None: + super().__init__( + comparator=comparator, attribute=attribute, value=value, **kwargs + ) + + +class Operation(FilterDirective): + """A logical operation over other directives.""" + + operator: Operator + arguments: List[FilterDirective] + + def __init__( + self, operator: Operator, arguments: List[FilterDirective], **kwargs: Any + ): + super().__init__(operator=operator, arguments=arguments, **kwargs) + + +class StructuredQuery(Expr): + """A structured query.""" + + query: str + """Query string.""" + filter: Optional[FilterDirective] + """Filtering expression.""" + limit: Optional[int] + """Limit on the number of results.""" + + def __init__( + self, + query: str, + filter: Optional[FilterDirective], + limit: Optional[int] = None, + **kwargs: Any, + ): + super().__init__(query=query, filter=filter, limit=limit, **kwargs) diff --git a/libs/langchain/langchain/chains/query_constructor/base.py b/libs/langchain/langchain/chains/query_constructor/base.py index a0cd9ad88543f..9b7e3d37daef6 100644 --- a/libs/langchain/langchain/chains/query_constructor/base.py +++ b/libs/langchain/langchain/chains/query_constructor/base.py @@ -11,9 +11,7 @@ from langchain_core.prompts import BasePromptTemplate from langchain_core.prompts.few_shot import FewShotPromptTemplate from langchain_core.runnables import Runnable - -from langchain.chains.llm import LLMChain -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, FilterDirective, @@ -21,6 +19,8 @@ Operator, StructuredQuery, ) + +from langchain.chains.llm import LLMChain from langchain.chains.query_constructor.parser import get_parser from langchain.chains.query_constructor.prompt import ( DEFAULT_EXAMPLES, diff --git a/libs/langchain/langchain/chains/query_constructor/ir.py b/libs/langchain/langchain/chains/query_constructor/ir.py index eb61cea6b507f..6c854ae59d4b2 100644 --- a/libs/langchain/langchain/chains/query_constructor/ir.py +++ b/libs/langchain/langchain/chains/query_constructor/ir.py @@ -1,143 +1,22 @@ """Internal representation of a structured query language.""" -from __future__ import annotations - -from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, List, Optional, Sequence, Union - -from langchain_core.pydantic_v1 import BaseModel - - -class Visitor(ABC): - """Defines interface for IR translation using visitor pattern.""" - - allowed_comparators: Optional[Sequence[Comparator]] = None - allowed_operators: Optional[Sequence[Operator]] = None - - def _validate_func(self, func: Union[Operator, Comparator]) -> None: - if isinstance(func, Operator) and self.allowed_operators is not None: - if func not in self.allowed_operators: - raise ValueError( - f"Received disallowed operator {func}. Allowed " - f"comparators are {self.allowed_operators}" - ) - if isinstance(func, Comparator) and self.allowed_comparators is not None: - if func not in self.allowed_comparators: - raise ValueError( - f"Received disallowed comparator {func}. Allowed " - f"comparators are {self.allowed_comparators}" - ) - - @abstractmethod - def visit_operation(self, operation: Operation) -> Any: - """Translate an Operation.""" - - @abstractmethod - def visit_comparison(self, comparison: Comparison) -> Any: - """Translate a Comparison.""" - - @abstractmethod - def visit_structured_query(self, structured_query: StructuredQuery) -> Any: - """Translate a StructuredQuery.""" - - -def _to_snake_case(name: str) -> str: - """Convert a name into snake_case.""" - snake_case = "" - for i, char in enumerate(name): - if char.isupper() and i != 0: - snake_case += "_" + char.lower() - else: - snake_case += char.lower() - return snake_case - - -class Expr(BaseModel): - """Base class for all expressions.""" - - def accept(self, visitor: Visitor) -> Any: - """Accept a visitor. - - Args: - visitor: visitor to accept - - Returns: - result of visiting - """ - return getattr(visitor, f"visit_{_to_snake_case(self.__class__.__name__)}")( - self - ) - - -class Operator(str, Enum): - """Enumerator of the operations.""" - - AND = "and" - OR = "or" - NOT = "not" - - -class Comparator(str, Enum): - """Enumerator of the comparison operators.""" - - EQ = "eq" - NE = "ne" - GT = "gt" - GTE = "gte" - LT = "lt" - LTE = "lte" - CONTAIN = "contain" - LIKE = "like" - IN = "in" - NIN = "nin" - - -class FilterDirective(Expr, ABC): - """A filtering expression.""" - - -class Comparison(FilterDirective): - """A comparison to a value.""" - - comparator: Comparator - attribute: str - value: Any - - def __init__( - self, comparator: Comparator, attribute: str, value: Any, **kwargs: Any - ) -> None: - super().__init__( - comparator=comparator, attribute=attribute, value=value, **kwargs - ) - - -class Operation(FilterDirective): - """A logical operation over other directives.""" - - operator: Operator - arguments: List[FilterDirective] - - def __init__( - self, operator: Operator, arguments: List[FilterDirective], **kwargs: Any - ): - super().__init__(operator=operator, arguments=arguments, **kwargs) - - -class StructuredQuery(Expr): - """A structured query.""" - - query: str - """Query string.""" - filter: Optional[FilterDirective] - """Filtering expression.""" - limit: Optional[int] - """Limit on the number of results.""" - - def __init__( - self, - query: str, - filter: Optional[FilterDirective], - limit: Optional[int] = None, - **kwargs: Any, - ): - super().__init__(query=query, filter=filter, limit=limit, **kwargs) +from langchain_core.structured_query import ( + Comparator, + Comparison, + Expr, + FilterDirective, + Operation, + Operator, + StructuredQuery, + Visitor, +) + +__all__ = [ + "Visitor", + "Expr", + "Operator", + "Comparator", + "FilterDirective", + "Comparison", + "Operation", + "StructuredQuery", +] diff --git a/libs/langchain/langchain/chains/query_constructor/parser.py b/libs/langchain/langchain/chains/query_constructor/parser.py index 26c5360d59feb..04e457ddb9b9f 100644 --- a/libs/langchain/langchain/chains/query_constructor/parser.py +++ b/libs/langchain/langchain/chains/query_constructor/parser.py @@ -17,7 +17,7 @@ def v_args(*args: Any, **kwargs: Any) -> Any: # type: ignore Transformer = object # type: ignore Lark = object # type: ignore -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, FilterDirective, diff --git a/libs/langchain/langchain/retrievers/self_query/astradb.py b/libs/langchain/langchain/retrievers/self_query/astradb.py index 0b8d3ab800f7c..006972935ffc2 100644 --- a/libs/langchain/langchain/retrievers/self_query/astradb.py +++ b/libs/langchain/langchain/retrievers/self_query/astradb.py @@ -1,7 +1,7 @@ """Logic for converting internal query language to a valid AstraDB query.""" from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/base.py b/libs/langchain/langchain/retrievers/self_query/base.py index 14773ee1e57e7..3295a593b8efa 100644 --- a/libs/langchain/langchain/retrievers/self_query/base.py +++ b/libs/langchain/langchain/retrievers/self_query/base.py @@ -38,10 +38,10 @@ from langchain_core.pydantic_v1 import Field, root_validator from langchain_core.retrievers import BaseRetriever from langchain_core.runnables import Runnable +from langchain_core.structured_query import StructuredQuery, Visitor from langchain_core.vectorstores import VectorStore from langchain.chains.query_constructor.base import load_query_constructor_runnable -from langchain.chains.query_constructor.ir import StructuredQuery, Visitor from langchain.chains.query_constructor.schema import AttributeInfo from langchain.retrievers.self_query.astradb import AstraDBTranslator from langchain.retrievers.self_query.chroma import ChromaTranslator diff --git a/libs/langchain/langchain/retrievers/self_query/chroma.py b/libs/langchain/langchain/retrievers/self_query/chroma.py index 8c9a79b12d35e..6f766e7e138a9 100644 --- a/libs/langchain/langchain/retrievers/self_query/chroma.py +++ b/libs/langchain/langchain/retrievers/self_query/chroma.py @@ -1,6 +1,6 @@ from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/dashvector.py b/libs/langchain/langchain/retrievers/self_query/dashvector.py index 24ae50239adf8..c1d63d1aaeacb 100644 --- a/libs/langchain/langchain/retrievers/self_query/dashvector.py +++ b/libs/langchain/langchain/retrievers/self_query/dashvector.py @@ -1,7 +1,7 @@ """Logic for converting internal query language to a valid DashVector query.""" from typing import Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/databricks_vector_search.py b/libs/langchain/langchain/retrievers/self_query/databricks_vector_search.py index f8ef053f5a55c..78dc17c7fc7e8 100644 --- a/libs/langchain/langchain/retrievers/self_query/databricks_vector_search.py +++ b/libs/langchain/langchain/retrievers/self_query/databricks_vector_search.py @@ -2,7 +2,7 @@ from itertools import chain from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/deeplake.py b/libs/langchain/langchain/retrievers/self_query/deeplake.py index 030933b32e8b0..d7e2ab87d6a3b 100644 --- a/libs/langchain/langchain/retrievers/self_query/deeplake.py +++ b/libs/langchain/langchain/retrievers/self_query/deeplake.py @@ -1,7 +1,7 @@ """Logic for converting internal query language to a valid Chroma query.""" from typing import Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/dingo.py b/libs/langchain/langchain/retrievers/self_query/dingo.py index 76e9ef862d3ad..6c2402f65c913 100644 --- a/libs/langchain/langchain/retrievers/self_query/dingo.py +++ b/libs/langchain/langchain/retrievers/self_query/dingo.py @@ -1,6 +1,6 @@ from typing import Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/elasticsearch.py b/libs/langchain/langchain/retrievers/self_query/elasticsearch.py index 7c2f7671a5c9b..d07c284b1256d 100644 --- a/libs/langchain/langchain/retrievers/self_query/elasticsearch.py +++ b/libs/langchain/langchain/retrievers/self_query/elasticsearch.py @@ -1,6 +1,6 @@ from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/milvus.py b/libs/langchain/langchain/retrievers/self_query/milvus.py index dbc61f6f71203..6fb1cc5c4e4ab 100644 --- a/libs/langchain/langchain/retrievers/self_query/milvus.py +++ b/libs/langchain/langchain/retrievers/self_query/milvus.py @@ -1,7 +1,7 @@ """Logic for converting internal query language to a valid Milvus query.""" from typing import Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/mongodb_atlas.py b/libs/langchain/langchain/retrievers/self_query/mongodb_atlas.py index a10e7b58fa933..ebef2163beb0a 100644 --- a/libs/langchain/langchain/retrievers/self_query/mongodb_atlas.py +++ b/libs/langchain/langchain/retrievers/self_query/mongodb_atlas.py @@ -1,7 +1,7 @@ """Logic for converting internal query language to a valid MongoDB Atlas query.""" from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/myscale.py b/libs/langchain/langchain/retrievers/self_query/myscale.py index 8b1afaf8b9de6..50a74c568b6ea 100644 --- a/libs/langchain/langchain/retrievers/self_query/myscale.py +++ b/libs/langchain/langchain/retrievers/self_query/myscale.py @@ -1,7 +1,7 @@ import re from typing import Any, Callable, Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/opensearch.py b/libs/langchain/langchain/retrievers/self_query/opensearch.py index bb27cddd0b481..e01ec66639ea5 100644 --- a/libs/langchain/langchain/retrievers/self_query/opensearch.py +++ b/libs/langchain/langchain/retrievers/self_query/opensearch.py @@ -1,6 +1,6 @@ from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/pgvector.py b/libs/langchain/langchain/retrievers/self_query/pgvector.py index ebe5bf42ffc43..5fea65b01c810 100644 --- a/libs/langchain/langchain/retrievers/self_query/pgvector.py +++ b/libs/langchain/langchain/retrievers/self_query/pgvector.py @@ -1,6 +1,6 @@ from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/pinecone.py b/libs/langchain/langchain/retrievers/self_query/pinecone.py index 80c401dd95fe1..99c42f393bf93 100644 --- a/libs/langchain/langchain/retrievers/self_query/pinecone.py +++ b/libs/langchain/langchain/retrievers/self_query/pinecone.py @@ -1,6 +1,6 @@ from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/qdrant.py b/libs/langchain/langchain/retrievers/self_query/qdrant.py index c99287751a158..f4c3298b6677d 100644 --- a/libs/langchain/langchain/retrievers/self_query/qdrant.py +++ b/libs/langchain/langchain/retrievers/self_query/qdrant.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/redis.py b/libs/langchain/langchain/retrievers/self_query/redis.py index cefe576182e4a..56d3a17b555d5 100644 --- a/libs/langchain/langchain/retrievers/self_query/redis.py +++ b/libs/langchain/langchain/retrievers/self_query/redis.py @@ -12,8 +12,7 @@ RedisText, ) from langchain_community.vectorstores.redis.schema import RedisModel - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/supabase.py b/libs/langchain/langchain/retrievers/self_query/supabase.py index 0ac0f183d787a..63794cf3787cf 100644 --- a/libs/langchain/langchain/retrievers/self_query/supabase.py +++ b/libs/langchain/langchain/retrievers/self_query/supabase.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/tencentvectordb.py b/libs/langchain/langchain/retrievers/self_query/tencentvectordb.py index c01f2b3e9bf31..9bed9c1cddb2b 100644 --- a/libs/langchain/langchain/retrievers/self_query/tencentvectordb.py +++ b/libs/langchain/langchain/retrievers/self_query/tencentvectordb.py @@ -2,7 +2,7 @@ from typing import Optional, Sequence, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/timescalevector.py b/libs/langchain/langchain/retrievers/self_query/timescalevector.py index 3d417578fe577..bfac120bded7e 100644 --- a/libs/langchain/langchain/retrievers/self_query/timescalevector.py +++ b/libs/langchain/langchain/retrievers/self_query/timescalevector.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/vectara.py b/libs/langchain/langchain/retrievers/self_query/vectara.py index 02d64f04708bb..24886a1af99c4 100644 --- a/libs/langchain/langchain/retrievers/self_query/vectara.py +++ b/libs/langchain/langchain/retrievers/self_query/vectara.py @@ -1,6 +1,6 @@ from typing import Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/langchain/retrievers/self_query/weaviate.py b/libs/langchain/langchain/retrievers/self_query/weaviate.py index 1db80b9ba00f5..2e5e3e691e2e9 100644 --- a/libs/langchain/langchain/retrievers/self_query/weaviate.py +++ b/libs/langchain/langchain/retrievers/self_query/weaviate.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Dict, Tuple, Union -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, diff --git a/libs/langchain/tests/unit_tests/chains/query_constructor/test_parser.py b/libs/langchain/tests/unit_tests/chains/query_constructor/test_parser.py index 0b83e6ef01a51..db2c7ca1cd0c5 100644 --- a/libs/langchain/tests/unit_tests/chains/query_constructor/test_parser.py +++ b/libs/langchain/tests/unit_tests/chains/query_constructor/test_parser.py @@ -3,13 +3,13 @@ import lark import pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, ) + from langchain.chains.query_constructor.parser import get_parser DEFAULT_PARSER = get_parser() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_astradb.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_astradb.py index 78fc47603a434..8112871607152 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_astradb.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_astradb.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.astradb import AstraDBTranslator DEFAULT_TRANSLATOR = AstraDBTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_base.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_base.py index 1eeab1dfb8eac..03c5a6fc00a7f 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_base.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_base.py @@ -6,8 +6,7 @@ CallbackManagerForRetrieverRun, ) from langchain_core.documents import Document - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, @@ -15,6 +14,7 @@ StructuredQuery, Visitor, ) + from langchain.chains.query_constructor.schema import AttributeInfo from langchain.retrievers import SelfQueryRetriever from tests.unit_tests.indexes.test_indexing import InMemoryVectorStore diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_chroma.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_chroma.py index 47c74cb5f1986..45fed14f564f3 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_chroma.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_chroma.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.chroma import ChromaTranslator DEFAULT_TRANSLATOR = ChromaTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_dashvector.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_dashvector.py index 137de85fdc0ed..29c50c36f02c6 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_dashvector.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_dashvector.py @@ -1,13 +1,13 @@ from typing import Any, Tuple import pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, ) + from langchain.retrievers.self_query.dashvector import DashvectorTranslator DEFAULT_TRANSLATOR = DashvectorTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_databricks_vector_search.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_databricks_vector_search.py index 8d5937ab9d5e3..4b1728ce41ae2 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_databricks_vector_search.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_databricks_vector_search.py @@ -1,14 +1,14 @@ from typing import Any, Dict, Tuple import pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.databricks_vector_search import ( DatabricksVectorSearchTranslator, ) diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_deeplake.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_deeplake.py index 055d434d3011d..bbc61f7bdea5f 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_deeplake.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_deeplake.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.deeplake import DeepLakeTranslator DEFAULT_TRANSLATOR = DeepLakeTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_dingo.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_dingo.py index fa6e52b37fa0b..b853ff081f113 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_dingo.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_dingo.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.dingo import DingoDBTranslator DEFAULT_TRANSLATOR = DingoDBTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py index 519f366f03146..56a03283773f4 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_elasticsearch.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.elasticsearch import ElasticsearchTranslator DEFAULT_TRANSLATOR = ElasticsearchTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_milvus.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_milvus.py index 4a96c18e274a6..9e5e994908d46 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_milvus.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_milvus.py @@ -1,14 +1,14 @@ from typing import Any, Dict, Tuple import pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.milvus import MilvusTranslator DEFAULT_TRANSLATOR = MilvusTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_mongodb_atlas.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_mongodb_atlas.py index 9eceec6b70f4d..6827c566837f4 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_mongodb_atlas.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_mongodb_atlas.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.mongodb_atlas import MongoDBAtlasTranslator DEFAULT_TRANSLATOR = MongoDBAtlasTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_myscale.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_myscale.py index ce54c83349de1..ca8b878ea8688 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_myscale.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_myscale.py @@ -1,14 +1,14 @@ from typing import Any, Dict, Tuple import pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.myscale import MyScaleTranslator DEFAULT_TRANSLATOR = MyScaleTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_opensearch.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_opensearch.py index 93b6629ecb01e..0b18f1180ec2a 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_opensearch.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_opensearch.py @@ -1,10 +1,11 @@ -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.opensearch import OpenSearchTranslator DEFAULT_TRANSLATOR = OpenSearchTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_pgvector.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_pgvector.py index 451508bdab3d1..43bdae894993d 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_pgvector.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_pgvector.py @@ -1,14 +1,14 @@ from typing import Dict, Tuple import pytest as pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.pgvector import PGVectorTranslator DEFAULT_TRANSLATOR = PGVectorTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_pinecone.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_pinecone.py index 7e818dcec5b61..cf175667b349a 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_pinecone.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_pinecone.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.pinecone import PineconeTranslator DEFAULT_TRANSLATOR = PineconeTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_redis.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_redis.py index 490e247f93f6d..44801f6d8e165 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_redis.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_redis.py @@ -13,14 +13,14 @@ TagFieldSchema, TextFieldSchema, ) - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.redis import RedisTranslator diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_supabase.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_supabase.py index de9b04fabf8fe..8fc45dd2ee17a 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_supabase.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_supabase.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.supabase import SupabaseVectorTranslator DEFAULT_TRANSLATOR = SupabaseVectorTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_tencentvectordb.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_tencentvectordb.py index a634689caadec..d7ee4a8c301b9 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_tencentvectordb.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_tencentvectordb.py @@ -1,10 +1,11 @@ -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.tencentvectordb import TencentVectorDBTranslator diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_timescalevector.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_timescalevector.py index 35dd328e06ac4..e813d63e2ed47 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_timescalevector.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_timescalevector.py @@ -1,14 +1,14 @@ from typing import Dict, Tuple import pytest as pytest - -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.timescalevector import TimescaleVectorTranslator DEFAULT_TRANSLATOR = TimescaleVectorTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_vectara.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_vectara.py index 05c15f26ac74d..fe41cdb214877 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_vectara.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_vectara.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.vectara import VectaraTranslator DEFAULT_TRANSLATOR = VectaraTranslator() diff --git a/libs/langchain/tests/unit_tests/retrievers/self_query/test_weaviate.py b/libs/langchain/tests/unit_tests/retrievers/self_query/test_weaviate.py index a6489a203aac0..999c7c6fb8e16 100644 --- a/libs/langchain/tests/unit_tests/retrievers/self_query/test_weaviate.py +++ b/libs/langchain/tests/unit_tests/retrievers/self_query/test_weaviate.py @@ -1,12 +1,13 @@ from typing import Dict, Tuple -from langchain.chains.query_constructor.ir import ( +from langchain_core.structured_query import ( Comparator, Comparison, Operation, Operator, StructuredQuery, ) + from langchain.retrievers.self_query.weaviate import WeaviateTranslator DEFAULT_TRANSLATOR = WeaviateTranslator() From 435bc5612b7f648291aaea0de890daba9d5e9b47 Mon Sep 17 00:00:00 2001 From: YISH Date: Fri, 26 Apr 2024 00:45:52 +0800 Subject: [PATCH 06/36] openai[patch]: Allow disablling safe_len_embeddings(OpenAIEmbeddings) (#19743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI API compatible server may not support `safe_len_embedding`， use `disable_safe_len_embeddings=True` to disable it. --------- Co-authored-by: Bagatur --- .../langchain_openai/embeddings/base.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libs/partners/openai/langchain_openai/embeddings/base.py b/libs/partners/openai/langchain_openai/embeddings/base.py index b64388055bd25..8ba966c47e29f 100644 --- a/libs/partners/openai/langchain_openai/embeddings/base.py +++ b/libs/partners/openai/langchain_openai/embeddings/base.py @@ -129,6 +129,9 @@ class OpenAIEmbeddings(BaseModel, Embeddings): http_async_client: Union[Any, None] = None """Optional httpx.AsyncClient. Only used for async invocations. Must specify http_client as well if you'd like a custom client for sync invocations.""" + check_embedding_ctx_length: bool = True + """Whether to check the token length of inputs and automatically split inputs + longer than embedding_ctx_length.""" class Config: """Configuration for this pydantic object.""" @@ -511,6 +514,18 @@ def embed_documents( Returns: List of embeddings, one for each text. """ + if not self.check_embedding_ctx_length: + embeddings: List[List[float]] = [] + for text in texts: + response = self.client.create( + input=text, + **self._invocation_params, + ) + if not isinstance(response, dict): + response = response.dict() + embeddings.extend(r["embedding"] for r in response["data"]) + return embeddings + # NOTE: to keep things simple, we assume the list may contain texts longer # than the maximum context and use length-safe embedding function. engine = cast(str, self.deployment) @@ -529,6 +544,18 @@ async def aembed_documents( Returns: List of embeddings, one for each text. """ + if not self.check_embedding_ctx_length: + embeddings: List[List[float]] = [] + for text in texts: + response = await self.async_client.create( + input=text, + **self._invocation_params, + ) + if not isinstance(response, dict): + response = response.dict() + embeddings.extend(r["embedding"] for r in response["data"]) + return embeddings + # NOTE: to keep things simple, we assume the list may contain texts longer # than the maximum context and use length-safe embedding function. engine = cast(str, self.deployment) From 123f19e4fecfa5d2bec06b404be22e0b1ffdf6be Mon Sep 17 00:00:00 2001 From: Jason_Chen <820542443@qq.com> Date: Fri, 26 Apr 2024 01:04:04 +0800 Subject: [PATCH 07/36] community[patch]: add BeautifulSoupTransformer remove_unwanted_classnames method (#20467) Add the remove_unwanted_classnames method to the BeautifulSoupTransformer class, which can filter more effectively. --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> Co-authored-by: Bagatur --- .../beautiful_soup_transformer.py | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/libs/community/langchain_community/document_transformers/beautiful_soup_transformer.py b/libs/community/langchain_community/document_transformers/beautiful_soup_transformer.py index b70af98789851..901aed2bdf270 100644 --- a/libs/community/langchain_community/document_transformers/beautiful_soup_transformer.py +++ b/libs/community/langchain_community/document_transformers/beautiful_soup_transformer.py @@ -1,4 +1,4 @@ -from typing import Any, Iterator, List, Sequence, cast +from typing import Any, Iterator, List, Sequence, Tuple, Union, cast from langchain_core.documents import BaseDocumentTransformer, Document @@ -33,10 +33,11 @@ def __init__(self) -> None: def transform_documents( self, documents: Sequence[Document], - unwanted_tags: List[str] = ["script", "style"], - tags_to_extract: List[str] = ["p", "li", "div", "a"], + unwanted_tags: Union[List[str], Tuple[str, ...]] = ("script", "style"), + tags_to_extract: Union[List[str], Tuple[str, ...]] = ("p", "li", "div", "a"), remove_lines: bool = True, *, + unwanted_classnames: Union[Tuple[str, ...], List[str]] = (), remove_comments: bool = False, **kwargs: Any, ) -> Sequence[Document]: @@ -48,6 +49,7 @@ def transform_documents( unwanted_tags: A list of tags to be removed from the HTML. tags_to_extract: A list of tags whose content will be extracted. remove_lines: If set to True, unnecessary lines will be removed. + unwanted_classnames: A list of class names to be removed from the HTML remove_comments: If set to True, comments will be removed. Returns: @@ -56,6 +58,10 @@ def transform_documents( for doc in documents: cleaned_content = doc.page_content + cleaned_content = self.remove_unwanted_classnames( + cleaned_content, unwanted_classnames + ) + cleaned_content = self.remove_unwanted_tags(cleaned_content, unwanted_tags) cleaned_content = self.extract_tags( @@ -70,7 +76,31 @@ def transform_documents( return documents @staticmethod - def remove_unwanted_tags(html_content: str, unwanted_tags: List[str]) -> str: + def remove_unwanted_classnames( + html_content: str, unwanted_classnames: Union[List[str], Tuple[str, ...]] + ) -> str: + """ + Remove unwanted classname from a given HTML content. + + Args: + html_content: The original HTML content string. + unwanted_classnames: A list of classnames to be removed from the HTML. + + Returns: + A cleaned HTML string with unwanted classnames removed. + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html_content, "html.parser") + for classname in unwanted_classnames: + for element in soup.find_all(class_=classname): + element.decompose() + return str(soup) + + @staticmethod + def remove_unwanted_tags( + html_content: str, unwanted_tags: Union[List[str], Tuple[str, ...]] + ) -> str: """ Remove unwanted tags from a given HTML content. @@ -91,7 +121,10 @@ def remove_unwanted_tags(html_content: str, unwanted_tags: List[str]) -> str: @staticmethod def extract_tags( - html_content: str, tags: List[str], *, remove_comments: bool = False + html_content: str, + tags: Union[List[str], Tuple[str, ...]], + *, + remove_comments: bool = False, ) -> str: """ Extract specific tags from a given HTML content. From 033d3102c444e2b14627c85864ca97216b0f679a Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Thu, 25 Apr 2024 10:18:29 -0700 Subject: [PATCH 08/36] multiple: remove external repo mds (#20896) api docs build doesn't tolerate them --- libs/partners/astradb.md | 3 --- libs/partners/cohere.md | 3 --- libs/partners/elasticsearch.md | 5 ----- libs/partners/google_genai.md | 3 --- libs/partners/google_vertexai.md | 3 --- libs/partners/nvidia_ai_endpoints.md | 3 --- libs/partners/nvidia_trt.md | 3 --- 7 files changed, 23 deletions(-) delete mode 100644 libs/partners/astradb.md delete mode 100644 libs/partners/cohere.md delete mode 100644 libs/partners/elasticsearch.md delete mode 100644 libs/partners/google_genai.md delete mode 100644 libs/partners/google_vertexai.md delete mode 100644 libs/partners/nvidia_ai_endpoints.md delete mode 100644 libs/partners/nvidia_trt.md diff --git a/libs/partners/astradb.md b/libs/partners/astradb.md deleted file mode 100644 index 62566a5677632..0000000000000 --- a/libs/partners/astradb.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-datastax/tree/main/libs/astradb \ No newline at end of file diff --git a/libs/partners/cohere.md b/libs/partners/cohere.md deleted file mode 100644 index 83896334383f2..0000000000000 --- a/libs/partners/cohere.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-cohere/tree/main/libs/cohere \ No newline at end of file diff --git a/libs/partners/elasticsearch.md b/libs/partners/elasticsearch.md deleted file mode 100644 index 8bcc4cdbb47a6..0000000000000 --- a/libs/partners/elasticsearch.md +++ /dev/null @@ -1,5 +0,0 @@ -# langchain-elasticsearch - -This package has moved! - -https://github.com/langchain-ai/langchain-elastic/tree/main/libs/elasticsearch diff --git a/libs/partners/google_genai.md b/libs/partners/google_genai.md deleted file mode 100644 index 088b699ee2e71..0000000000000 --- a/libs/partners/google_genai.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-google/tree/main/libs/genai \ No newline at end of file diff --git a/libs/partners/google_vertexai.md b/libs/partners/google_vertexai.md deleted file mode 100644 index 2ac1ce42e160a..0000000000000 --- a/libs/partners/google_vertexai.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-google/tree/main/libs/vertexai \ No newline at end of file diff --git a/libs/partners/nvidia_ai_endpoints.md b/libs/partners/nvidia_ai_endpoints.md deleted file mode 100644 index 3e19cc333d013..0000000000000 --- a/libs/partners/nvidia_ai_endpoints.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-nvidia/tree/main/libs/ai-endpoints \ No newline at end of file diff --git a/libs/partners/nvidia_trt.md b/libs/partners/nvidia_trt.md deleted file mode 100644 index 8a2546ff82d90..0000000000000 --- a/libs/partners/nvidia_trt.md +++ /dev/null @@ -1,3 +0,0 @@ -This package has moved! - -https://github.com/langchain-ai/langchain-nvidia/tree/main/libs/trt \ No newline at end of file From 47f8627f98de30ae36a31f153145271635235584 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Thu, 25 Apr 2024 19:27:10 +0200 Subject: [PATCH 09/36] community[mionr]: add Jina Reranker in retrievers module (#19406) - **Description:** Adapt JinaEmbeddings to run with the new Jina AI Rerank API - **Twitter handle:** https://twitter.com/JinaAI_ - [ ] **Add tests and docs**: If you're adding a new integration, please include 1. a test for the integration, preferably unit tests that do not rely on network access, 2. an example notebook showing its use. It lives in `docs/docs/integrations` directory. - [ ] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> Co-authored-by: Bagatur --- .../document_transformers/jina_rerank.ipynb | 254 ++++++++++++++++++ .../document_compressors/__init__.py | 4 + .../document_compressors/jina_rerank.py | 125 +++++++++ .../document_compressors/test_imports.py | 2 +- 4 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 docs/docs/integrations/document_transformers/jina_rerank.ipynb create mode 100644 libs/community/langchain_community/document_compressors/jina_rerank.py diff --git a/docs/docs/integrations/document_transformers/jina_rerank.ipynb b/docs/docs/integrations/document_transformers/jina_rerank.ipynb new file mode 100644 index 0000000000000..beebf1ccb3255 --- /dev/null +++ b/docs/docs/integrations/document_transformers/jina_rerank.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f6ff09ab-c736-4a18-a717-563b4e29d22d", + "metadata": {}, + "source": [ + "# Jina Reranker" + ] + }, + { + "cell_type": "markdown", + "id": "1288789a-4c30-4fc3-90c7-dd1741a2550b", + "metadata": {}, + "source": [ + "This notebook shows how to use Jina Reranker for document compression and retrieval." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0e4d52e-3968-4f8b-9865-a886f27e5feb", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -qU langchain langchain-openai langchain-community langchain-text-splitters langchainhub\n", + "\n", + "%pip install --upgrade --quiet faiss\n", + "\n", + "# OR (depending on Python version)\n", + "\n", + "%pip install --upgrade --quiet faiss_cpu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1fc07a6-8e01-4aa5-8ed4-ca2b0bfca70c", + "metadata": {}, + "outputs": [], + "source": [ + "# Helper function for printing docs\n", + "\n", + "\n", + "def pretty_print_docs(docs):\n", + " print(\n", + " f\"\\n{'-' * 100}\\n\".join(\n", + " [f\"Document {i+1}:\\n\\n\" + d.page_content for i, d in enumerate(docs)]\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "d8ec4823-fdc1-4339-8a25-da598a1e2a4c", + "metadata": {}, + "source": [ + "## Set up the base vector store retriever" + ] + }, + { + "cell_type": "markdown", + "id": "9db25269-e798-496f-8fb9-2bb280735118", + "metadata": {}, + "source": [ + "Let's start by initializing a simple vector store retriever and storing the 2023 State of the Union speech (in chunks). We can set up the retriever to retrieve a high number (20) of docs." + ] + }, + { + "cell_type": "markdown", + "id": "ce01a2b5-d7f4-4902-9156-9a3a86704f40", + "metadata": {}, + "source": [ + "##### Set the Jina and OpenAI API keys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6692d5c5-c84a-4d42-8dd8-5ce90ff56d20", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass()\n", + "os.environ[\"JINA_API_KEY\"] = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "981159af-fa3c-4f75-adb4-1a4de1950f2f", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import TextLoader\n", + "from langchain_community.embeddings import JinaEmbeddings\n", + "from langchain_community.vectorstores import FAISS\n", + "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", + "\n", + "documents = TextLoader(\n", + " \"../../modules/state_of_the_union.txt\",\n", + ").load()\n", + "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)\n", + "texts = text_splitter.split_documents(documents)\n", + "\n", + "embedding = JinaEmbeddings(model_name=\"jina-embeddings-v2-base-en\")\n", + "retriever = FAISS.from_documents(texts, embedding).as_retriever(search_kwargs={\"k\": 20})\n", + "\n", + "query = \"What did the president say about Ketanji Brown Jackson\"\n", + "docs = retriever.get_relevant_documents(query)\n", + "pretty_print_docs(docs)" + ] + }, + { + "cell_type": "markdown", + "id": "b5a514b7-027a-4dd4-9cfc-63fb4d50aa66", + "metadata": {}, + "source": [ + "## Doing reranking with JinaRerank" + ] + }, + { + "cell_type": "markdown", + "id": "bdd9e0ca-d728-42cb-88ad-459fb8a56b33", + "metadata": {}, + "source": [ + "Now let's wrap our base retriever with a ContextualCompressionRetriever, using Jina Reranker as a compressor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3000019e-cc0d-4365-91d0-72247ee4d624", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.retrievers import ContextualCompressionRetriever\n", + "from langchain_community.document_compressors import JinaRerank\n", + "\n", + "compressor = JinaRerank()\n", + "compression_retriever = ContextualCompressionRetriever(\n", + " base_compressor=compressor, base_retriever=retriever\n", + ")\n", + "\n", + "compressed_docs = compression_retriever.get_relevant_documents(\n", + " \"What did the president say about Ketanji Jackson Brown\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f314f74c-48a9-4243-8d3c-2b7f820e1e40", + "metadata": {}, + "outputs": [], + "source": [ + "pretty_print_docs(compressed_docs)" + ] + }, + { + "cell_type": "markdown", + "id": "87164f04-194b-4138-8d94-f179f6f34a31", + "metadata": {}, + "source": [ + "## QA reranking with Jina Reranker" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2b4ab60b-5a26-4cfb-9b58-3dc2d83b772b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "================================\u001b[1m System Message \u001b[0m================================\n", + "\n", + "Answer any use questions based solely on the context below:\n", + "\n", + "\n", + "\u001b[33;1m\u001b[1;3m{context}\u001b[0m\n", + "\n", + "\n", + "=============================\u001b[1m Messages Placeholder \u001b[0m=============================\n", + "\n", + "\u001b[33;1m\u001b[1;3m{chat_history}\u001b[0m\n", + "\n", + "================================\u001b[1m Human Message \u001b[0m=================================\n", + "\n", + "\u001b[33;1m\u001b[1;3m{input}\u001b[0m\n" + ] + } + ], + "source": [ + "from langchain import hub\n", + "from langchain.chains import create_retrieval_chain\n", + "from langchain.chains.combine_documents import create_stuff_documents_chain\n", + "\n", + "retrieval_qa_chat_prompt = hub.pull(\"langchain-ai/retrieval-qa-chat\")\n", + "retrieval_qa_chat_prompt.pretty_print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72af3eb3-b644-4b5f-bf5f-f1dc43c96882", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_openai import ChatOpenAI\n", + "\n", + "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n", + "combine_docs_chain = create_stuff_documents_chain(llm, retrieval_qa_chat_prompt)\n", + "chain = create_retrieval_chain(compression_retriever, combine_docs_chain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "126401a7-c545-4de0-92dc-e9bc1001a6ba", + "metadata": {}, + "outputs": [], + "source": [ + "chain.invoke({\"input\": query})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "poetry-venv-2", + "language": "python", + "name": "poetry-venv-2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/libs/community/langchain_community/document_compressors/__init__.py b/libs/community/langchain_community/document_compressors/__init__.py index 4b7cb93e9429a..ca9822d4baea2 100644 --- a/libs/community/langchain_community/document_compressors/__init__.py +++ b/libs/community/langchain_community/document_compressors/__init__.py @@ -2,6 +2,9 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from langchain_community.document_compressors.jina_rerank import ( + JinaRerank, # noqa: F401 + ) from langchain_community.document_compressors.llmlingua_filter import ( LLMLinguaCompressor, # noqa: F401 ) @@ -14,6 +17,7 @@ _module_lookup = { "LLMLinguaCompressor": "langchain_community.document_compressors.llmlingua_filter", "OpenVINOReranker": "langchain_community.document_compressors.openvino_rerank", + "JinaRerank": "langchain_community.document_compressors.jina_rerank", } diff --git a/libs/community/langchain_community/document_compressors/jina_rerank.py b/libs/community/langchain_community/document_compressors/jina_rerank.py new file mode 100644 index 0000000000000..0e1f40a822321 --- /dev/null +++ b/libs/community/langchain_community/document_compressors/jina_rerank.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Sequence, Union + +import requests +from langchain_core.callbacks import Callbacks +from langchain_core.documents import BaseDocumentCompressor, Document +from langchain_core.pydantic_v1 import Extra, root_validator +from langchain_core.utils import get_from_dict_or_env + +JINA_API_URL: str = "https://api.jina.ai/v1/rerank" + + +class JinaRerank(BaseDocumentCompressor): + """Document compressor that uses `Jina Rerank API`.""" + + session: Any = None + """Requests session to communicate with API.""" + top_n: Optional[int] = 3 + """Number of documents to return.""" + model: str = "jina-reranker-v1-base-en" + """Model to use for reranking.""" + jina_api_key: Optional[str] = None + """Jina API key. Must be specified directly or via environment variable + JINA_API_KEY.""" + user_agent: str = "langchain" + """Identifier for the application making the request.""" + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + arbitrary_types_allowed = True + + @root_validator(pre=True) + def validate_environment(cls, values: Dict) -> Dict: + """Validate that api key exists in environment.""" + jina_api_key = get_from_dict_or_env(values, "jina_api_key", "JINA_API_KEY") + user_agent = values.get("user_agent", "langchain") + session = requests.Session() + session.headers.update( + { + "Authorization": f"Bearer {jina_api_key}", + "Accept-Encoding": "identity", + "Content-type": "application/json", + "user-agent": user_agent, + } + ) + values["session"] = session + return values + + def rerank( + self, + documents: Sequence[Union[str, Document, dict]], + query: str, + *, + model: Optional[str] = None, + top_n: Optional[int] = -1, + max_chunks_per_doc: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """Returns an ordered list of documents ordered by their relevance to the provided query. + + Args: + query: The query to use for reranking. + documents: A sequence of documents to rerank. + model: The model to use for re-ranking. Default to self.model. + top_n : The number of results to return. If None returns all results. + Defaults to self.top_n. + max_chunks_per_doc : The maximum number of chunks derived from a document. + """ # noqa: E501 + if len(documents) == 0: # to avoid empty api call + return [] + docs = [ + doc.page_content if isinstance(doc, Document) else doc for doc in documents + ] + model = model or self.model + top_n = top_n if (top_n is None or top_n > 0) else self.top_n + data = { + "query": query, + "documents": docs, + "model": model, + "top_n": top_n, + } + + resp = self.session.post( + JINA_API_URL, + json=data, + ).json() + + if "results" not in resp: + raise RuntimeError(resp["detail"]) + + results = resp["results"] + result_dicts = [] + for res in results: + result_dicts.append( + {"index": res["index"], "relevance_score": res["relevance_score"]} + ) + return result_dicts + + def compress_documents( + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, + ) -> Sequence[Document]: + """ + Compress documents using Jina's Rerank API. + + Args: + documents: A sequence of documents to compress. + query: The query to use for compressing the documents. + callbacks: Callbacks to run during the compression process. + + Returns: + A sequence of compressed documents. + """ + compressed = [] + for res in self.rerank(documents, query): + doc = documents[res["index"]] + doc_copy = Document(doc.page_content, metadata=deepcopy(doc.metadata)) + doc_copy.metadata["relevance_score"] = res["relevance_score"] + compressed.append(doc_copy) + return compressed diff --git a/libs/community/tests/unit_tests/document_compressors/test_imports.py b/libs/community/tests/unit_tests/document_compressors/test_imports.py index 2c928857abb24..d6451cdd88ed3 100644 --- a/libs/community/tests/unit_tests/document_compressors/test_imports.py +++ b/libs/community/tests/unit_tests/document_compressors/test_imports.py @@ -1,6 +1,6 @@ from langchain_community.document_compressors import __all__, _module_lookup -EXPECTED_ALL = ["LLMLinguaCompressor", "OpenVINOReranker"] +EXPECTED_ALL = ["LLMLinguaCompressor", "OpenVINOReranker", "JinaRerank"] def test_all_imports() -> None: From 8bbd8811be664e4eac30fa1fd0dd37eddec84174 Mon Sep 17 00:00:00 2001 From: Pavlo Paliychuk Date: Thu, 25 Apr 2024 13:41:08 -0400 Subject: [PATCH 10/36] docs: Fix misplaced zep cloud example links (#20867) Thank you for contributing to LangChain! - [x] **PR title**: Fix misplaced zep cloud example links - [x] **PR message**: - **Description:** Fixes misplaced links for vector store and memory zep cloud examples - [x] **Add tests and docs**: If you're adding a new integration, please include 1. a test for the integration, preferably unit tests that do not rely on network access, 2. an example notebook showing its use. It lives in `docs/docs/integrations` directory. - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ Additional guidelines: - Make sure optional dependencies are imported within a function. - Please do not add dependencies to pyproject.toml files (even optional ones) unless they are required for unit tests. - Most PRs should not touch more than one package. - Changes should be backwards compatible. - If you are adding something to community, do not re-import it in langchain. If no one reviews your PR within a few days, please @-mention one of baskaryan, efriis, eyurtsev, hwchase17. --- docs/docs/integrations/memory/zep_memory.ipynb | 4 ++-- docs/docs/integrations/providers/zep.mdx | 4 ++-- docs/docs/integrations/retrievers/zep_memorystore.ipynb | 3 +-- docs/docs/integrations/vectorstores/zep.ipynb | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/docs/integrations/memory/zep_memory.ipynb b/docs/docs/integrations/memory/zep_memory.ipynb index 623191e7c14e2..c7a1f778c5b6b 100644 --- a/docs/docs/integrations/memory/zep_memory.ipynb +++ b/docs/docs/integrations/memory/zep_memory.ipynb @@ -6,11 +6,11 @@ "# Zep\n", "> Recall, understand, and extract data from chat histories. Power personalized AI experiences.\n", "\n", - ">[Zep](http://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", + ">[Zep](https://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", "> With Zep, you can provide AI assistants with the ability to recall past conversations, no matter how distant,\n", "> while also reducing hallucinations, latency, and cost.\n", "\n", - "> Interested in Zep Cloud? See [Zep Cloud Installation Guide](https://help.getzep.com/sdks) and [Zep Cloud Memory Example](https://help.getzep.com/langchain/examples/vectorstore-example)\n", + "> Interested in Zep Cloud? See [Zep Cloud Installation Guide](https://help.getzep.com/sdks) and [Zep Cloud Memory Example](https://help.getzep.com/langchain/examples/messagehistory-example)\n", "\n", "## Open Source Installation and Setup\n", "\n", diff --git a/docs/docs/integrations/providers/zep.mdx b/docs/docs/integrations/providers/zep.mdx index c86787769a613..220a35c910e35 100644 --- a/docs/docs/integrations/providers/zep.mdx +++ b/docs/docs/integrations/providers/zep.mdx @@ -1,7 +1,7 @@ # Zep > Recall, understand, and extract data from chat histories. Power personalized AI experiences. ->[Zep](http://www.getzep.com) is a long-term memory service for AI Assistant apps. +>[Zep](https://www.getzep.com) is a long-term memory service for AI Assistant apps. > With Zep, you can provide AI assistants with the ability to recall past conversations, no matter how distant, > while also reducing hallucinations, latency, and cost. @@ -22,7 +22,7 @@ Zep allows you to be more intentional about constructing your prompt: - and/or relevant Business data from Zep Document Collections. ## What is Zep Cloud? -[Zep Cloud](http://www.getzep.com) is a managed service with Zep Open Source at its core. +[Zep Cloud](https://www.getzep.com) is a managed service with Zep Open Source at its core. In addition to Zep Open Source's memory management features, Zep Cloud offers: - **Fact Extraction**: Automatically build fact tables from conversations, without having to define a data schema upfront. - **Dialog Classification**: Instantly and accurately classify chat dialog. Understand user intent and emotion, segment users, and more. Route chains based on semantic context, and trigger events. diff --git a/docs/docs/integrations/retrievers/zep_memorystore.ipynb b/docs/docs/integrations/retrievers/zep_memorystore.ipynb index aaa8a053fd1ac..65733bfea2794 100644 --- a/docs/docs/integrations/retrievers/zep_memorystore.ipynb +++ b/docs/docs/integrations/retrievers/zep_memorystore.ipynb @@ -8,7 +8,7 @@ "\n", "> Recall, understand, and extract data from chat histories. Power personalized AI experiences.\n", "\n", - "> [Zep](http://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", + "> [Zep](https://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", "> With Zep, you can provide AI assistants with the ability to recall past conversations, no matter how distant,\n", "> while also reducing hallucinations, latency, and cost.\n", "\n", @@ -17,7 +17,6 @@ "## Open Source Installation and Setup\n", "\n", "> Zep Open Source project: [https://github.com/getzep/zep](https://github.com/getzep/zep)\n", - ">\n", "> Zep Open Source Docs: [https://docs.getzep.com/](https://docs.getzep.com/)" ], "metadata": { diff --git a/docs/docs/integrations/vectorstores/zep.ipynb b/docs/docs/integrations/vectorstores/zep.ipynb index 917fb52a6a74e..191f449e01310 100644 --- a/docs/docs/integrations/vectorstores/zep.ipynb +++ b/docs/docs/integrations/vectorstores/zep.ipynb @@ -6,11 +6,11 @@ "# Zep\n", "> Recall, understand, and extract data from chat histories. Power personalized AI experiences.\n", "\n", - ">[Zep](http://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", + "> [Zep](https://www.getzep.com) is a long-term memory service for AI Assistant apps.\n", "> With Zep, you can provide AI assistants with the ability to recall past conversations, no matter how distant,\n", "> while also reducing hallucinations, latency, and cost.\n", "\n", - "> Interested in Zep Cloud? See [Zep Cloud Installation Guide](https://help.getzep.com/sdks) and [Zep Cloud Vector Store example](https://help.getzep.com/langchain/examples/messagehistory-example)\n", + "> Interested in Zep Cloud? See [Zep Cloud Installation Guide](https://help.getzep.com/sdks) and [Zep Cloud Vector Store example](https://help.getzep.com/langchain/examples/vectorstore-example)\n", "\n", "## Open Source Installation and Setup\n", "\n", @@ -32,7 +32,7 @@ "metadata": { "collapsed": false }, - "id": "182f1b4f6cc28385" + "id": "550edc01b00149cd" }, { "cell_type": "markdown", From 9fdaee08efa746eed98c4b3f2cf270b95c5ae1d2 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Thu, 25 Apr 2024 10:41:19 -0700 Subject: [PATCH 11/36] upstage: release 0.1.2 (#20898) --- libs/partners/upstage/poetry.lock | 8 ++++---- libs/partners/upstage/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/partners/upstage/poetry.lock b/libs/partners/upstage/poetry.lock index 7c9da260553c2..02aaa3ce5a8fe 100644 --- a/libs/partners/upstage/poetry.lock +++ b/libs/partners/upstage/poetry.lock @@ -340,7 +340,7 @@ files = [ [[package]] name = "langchain-core" -version = "0.1.45" +version = "0.1.46" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" @@ -548,13 +548,13 @@ files = [ [[package]] name = "openai" -version = "1.23.4" +version = "1.23.6" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.23.4-py3-none-any.whl", hash = "sha256:ecb72dcb415c8a1f1b6ef2fe32f8fc9a0942727b6365e8caedf916db5c19b180"}, - {file = "openai-1.23.4.tar.gz", hash = "sha256:72c5a2ab2cda5727b6897f9d079aec16ceccf7dd2e0e0c84a21f7304d5484563"}, + {file = "openai-1.23.6-py3-none-any.whl", hash = "sha256:f406c76ba279d16b9aca5a89cee0d968488e39f671f4dc6f0d690ac3c6f6fca1"}, + {file = "openai-1.23.6.tar.gz", hash = "sha256:612de2d54cf580920a1156273f84aada6b3dca26d048f62eb5364a4314d7f449"}, ] [package.dependencies] diff --git a/libs/partners/upstage/pyproject.toml b/libs/partners/upstage/pyproject.toml index a6617c19bd9c3..4662729112261 100644 --- a/libs/partners/upstage/pyproject.toml +++ b/libs/partners/upstage/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langchain-upstage" -version = "0.1.1" +version = "0.1.2" description = "An integration package connecting Upstage and LangChain" authors = [] readme = "README.md" From fd186419597c05b9bd2410e378b9c5dcf903e9ef Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:41:36 +0200 Subject: [PATCH 12/36] docs: Use voyage-law-2 in the examples (#20784) Thank you for contributing to LangChain! - [x] **PR title**: "package: description" - Where "package" is whichever of langchain, community, core, experimental, etc. is being modified. Use "docs: ..." for purely docs changes, "templates: ..." for template changes, "infra: ..." for CI changes. - Example: "community: add foobar LLM" **Description:** In VoyageAI text-embedding examples use voyage-law-2 model - [x] **Add tests and docs**: If you're adding a new integration, please include 1. a test for the integration, preferably unit tests that do not rely on network access, 2. an example notebook showing its use. It lives in `docs/docs/integrations` directory. - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ Additional guidelines: - Make sure optional dependencies are imported within a function. - Please do not add dependencies to pyproject.toml files (even optional ones) unless they are required for unit tests. - Most PRs should not touch more than one package. - Changes should be backwards compatible. - If you are adding something to community, do not re-import it in langchain. If no one reviews your PR within a few days, please @-mention one of baskaryan, efriis, eyurtsev, hwchase17. --- .../document_transformers/voyageai-reranker.ipynb | 10 ++++++++-- docs/docs/integrations/text_embedding/voyageai.ipynb | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/docs/integrations/document_transformers/voyageai-reranker.ipynb b/docs/docs/integrations/document_transformers/voyageai-reranker.ipynb index 0ffd6afb0d6a5..8e32d3997bdf8 100644 --- a/docs/docs/integrations/document_transformers/voyageai-reranker.ipynb +++ b/docs/docs/integrations/document_transformers/voyageai-reranker.ipynb @@ -84,7 +84,13 @@ }, "source": [ "## Set up the base vector store retriever\n", - "Let's start by initializing a simple vector store retriever and storing the 2023 State of the Union speech (in chunks). We can set up the retriever to retrieve a high number (20) of docs." + "Let's start by initializing a simple vector store retriever and storing the 2023 State of the Union speech (in chunks). We can set up the retriever to retrieve a high number (20) of docs. You can use any of the following Embeddings models: ([source](https://docs.voyageai.com/docs/embeddings)):\n", + "\n", + "- `voyage-large-2` (default)\n", + "- `voyage-code-2`\n", + "- `voyage-2`\n", + "- `voyage-law-2`\n", + "- `voyage-lite-02-instruct`" ] }, { @@ -316,7 +322,7 @@ "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)\n", "texts = text_splitter.split_documents(documents)\n", "retriever = FAISS.from_documents(\n", - " texts, VoyageAIEmbeddings(model=\"voyage-2\")\n", + " texts, VoyageAIEmbeddings(model=\"voyage-law-2\")\n", ").as_retriever(search_kwargs={\"k\": 20})\n", "\n", "query = \"What did the president say about Ketanji Brown Jackson\"\n", diff --git a/docs/docs/integrations/text_embedding/voyageai.ipynb b/docs/docs/integrations/text_embedding/voyageai.ipynb index 83cc9f38c18e7..adc53b7403ba9 100644 --- a/docs/docs/integrations/text_embedding/voyageai.ipynb +++ b/docs/docs/integrations/text_embedding/voyageai.ipynb @@ -27,7 +27,13 @@ "id": "137cfde9-b88c-409a-9394-a9e31a6bf30d", "metadata": {}, "source": [ - "Voyage AI utilizes API keys to monitor usage and manage permissions. To obtain your key, create an account on our [homepage](https://www.voyageai.com). Then, create a VoyageEmbeddings model with your API key. Please refer to the documentation for further details on the available models: https://docs.voyageai.com/embeddings/" + "Voyage AI utilizes API keys to monitor usage and manage permissions. To obtain your key, create an account on our [homepage](https://www.voyageai.com). Then, create a VoyageEmbeddings model with your API key. You can use any of the following models: ([source](https://docs.voyageai.com/docs/embeddings)):\n", + "\n", + "- `voyage-large-2` (default)\n", + "- `voyage-code-2`\n", + "- `voyage-2`\n", + "- `voyage-law-2`\n", + "- `voyage-lite-02-instruct`" ] }, { @@ -38,7 +44,7 @@ "outputs": [], "source": [ "embeddings = VoyageAIEmbeddings(\n", - " voyage_api_key=\"[ Your Voyage API key ]\", model=\"voyage-2\"\n", + " voyage_api_key=\"[ Your Voyage API key ]\", model=\"voyage-law-2\"\n", ")" ] }, From 73551518e1d7d9f3d54b12e9b1faca37484b616e Mon Sep 17 00:00:00 2001 From: samanhappy Date: Fri, 26 Apr 2024 01:42:06 +0800 Subject: [PATCH 13/36] docs: Fix broken link in agents.ipynb (#20872) --- docs/docs/use_cases/tool_use/agents.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/use_cases/tool_use/agents.ipynb b/docs/docs/use_cases/tool_use/agents.ipynb index 797a05c69d4d3..366918a04f6c9 100644 --- a/docs/docs/use_cases/tool_use/agents.ipynb +++ b/docs/docs/use_cases/tool_use/agents.ipynb @@ -23,7 +23,7 @@ "\n", "LangChain comes with a number of built-in agents that are optimized for different use cases. Read about all the [agent types here](/docs/modules/agents/agent_types/).\n", "\n", - "We'll use the [tool calling agent](/docs/modules/agents/agent_types/tool_calling/), which is generally the most reliable kind and the recommended one for most use cases. \"Tool calling\" in this case refers to a specific type of model API that allows for explicitly passing tool definitions to models and getting explicit tool invocations out. For more on tool calling models see [this guide].(/docs/modules/model_io/chat/function_calling/)\n", + "We'll use the [tool calling agent](/docs/modules/agents/agent_types/tool_calling/), which is generally the most reliable kind and the recommended one for most use cases. \"Tool calling\" in this case refers to a specific type of model API that allows for explicitly passing tool definitions to models and getting explicit tool invocations out. For more on tool calling models see [this guide](/docs/modules/model_io/chat/function_calling/).\n", "\n", "![agent](../../../static/img/tool_agent.svg)" ] From b58bb7664c48313932c91f3b97ce3667f1f9ee8a Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Fri, 26 Apr 2024 02:29:41 +0800 Subject: [PATCH 14/36] community[patch]: add HTTP response headers Content-Type to metadata of RecursiveUrlLoader document (#20875) **Description:** The RecursiveUrlLoader loader offers a link_regex parameter that can filter out URLs. However, this filtering capability is limited, and if the internal links of the website change, unexpected resources may be loaded. These resources, such as font files, can cause problems in subsequent embedding processing. > https://blog.langchain.dev/assets/fonts/source-sans-pro-v21-latin-ext_latin-regular.woff2?v=0312715cbf We can add the Content-Type in the HTTP response headers to the document metadata so developers can choose which resources to use. This allows developers to make their own choices. For example, the following may be a good choice for text knowledge. - text/plain - simple text file - text/html - HTML web page - text/xml - XML format file - text/json - JSON format data - application/pdf - PDF file - application/msword - Word document and ignore the following - text/css - CSS stylesheet - text/javascript - JavaScript script - application/octet-stream - binary data - image/jpeg - JPEG image - image/png - PNG image - image/gif - GIF image - image/svg+xml - SVG image - audio/mpeg - MPEG audio files - video/mp4 - MP4 video file - application/font-woff - WOFF font file - application/font-ttf - TTF font file - application/zip - ZIP compressed file - application/octet-stream - binary data **Twitter handle:** @coolbeevip --------- Co-authored-by: Bagatur --- .../document_loaders/recursive_url_loader.py | 69 +++++++++++++++---- .../test_recursive_url_loader.py | 16 ++++- 2 files changed, 69 insertions(+), 16 deletions(-) diff --git a/libs/community/langchain_community/document_loaders/recursive_url_loader.py b/libs/community/langchain_community/document_loaders/recursive_url_loader.py index 6231c7af8d665..687b02ccd69be 100644 --- a/libs/community/langchain_community/document_loaders/recursive_url_loader.py +++ b/libs/community/langchain_community/document_loaders/recursive_url_loader.py @@ -1,10 +1,10 @@ from __future__ import annotations import asyncio +import inspect import logging import re from typing import ( - TYPE_CHECKING, Callable, Iterator, List, @@ -12,23 +12,25 @@ Sequence, Set, Union, + cast, ) +import aiohttp import requests from langchain_core.documents import Document from langchain_core.utils.html import extract_sub_links from langchain_community.document_loaders.base import BaseLoader -if TYPE_CHECKING: - import aiohttp - logger = logging.getLogger(__name__) -def _metadata_extractor(raw_html: str, url: str) -> dict: +def _metadata_extractor( + raw_html: str, url: str, response: Union[requests.Response, aiohttp.ClientResponse] +) -> dict: """Extract metadata from raw html using BeautifulSoup.""" - metadata = {"source": url} + content_type = getattr(response, "headers").get("Content-Type", "") + metadata = {"source": url, "content_type": content_type} try: from bs4 import BeautifulSoup @@ -86,7 +88,7 @@ def __init__( max_depth: Optional[int] = 2, use_async: Optional[bool] = None, extractor: Optional[Callable[[str], str]] = None, - metadata_extractor: Optional[Callable[[str, str], dict]] = None, + metadata_extractor: Optional[_MetadataExtractorType] = None, exclude_dirs: Optional[Sequence[str]] = (), timeout: Optional[int] = 10, prevent_outside: bool = True, @@ -108,10 +110,22 @@ def __init__( extractor: A function to extract document contents from raw html. When extract function returns an empty string, the document is ignored. - metadata_extractor: A function to extract metadata from raw html and the - source url (args in that order). Default extractor will attempt - to use BeautifulSoup4 to extract the title, description and language - of the page. + metadata_extractor: A function to extract metadata from args: raw html, the + source url, and the requests.Response/aiohttp.ClientResponse object + (args in that order). + Default extractor will attempt to use BeautifulSoup4 to extract the + title, description and language of the page. + ..code-block:: python + + import requests + import aiohttp + + def simple_metadata_extractor( + raw_html: str, url: str, response: Union[requests.Response, aiohttp.ClientResponse] + ) -> dict: + content_type = getattr(response, "headers").get("Content-Type", "") + return {"source": url, "content_type": content_type} + exclude_dirs: A list of subdirectories to exclude. timeout: The timeout for the requests, in the unit of seconds. If None then connection will not timeout. @@ -123,17 +137,18 @@ def __init__( continue_on_failure: If True, continue if getting or parsing a link raises an exception. Otherwise, raise the exception. base_url: The base url to check for outside links against. - """ + """ # noqa: E501 self.url = url self.max_depth = max_depth if max_depth is not None else 2 self.use_async = use_async if use_async is not None else False self.extractor = extractor if extractor is not None else lambda x: x - self.metadata_extractor = ( + metadata_extractor = ( metadata_extractor if metadata_extractor is not None else _metadata_extractor ) + self.metadata_extractor = _wrap_metadata_extractor(metadata_extractor) self.exclude_dirs = exclude_dirs if exclude_dirs is not None else () if any(url.startswith(exclude_dir) for exclude_dir in self.exclude_dirs): @@ -184,7 +199,7 @@ def _get_child_links_recursive( if content: yield Document( page_content=content, - metadata=self.metadata_extractor(response.text, url), + metadata=self.metadata_extractor(response.text, url, response), ) # Store the visited links and recursively visit the children @@ -270,7 +285,7 @@ async def _async_get_child_links_recursive( results.append( Document( page_content=content, - metadata=self.metadata_extractor(text, url), + metadata=self.metadata_extractor(text, url, response), ) ) if depth < self.max_depth - 1: @@ -318,3 +333,27 @@ def lazy_load(self) -> Iterator[Document]: return iter(results or []) else: return self._get_child_links_recursive(self.url, visited) + + +_MetadataExtractorType1 = Callable[[str, str], dict] +_MetadataExtractorType2 = Callable[ + [str, str, Union[requests.Response, aiohttp.ClientResponse]], dict +] +_MetadataExtractorType = Union[_MetadataExtractorType1, _MetadataExtractorType2] + + +def _wrap_metadata_extractor( + metadata_extractor: _MetadataExtractorType, +) -> _MetadataExtractorType2: + if len(inspect.signature(metadata_extractor).parameters) == 3: + return cast(_MetadataExtractorType2, metadata_extractor) + else: + + def _metadata_extractor_wrapper( + raw_html: str, + url: str, + response: Union[requests.Response, aiohttp.ClientResponse], + ) -> dict: + return cast(_MetadataExtractorType1, metadata_extractor)(raw_html, url) + + return _metadata_extractor_wrapper diff --git a/libs/community/tests/integration_tests/document_loaders/test_recursive_url_loader.py b/libs/community/tests/integration_tests/document_loaders/test_recursive_url_loader.py index ff8083bcd5098..1f359932227be 100644 --- a/libs/community/tests/integration_tests/document_loaders/test_recursive_url_loader.py +++ b/libs/community/tests/integration_tests/document_loaders/test_recursive_url_loader.py @@ -35,7 +35,7 @@ def test_sync_recursive_url_loader() -> None: url, extractor=lambda _: "placeholder", use_async=False, max_depth=2 ) docs = loader.load() - assert len(docs) == 25 + assert len(docs) == 24 assert docs[0].page_content == "placeholder" @@ -55,3 +55,17 @@ def test_loading_invalid_url() -> None: ) docs = loader.load() assert len(docs) == 0 + + +def test_sync_async_metadata_necessary_properties() -> None: + url = "https://docs.python.org/3.9/" + loader = RecursiveUrlLoader(url, use_async=False, max_depth=2) + async_loader = RecursiveUrlLoader(url, use_async=False, max_depth=2) + docs = loader.load() + async_docs = async_loader.load() + for doc in docs: + assert "source" in doc.metadata + assert "content_type" in doc.metadata + for doc in async_docs: + assert "source" in doc.metadata + assert "content_type" in doc.metadata From 648a59574b1ba3cfb30020a96362c9e66bade82a Mon Sep 17 00:00:00 2001 From: Tomaz Bratanic Date: Thu, 25 Apr 2024 20:30:22 +0200 Subject: [PATCH 15/36] community[patch]: Support passing graph object to Neo4j integrations (#20876) For driver connection reusage, we introduce passing the graph object to neo4j integrations --- .../chat_message_histories/neo4j.py | 57 ++++++++++------ .../vectorstores/neo4j_vector.py | 68 ++++++++++--------- .../chat_message_histories/test_neo4j.py | 66 ++++++++++++++++++ .../vectorstores/test_neo4jvector.py | 18 +++++ 4 files changed, 157 insertions(+), 52 deletions(-) create mode 100644 libs/community/tests/integration_tests/chat_message_histories/test_neo4j.py diff --git a/libs/community/langchain_community/chat_message_histories/neo4j.py b/libs/community/langchain_community/chat_message_histories/neo4j.py index d64b1e5ed6eec..972f284b04283 100644 --- a/libs/community/langchain_community/chat_message_histories/neo4j.py +++ b/libs/community/langchain_community/chat_message_histories/neo4j.py @@ -2,7 +2,9 @@ from langchain_core.chat_history import BaseChatMessageHistory from langchain_core.messages import BaseMessage, messages_from_dict -from langchain_core.utils import get_from_env +from langchain_core.utils import get_from_dict_or_env + +from langchain_community.graphs import Neo4jGraph class Neo4jChatMessageHistory(BaseChatMessageHistory): @@ -17,6 +19,8 @@ def __init__( database: str = "neo4j", node_label: str = "Session", window: int = 3, + *, + graph: Optional[Neo4jGraph] = None, ): try: import neo4j @@ -30,30 +34,41 @@ def __init__( if not session_id: raise ValueError("Please ensure that the session_id parameter is provided") - url = get_from_env("url", "NEO4J_URI", url) - username = get_from_env("username", "NEO4J_USERNAME", username) - password = get_from_env("password", "NEO4J_PASSWORD", password) - database = get_from_env("database", "NEO4J_DATABASE", database) + # Graph object takes precedent over env or input params + if graph: + self._driver = graph._driver + self._database = graph._database + else: + # Handle if the credentials are environment variables + url = get_from_dict_or_env({"url": url}, "url", "NEO4J_URI") + username = get_from_dict_or_env( + {"username": username}, "username", "NEO4J_USERNAME" + ) + password = get_from_dict_or_env( + {"password": password}, "password", "NEO4J_PASSWORD" + ) + database = get_from_dict_or_env( + {"database": database}, "database", "NEO4J_DATABASE", "neo4j" + ) - self._driver = neo4j.GraphDatabase.driver(url, auth=(username, password)) - self._database = database + self._driver = neo4j.GraphDatabase.driver(url, auth=(username, password)) + self._database = database + # Verify connection + try: + self._driver.verify_connectivity() + except neo4j.exceptions.ServiceUnavailable: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the url is correct" + ) + except neo4j.exceptions.AuthError: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the username and password are correct" + ) self._session_id = session_id self._node_label = node_label self._window = window - - # Verify connection - try: - self._driver.verify_connectivity() - except neo4j.exceptions.ServiceUnavailable: - raise ValueError( - "Could not connect to Neo4j database. " - "Please ensure that the url is correct" - ) - except neo4j.exceptions.AuthError: - raise ValueError( - "Could not connect to Neo4j database. " - "Please ensure that the username and password are correct" - ) # Create session node self._driver.execute_query( f"MERGE (s:`{self._node_label}` {{id:$session_id}})", diff --git a/libs/community/langchain_community/vectorstores/neo4j_vector.py b/libs/community/langchain_community/vectorstores/neo4j_vector.py index 3a9c57de42a98..d09145b1b7f34 100644 --- a/libs/community/langchain_community/vectorstores/neo4j_vector.py +++ b/libs/community/langchain_community/vectorstores/neo4j_vector.py @@ -20,6 +20,7 @@ from langchain_core.utils import get_from_dict_or_env from langchain_core.vectorstores import VectorStore +from langchain_community.graphs import Neo4jGraph from langchain_community.vectorstores.utils import DistanceStrategy DEFAULT_DISTANCE_STRATEGY = DistanceStrategy.COSINE @@ -483,6 +484,7 @@ def __init__( retrieval_query: str = "", relevance_score_fn: Optional[Callable[[float], float]] = None, index_type: IndexType = DEFAULT_INDEX_TYPE, + graph: Optional[Neo4jGraph] = None, ) -> None: try: import neo4j @@ -501,40 +503,44 @@ def __init__( "distance_strategy must be either 'EUCLIDEAN_DISTANCE' or 'COSINE'" ) - # Handle if the credentials are environment variables - - # Support URL for backwards compatibility - if not url: - url = os.environ.get("NEO4J_URL") - - url = get_from_dict_or_env({"url": url}, "url", "NEO4J_URI") - username = get_from_dict_or_env( - {"username": username}, "username", "NEO4J_USERNAME" - ) - password = get_from_dict_or_env( - {"password": password}, "password", "NEO4J_PASSWORD" - ) - database = get_from_dict_or_env( - {"database": database}, "database", "NEO4J_DATABASE", "neo4j" - ) - - self._driver = neo4j.GraphDatabase.driver(url, auth=(username, password)) - self._database = database - self.schema = "" - # Verify connection - try: - self._driver.verify_connectivity() - except neo4j.exceptions.ServiceUnavailable: - raise ValueError( - "Could not connect to Neo4j database. " - "Please ensure that the url is correct" + # Graph object takes precedent over env or input params + if graph: + self._driver = graph._driver + self._database = graph._database + else: + # Handle if the credentials are environment variables + # Support URL for backwards compatibility + if not url: + url = os.environ.get("NEO4J_URL") + + url = get_from_dict_or_env({"url": url}, "url", "NEO4J_URI") + username = get_from_dict_or_env( + {"username": username}, "username", "NEO4J_USERNAME" ) - except neo4j.exceptions.AuthError: - raise ValueError( - "Could not connect to Neo4j database. " - "Please ensure that the username and password are correct" + password = get_from_dict_or_env( + {"password": password}, "password", "NEO4J_PASSWORD" + ) + database = get_from_dict_or_env( + {"database": database}, "database", "NEO4J_DATABASE", "neo4j" ) + self._driver = neo4j.GraphDatabase.driver(url, auth=(username, password)) + self._database = database + # Verify connection + try: + self._driver.verify_connectivity() + except neo4j.exceptions.ServiceUnavailable: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the url is correct" + ) + except neo4j.exceptions.AuthError: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the username and password are correct" + ) + + self.schema = "" # Verify if the version support vector index self._is_enterprise = False self.verify_version() diff --git a/libs/community/tests/integration_tests/chat_message_histories/test_neo4j.py b/libs/community/tests/integration_tests/chat_message_histories/test_neo4j.py new file mode 100644 index 0000000000000..5ab1af54607e2 --- /dev/null +++ b/libs/community/tests/integration_tests/chat_message_histories/test_neo4j.py @@ -0,0 +1,66 @@ +import os + +from langchain_core.messages import AIMessage, HumanMessage + +from langchain_community.chat_message_histories import Neo4jChatMessageHistory +from langchain_community.graphs import Neo4jGraph + + +def test_add_messages() -> None: + """Basic testing: adding messages to the Neo4jChatMessageHistory.""" + assert os.environ.get("NEO4J_URI") is not None + assert os.environ.get("NEO4J_USERNAME") is not None + assert os.environ.get("NEO4J_PASSWORD") is not None + message_store = Neo4jChatMessageHistory("23334") + message_store.clear() + assert len(message_store.messages) == 0 + message_store.add_user_message("Hello! Language Chain!") + message_store.add_ai_message("Hi Guys!") + + # create another message store to check if the messages are stored correctly + message_store_another = Neo4jChatMessageHistory("46666") + message_store_another.clear() + assert len(message_store_another.messages) == 0 + message_store_another.add_user_message("Hello! Bot!") + message_store_another.add_ai_message("Hi there!") + message_store_another.add_user_message("How's this pr going?") + + # Now check if the messages are stored in the database correctly + assert len(message_store.messages) == 2 + assert isinstance(message_store.messages[0], HumanMessage) + assert isinstance(message_store.messages[1], AIMessage) + assert message_store.messages[0].content == "Hello! Language Chain!" + assert message_store.messages[1].content == "Hi Guys!" + + assert len(message_store_another.messages) == 3 + assert isinstance(message_store_another.messages[0], HumanMessage) + assert isinstance(message_store_another.messages[1], AIMessage) + assert isinstance(message_store_another.messages[2], HumanMessage) + assert message_store_another.messages[0].content == "Hello! Bot!" + assert message_store_another.messages[1].content == "Hi there!" + assert message_store_another.messages[2].content == "How's this pr going?" + + # Now clear the first history + message_store.clear() + assert len(message_store.messages) == 0 + assert len(message_store_another.messages) == 3 + message_store_another.clear() + assert len(message_store.messages) == 0 + assert len(message_store_another.messages) == 0 + + +def test_add_messages_graph_object() -> None: + """Basic testing: Passing driver through graph object.""" + assert os.environ.get("NEO4J_URI") is not None + assert os.environ.get("NEO4J_USERNAME") is not None + assert os.environ.get("NEO4J_PASSWORD") is not None + graph = Neo4jGraph() + # rewrite env for testing + os.environ["NEO4J_USERNAME"] = "foo" + message_store = Neo4jChatMessageHistory("23334", graph=graph) + message_store.clear() + assert len(message_store.messages) == 0 + message_store.add_user_message("Hello! Language Chain!") + message_store.add_ai_message("Hi Guys!") + # Now check if the messages are stored in the database correctly + assert len(message_store.messages) == 2 diff --git a/libs/community/tests/integration_tests/vectorstores/test_neo4jvector.py b/libs/community/tests/integration_tests/vectorstores/test_neo4jvector.py index 87db0737705bf..61c5e1117c207 100644 --- a/libs/community/tests/integration_tests/vectorstores/test_neo4jvector.py +++ b/libs/community/tests/integration_tests/vectorstores/test_neo4jvector.py @@ -4,6 +4,7 @@ from langchain_core.documents import Document +from langchain_community.graphs import Neo4jGraph from langchain_community.vectorstores.neo4j_vector import ( Neo4jVector, SearchType, @@ -902,3 +903,20 @@ def test_neo4jvector_relationship_index_retrieval() -> None: assert output == [Document(page_content="foo-text", metadata={"foo": "bar"})] drop_vector_indexes(docsearch) + + +def test_neo4jvector_passing_graph_object() -> None: + """Test end to end construction and search with passing graph object.""" + graph = Neo4jGraph() + # Rewrite env vars to make sure it fails if env is used + os.environ["NEO4J_URI"] = "foo" + docsearch = Neo4jVector.from_texts( + texts=texts, + embedding=FakeEmbeddingsWithOsDimension(), + graph=graph, + pre_delete_collection=True, + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo")] + + drop_vector_indexes(docsearch) From 25b39103e1d3b436fa196d50578748f69ea2a4d8 Mon Sep 17 00:00:00 2001 From: merdan <48309329+merdan-9@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:47:22 +0800 Subject: [PATCH 16/36] docs: hide model import in multiple_tools.ipynb (#20883) **Description:** This PR removes an unnecessary code snippet from the documentation. The snippet in question is not relevant to the content and does not contribute to the overall understanding of the topic. It contained redundant imports and unused code, potentially causing confusion for readers. **Issue:** There is no specific issue number associated with this change. **Dependencies:** No additional dependencies are required for this change. --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> Co-authored-by: Bagatur --- docs/docs/use_cases/tool_use/multiple_tools.ipynb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/docs/use_cases/tool_use/multiple_tools.ipynb b/docs/docs/use_cases/tool_use/multiple_tools.ipynb index cdc27dcb3c1d8..bd8bfeb7e6c32 100644 --- a/docs/docs/use_cases/tool_use/multiple_tools.ipynb +++ b/docs/docs/use_cases/tool_use/multiple_tools.ipynb @@ -136,6 +136,9 @@ "metadata": {}, "outputs": [], "source": [ + "# | echo: false\n", + "# | output: false\n", + "\n", "from langchain_anthropic import ChatAnthropic\n", "\n", "llm = ChatAnthropic(model=\"claude-3-sonnet-20240229\", temperature=0)" From 18d53da9555372d8864bdb790d1e34678fd3dc63 Mon Sep 17 00:00:00 2001 From: ccurme Date: Thu, 25 Apr 2024 14:50:39 -0400 Subject: [PATCH 17/36] core, community: deprecate tool.__call__ (#20900) Does not update docs. --- .../tools/edenai/test_audio_speech_to_text.py | 2 +- .../tools/edenai/test_audio_text_to_speech.py | 2 +- .../tools/edenai/test_image_explicitcontent.py | 2 +- .../tools/edenai/test_image_objectdetection.py | 2 +- .../tools/edenai/test_ocr_identityparser.py | 2 +- .../tools/edenai/test_ocr_invoiceparser.py | 4 +++- .../tools/edenai/test_text_moderation.py | 2 +- .../tests/integration_tests/utilities/test_arxiv.py | 2 +- .../utilities/test_duckduckdgo_search_api.py | 4 ++-- .../tests/integration_tests/utilities/test_pubmed.py | 2 +- .../tests/unit_tests/tools/eden_ai/test_tools.py | 2 +- libs/core/langchain_core/tools.py | 2 ++ libs/core/tests/unit_tests/test_tools.py | 8 ++++---- 13 files changed, 20 insertions(+), 16 deletions(-) diff --git a/libs/community/tests/integration_tests/tools/edenai/test_audio_speech_to_text.py b/libs/community/tests/integration_tests/tools/edenai/test_audio_speech_to_text.py index 6dc5b160ba6e1..000fc669cf229 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_audio_speech_to_text.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_audio_speech_to_text.py @@ -15,7 +15,7 @@ def test_edenai_call() -> None: """Test simple call to edenai's speech to text endpoint.""" speech2text = EdenAiSpeechToTextTool(providers=["amazon"]) - output = speech2text( + output = speech2text.invoke( "https://audio-samples.github.io/samples/mp3/blizzard_unconditional/sample-0.mp3" ) diff --git a/libs/community/tests/integration_tests/tools/edenai/test_audio_text_to_speech.py b/libs/community/tests/integration_tests/tools/edenai/test_audio_text_to_speech.py index 780785e93f6b7..31c42ccf42a19 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_audio_text_to_speech.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_audio_text_to_speech.py @@ -19,7 +19,7 @@ def test_edenai_call() -> None: providers=["amazon"], language="en", voice="MALE" ) - output = text2speech("hello") + output = text2speech.invoke("hello") parsed_url = urlparse(output) assert text2speech.name == "edenai_text_to_speech" diff --git a/libs/community/tests/integration_tests/tools/edenai/test_image_explicitcontent.py b/libs/community/tests/integration_tests/tools/edenai/test_image_explicitcontent.py index 8ff311d55475b..9f76ebdc9bd5c 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_image_explicitcontent.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_image_explicitcontent.py @@ -15,7 +15,7 @@ def test_edenai_call() -> None: """Test simple call to edenai's image moderation endpoint.""" image_moderation = EdenAiExplicitImageTool(providers=["amazon"]) - output = image_moderation("https://static.javatpoint.com/images/objects.jpg") + output = image_moderation.invoke("https://static.javatpoint.com/images/objects.jpg") assert image_moderation.name == "edenai_image_explicit_content_detection" assert image_moderation.feature == "image" diff --git a/libs/community/tests/integration_tests/tools/edenai/test_image_objectdetection.py b/libs/community/tests/integration_tests/tools/edenai/test_image_objectdetection.py index 5f23fe9476844..d68b0193042a0 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_image_objectdetection.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_image_objectdetection.py @@ -15,7 +15,7 @@ def test_edenai_call() -> None: """Test simple call to edenai's object detection endpoint.""" object_detection = EdenAiObjectDetectionTool(providers=["google"]) - output = object_detection("https://static.javatpoint.com/images/objects.jpg") + output = object_detection.invoke("https://static.javatpoint.com/images/objects.jpg") assert object_detection.name == "edenai_object_detection" assert object_detection.feature == "image" diff --git a/libs/community/tests/integration_tests/tools/edenai/test_ocr_identityparser.py b/libs/community/tests/integration_tests/tools/edenai/test_ocr_identityparser.py index 02f659fa64686..c0234a673fe4e 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_ocr_identityparser.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_ocr_identityparser.py @@ -15,7 +15,7 @@ def test_edenai_call() -> None: """Test simple call to edenai's identity parser endpoint.""" id_parser = EdenAiParsingIDTool(providers=["amazon"], language="en") - output = id_parser( + output = id_parser.invoke( "https://www.citizencard.com/images/citizencard-uk-id-card-2023.jpg" ) diff --git a/libs/community/tests/integration_tests/tools/edenai/test_ocr_invoiceparser.py b/libs/community/tests/integration_tests/tools/edenai/test_ocr_invoiceparser.py index 9bd8493d85e9b..c8a62b8e61884 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_ocr_invoiceparser.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_ocr_invoiceparser.py @@ -15,7 +15,9 @@ def test_edenai_call() -> None: """Test simple call to edenai's invoice parser endpoint.""" invoice_parser = EdenAiParsingInvoiceTool(providers=["amazon"], language="en") - output = invoice_parser("https://app.edenai.run/assets/img/data_1.72e3bdcc.png") + output = invoice_parser.invoke( + "https://app.edenai.run/assets/img/data_1.72e3bdcc.png" + ) assert invoice_parser.name == "edenai_invoice_parsing" assert invoice_parser.feature == "ocr" diff --git a/libs/community/tests/integration_tests/tools/edenai/test_text_moderation.py b/libs/community/tests/integration_tests/tools/edenai/test_text_moderation.py index e929db0bb7294..0a6d2e9c28197 100644 --- a/libs/community/tests/integration_tests/tools/edenai/test_text_moderation.py +++ b/libs/community/tests/integration_tests/tools/edenai/test_text_moderation.py @@ -16,7 +16,7 @@ def test_edenai_call() -> None: text_moderation = EdenAiTextModerationTool(providers=["openai"], language="en") - output = text_moderation("i hate you") + output = text_moderation.invoke("i hate you") assert text_moderation.name == "edenai_explicit_content_detection_text" assert text_moderation.feature == "text" diff --git a/libs/community/tests/integration_tests/utilities/test_arxiv.py b/libs/community/tests/integration_tests/utilities/test_arxiv.py index a0799c9c3d67b..c11ae32732814 100644 --- a/libs/community/tests/integration_tests/utilities/test_arxiv.py +++ b/libs/community/tests/integration_tests/utilities/test_arxiv.py @@ -151,7 +151,7 @@ def _load_arxiv_from_universal_entry(**kwargs: Any) -> BaseTool: def test_load_arxiv_from_universal_entry() -> None: arxiv_tool = _load_arxiv_from_universal_entry() - output = arxiv_tool("Caprice Stanley") + output = arxiv_tool.invoke("Caprice Stanley") assert ( "On Mixing Behavior of a Family of Random Walks" in output ), "failed to fetch a valid result" diff --git a/libs/community/tests/integration_tests/utilities/test_duckduckdgo_search_api.py b/libs/community/tests/integration_tests/utilities/test_duckduckdgo_search_api.py index d8e20b81a21be..220dc048a54e6 100644 --- a/libs/community/tests/integration_tests/utilities/test_duckduckdgo_search_api.py +++ b/libs/community/tests/integration_tests/utilities/test_duckduckdgo_search_api.py @@ -20,7 +20,7 @@ def ddg_installed() -> bool: def test_ddg_search_tool() -> None: keywords = "Bella Ciao" tool = DuckDuckGoSearchRun() - result = tool(keywords) + result = tool.invoke(keywords) print(result) # noqa: T201 assert len(result.split()) > 20 @@ -29,6 +29,6 @@ def test_ddg_search_tool() -> None: def test_ddg_search_news_tool() -> None: keywords = "Tesla" tool = DuckDuckGoSearchResults(source="news") - result = tool(keywords) + result = tool.invoke(keywords) print(result) # noqa: T201 assert len(result.split()) > 20 diff --git a/libs/community/tests/integration_tests/utilities/test_pubmed.py b/libs/community/tests/integration_tests/utilities/test_pubmed.py index e3b67f2f62856..67bd85ec89708 100644 --- a/libs/community/tests/integration_tests/utilities/test_pubmed.py +++ b/libs/community/tests/integration_tests/utilities/test_pubmed.py @@ -147,7 +147,7 @@ def test_load_pupmed_from_universal_entry() -> None: "Examining the Validity of ChatGPT in Identifying " "Relevant Nephrology Literature" ) - output = pubmed_tool(search_string) + output = pubmed_tool.invoke(search_string) test_string = ( "Examining the Validity of ChatGPT in Identifying " "Relevant Nephrology Literature: Findings and Implications" diff --git a/libs/community/tests/unit_tests/tools/eden_ai/test_tools.py b/libs/community/tests/unit_tests/tools/eden_ai/test_tools.py index 773a88102a163..3a1b4ca86af1a 100644 --- a/libs/community/tests/unit_tests/tools/eden_ai/test_tools.py +++ b/libs/community/tests/unit_tests/tools/eden_ai/test_tools.py @@ -100,6 +100,6 @@ def test_parse_response_format(mock_post: MagicMock) -> None: ] mock_post.return_value = mock_response - result = tool("some query") + result = tool.invoke("some query") assert result == 'nsfw_likelihood: 5\n"offensive": 4\n"hate_speech": 5' diff --git a/libs/core/langchain_core/tools.py b/libs/core/langchain_core/tools.py index 0e576a39d2645..38119ceb705db 100644 --- a/libs/core/langchain_core/tools.py +++ b/libs/core/langchain_core/tools.py @@ -29,6 +29,7 @@ from inspect import signature from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Type, Union +from langchain_core._api import deprecated from langchain_core.callbacks import ( AsyncCallbackManager, AsyncCallbackManagerForToolRun, @@ -559,6 +560,7 @@ async def arun( ) return observation + @deprecated("0.1.47", alternative="invoke", removal="0.3.0") def __call__(self, tool_input: str, callbacks: Callbacks = None) -> str: """Make tool callable.""" return self.run(tool_input, callbacks=callbacks) diff --git a/libs/core/tests/unit_tests/test_tools.py b/libs/core/tests/unit_tests/test_tools.py index df52ae59ca067..a424d6ecf6026 100644 --- a/libs/core/tests/unit_tests/test_tools.py +++ b/libs/core/tests/unit_tests/test_tools.py @@ -39,7 +39,7 @@ def search_api(query: str) -> str: assert isinstance(search_api, BaseTool) assert search_api.name == "search_api" assert not search_api.return_direct - assert search_api("test") == "API result" + assert search_api.invoke("test") == "API result" class _MockSchema(BaseModel): @@ -562,7 +562,7 @@ def search_api(query: str) -> str: def test_create_tool_positional_args() -> None: """Test that positional arguments are allowed.""" test_tool = Tool("test_name", lambda x: x, "test_description") - assert test_tool("foo") == "foo" + assert test_tool.invoke("foo") == "foo" assert test_tool.name == "test_name" assert test_tool.description == "test_description" assert test_tool.is_single_input @@ -572,7 +572,7 @@ def test_create_tool_keyword_args() -> None: """Test that keyword arguments are allowed.""" test_tool = Tool(name="test_name", func=lambda x: x, description="test_description") assert test_tool.is_single_input - assert test_tool("foo") == "foo" + assert test_tool.invoke("foo") == "foo" assert test_tool.name == "test_name" assert test_tool.description == "test_description" @@ -590,7 +590,7 @@ async def _test_func(x: str) -> str: coroutine=_test_func, ) assert test_tool.is_single_input - assert test_tool("foo") == "foo" + assert test_tool.invoke("foo") == "foo" assert test_tool.name == "test_name" assert test_tool.description == "test_description" assert test_tool.coroutine is not None From 97ba189ef95e7e7baa062839f666bb1ce01f4326 Mon Sep 17 00:00:00 2001 From: ccurme Date: Thu, 25 Apr 2024 15:05:43 -0400 Subject: [PATCH 18/36] docs: update chat model feature table (#20899) --- docs/scripts/model_feat_table.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/scripts/model_feat_table.py b/docs/scripts/model_feat_table.py index a0f6d7df25ff7..e56fee80d0cfa 100644 --- a/docs/scripts/model_feat_table.py +++ b/docs/scripts/model_feat_table.py @@ -51,12 +51,12 @@ "package": "langchain-google-vertexai", }, "ChatGroq": { - "tool_calling": "partial", + "tool_calling": True, "structured_output": True, "package": "langchain-groq", }, "ChatCohere": { - "tool_calling": "partial", + "tool_calling": True, "structured_output": True, "package": "langchain-cohere", }, @@ -98,8 +98,7 @@ - *Batch* support defaults to calling the underlying ChatModel in parallel for each input by making use of a thread pool executor (in the sync batch case) or `asyncio.gather` (in the async batch case). The concurrency can be controlled with the `max_concurrency` key in `RunnableConfig`. Each ChatModel integration can optionally provide native implementations to truly enable async or streaming. -The table shows, for each integration, which features have been implemented with native support. -Yellow circles (🟡) indicates partial support - for example, if the model supports tool calling but not tool messages for agents. +The table shows, for each integration, which features have been implemented with native support. {table} From 4417aa1cff645843c70c708639df71e352bc082b Mon Sep 17 00:00:00 2001 From: ccurme Date: Thu, 25 Apr 2024 15:23:29 -0400 Subject: [PATCH 19/36] mistral, openai: support custom tokenizers in chat models (#20901) --- .../mistralai/tests/unit_tests/test_chat_models.py | 10 +++++++++- .../openai/langchain_openai/chat_models/base.py | 2 ++ .../openai/tests/unit_tests/chat_models/test_base.py | 10 +++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/libs/partners/mistralai/tests/unit_tests/test_chat_models.py b/libs/partners/mistralai/tests/unit_tests/test_chat_models.py index ab70d02d45baf..63713f227313e 100644 --- a/libs/partners/mistralai/tests/unit_tests/test_chat_models.py +++ b/libs/partners/mistralai/tests/unit_tests/test_chat_models.py @@ -1,7 +1,7 @@ """Test MistralAI Chat API wrapper.""" import os -from typing import Any, AsyncGenerator, Dict, Generator, cast +from typing import Any, AsyncGenerator, Dict, Generator, List, cast from unittest.mock import patch import pytest @@ -190,3 +190,11 @@ def test__convert_dict_to_message_tool_call() -> None: ) assert result == expected_output assert _convert_message_to_mistral_chat_message(expected_output) == message + + +def test_custom_token_counting() -> None: + def token_encoder(text: str) -> List[int]: + return [1, 2, 3] + + llm = ChatMistralAI(custom_get_token_ids=token_encoder) + assert llm.get_token_ids("foo") == [1, 2, 3] diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index e68377c2a85d2..334aa96885e7c 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -703,6 +703,8 @@ def _get_encoding_model(self) -> Tuple[str, tiktoken.Encoding]: def get_token_ids(self, text: str) -> List[int]: """Get the tokens present in the text with tiktoken package.""" + if self.custom_get_token_ids is not None: + return self.custom_get_token_ids(text) # tiktoken NOT supported for Python 3.7 or below if sys.version_info[1] <= 7: return super().get_token_ids(text) diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py index 9665af8f64432..e4cc2cfddf85e 100644 --- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py @@ -1,7 +1,7 @@ """Test OpenAI Chat API wrapper.""" import json -from typing import Any +from typing import Any, List from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -279,3 +279,11 @@ def test_openai_invoke_name(mock_completion: dict) -> None: # check return type has name assert res.content == "Bar Baz" assert res.name == "Erick" + + +def test_custom_token_counting() -> None: + def token_encoder(text: str) -> List[int]: + return [1, 2, 3] + + llm = ChatOpenAI(custom_get_token_ids=token_encoder) + assert llm.get_token_ids("foo") == [1, 2, 3] From b0d7bab1d7e1fd954834222e4b0c3298b316970e Mon Sep 17 00:00:00 2001 From: Andres Algaba Date: Thu, 25 Apr 2024 21:42:03 +0200 Subject: [PATCH 20/36] community[patch]: deprecate persist method in Chroma (#20855) Thank you for contributing to LangChain! - [x] **PR title** - [x] **PR message**: - **Description:** Deprecate persist method in Chroma no longer exists in Chroma 0.4.x - **Issue:** #20851 - **Dependencies:** None - **Twitter handle:** AndresAlgaba1 - [x] **Add tests and docs**: If you're adding a new integration, please include 1. a test for the integration, preferably unit tests that do not rely on network access, 2. an example notebook showing its use. It lives in `docs/docs/integrations` directory. - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> --- .../langchain_community/vectorstores/chroma.py | 12 ++++++++++++ .../integration_tests/vectorstores/test_chroma.py | 2 -- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/vectorstores/chroma.py b/libs/community/langchain_community/vectorstores/chroma.py index 7723285fafa6c..134f2f9acd779 100644 --- a/libs/community/langchain_community/vectorstores/chroma.py +++ b/libs/community/langchain_community/vectorstores/chroma.py @@ -16,6 +16,7 @@ ) import numpy as np +from langchain_core._api import deprecated from langchain_core.documents import Document from langchain_core.embeddings import Embeddings from langchain_core.utils import xor_args @@ -610,11 +611,22 @@ def get( return self._collection.get(**kwargs) + @deprecated( + since="0.1.17", + message=( + "Since Chroma 0.4.x the manual persistence method is no longer " + "supported as docs are automatically persisted." + ), + removal="0.3.0", + ) def persist(self) -> None: """Persist the collection. This can be used to explicitly persist the data to disk. It will also be called automatically when the object is destroyed. + + Since Chroma 0.4.x the manual persistence method is no longer + supported as docs are automatically persisted. """ if self._persist_directory is None: raise ValueError( diff --git a/libs/community/tests/integration_tests/vectorstores/test_chroma.py b/libs/community/tests/integration_tests/vectorstores/test_chroma.py index 0a0cf529d08e6..b6549626a6caf 100644 --- a/libs/community/tests/integration_tests/vectorstores/test_chroma.py +++ b/libs/community/tests/integration_tests/vectorstores/test_chroma.py @@ -136,8 +136,6 @@ def test_chroma_with_persistence() -> None: output = docsearch.similarity_search("foo", k=1) assert output == [Document(page_content="foo")] - docsearch.persist() - # Get a new VectorStore from the persisted directory docsearch = Chroma( collection_name=collection_name, From 5ec5e90c9979fa1b7190d06a054d3cbed248dee5 Mon Sep 17 00:00:00 2001 From: davidefantiniIntel <115252273+davidefantiniIntel@users.noreply.github.com> Date: Thu, 25 Apr 2024 20:44:53 +0100 Subject: [PATCH 21/36] community: fix tqdm import (#20263) Description: Fix tqdm import in QuantizedBiEncoderEmbeddings --- libs/community/langchain_community/embeddings/optimum_intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/embeddings/optimum_intel.py b/libs/community/langchain_community/embeddings/optimum_intel.py index 56d4a6f20ac6f..032268912195c 100644 --- a/libs/community/langchain_community/embeddings/optimum_intel.py +++ b/libs/community/langchain_community/embeddings/optimum_intel.py @@ -178,7 +178,7 @@ def embed_documents(self, texts: List[str]) -> List[List[float]]: "Unable to import pandas, please install with `pip install -U pandas`." ) from e try: - import tqdm + from tqdm import tqdm except ImportError as e: raise ImportError( "Unable to import tqdm, please install with `pip install -U tqdm`." From feca792a886e9d0de90df17bc4db728efc74a222 Mon Sep 17 00:00:00 2001 From: Jingpan Xiong <71321890+klaus-xiong@users.noreply.github.com> Date: Fri, 26 Apr 2024 03:49:29 +0800 Subject: [PATCH 22/36] community[minor]: Add relyt vector database (#20316) Co-authored-by: kaka Co-authored-by: Bagatur Co-authored-by: jingsi Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> --- .../integrations/vectorstores/relyt.ipynb | 166 ++++++ .../vectorstores/__init__.py | 5 + .../langchain_community/vectorstores/relyt.py | 518 ++++++++++++++++++ .../vectorstores/test_relyt.py | 167 ++++++ .../unit_tests/vectorstores/test_imports.py | 1 + .../vectorstores/test_indexing_docs.py | 1 + .../vectorstores/test_public_api.py | 1 + 7 files changed, 859 insertions(+) create mode 100644 docs/docs/integrations/vectorstores/relyt.ipynb create mode 100644 libs/community/langchain_community/vectorstores/relyt.py create mode 100644 libs/community/tests/integration_tests/vectorstores/test_relyt.py diff --git a/docs/docs/integrations/vectorstores/relyt.ipynb b/docs/docs/integrations/vectorstores/relyt.ipynb new file mode 100644 index 0000000000000..b6956f5a449fd --- /dev/null +++ b/docs/docs/integrations/vectorstores/relyt.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Relyt\n", + "\n", + ">[Relyt](https://docs.relyt.cn/docs/vector-engine/use/) is a cloud native data warehousing service that is designed to analyze large volumes of data online.\n", + "\n", + ">`Relyt` is compatible with the ANSI SQL 2003 syntax and the PostgreSQL and Oracle database ecosystems. Relyt also supports row store and column store. Relyt processes petabytes of data offline at a high performance level and supports highly concurrent online queries.\n", + "\n", + "This notebook shows how to use functionality related to the `Relyt` vector database.\n", + "To run, you should have an [Relyt](https://docs.relyt.cn/) instance up and running:\n", + "- Using [Relyt Vector Database](https://docs.relyt.cn/docs/vector-engine/use/). Click here to fast deploy it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install \"pgvecto_rs[sdk]\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import TextLoader\n", + "from langchain_community.embeddings.fake import FakeEmbeddings\n", + "from langchain_community.vectorstores import Relyt\n", + "from langchain_text_splitters import CharacterTextSplitter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Split documents and get embeddings by call community API" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", + "documents = loader.load()\n", + "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", + "docs = text_splitter.split_documents(documents)\n", + "\n", + "embeddings = FakeEmbeddings(size=1536)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Connect to Relyt by setting related ENVIRONMENTS.\n", + "```\n", + "export PG_HOST={your_relyt_hostname}\n", + "export PG_PORT={your_relyt_port} # Optional, default is 5432\n", + "export PG_DATABASE={your_database} # Optional, default is postgres\n", + "export PG_USER={database_username}\n", + "export PG_PASSWORD={database_password}\n", + "```\n", + "\n", + "Then store your embeddings and documents into Relyt" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "connection_string = Relyt.connection_string_from_db_params(\n", + " driver=os.environ.get(\"PG_DRIVER\", \"psycopg2cffi\"),\n", + " host=os.environ.get(\"PG_HOST\", \"localhost\"),\n", + " port=int(os.environ.get(\"PG_PORT\", \"5432\")),\n", + " database=os.environ.get(\"PG_DATABASE\", \"postgres\"),\n", + " user=os.environ.get(\"PG_USER\", \"postgres\"),\n", + " password=os.environ.get(\"PG_PASSWORD\", \"postgres\"),\n", + ")\n", + "\n", + "vector_db = Relyt.from_documents(\n", + " docs,\n", + " embeddings,\n", + " connection_string=connection_string,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Query and retrieve data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What did the president say about Ketanji Brown Jackson\"\n", + "docs = vector_db.similarity_search(query)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n", + "\n", + "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n", + "\n", + "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n", + "\n", + "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n" + ] + } + ], + "source": [ + "print(docs[0].page_content)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/libs/community/langchain_community/vectorstores/__init__.py b/libs/community/langchain_community/vectorstores/__init__.py index fe0c1b460014c..33e13b7d99a59 100644 --- a/libs/community/langchain_community/vectorstores/__init__.py +++ b/libs/community/langchain_community/vectorstores/__init__.py @@ -196,6 +196,9 @@ from langchain_community.vectorstores.redis import ( Redis, # noqa: F401 ) + from langchain_community.vectorstores.relyt import ( + Relyt, # noqa: F401 + ) from langchain_community.vectorstores.rocksetdb import ( Rockset, # noqa: F401 ) @@ -344,6 +347,7 @@ "Pinecone", "Qdrant", "Redis", + "Relyt", "Rockset", "SKLearnVectorStore", "SQLiteVSS", @@ -437,6 +441,7 @@ "Pinecone": "langchain_community.vectorstores.pinecone", "Qdrant": "langchain_community.vectorstores.qdrant", "Redis": "langchain_community.vectorstores.redis", + "Relyt": "langchain_community.vectorstores.relyt", "Rockset": "langchain_community.vectorstores.rocksetdb", "SKLearnVectorStore": "langchain_community.vectorstores.sklearn", "SQLiteVSS": "langchain_community.vectorstores.sqlitevss", diff --git a/libs/community/langchain_community/vectorstores/relyt.py b/libs/community/langchain_community/vectorstores/relyt.py new file mode 100644 index 0000000000000..691fff3b4f216 --- /dev/null +++ b/libs/community/langchain_community/vectorstores/relyt.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import logging +import uuid +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Type + +from sqlalchemy import Column, String, Table, create_engine, insert, text +from sqlalchemy.dialects.postgresql import JSON, TEXT + +try: + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base + +from langchain_core.documents import Document +from langchain_core.embeddings import Embeddings +from langchain_core.utils import get_from_dict_or_env +from langchain_core.vectorstores import VectorStore + +_LANGCHAIN_DEFAULT_EMBEDDING_DIM = 1536 +_LANGCHAIN_DEFAULT_COLLECTION_NAME = "langchain_document" + +Base = declarative_base() # type: Any + + +class Relyt(VectorStore): + """`Relyt` (distributed PostgreSQL) vector store. + + Relyt is a distributed full postgresql syntax cloud-native database. + - `connection_string` is a postgres connection string. + - `embedding_function` any embedding function implementing + `langchain.embeddings.base.Embeddings` interface. + - `collection_name` is the name of the collection to use. (default: langchain) + - NOTE: This is not the name of the table, but the name of the collection. + The tables will be created when initializing the store (if not exists) + So, make sure the user has the right permissions to create tables. + - `pre_delete_collection` if True, will delete the collection if it exists. + (default: False) + - Useful for testing. + + """ + + def __init__( + self, + connection_string: str, + embedding_function: Embeddings, + embedding_dimension: int = _LANGCHAIN_DEFAULT_EMBEDDING_DIM, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + pre_delete_collection: bool = False, + logger: Optional[logging.Logger] = None, + engine_args: Optional[dict] = None, + ) -> None: + """Initialize a PGVecto_rs vectorstore. + + Args: + embedding: Embeddings to use. + dimension: Dimension of the embeddings. + db_url: Database URL. + collection_name: Name of the collection. + new_table: Whether to create a new table or connect to an existing one. + If true, the table will be dropped if exists, then recreated. + Defaults to False. + """ + try: + from pgvecto_rs.sdk import PGVectoRs + + PGVectoRs( + db_url=connection_string, + collection_name=collection_name, + dimension=embedding_dimension, + recreate=pre_delete_collection, + ) + except ImportError as e: + raise ImportError( + "Unable to import pgvector_rs.sdk , please install with " + '`pip install "pgvecto_rs[sdk]"`.' + ) from e + + self.connection_string = connection_string + self.embedding_function = embedding_function + self.embedding_dimension = embedding_dimension + self.collection_name = collection_name + self.pre_delete_collection = pre_delete_collection + self.logger = logger or logging.getLogger(__name__) + self.__post_init__(engine_args) + + def __post_init__( + self, + engine_args: Optional[dict] = None, + ) -> None: + """ + Initialize the store. + """ + + _engine_args = engine_args or {} + + if ( + "pool_recycle" not in _engine_args + ): # Check if pool_recycle is not in _engine_args + _engine_args[ + "pool_recycle" + ] = 3600 # Set pool_recycle to 3600s if not present + + self.engine = create_engine(self.connection_string, **_engine_args) + self.create_collection() + + @property + def embeddings(self) -> Embeddings: + return self.embedding_function + + def _select_relevance_score_fn(self) -> Callable[[float], float]: + return self._euclidean_relevance_score_fn + + def create_table_if_not_exists(self) -> None: + # Define the dynamic table + """ + Table( + self.collection_name, + Base.metadata, + Column("id", TEXT, primary_key=True, default=uuid.uuid4), + Column("embedding", Vector(self.embedding_dimension)), + Column("document", String, nullable=True), + Column("metadata", JSON, nullable=True), + extend_existing=True, + ) + """ + with self.engine.connect() as conn: + with conn.begin(): + # create vectors + conn.execute(text("CREATE EXTENSION IF NOT EXISTS vectors")) + conn.execute(text('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"')) + + # Create the table + # Base.metadata.create_all(conn) + table_name = f"{self.collection_name}" + table_query = text( + f""" + SELECT 1 + FROM pg_class + WHERE relname = '{table_name}'; + """ + ) + result = conn.execute(table_query).scalar() + if not result: + table_statement = text( + f""" + CREATE TABLE {table_name} ( + id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(), + embedding vector({self.embedding_dimension}), + document TEXT, + metadata JSON + ) USING heap; + """ + ) + conn.execute(table_statement) + + # Check if the index exists + index_name = f"{self.collection_name}_embedding_idx" + index_query = text( + f""" + SELECT 1 + FROM pg_indexes + WHERE indexname = '{index_name}'; + """ + ) + result = conn.execute(index_query).scalar() + + # Create the index if it doesn't exist + if not result: + index_statement = text( + f""" + CREATE INDEX {index_name} + ON {self.collection_name} + USING vectors (embedding vector_l2_ops) + WITH (options = $$ + optimizing.optimizing_threads = 30 + segment.max_growing_segment_size = 600 + segment.max_sealed_segment_size = 30000000 + [indexing.hnsw] + m=30 + ef_construction=500 + $$); + """ + ) + conn.execute(index_statement) + + def create_collection(self) -> None: + if self.pre_delete_collection: + self.delete_collection() + self.create_table_if_not_exists() + + def delete_collection(self) -> None: + self.logger.debug("Trying to delete collection") + drop_statement = text(f"DROP TABLE IF EXISTS {self.collection_name};") + with self.engine.connect() as conn: + with conn.begin(): + conn.execute(drop_statement) + + def add_texts( + self, + texts: Iterable[str], + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + batch_size: int = 500, + **kwargs: Any, + ) -> List[str]: + """Run more texts through the embeddings and add to the vectorstore. + + Args: + texts: Iterable of strings to add to the vectorstore. + metadatas: Optional list of metadatas associated with the texts. + kwargs: vectorstore specific parameters + + Returns: + List of ids from adding the texts into the vectorstore. + """ + from pgvecto_rs.sqlalchemy import Vector + + if ids is None: + ids = [str(uuid.uuid1()) for _ in texts] + + embeddings = self.embedding_function.embed_documents(list(texts)) + + if not metadatas: + metadatas = [{} for _ in texts] + + # Define the table schema + chunks_table = Table( + self.collection_name, + Base.metadata, + Column("id", TEXT, primary_key=True), + Column("embedding", Vector(self.embedding_dimension)), + Column("document", String, nullable=True), + Column("metadata", JSON, nullable=True), + extend_existing=True, + ) + + chunks_table_data = [] + with self.engine.connect() as conn: + with conn.begin(): + for document, metadata, chunk_id, embedding in zip( + texts, metadatas, ids, embeddings + ): + chunks_table_data.append( + { + "id": chunk_id, + "embedding": embedding, + "document": document, + "metadata": metadata, + } + ) + + # Execute the batch insert when the batch size is reached + if len(chunks_table_data) == batch_size: + conn.execute(insert(chunks_table).values(chunks_table_data)) + # Clear the chunks_table_data list for the next batch + chunks_table_data.clear() + + # Insert any remaining records that didn't make up a full batch + if chunks_table_data: + conn.execute(insert(chunks_table).values(chunks_table_data)) + + return ids + + def similarity_search( + self, + query: str, + k: int = 4, + filter: Optional[dict] = None, + **kwargs: Any, + ) -> List[Document]: + """Run similarity search with AnalyticDB with distance. + + Args: + query (str): Query text to search for. + k (int): Number of results to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query. + """ + embedding = self.embedding_function.embed_query(text=query) + return self.similarity_search_by_vector( + embedding=embedding, + k=k, + filter=filter, + ) + + def similarity_search_with_score( + self, + query: str, + k: int = 4, + filter: Optional[dict] = None, + ) -> List[Tuple[Document, float]]: + """Return docs most similar to query. + + Args: + query: Text to look up documents similar to. + k: Number of Documents to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query and score for each + """ + embedding = self.embedding_function.embed_query(query) + docs = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, filter=filter + ) + return docs + + def similarity_search_with_score_by_vector( + self, + embedding: List[float], + k: int = 4, + filter: Optional[dict] = None, + ) -> List[Tuple[Document, float]]: + # Add the filter if provided + try: + from sqlalchemy.engine import Row + except ImportError: + raise ImportError( + "Could not import Row from sqlalchemy.engine. " + "Please 'pip install sqlalchemy>=1.4'." + ) + + filter_condition = "" + if filter is not None: + conditions = [ + f"metadata->>{key!r} = {value!r}" for key, value in filter.items() + ] + filter_condition = f"WHERE {' AND '.join(conditions)}" + + # Define the base query + sql_query = f""" + set vectors.enable_search_growing = on; + set vectors.enable_search_write = on; + SELECT document, metadata, embedding <-> :embedding as distance + FROM {self.collection_name} + {filter_condition} + ORDER BY embedding <-> :embedding + LIMIT :k + """ + + # Set up the query parameters + embedding_str = ", ".join(format(x) for x in embedding) + embedding_str = "[" + embedding_str + "]" + params = {"embedding": embedding_str, "k": k} + + # Execute the query and fetch the results + with self.engine.connect() as conn: + results: Sequence[Row] = conn.execute(text(sql_query), params).fetchall() + + documents_with_scores = [ + ( + Document( + page_content=result.document, + metadata=result.metadata, + ), + result.distance if self.embedding_function is not None else None, + ) + for result in results + ] + return documents_with_scores + + def similarity_search_by_vector( + self, + embedding: List[float], + k: int = 4, + filter: Optional[dict] = None, + **kwargs: Any, + ) -> List[Document]: + """Return docs most similar to embedding vector. + + Args: + embedding: Embedding to look up documents similar to. + k: Number of Documents to return. Defaults to 4. + filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. + + Returns: + List of Documents most similar to the query vector. + """ + docs_and_scores = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, filter=filter + ) + return [doc for doc, _ in docs_and_scores] + + def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]: + """Delete by vector IDs. + + Args: + ids: List of ids to delete. + """ + from pgvecto_rs.sqlalchemy import Vector + + if ids is None: + raise ValueError("No ids provided to delete.") + + # Define the table schema + chunks_table = Table( + self.collection_name, + Base.metadata, + Column("id", TEXT, primary_key=True), + Column("embedding", Vector(self.embedding_dimension)), + Column("document", String, nullable=True), + Column("metadata", JSON, nullable=True), + extend_existing=True, + ) + + try: + with self.engine.connect() as conn: + with conn.begin(): + delete_condition = chunks_table.c.id.in_(ids) + conn.execute(chunks_table.delete().where(delete_condition)) + return True + except Exception as e: + print("Delete operation failed:", str(e)) # noqa: T201 + return False + + @classmethod + def from_texts( + cls: Type[Relyt], + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + embedding_dimension: int = _LANGCHAIN_DEFAULT_EMBEDDING_DIM, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + ids: Optional[List[str]] = None, + pre_delete_collection: bool = False, + engine_args: Optional[dict] = None, + **kwargs: Any, + ) -> Relyt: + """ + Return VectorStore initialized from texts and embeddings. + Postgres Connection string is required + Either pass it as a parameter + or set the PG_CONNECTION_STRING environment variable. + """ + + connection_string = cls.get_connection_string(kwargs) + + store = cls( + connection_string=connection_string, + collection_name=collection_name, + embedding_function=embedding, + embedding_dimension=embedding_dimension, + pre_delete_collection=pre_delete_collection, + engine_args=engine_args, + ) + + store.add_texts(texts=texts, metadatas=metadatas, ids=ids, **kwargs) + return store + + @classmethod + def get_connection_string(cls, kwargs: Dict[str, Any]) -> str: + connection_string: str = get_from_dict_or_env( + data=kwargs, + key="connection_string", + env_key="PG_CONNECTION_STRING", + ) + + if not connection_string: + raise ValueError( + "Postgres connection string is required" + "Either pass it as a parameter" + "or set the PG_CONNECTION_STRING environment variable." + ) + + return connection_string + + @classmethod + def from_documents( + cls: Type[Relyt], + documents: List[Document], + embedding: Embeddings, + embedding_dimension: int = _LANGCHAIN_DEFAULT_EMBEDDING_DIM, + collection_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME, + ids: Optional[List[str]] = None, + pre_delete_collection: bool = False, + engine_args: Optional[dict] = None, + **kwargs: Any, + ) -> Relyt: + """ + Return VectorStore initialized from documents and embeddings. + Postgres Connection string is required + Either pass it as a parameter + or set the PG_CONNECTION_STRING environment variable. + """ + + texts = [d.page_content for d in documents] + metadatas = [d.metadata for d in documents] + connection_string = cls.get_connection_string(kwargs) + + kwargs["connection_string"] = connection_string + + return cls.from_texts( + texts=texts, + pre_delete_collection=pre_delete_collection, + embedding=embedding, + embedding_dimension=embedding_dimension, + metadatas=metadatas, + ids=ids, + collection_name=collection_name, + engine_args=engine_args, + **kwargs, + ) + + @classmethod + def connection_string_from_db_params( + cls, + driver: str, + host: str, + port: int, + database: str, + user: str, + password: str, + ) -> str: + """Return connection string from database parameters.""" + return f"postgresql+{driver}://{user}:{password}@{host}:{port}/{database}" diff --git a/libs/community/tests/integration_tests/vectorstores/test_relyt.py b/libs/community/tests/integration_tests/vectorstores/test_relyt.py new file mode 100644 index 0000000000000..51f76d1c0d8fc --- /dev/null +++ b/libs/community/tests/integration_tests/vectorstores/test_relyt.py @@ -0,0 +1,167 @@ +"""Test Relyt functionality.""" +import os +from typing import List + +from langchain_core.documents import Document + +from langchain_community.vectorstores.relyt import Relyt +from tests.integration_tests.vectorstores.fake_embeddings import FakeEmbeddings + +CONNECTION_STRING = Relyt.connection_string_from_db_params( + driver=os.environ.get("PG_DRIVER", "psycopg2cffi"), + host=os.environ.get("PG_HOST", "localhost"), + port=int(os.environ.get("PG_PORT", "5432")), + database=os.environ.get("PG_DATABASE", "postgres"), + user=os.environ.get("PG_USER", "postgres"), + password=os.environ.get("PG_PASSWORD", "postgres"), +) + + +ADA_TOKEN_COUNT = 1536 + + +class FakeEmbeddingsWithAdaDimension(FakeEmbeddings): + """Fake embeddings functionality for testing.""" + + def embed_documents(self, texts: List[str]) -> List[List[float]]: + """Return simple embeddings.""" + return [ + [float(1.0)] * (ADA_TOKEN_COUNT - 1) + [float(i)] for i in range(len(texts)) + ] + + def embed_query(self, text: str) -> List[float]: + """Return simple embeddings.""" + return [float(1.0)] * (ADA_TOKEN_COUNT - 1) + [float(0.0)] + + +def test_relyt() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection", + embedding=FakeEmbeddingsWithAdaDimension(), + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo")] + + +def test_relyt_with_engine_args() -> None: + engine_args = {"pool_recycle": 3600, "pool_size": 50} + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection", + embedding=FakeEmbeddingsWithAdaDimension(), + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + engine_args=engine_args, + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo")] + + +def test_relyt_with_metadatas() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo", metadata={"page": "0"})] + + +def test_relyt_with_metadatas_with_scores() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search_with_score("foo", k=1) + assert output == [(Document(page_content="foo", metadata={"page": "0"}), 0.0)] + + +def test_relyt_with_filter_match() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection_filter", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "0"}) + assert output == [(Document(page_content="foo", metadata={"page": "0"}), 0.0)] + + +def test_relyt_with_filter_distant_match() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection_filter", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "2"}) + print(output) # noqa: T201 + assert output == [(Document(page_content="baz", metadata={"page": "2"}), 4.0)] + + +def test_relyt_with_filter_no_match() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection_filter", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + pre_delete_collection=True, + ) + output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "5"}) + assert output == [] + + +def test_relyt_delete() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + ids = ["fooid", "barid", "bazid"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = Relyt.from_texts( + texts=texts, + collection_name="test_collection_delete", + embedding=FakeEmbeddingsWithAdaDimension(), + metadatas=metadatas, + connection_string=CONNECTION_STRING, + ids=ids, + pre_delete_collection=True, + ) + output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "2"}) + print(output) # noqa: T201 + assert output == [(Document(page_content="baz", metadata={"page": "2"}), 4.0)] + docsearch.delete(ids=ids) + output = docsearch.similarity_search_with_score("foo", k=1, filter={"page": "2"}) + assert output == [] diff --git a/libs/community/tests/unit_tests/vectorstores/test_imports.py b/libs/community/tests/unit_tests/vectorstores/test_imports.py index 97a26daa1deb8..62231de322ea3 100644 --- a/libs/community/tests/unit_tests/vectorstores/test_imports.py +++ b/libs/community/tests/unit_tests/vectorstores/test_imports.py @@ -66,6 +66,7 @@ "Pinecone", "Qdrant", "Redis", + "Relyt", "Rockset", "SKLearnVectorStore", "SQLiteVSS", diff --git a/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py b/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py index c0e29eb995173..9d639a34de89c 100644 --- a/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py +++ b/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py @@ -76,6 +76,7 @@ def check_compatibility(vector_store: VectorStore) -> bool: "Pinecone", "Qdrant", "Redis", + "Relyt", "Rockset", "ScaNN", "SemaDB", diff --git a/libs/community/tests/unit_tests/vectorstores/test_public_api.py b/libs/community/tests/unit_tests/vectorstores/test_public_api.py index 96f62992de912..314188c913bf5 100644 --- a/libs/community/tests/unit_tests/vectorstores/test_public_api.py +++ b/libs/community/tests/unit_tests/vectorstores/test_public_api.py @@ -61,6 +61,7 @@ "Pinecone", "Qdrant", "Redis", + "Relyt", "Rockset", "SKLearnVectorStore", "ScaNN", From b5d5c97f2cf0fe8d2ea1b87a7575a1364265dda9 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Thu, 25 Apr 2024 15:51:33 -0400 Subject: [PATCH 23/36] cli[minor]: Add __version__ (#20903) Add __version__ to cli --- libs/cli/langchain_cli/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/cli/langchain_cli/__init__.py b/libs/cli/langchain_cli/__init__.py index e69de29bb2d1d..c8bb365cacd4f 100644 --- a/libs/cli/langchain_cli/__init__.py +++ b/libs/cli/langchain_cli/__init__.py @@ -0,0 +1,8 @@ +from importlib import metadata + +try: + __version__ = metadata.version(__package__) +except metadata.PackageNotFoundError: + # Case where package metadata is not available. + __version__ = "" +del metadata # optional, avoids polluting the results of dir(__package__) From 3c702a772b585fee48990e1f275b29c8132bf3c0 Mon Sep 17 00:00:00 2001 From: Rahul Triptahi Date: Fri, 26 Apr 2024 01:25:33 +0530 Subject: [PATCH 24/36] community[patch]: Add semantic info to metadata, classified by pebblo-server. (#20468) Description: Add support for Semantic topics and entities. Classification done by pebblo-server is not used to enhance metadata of Documents loaded by document loaders. Dependencies: None Documentation: Updated. Signed-off-by: Rahul Tripathi Co-authored-by: Rahul Tripathi --- .../document_loaders/pebblo.ipynb | 37 ++++- .../document_loaders/pebblo.py | 156 +++++++++++++++--- .../langchain_community/utilities/pebblo.py | 5 + 3 files changed, 174 insertions(+), 24 deletions(-) diff --git a/docs/docs/integrations/document_loaders/pebblo.ipynb b/docs/docs/integrations/document_loaders/pebblo.ipynb index e444c426cd23b..177a11fbab9f2 100644 --- a/docs/docs/integrations/document_loaders/pebblo.ipynb +++ b/docs/docs/integrations/document_loaders/pebblo.ipynb @@ -69,7 +69,7 @@ "source": [ "### Send semantic topics and identities to Pebblo cloud server\n", "\n", - "To send semantic data to pebblo-cloud, pass api-key to PebbloSafeLoader as an argument or alternatively, put the api-ket in `PEBBLO_API_KEY` environment variable." + "To send semantic data to pebblo-cloud, pass api-key to PebbloSafeLoader as an argument or alternatively, put the api-key in `PEBBLO_API_KEY` environment variable." ] }, { @@ -91,6 +91,41 @@ "documents = loader.load()\n", "print(documents)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add semantic topics and identities to loaded metadata\n", + "\n", + "To add semantic topics and sematic entities to metadata of loaded documents, set load_semantic to True as an argument or alternatively, define a new environment variable `PEBBLO_LOAD_SEMANTIC`, and setting it to True." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders.csv_loader import CSVLoader\n", + "from langchain_community.document_loaders import PebbloSafeLoader\n", + "\n", + "loader = PebbloSafeLoader(\n", + " CSVLoader(\"data/corp_sens_data.csv\"),\n", + " name=\"acme-corp-rag-1\", # App name (Mandatory)\n", + " owner=\"Joe Smith\", # Owner (Optional)\n", + " description=\"Support productivity RAG application\", # Description (Optional)\n", + " api_key=\"my-api-key\", # API key (Optional, can be set in the environment variable PEBBLO_API_KEY)\n", + " load_semantic=True, # Load semantic data (Optional, default is False, can be set in the environment variable PEBBLO_LOAD_SEMANTIC)\n", + ")\n", + "documents = loader.load()\n", + "print(documents[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/libs/community/langchain_community/document_loaders/pebblo.py b/libs/community/langchain_community/document_loaders/pebblo.py index 5cdeffaac4e8e..8b348826293cc 100644 --- a/libs/community/langchain_community/document_loaders/pebblo.py +++ b/libs/community/langchain_community/document_loaders/pebblo.py @@ -5,9 +5,9 @@ import os import uuid from http import HTTPStatus -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Optional, Union -import requests +import requests # type: ignore from langchain_core.documents import Document from langchain_community.document_loaders.base import BaseLoader @@ -19,6 +19,7 @@ PLUGIN_VERSION, App, Doc, + IndexedDocument, get_full_path, get_loader_full_path, get_loader_type, @@ -43,6 +44,7 @@ def __init__( owner: str = "", description: str = "", api_key: Optional[str] = None, + load_semantic: bool = False, ): if not name or not isinstance(name, str): raise NameError("Must specify a valid name.") @@ -50,15 +52,17 @@ def __init__( self.api_key = os.environ.get("PEBBLO_API_KEY") or api_key self.load_id = str(uuid.uuid4()) self.loader = langchain_loader + self.load_semantic = os.environ.get("PEBBLO_LOAD_SEMANTIC") or load_semantic self.owner = owner self.description = description self.source_path = get_loader_full_path(self.loader) self.source_owner = PebbloSafeLoader.get_file_owner_from_path(self.source_path) self.docs: List[Document] = [] + self.docs_with_id: Union[List[IndexedDocument], List[Document], List] = [] loader_name = str(type(self.loader)).split(".")[-1].split("'")[0] self.source_type = get_loader_type(loader_name) self.source_path_size = self.get_source_size(self.source_path) - self.source_aggr_size = 0 + self.source_aggregate_size = 0 self.loader_details = { "loader": loader_name, "source_path": self.source_path, @@ -80,7 +84,15 @@ def load(self) -> List[Document]: list: Documents fetched from load method of the wrapped `loader`. """ self.docs = self.loader.load() - self._send_loader_doc(loading_end=True) + if not self.load_semantic: + self._classify_doc(self.docs, loading_end=True) + return self.docs + self.docs_with_id = self._index_docs() + classified_docs = self._classify_doc(self.docs_with_id, loading_end=True) + self.docs_with_id = self._add_semantic_to_docs( + self.docs_with_id, classified_docs + ) + self.docs = self._unindex_docs(self.docs_with_id) # type: ignore return self.docs def lazy_load(self) -> Iterator[Document]: @@ -104,13 +116,19 @@ def lazy_load(self) -> Iterator[Document]: doc = next(doc_iterator) except StopIteration: self.docs = [] - self._send_loader_doc(loading_end=True) break - self.docs = [ - doc, - ] - self._send_loader_doc() - yield doc + self.docs = list((doc,)) + if not self.load_semantic: + self._classify_doc(self.docs, loading_end=True) + yield self.docs[0] + else: + self.docs_with_id = self._index_docs() + classified_doc = self._classify_doc(self.docs) + self.docs_with_id = self._add_semantic_to_docs( + self.docs_with_id, classified_doc + ) + self.docs = self._unindex_docs(self.docs_with_id) # type: ignore + yield self.docs[0] @classmethod def set_discover_sent(cls) -> None: @@ -120,16 +138,23 @@ def set_discover_sent(cls) -> None: def set_loader_sent(cls) -> None: cls._loader_sent = True - def _send_loader_doc(self, loading_end: bool = False) -> list: + def _classify_doc(self, loaded_docs: list, loading_end: bool = False) -> list: """Send documents fetched from loader to pebblo-server. Then send classified documents to Daxa cloud(If api_key is present). Internal method. Args: + + loaded_docs (list): List of documents fetched from loader's load operation. loading_end (bool, optional): Flag indicating the halt of data - loading by loader. Defaults to False. + loading by loader. Defaults to False. """ - headers = {"Accept": "application/json", "Content-Type": "application/json"} - doc_content = [doc.dict() for doc in self.docs] + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + if loading_end is True: + PebbloSafeLoader.set_loader_sent() + doc_content = [doc.dict() for doc in loaded_docs] docs = [] for doc in doc_content: doc_authorized_identities = doc.get("metadata", {}).get( @@ -144,11 +169,13 @@ def _send_loader_doc(self, loading_end: bool = False) -> list: doc_source_size = self.get_source_size(doc_source_path) page_content = str(doc.get("page_content")) page_content_size = self.calculate_content_size(page_content) - self.source_aggr_size += page_content_size + self.source_aggregate_size += page_content_size + doc_id = doc.get("id", None) or 0 docs.append( { "doc": page_content, "source_path": doc_source_path, + "id": doc_id, "last_modified": doc.get("metadata", {}).get("last_modified"), "file_owner": doc_source_owner, **( @@ -176,7 +203,9 @@ def _send_loader_doc(self, loading_end: bool = False) -> list: if loading_end is True: payload["loading_end"] = "true" if "loader_details" in payload: - payload["loader_details"]["source_aggr_size"] = self.source_aggr_size + payload["loader_details"]["source_aggregate_size"] = ( # noqa + self.source_aggregate_size + ) payload = Doc(**payload).dict(exclude_unset=True) load_doc_url = f"{CLASSIFIER_URL}{LOADER_DOC_URL}" classified_docs = [] @@ -202,11 +231,9 @@ def _send_loader_doc(self, loading_end: bool = False) -> list: except requests.exceptions.RequestException: logger.warning("Unable to reach pebblo server.") except Exception as e: - logger.warning("An Exception caught in _send_loader_doc: %s", e) - + logger.warning("An Exception caught in _send_loader_doc: local %s", e) if self.api_key: if not classified_docs: - logger.warning("No classified docs to send to pebblo-cloud.") return classified_docs try: payload["docs"] = classified_docs @@ -234,7 +261,7 @@ def _send_loader_doc(self, loading_end: bool = False) -> list: except requests.exceptions.RequestException: logger.warning("Unable to reach Pebblo cloud server.") except Exception as e: - logger.warning("An Exception caught in _send_loader_doc: %s", e) + logger.warning("An Exception caught in _send_loader_doc: cloud %s", e) if loading_end is True: PebbloSafeLoader.set_loader_sent() @@ -270,6 +297,12 @@ def _send_discover(self) -> None: pebblo_resp = requests.post( app_discover_url, headers=headers, json=payload, timeout=20 ) + if self.api_key: + pebblo_cloud_url = f"{PEBBLO_CLOUD_URL}/v1/discover" + headers.update({"x-api-key": self.api_key}) + _ = requests.post( + pebblo_cloud_url, headers=headers, json=payload, timeout=20 + ) logger.debug( "send_discover[local]: request url %s, body %s len %s\ response status %s body %s", @@ -287,8 +320,8 @@ def _send_discover(self) -> None: ) except requests.exceptions.RequestException: logger.warning("Unable to reach pebblo server.") - except Exception: - logger.warning("An Exception caught in _send_discover.") + except Exception as e: + logger.warning("An Exception caught in _send_discover: local %s", e) if self.api_key: try: @@ -316,7 +349,7 @@ def _send_discover(self) -> None: except requests.exceptions.RequestException: logger.warning("Unable to reach Pebblo cloud server.") except Exception as e: - logger.warning("An Exception caught in _send_discover: %s", e) + logger.warning("An Exception caught in _send_discover: cloud %s", e) def _get_app_details(self) -> App: """Fetch app details. Internal method. @@ -378,3 +411,80 @@ def get_source_size(self, source_path: str) -> int: total_size += os.path.getsize(fp) size = total_size return size + + def _index_docs(self) -> List[IndexedDocument]: + """ + Indexes the documents and returns a list of IndexedDocument objects. + + Returns: + List[IndexedDocument]: A list of IndexedDocument objects with unique IDs. + """ + docs_with_id = [ + IndexedDocument(id=hex(i)[2:], **doc.dict()) + for i, doc in enumerate(self.docs) + ] + return docs_with_id + + def _add_semantic_to_docs( + self, docs_with_id: List[IndexedDocument], classified_docs: List[dict] + ) -> List[Document]: + """ + Adds semantic metadata to the given list of documents. + + Args: + docs_with_id (List[IndexedDocument]): A list of IndexedDocument objects + containing the documents with their IDs. + classified_docs (List[dict]): A list of dictionaries containing the + classified documents. + + Returns: + List[Document]: A list of Document objects with added semantic metadata. + """ + indexed_docs = { + doc.id: Document(page_content=doc.page_content, metadata=doc.metadata) + for doc in docs_with_id + } + + for classified_doc in classified_docs: + doc_id = classified_doc.get("id") + if doc_id in indexed_docs: + self._add_semantic_to_doc(indexed_docs[doc_id], classified_doc) + + semantic_metadata_docs = [doc for doc in indexed_docs.values()] + + return semantic_metadata_docs + + def _unindex_docs(self, docs_with_id: List[IndexedDocument]) -> List[Document]: + """ + Converts a list of IndexedDocument objects to a list of Document objects. + + Args: + docs_with_id (List[IndexedDocument]): A list of IndexedDocument objects. + + Returns: + List[Document]: A list of Document objects. + """ + docs = [ + Document(page_content=doc.page_content, metadata=doc.metadata) + for i, doc in enumerate(docs_with_id) + ] + return docs + + def _add_semantic_to_doc(self, doc: Document, classified_doc: dict) -> Document: + """ + Adds semantic metadata to the given document in-place. + + Args: + doc (Document): A Document object. + classified_doc (dict): A dictionary containing the classified document. + + Returns: + Document: The Document object with added semantic metadata. + """ + doc.metadata["pebblo_semantic_entities"] = list( + classified_doc.get("entities", {}).keys() + ) + doc.metadata["pebblo_semantic_topics"] = list( + classified_doc.get("topics", {}).keys() + ) + return doc diff --git a/libs/community/langchain_community/utilities/pebblo.py b/libs/community/langchain_community/utilities/pebblo.py index da65a5835dde8..df799c7fe00a3 100644 --- a/libs/community/langchain_community/utilities/pebblo.py +++ b/libs/community/langchain_community/utilities/pebblo.py @@ -6,6 +6,7 @@ import platform from typing import Optional, Tuple +from langchain_core.documents import Document from langchain_core.env import get_runtime_environment from langchain_core.pydantic_v1 import BaseModel @@ -61,6 +62,10 @@ logger = logging.getLogger(__name__) +class IndexedDocument(Document): + id: str + + class Runtime(BaseModel): """Pebblo Runtime. From 471718c8d40d86c9a362906b89de531af88ffa47 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Fri, 26 Apr 2024 03:58:18 +0800 Subject: [PATCH 25/36] community[patch]: add more data types support to ipex-llm llm integration (#20833) - **Description**: - **add support for more data types**: by default `IpexLLM` will load the model in int4 format. This PR adds more data types support such as `sym_in5`, `sym_int8`, etc. Data formats like NF3, NF4, FP4 and FP8 are only supported on GPU and will be added in future PR. - Fix a small issue in saving/loading, update api docs - **Dependencies**: `ipex-llm` library - **Document**: In `docs/docs/integrations/llms/ipex_llm.ipynb`, added instructions for saving/loading low-bit model. - **Tests**: added new test cases to `libs/community/tests/integration_tests/llms/test_ipex_llm.py`, added config params. - **Contribution maintainer**: @shane-huang --- docs/docs/integrations/llms/ipex_llm.ipynb | 129 ++++++++++++--- .../langchain_community/llms/bigdl_llm.py | 35 +++- .../langchain_community/llms/ipex_llm.py | 153 ++++++++++++------ .../integration_tests/llms/test_bigdl_llm.py | 28 +++- .../integration_tests/llms/test_ipex_llm.py | 81 ++++++++-- 5 files changed, 342 insertions(+), 84 deletions(-) diff --git a/docs/docs/integrations/llms/ipex_llm.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb index 25519b2d92721..32d768d8070f6 100644 --- a/docs/docs/integrations/llms/ipex_llm.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -6,9 +6,9 @@ "source": [ "# IPEX-LLM\n", "\n", - "> [IPEX-LLM](https://github.com/intel-analytics/ipex-llm/) is a low-bit LLM optimization library on Intel XPU (Xeon/Core/Flex/Arc/Max). It can make LLMs run extremely fast and consume much less memory on Intel platforms. It is open sourced under Apache 2.0 License.\n", + "> [IPEX-LLM](https://github.com/intel-analytics/ipex-llm/) is a PyTorch library for running LLM on Intel CPU and GPU (e.g., local PC with iGPU, discrete GPU such as Arc, Flex and Max) with very low latency. \n", "\n", - "This example goes over how to use LangChain to interact with IPEX-LLM for text generation. \n" + "This example goes over how to use LangChain to interact with `ipex-llm` for text generation. \n" ] }, { @@ -49,7 +49,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Usage" + "## Basic Usage" ] }, { @@ -58,9 +58,20 @@ "metadata": {}, "outputs": [], "source": [ + "import warnings\n", + "\n", "from langchain.chains import LLMChain\n", "from langchain_community.llms import IpexLLM\n", - "from langchain_core.prompts import PromptTemplate" + "from langchain_core.prompts import PromptTemplate\n", + "\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning, message=\".*padding_mask.*\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the prompt template for your model. In this example, we use the [vicuna-1.5](https://huggingface.co/lmsys/vicuna-7b-v1.5) model. If you're working with a different model, choose a proper template accordingly." ] }, { @@ -77,7 +88,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Load Model: " + "Load the model locally using IpexLLM using `IpexLLM.from_model_id`. It will load the model directly in its Huggingface format and convert it automatically to low-bit format for inference." ] }, { @@ -88,7 +99,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "27c08180714a44c7ab766624d5054163", + "model_id": "897501860fe4452b836f816c72d955dd", "version_major": 2, "version_minor": 0 }, @@ -103,7 +114,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-03-27 00:58:43,670 - INFO - Converting the current model to sym_int4 format......\n" + "2024-04-24 21:20:12,461 - INFO - Converting the current model to sym_int4 format......\n" ] } ], @@ -130,13 +141,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n", + "/opt/anaconda3/envs/shane-langchain-3.11/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `LLMChain` was deprecated in LangChain 0.1.17 and will be removed in 0.3.0. Use RunnableSequence, e.g., `prompt | llm` instead.\n", " warn_deprecated(\n", - "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", - " warnings.warn(\n", - "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", - " warnings.warn(\n", - "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", + "/opt/anaconda3/envs/shane-langchain-3.11/lib/python3.11/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", " warnings.warn(\n" ] }, @@ -144,10 +151,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "AI stands for \"Artificial Intelligence.\" It refers to the development of computer systems that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, decision-making, and language translation. AI can be achieved through a combination of techniques such as machine learning, natural language processing, computer vision, and robotics. The ultimate goal of AI research is to create machines that can think and learn like humans, and can even exceed human capabilities in certain areas.\n" ] } @@ -156,15 +159,99 @@ "llm_chain = LLMChain(prompt=prompt, llm=llm)\n", "\n", "question = \"What is AI?\"\n", - "output = llm_chain.run(question)" + "output = llm_chain.invoke(question)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save/Load Low-bit Model\n", + "Alternatively, you might save the low-bit model to disk once and use `from_model_id_low_bit` instead of `from_model_id` to reload it for later use - even across different machines. It is space-efficient, as the low-bit model demands significantly less disk space than the original model. And `from_model_id_low_bit` is also more efficient than `from_model_id` in terms of speed and memory usage, as it skips the model conversion step." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To save the low-bit model, use `save_low_bit` as follows." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "saved_lowbit_model_path = \"./vicuna-7b-1.5-low-bit\" # path to save low-bit model\n", + "llm.model.save_low_bit(saved_lowbit_model_path)\n", + "del llm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load the model from saved lowbit model path as follows. \n", + "> Note that the saved path for the low-bit model only includes the model itself but not the tokenizers. If you wish to have everything in one place, you will need to manually download or copy the tokenizer files from the original model's directory to the location where the low-bit model is saved." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-04-24 21:20:35,874 - INFO - Converting the current model to sym_int4 format......\n" + ] + } + ], + "source": [ + "llm_lowbit = IpexLLM.from_model_id_low_bit(\n", + " model_id=saved_lowbit_model_path,\n", + " tokenizer_id=\"lmsys/vicuna-7b-v1.5\",\n", + " # tokenizer_name=saved_lowbit_model_path, # copy the tokenizers to saved path if you want to use it this way\n", + " model_kwargs={\"temperature\": 0, \"max_length\": 64, \"trust_remote_code\": True},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the loaded model in Chains:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/shane-langchain-3.11/lib/python3.11/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AI stands for \"Artificial Intelligence.\" It refers to the development of computer systems that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, decision-making, and language translation. AI can be achieved through a combination of techniques such as machine learning, natural language processing, computer vision, and robotics. The ultimate goal of AI research is to create machines that can think and learn like humans, and can even exceed human capabilities in certain areas.\n" + ] + } + ], + "source": [ + "llm_chain = LLMChain(prompt=prompt, llm=llm_lowbit)\n", + "\n", + "question = \"What is AI?\"\n", + "output = llm_chain.invoke(question)" + ] } ], "metadata": { @@ -183,7 +270,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py index f225587a73331..3181e5f540eeb 100644 --- a/libs/community/langchain_community/llms/bigdl_llm.py +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -23,6 +23,10 @@ def from_model_id( cls, model_id: str, model_kwargs: Optional[dict] = None, + *, + tokenizer_id: Optional[str] = None, + load_in_4bit: bool = True, + load_in_low_bit: Optional[str] = None, **kwargs: Any, ) -> LLM: """ @@ -31,6 +35,8 @@ def from_model_id( Args: model_id: Path for the huggingface repo id to be downloaded or the huggingface checkpoint folder. + tokenizer_id: Path for the huggingface repo id to be downloaded or + the huggingface checkpoint folder which contains the tokenizer. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. @@ -52,12 +58,27 @@ def from_model_id( "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." ) + if load_in_low_bit is not None: + logger.warning( + """`load_in_low_bit` option is not supported in BigdlLLM and + is ignored. For more data types support with `load_in_low_bit`, + use IpexLLM instead.""" + ) + + if not load_in_4bit: + raise ValueError( + "BigdlLLM only supports loading in 4-bit mode, " + "i.e. load_in_4bit = True. " + "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + ) + _model_kwargs = model_kwargs or {} + _tokenizer_id = tokenizer_id or model_id try: - tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + tokenizer = AutoTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) except Exception: - tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + tokenizer = LlamaTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) try: model = AutoModelForCausalLM.from_pretrained( @@ -86,6 +107,8 @@ def from_model_id_low_bit( cls, model_id: str, model_kwargs: Optional[dict] = None, + *, + tokenizer_id: Optional[str] = None, **kwargs: Any, ) -> LLM: """ @@ -94,6 +117,8 @@ def from_model_id_low_bit( Args: model_id: Path for the bigdl-llm transformers low-bit model folder. + tokenizer_id: Path for the huggingface repo id or local model folder + which contains the tokenizer. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. @@ -117,10 +142,12 @@ def from_model_id_low_bit( ) _model_kwargs = model_kwargs or {} + _tokenizer_id = tokenizer_id or model_id + try: - tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + tokenizer = AutoTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) except Exception: - tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + tokenizer = LlamaTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) try: model = AutoModelForCausalLM.load_low_bit(model_id, **_model_kwargs) diff --git a/libs/community/langchain_community/llms/ipex_llm.py b/libs/community/langchain_community/llms/ipex_llm.py index ed03770180691..a8e60fe6b1f4a 100644 --- a/libs/community/langchain_community/llms/ipex_llm.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -42,6 +42,10 @@ def from_model_id( cls, model_id: str, model_kwargs: Optional[dict] = None, + *, + tokenizer_id: Optional[str] = None, + load_in_4bit: bool = True, + load_in_low_bit: Optional[str] = None, **kwargs: Any, ) -> LLM: """ @@ -50,52 +54,29 @@ def from_model_id( Args: model_id: Path for the huggingface repo id to be downloaded or the huggingface checkpoint folder. + tokenizer_id: Path for the huggingface repo id to be downloaded or + the huggingface checkpoint folder which contains the tokenizer. + load_in_4bit: "Whether to load model in 4bit. + Unused if `load_in_low_bit` is not None. + load_in_low_bit: Which low bit precisions to use when loading model. + Example values: 'sym_int4', 'asym_int4', 'fp4', 'nf4', 'fp8', etc. + Overrides `load_in_4bit` if specified. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. Returns: An object of IpexLLM. - """ - try: - from ipex_llm.transformers import ( - AutoModel, - AutoModelForCausalLM, - ) - from transformers import AutoTokenizer, LlamaTokenizer - except ImportError: - raise ValueError( - "Could not import ipex-llm or transformers. " - "Please install it with `pip install --pre --upgrade ipex-llm[all]`." - ) - - _model_kwargs = model_kwargs or {} - - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) - except Exception: - tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) - - try: - model = AutoModelForCausalLM.from_pretrained( - model_id, load_in_4bit=True, **_model_kwargs - ) - except Exception: - model = AutoModel.from_pretrained( - model_id, load_in_4bit=True, **_model_kwargs - ) - - if "trust_remote_code" in _model_kwargs: - _model_kwargs = { - k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" - } + """ - return cls( + return cls._load_model( model_id=model_id, - model=model, - tokenizer=tokenizer, - model_kwargs=_model_kwargs, - **kwargs, + tokenizer_id=tokenizer_id, + low_bit_model=False, + load_in_4bit=load_in_4bit, + load_in_low_bit=load_in_low_bit, + model_kwargs=model_kwargs, + kwargs=kwargs, ) @classmethod @@ -103,6 +84,8 @@ def from_model_id_low_bit( cls, model_id: str, model_kwargs: Optional[dict] = None, + *, + tokenizer_id: Optional[str] = None, **kwargs: Any, ) -> LLM: """ @@ -111,12 +94,36 @@ def from_model_id_low_bit( Args: model_id: Path for the ipex-llm transformers low-bit model folder. + tokenizer_id: Path for the huggingface repo id or local model folder + which contains the tokenizer. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. Returns: An object of IpexLLM. """ + + return cls._load_model( + model_id=model_id, + tokenizer_id=tokenizer_id, + low_bit_model=True, + load_in_4bit=False, # not used for low-bit model + load_in_low_bit=None, # not used for low-bit model + model_kwargs=model_kwargs, + kwargs=kwargs, + ) + + @classmethod + def _load_model( + cls, + model_id: str, + tokenizer_id: Optional[str] = None, + load_in_4bit: bool = False, + load_in_low_bit: Optional[str] = None, + low_bit_model: bool = False, + model_kwargs: Optional[dict] = None, + kwargs: Optional[dict] = None, + ) -> Any: try: from ipex_llm.transformers import ( AutoModel, @@ -126,26 +133,62 @@ def from_model_id_low_bit( except ImportError: raise ValueError( - "Could not import ipex-llm or transformers. " - "Please install it with `pip install --pre --upgrade ipex-llm[all]`." + "Could not import ipex-llm. " + "Please install `ipex-llm` properly following installation guides: " + "https://github.com/intel-analytics/ipex-llm?tab=readme-ov-file#install-ipex-llm." ) _model_kwargs = model_kwargs or {} - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) - except Exception: - tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + kwargs = kwargs or {} + + _tokenizer_id = tokenizer_id or model_id try: - model = AutoModelForCausalLM.load_low_bit(model_id, **_model_kwargs) + tokenizer = AutoTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) except Exception: - model = AutoModel.load_low_bit(model_id, **_model_kwargs) + tokenizer = LlamaTokenizer.from_pretrained(_tokenizer_id, **_model_kwargs) + # restore model_kwargs if "trust_remote_code" in _model_kwargs: _model_kwargs = { k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" } + # load model with AutoModelForCausalLM and falls back to AutoModel on failure. + load_kwargs = { + "use_cache": True, + "trust_remote_code": True, + } + + if not low_bit_model: + if load_in_low_bit is not None: + load_function_name = "from_pretrained" + load_kwargs["load_in_low_bit"] = load_in_low_bit # type: ignore + else: + load_function_name = "from_pretrained" + load_kwargs["load_in_4bit"] = load_in_4bit + else: + load_function_name = "load_low_bit" + + try: + # Attempt to load with AutoModelForCausalLM + model = cls._load_model_general( + AutoModelForCausalLM, + load_function_name=load_function_name, + model_id=model_id, + load_kwargs=load_kwargs, + model_kwargs=_model_kwargs, + ) + except Exception: + # Fallback to AutoModel if there's an exception + model = cls._load_model_general( + AutoModel, + load_function_name=load_function_name, + model_id=model_id, + load_kwargs=load_kwargs, + model_kwargs=_model_kwargs, + ) + return cls( model_id=model_id, model=model, @@ -154,6 +197,24 @@ def from_model_id_low_bit( **kwargs, ) + @staticmethod + def _load_model_general( + model_class: Any, + load_function_name: str, + model_id: str, + load_kwargs: dict, + model_kwargs: dict, + ) -> Any: + """General function to attempt to load a model.""" + try: + load_function = getattr(model_class, load_function_name) + return load_function(model_id, **{**load_kwargs, **model_kwargs}) + except Exception as e: + logger.error( + f"Failed to load model using " + f"{model_class.__name__}.{load_function_name}: {e}" + ) + @property def _identifying_params(self) -> Mapping[str, Any]: """Get the identifying parameters.""" diff --git a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py index d214df8429cb3..8d3340ce2e3be 100644 --- a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py +++ b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py @@ -1,23 +1,43 @@ """Test BigdlLLM""" +import os + +import pytest from langchain_core.outputs import LLMResult from langchain_community.llms.bigdl_llm import BigdlLLM +model_ids_to_test = os.getenv("TEST_BIGDLLLM_MODEL_IDS") or "" +skip_if_no_model_ids = pytest.mark.skipif( + not model_ids_to_test, + reason="TEST_BIGDLLLM_MODEL_IDS environment variable not set.", +) +model_ids_to_test = [model_id.strip() for model_id in model_ids_to_test.split(",")] # type: ignore + -def test_call() -> None: +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_call(model_id: str) -> None: """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( - model_id="lmsys/vicuna-7b-v1.5", + model_id=model_id, model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) output = llm.invoke("Hello!") assert isinstance(output, str) -def test_generate() -> None: +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_generate(model_id: str) -> None: """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( - model_id="lmsys/vicuna-7b-v1.5", + model_id=model_id, model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) output = llm.generate(["Hello!"]) diff --git a/libs/community/tests/integration_tests/llms/test_ipex_llm.py b/libs/community/tests/integration_tests/llms/test_ipex_llm.py index a98bbf14be7c2..163458029c5d5 100644 --- a/libs/community/tests/integration_tests/llms/test_ipex_llm.py +++ b/libs/community/tests/integration_tests/llms/test_ipex_llm.py @@ -1,25 +1,88 @@ """Test IPEX LLM""" +import os +from typing import Any + +import pytest from langchain_core.outputs import LLMResult -from langchain_community.llms.ipex_llm import IpexLLM +from langchain_community.llms import IpexLLM + +model_ids_to_test = os.getenv("TEST_IPEXLLM_MODEL_IDS") or "" +skip_if_no_model_ids = pytest.mark.skipif( + not model_ids_to_test, reason="TEST_IPEXLLM_MODEL_IDS environment variable not set." +) +model_ids_to_test = [model_id.strip() for model_id in model_ids_to_test.split(",")] # type: ignore -def test_call() -> None: - """Test valid call to ipex-llm.""" +def load_model(model_id: str) -> Any: llm = IpexLLM.from_model_id( - model_id="lmsys/vicuna-7b-v1.5", + model_id=model_id, model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) - output = llm.invoke("Hello!") - assert isinstance(output, str) + return llm -def test_generate() -> None: - """Test valid call to ipex-llm.""" +def load_model_more_types(model_id: str, load_in_low_bit: str) -> Any: llm = IpexLLM.from_model_id( - model_id="lmsys/vicuna-7b-v1.5", + model_id=model_id, + load_in_low_bit=load_in_low_bit, model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) + return llm + + +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_call(model_id: str) -> None: + """Test valid call.""" + llm = load_model(model_id) + output = llm.invoke("Hello!") + assert isinstance(output, str) + + +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_asym_int4(model_id: str) -> None: + """Test asym int4 data type.""" + llm = load_model_more_types(model_id=model_id, load_in_low_bit="asym_int4") + output = llm.invoke("Hello!") + assert isinstance(output, str) + + +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_generate(model_id: str) -> None: + """Test valid generate.""" + llm = load_model(model_id) output = llm.generate(["Hello!"]) assert isinstance(output, LLMResult) assert isinstance(output.generations, list) + + +@skip_if_no_model_ids +@pytest.mark.parametrize( + "model_id", + model_ids_to_test, +) +def test_save_load_lowbit(model_id: str) -> None: + """Test save and load lowbit model.""" + saved_lowbit_path = "/tmp/saved_model" + llm = load_model(model_id) + llm.model.save_low_bit(saved_lowbit_path) + del llm + loaded_llm = IpexLLM.from_model_id_low_bit( + model_id=saved_lowbit_path, + tokenizer_id=model_id, + model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, + ) + output = loaded_llm.invoke("Hello!") + assert isinstance(output, str) From 88df1a7a778cad4eb5056cae116772d19680897c Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Thu, 25 Apr 2024 13:10:33 -0700 Subject: [PATCH 26/36] experimental[patch]: remove \n from AutoGPT feedback_tool exit check (#20132) --- .../langchain_experimental/autonomous_agents/autogpt/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/experimental/langchain_experimental/autonomous_agents/autogpt/agent.py b/libs/experimental/langchain_experimental/autonomous_agents/autogpt/agent.py index 84bfb4102ddf5..f28e42bb9988c 100644 --- a/libs/experimental/langchain_experimental/autonomous_agents/autogpt/agent.py +++ b/libs/experimental/langchain_experimental/autonomous_agents/autogpt/agent.py @@ -133,11 +133,11 @@ def run(self, goals: List[str]) -> str: f"Assistant Reply: {assistant_reply} " f"\nResult: {result} " ) if self.feedback_tool is not None: - feedback = f"\n{self.feedback_tool.run('Input: ')}" + feedback = f"{self.feedback_tool.run('Input: ')}" if feedback in {"q", "stop"}: print("EXITING") # noqa: T201 return "EXITING" - memory_to_add += feedback + memory_to_add += f"\n{feedback}" self.memory.add_documents([Document(page_content=memory_to_add)]) self.chat_history_memory.add_message(SystemMessage(content=result)) From 6d721cee1f99d0feaa1474c83761478a0e241533 Mon Sep 17 00:00:00 2001 From: Anish Chakraborty Date: Thu, 25 Apr 2024 22:10:56 +0200 Subject: [PATCH 27/36] core[patch]: improve comma separated list output parser to handle non-space separated list (#20434) - **Description:** Changes `lanchain_core.output_parsers.CommaSeparatedListOutputParser` to handle `,` as a delimiter alongside the previous implementation which used `, ` as delimiter. - **Issue:** Started noticing that some results returned by LLMs were not getting parsed correctly when the output contained `,` instead of `, `. - **Dependencies:** No - **Twitter handle:** not active on twitter. --- .../langchain_core/output_parsers/list.py | 4 ++-- .../output_parsers/test_list_parser.py | 21 ++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/libs/core/langchain_core/output_parsers/list.py b/libs/core/langchain_core/output_parsers/list.py index 36b9f7e9b776b..7fd96c94d805a 100644 --- a/libs/core/langchain_core/output_parsers/list.py +++ b/libs/core/langchain_core/output_parsers/list.py @@ -115,12 +115,12 @@ def get_lc_namespace(cls) -> List[str]: def get_format_instructions(self) -> str: return ( "Your response should be a list of comma separated values, " - "eg: `foo, bar, baz`" + "eg: `foo, bar, baz` or `foo,bar,baz`" ) def parse(self, text: str) -> List[str]: """Parse the output of an LLM call.""" - return text.strip().split(", ") + return [part.strip() for part in text.split(",")] @property def _type(self) -> str: diff --git a/libs/core/tests/unit_tests/output_parsers/test_list_parser.py b/libs/core/tests/unit_tests/output_parsers/test_list_parser.py index 710f9e52ceedc..ef3a00bc5f7ec 100644 --- a/libs/core/tests/unit_tests/output_parsers/test_list_parser.py +++ b/libs/core/tests/unit_tests/output_parsers/test_list_parser.py @@ -26,10 +26,29 @@ def test_single_item() -> None: assert list(parser.transform(iter([text]))) == [[a] for a in expected] +def test_multiple_items_with_spaces() -> None: + """Test that a string with multiple comma-separated items + with spaces is parsed to a list.""" + parser = CommaSeparatedListOutputParser() + text = "foo, bar, baz" + expected = ["foo", "bar", "baz"] + + assert parser.parse(text) == expected + assert add(parser.transform(t for t in text)) == expected + assert list(parser.transform(t for t in text)) == [[a] for a in expected] + assert list(parser.transform(t for t in text.splitlines(keepends=True))) == [ + [a] for a in expected + ] + assert list( + parser.transform(" " + t if i > 0 else t for i, t in enumerate(text.split(" "))) + ) == [[a] for a in expected] + assert list(parser.transform(iter([text]))) == [[a] for a in expected] + + def test_multiple_items() -> None: """Test that a string with multiple comma-separated items is parsed to a list.""" parser = CommaSeparatedListOutputParser() - text = "foo, bar, baz" + text = "foo,bar,baz" expected = ["foo", "bar", "baz"] assert parser.parse(text) == expected From 7b2d5ed1922ed103f33c9eef06351dee536bc26d Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Thu, 25 Apr 2024 13:16:39 -0700 Subject: [PATCH 28/36] experimental[patch]: return from HuggingGPT task executor task.run() exception (#20219) **Description:** Fixes a bug in the HuggingGPT task execution logic here: except Exception as e: self.status = "failed" self.message = str(e) self.status = "completed" self.save_product() where a caught exception effectively just sets `self.message` and can then throw an exception if, e.g., `self.product` is not defined. **Issue:** None that I'm aware of. **Dependencies:** None **Twitter handle:** https://twitter.com/michaeljschock Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> --- .../autonomous_agents/hugginggpt/task_executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt/task_executor.py b/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt/task_executor.py index 62a4c95da9583..c149a11cbd73c 100644 --- a/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt/task_executor.py +++ b/libs/experimental/langchain_experimental/autonomous_agents/hugginggpt/task_executor.py @@ -69,6 +69,8 @@ def run(self) -> str: except Exception as e: self.status = "failed" self.message = str(e) + return self.message + self.status = "completed" self.save_product() From 2febebd67db5a105a3b290282d15105b3757e7a4 Mon Sep 17 00:00:00 2001 From: am-kinetica <85610855+am-kinetica@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:09:00 +0530 Subject: [PATCH 29/36] community[minor]: Implemented Kinetica Document Loader and added notebooks (#20002) - [ ] **Kinetica Document Loader**: "community: a class to load Documents from Kinetica" - [ ] **Kinetica Document Loader**: - **Description:** implemented KineticaLoader in `kinetica_loader.py` - **Dependencies:** install the Kinetica API using `pip install gpudb==7.2.0.1 ` --- .../document_loaders/kinetica.ipynb | 125 +++++++++++++ docs/docs/integrations/providers/kinetica.mdx | 16 ++ .../integrations/retrievers/kinetica.ipynb | 171 ++++++++++++++++++ .../document_loaders/__init__.py | 1 + .../document_loaders/kinetica_loader.py | 103 +++++++++++ .../document_loaders/test_imports.py | 1 + 6 files changed, 417 insertions(+) create mode 100644 docs/docs/integrations/document_loaders/kinetica.ipynb create mode 100644 docs/docs/integrations/retrievers/kinetica.ipynb create mode 100644 libs/community/langchain_community/document_loaders/kinetica_loader.py diff --git a/docs/docs/integrations/document_loaders/kinetica.ipynb b/docs/docs/integrations/document_loaders/kinetica.ipynb new file mode 100644 index 0000000000000..0176557308ff0 --- /dev/null +++ b/docs/docs/integrations/document_loaders/kinetica.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Kinetica\n", + "\n", + "This notebooks goes over how to load documents from Kinetica" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install gpudb==7.2.0.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders.kinetica_loader import KineticaLoader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Loading Environment Variables\n", + "import os\n", + "\n", + "from dotenv import load_dotenv\n", + "from langchain_community.vectorstores import (\n", + " KineticaSettings,\n", + ")\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Kinetica needs the connection to the database.\n", + "# This is how to set it up.\n", + "HOST = os.getenv(\"KINETICA_HOST\", \"http://127.0.0.1:9191\")\n", + "USERNAME = os.getenv(\"KINETICA_USERNAME\", \"\")\n", + "PASSWORD = os.getenv(\"KINETICA_PASSWORD\", \"\")\n", + "\n", + "\n", + "def create_config() -> KineticaSettings:\n", + " return KineticaSettings(host=HOST, username=USERNAME, password=PASSWORD)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders.kinetica_loader import KineticaLoader\n", + "\n", + "# The following `QUERY` is an example which will not run; this\n", + "# needs to be substituted with a valid `QUERY` that will return\n", + "# data and the `SCHEMA.TABLE` combination must exist in Kinetica.\n", + "\n", + "QUERY = \"select text, survey_id from SCHEMA.TABLE limit 10\"\n", + "kinetica_loader = KineticaLoader(\n", + " QUERY,\n", + " HOST,\n", + " USERNAME,\n", + " PASSWORD,\n", + ")\n", + "kinetica_documents = kinetica_loader.load()\n", + "print(kinetica_documents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders.kinetica_loader import KineticaLoader\n", + "\n", + "# The following `QUERY` is an example which will not run; this\n", + "# needs to be substituted with a valid `QUERY` that will return\n", + "# data and the `SCHEMA.TABLE` combination must exist in Kinetica.\n", + "\n", + "QUERY = \"select text, survey_id as source from SCHEMA.TABLE limit 10\"\n", + "snowflake_loader = KineticaLoader(\n", + " query=QUERY,\n", + " host=HOST,\n", + " username=USERNAME,\n", + " password=PASSWORD,\n", + " metadata_columns=[\"source\"],\n", + ")\n", + "kinetica_documents = snowflake_loader.load()\n", + "print(kinetica_documents)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/docs/integrations/providers/kinetica.mdx b/docs/docs/integrations/providers/kinetica.mdx index 317bdbf454326..29d039dfe745f 100644 --- a/docs/docs/integrations/providers/kinetica.mdx +++ b/docs/docs/integrations/providers/kinetica.mdx @@ -26,3 +26,19 @@ See [Kinetica Vectorsore API](/docs/integrations/vectorstores/kinetica) for usag from langchain_community.vectorstores import Kinetica ``` +## Document Loader + +The Kinetica Document loader can be used to load LangChain Documents from the +Kinetica database. + +See [Kinetica Document Loader](/docs/integrations/document_loaders/kinetica) for usage + +```python +from langchain_community.document_loaders.kinetica_loader import KineticaLoader +``` + +## Retriever + +The Kinetica Retriever can return documents given an unstructured query. + +See [Kinetica VectorStore based Retriever](/docs/integrations/retrievers/kinetica) for usage diff --git a/docs/docs/integrations/retrievers/kinetica.ipynb b/docs/docs/integrations/retrievers/kinetica.ipynb new file mode 100644 index 0000000000000..63f2fd16d5a12 --- /dev/null +++ b/docs/docs/integrations/retrievers/kinetica.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Kinetica Vectorstore based Retriever\n", + "\n", + ">[Kinetica](https://www.kinetica.com/) is a database with integrated support for vector similarity search\n", + "\n", + "It supports:\n", + "- exact and approximate nearest neighbor search\n", + "- L2 distance, inner product, and cosine distance\n", + "\n", + "This notebook shows how to use a retriever based on Kinetica vector store (`Kinetica`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Please ensure that this connector is installed in your working environment.\n", + "%pip install gpudb==7.2.0.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We want to use `OpenAIEmbeddings` so we have to get the OpenAI API Key." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Loading Environment Variables\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.docstore.document import Document\n", + "from langchain.text_splitter import CharacterTextSplitter\n", + "from langchain_community.document_loaders import TextLoader\n", + "from langchain_community.vectorstores import (\n", + " Kinetica,\n", + " KineticaSettings,\n", + ")\n", + "from langchain_openai import OpenAIEmbeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Kinetica needs the connection to the database.\n", + "# This is how to set it up.\n", + "HOST = os.getenv(\"KINETICA_HOST\", \"http://127.0.0.1:9191\")\n", + "USERNAME = os.getenv(\"KINETICA_USERNAME\", \"\")\n", + "PASSWORD = os.getenv(\"KINETICA_PASSWORD\", \"\")\n", + "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\", \"\")\n", + "\n", + "\n", + "def create_config() -> KineticaSettings:\n", + " return KineticaSettings(host=HOST, username=USERNAME, password=PASSWORD)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Retriever from vector store" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", + "documents = loader.load()\n", + "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", + "docs = text_splitter.split_documents(documents)\n", + "\n", + "embeddings = OpenAIEmbeddings()\n", + "\n", + "# The Kinetica Module will try to create a table with the name of the collection.\n", + "# So, make sure that the collection name is unique and the user has the permission to create a table.\n", + "\n", + "COLLECTION_NAME = \"state_of_the_union_test\"\n", + "connection = create_config()\n", + "\n", + "db = Kinetica.from_documents(\n", + " embedding=embeddings,\n", + " documents=docs,\n", + " collection_name=COLLECTION_NAME,\n", + " config=connection,\n", + ")\n", + "\n", + "# create retriever from the vector store\n", + "retriever = db.as_retriever(search_kwargs={\"k\": 2})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search with retriever" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = retriever.get_relevant_documents(\n", + " \"What did the president say about Ketanji Brown Jackson\"\n", + ")\n", + "print(docs[0].page_content)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/libs/community/langchain_community/document_loaders/__init__.py b/libs/community/langchain_community/document_loaders/__init__.py index fe52b4ff3bf5c..0f0d511000fad 100644 --- a/libs/community/langchain_community/document_loaders/__init__.py +++ b/libs/community/langchain_community/document_loaders/__init__.py @@ -781,6 +781,7 @@ "IuguLoader": "langchain_community.document_loaders.iugu", "JSONLoader": "langchain_community.document_loaders.json_loader", "JoplinLoader": "langchain_community.document_loaders.joplin", + "KineticaLoader": "langchain_community.document_loaders.kinetica_loader", "LakeFSLoader": "langchain_community.document_loaders.lakefs", "LarkSuiteDocLoader": "langchain_community.document_loaders.larksuite", "LLMSherpaFileLoader": "langchain_community.document_loaders.llmsherpa", diff --git a/libs/community/langchain_community/document_loaders/kinetica_loader.py b/libs/community/langchain_community/document_loaders/kinetica_loader.py new file mode 100644 index 0000000000000..d5cb1296e083c --- /dev/null +++ b/libs/community/langchain_community/document_loaders/kinetica_loader.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from langchain_core.documents import Document + +from langchain_community.document_loaders.base import BaseLoader + + +class KineticaLoader(BaseLoader): + """Load from `Kinetica` API. + + Each document represents one row of the result. The `page_content_columns` + are written into the `page_content` of the document. The `metadata_columns` + are written into the `metadata` of the document. By default, all columns + are written into the `page_content` and none into the `metadata`. + + """ + + def __init__( + self, + query: str, + host: str, + username: str, + password: str, + parameters: Optional[Dict[str, Any]] = None, + page_content_columns: Optional[List[str]] = None, + metadata_columns: Optional[List[str]] = None, + ): + """Initialize Kinetica document loader. + + Args: + query: The query to run in Kinetica. + parameters: Optional. Parameters to pass to the query. + page_content_columns: Optional. Columns written to Document `page_content`. + metadata_columns: Optional. Columns written to Document `metadata`. + """ + self.query = query + self.host = host + self.username = username + self.password = password + self.parameters = parameters + self.page_content_columns = page_content_columns + self.metadata_columns = metadata_columns if metadata_columns is not None else [] + + def _execute_query(self) -> List[Dict[str, Any]]: + try: + from gpudb import GPUdb, GPUdbSqlIterator + except ImportError: + raise ImportError( + "Could not import Kinetica python API. " + "Please install it with `pip install gpudb==7.2.0.1`." + ) + + try: + options = GPUdb.Options() + options.username = self.username + options.password = self.password + + conn = GPUdb(host=self.host, options=options) + + with GPUdbSqlIterator(conn, self.query) as records: + column_names = records.type_map.keys() + query_result = [dict(zip(column_names, record)) for record in records] + + except Exception as e: + print(f"An error occurred: {e}") # noqa: T201 + query_result = [] + + return query_result + + def _get_columns( + self, query_result: List[Dict[str, Any]] + ) -> Tuple[List[str], List[str]]: + page_content_columns = ( + self.page_content_columns if self.page_content_columns else [] + ) + metadata_columns = self.metadata_columns if self.metadata_columns else [] + if page_content_columns is None and query_result: + page_content_columns = list(query_result[0].keys()) + if metadata_columns is None: + metadata_columns = [] + return page_content_columns or [], metadata_columns + + def lazy_load(self) -> Iterator[Document]: + query_result = self._execute_query() + if isinstance(query_result, Exception): + print(f"An error occurred during the query: {query_result}") # noqa: T201 + return [] + page_content_columns, metadata_columns = self._get_columns(query_result) + if "*" in page_content_columns: + page_content_columns = list(query_result[0].keys()) + for row in query_result: + page_content = "\n".join( + f"{k}: {v}" for k, v in row.items() if k in page_content_columns + ) + metadata = {k: v for k, v in row.items() if k in metadata_columns} + doc = Document(page_content=page_content, metadata=metadata) + yield doc + + def load(self) -> List[Document]: + """Load data into document objects.""" + return list(self.lazy_load()) diff --git a/libs/community/tests/unit_tests/document_loaders/test_imports.py b/libs/community/tests/unit_tests/document_loaders/test_imports.py index 28c68459e76db..dc28a03e68648 100644 --- a/libs/community/tests/unit_tests/document_loaders/test_imports.py +++ b/libs/community/tests/unit_tests/document_loaders/test_imports.py @@ -89,6 +89,7 @@ "IuguLoader", "JSONLoader", "JoplinLoader", + "KineticaLoader", "LLMSherpaFileLoader", "LarkSuiteDocLoader", "LakeFSLoader", From 3ce27b36f3e2875ccb9400240a7f5ea0f1f8fd4b Mon Sep 17 00:00:00 2001 From: Dristy Srivastava <58721149+dristysrivastava@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:09:17 +0530 Subject: [PATCH 30/36] community[patch]: Add support for pebblo server and client version (#20269) **Description**: _PebbloSafeLoader_: Add support for pebblo server and client version **Documentation:** NA **Unit test:** NA **Issue:** NA **Dependencies:** None --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> --- .../langchain_community/document_loaders/pebblo.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libs/community/langchain_community/document_loaders/pebblo.py b/libs/community/langchain_community/document_loaders/pebblo.py index 8b348826293cc..9dd466e496a42 100644 --- a/libs/community/langchain_community/document_loaders/pebblo.py +++ b/libs/community/langchain_community/document_loaders/pebblo.py @@ -287,6 +287,7 @@ def calculate_content_size(page_content: str) -> int: def _send_discover(self) -> None: """Send app discovery payload to pebblo-server. Internal method.""" + pebblo_resp = None headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -326,6 +327,18 @@ def _send_discover(self) -> None: if self.api_key: try: headers.update({"x-api-key": self.api_key}) + if pebblo_resp: + pebblo_resp_docs = json.loads(pebblo_resp.text).get("docs") + payload.update( + { + "pebbloServerVersion": pebblo_resp_docs.get( + "pebbloServerVersion" + ), + "pebbloClientVersion": pebblo_resp_docs.get( + "pebbloClientVersion" + ), + } + ) pebblo_cloud_url = f"{PEBBLO_CLOUD_URL}{APP_DISCOVER_URL}" pebblo_cloud_response = requests.post( pebblo_cloud_url, headers=headers, json=payload, timeout=20 From 502a76d9ca1e6d2a347214841cf0c3132183c1e9 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 25 Apr 2024 13:42:01 -0700 Subject: [PATCH 31/36] community[patch]: Add initial tests for AzureSearch vector store (#17663) **Description:** AzureSearch vector store has no tests. This PR adds initial tests to validate the code can be imported and used. **Issue:** N/A **Dependencies:** azure-search-documents and azure-identity are added as optional dependencies for testing --------- Co-authored-by: Matt Gotteiner <[email protected]> Co-authored-by: Bagatur --- libs/community/poetry.lock | 97 +++++++++- libs/community/pyproject.toml | 4 + .../vectorstores/test_azure_search.py | 170 ++++++++++++++++++ 3 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 libs/community/tests/unit_tests/vectorstores/test_azure_search.py diff --git a/libs/community/poetry.lock b/libs/community/poetry.lock index cfd7fd4580966..392a96a4c1934 100644 --- a/libs/community/poetry.lock +++ b/libs/community/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aenum" @@ -552,6 +552,17 @@ azure-core = ">=1.30.0,<2.0.0" isodate = ">=0.6.1,<1.0.0" typing-extensions = ">=4.6.0" +[[package]] +name = "azure-common" +version = "1.1.28" +description = "Microsoft Azure Client Library for Python (Common)" +optional = true +python-versions = "*" +files = [ + {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, + {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, +] + [[package]] name = "azure-core" version = "1.30.1" @@ -571,6 +582,39 @@ typing-extensions = ">=4.6.0" [package.extras] aio = ["aiohttp (>=3.0)"] +[[package]] +name = "azure-identity" +version = "1.16.0" +description = "Microsoft Azure Identity Library for Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "azure-identity-1.16.0.tar.gz", hash = "sha256:6ff1d667cdcd81da1ceab42f80a0be63ca846629f518a922f7317a7e3c844e1b"}, + {file = "azure_identity-1.16.0-py3-none-any.whl", hash = "sha256:722fdb60b8fdd55fa44dc378b8072f4b419b56a5e54c0de391f644949f3a826f"}, +] + +[package.dependencies] +azure-core = ">=1.23.0" +cryptography = ">=2.5" +msal = ">=1.24.0" +msal-extensions = ">=0.3.0" + +[[package]] +name = "azure-search-documents" +version = "11.4.0" +description = "Microsoft Azure Cognitive Search Client Library for Python" +optional = true +python-versions = ">=3.7" +files = [ + {file = "azure-search-documents-11.4.0.tar.gz", hash = "sha256:599f269f106fb51e646ff426a218c21811575598e6a769b23fa4a0127c0f57e0"}, + {file = "azure_search_documents-11.4.0-py3-none-any.whl", hash = "sha256:e435266dc992a3450dc475309c9475f89a4bb0e9dac838140e609d9f1c7608ac"}, +] + +[package.dependencies] +azure-common = ">=1.1,<2.0" +azure-core = ">=1.28.0,<2.0.0" +isodate = ">=0.6.0" + [[package]] name = "babel" version = "2.14.0" @@ -3204,7 +3248,6 @@ files = [ {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:227b178b22a7f91ae88525810441791b1ca1fc71c86f03190911793be15cec3d"}, {file = "jq-1.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:780eb6383fbae12afa819ef676fc93e1548ae4b076c004a393af26a04b460742"}, {file = "jq-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08ded6467f4ef89fec35b2bf310f210f8cd13fbd9d80e521500889edf8d22441"}, - {file = "jq-1.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49e44ed677713f4115bd5bf2dbae23baa4cd503be350e12a1c1f506b0687848f"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:984f33862af285ad3e41e23179ac4795f1701822473e1a26bf87ff023e5a89ea"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42264fafc6166efb5611b5d4cb01058887d050a6c19334f6a3f8a13bb369df5"}, {file = "jq-1.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a67154f150aaf76cc1294032ed588436eb002097dd4fd1e283824bf753a05080"}, @@ -3715,7 +3758,7 @@ files = [ [[package]] name = "langchain-core" -version = "0.1.45" +version = "0.1.46" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" @@ -4204,6 +4247,25 @@ requests = ">=2.0.0,<3" [package.extras] broker = ["pymsalruntime (>=0.13.2,<0.15)"] +[[package]] +name = "msal-extensions" +version = "1.1.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +optional = true +python-versions = ">=3.7" +files = [ + {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"}, + {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"}, +] + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +packaging = "*" +portalocker = [ + {version = ">=1.0,<3", markers = "platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "platform_system == \"Windows\""}, +] + [[package]] name = "multidict" version = "6.0.5" @@ -5349,6 +5411,25 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "portalocker" +version = "2.8.2" +description = "Wraps the portalocker recipe for easy usage" +optional = true +python-versions = ">=3.8" +files = [ + {file = "portalocker-2.8.2-py3-none-any.whl", hash = "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"}, + {file = "portalocker-2.8.2.tar.gz", hash = "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"] + [[package]] name = "praw" version = "7.7.1" @@ -5529,6 +5610,8 @@ files = [ {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, @@ -5571,6 +5654,7 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -5579,6 +5663,8 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -6576,6 +6662,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -9229,9 +9316,9 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [extras] cli = ["typer"] -extended-testing = ["aiosqlite", "aleph-alpha-client", "anthropic", "arxiv", "assemblyai", "atlassian-python-api", "azure-ai-documentintelligence", "beautifulsoup4", "bibtexparser", "cassio", "chardet", "cloudpickle", "cloudpickle", "cohere", "databricks-vectorsearch", "datasets", "dgml-utils", "elasticsearch", "esprima", "faiss-cpu", "feedparser", "fireworks-ai", "friendli-client", "geopandas", "gitpython", "google-cloud-documentai", "gql", "gradientai", "hdbcli", "hologres-vector", "html2text", "httpx", "httpx-sse", "javelin-sdk", "jinja2", "jq", "jsonschema", "lxml", "markdownify", "motor", "msal", "mwparserfromhell", "mwxml", "newspaper3k", "numexpr", "nvidia-riva-client", "oci", "openai", "openapi-pydantic", "oracle-ads", "pandas", "pdfminer-six", "pgvector", "praw", "premai", "psychicapi", "py-trello", "pyjwt", "pymupdf", "pypdf", "pypdfium2", "pyspark", "rank-bm25", "rapidfuzz", "rapidocr-onnxruntime", "rdflib", "requests-toolbelt", "rspace_client", "scikit-learn", "sqlite-vss", "streamlit", "sympy", "telethon", "tidb-vector", "timescale-vector", "tqdm", "tree-sitter", "tree-sitter-languages", "upstash-redis", "vdms", "xata", "xmltodict"] +extended-testing = ["aiosqlite", "aleph-alpha-client", "anthropic", "arxiv", "assemblyai", "atlassian-python-api", "azure-ai-documentintelligence", "azure-identity", "azure-search-documents", "beautifulsoup4", "bibtexparser", "cassio", "chardet", "cloudpickle", "cloudpickle", "cohere", "databricks-vectorsearch", "datasets", "dgml-utils", "elasticsearch", "esprima", "faiss-cpu", "feedparser", "fireworks-ai", "friendli-client", "geopandas", "gitpython", "google-cloud-documentai", "gql", "gradientai", "hdbcli", "hologres-vector", "html2text", "httpx", "httpx-sse", "javelin-sdk", "jinja2", "jq", "jsonschema", "lxml", "markdownify", "motor", "msal", "mwparserfromhell", "mwxml", "newspaper3k", "numexpr", "nvidia-riva-client", "oci", "openai", "openapi-pydantic", "oracle-ads", "pandas", "pdfminer-six", "pgvector", "praw", "premai", "psychicapi", "py-trello", "pyjwt", "pymupdf", "pypdf", "pypdfium2", "pyspark", "rank-bm25", "rapidfuzz", "rapidocr-onnxruntime", "rdflib", "requests-toolbelt", "rspace_client", "scikit-learn", "sqlite-vss", "streamlit", "sympy", "telethon", "tidb-vector", "timescale-vector", "tqdm", "tree-sitter", "tree-sitter-languages", "upstash-redis", "vdms", "xata", "xmltodict"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "48ea73a94d06ae90f8f089017ae1bbcf9d37b2cc9957a44fb617785be0fe3236" +content-hash = "b066cbf8a1f02cae88c6c099e916d805fe6eb8685fd15c093d66cf52ea363fa5" diff --git a/libs/community/pyproject.toml b/libs/community/pyproject.toml index 43a4c185691ad..357d0244c60f6 100644 --- a/libs/community/pyproject.toml +++ b/libs/community/pyproject.toml @@ -94,6 +94,8 @@ hdbcli = {version = "^2.19.21", optional = true} oci = {version = "^2.119.1", optional = true} rdflib = {version = "7.0.0", optional = true} nvidia-riva-client = {version = "^2.14.0", optional = true} +azure-search-documents = {version = "11.4.0", optional = true} +azure-identity = {version = "^1.15.0", optional = true} tidb-vector = {version = ">=0.0.3,<1.0.0", optional = true} friendli-client = {version = "^1.2.4", optional = true} premai = {version = "^0.3.25", optional = true} @@ -268,6 +270,8 @@ extended_testing = [ "hdbcli", "oci", "rdflib", + "azure-search-documents", + "azure-identity", "tidb-vector", "cloudpickle", "friendli-client", diff --git a/libs/community/tests/unit_tests/vectorstores/test_azure_search.py b/libs/community/tests/unit_tests/vectorstores/test_azure_search.py new file mode 100644 index 0000000000000..25aabb8759d96 --- /dev/null +++ b/libs/community/tests/unit_tests/vectorstores/test_azure_search.py @@ -0,0 +1,170 @@ +import json +from typing import List, Optional +from unittest.mock import patch + +import pytest + +from langchain_community.vectorstores.azuresearch import AzureSearch +from tests.integration_tests.vectorstores.fake_embeddings import FakeEmbeddings + +DEFAULT_VECTOR_DIMENSION = 4 + + +class FakeEmbeddingsWithDimension(FakeEmbeddings): + """Fake embeddings functionality for testing.""" + + def __init__(self, dimension: int = DEFAULT_VECTOR_DIMENSION): + super().__init__() + self.dimension = dimension + + def embed_documents(self, embedding_texts: List[str]) -> List[List[float]]: + """Return simple embeddings.""" + return [ + [float(1.0)] * (self.dimension - 1) + [float(i)] + for i in range(len(embedding_texts)) + ] + + def embed_query(self, text: str) -> List[float]: + """Return simple embeddings.""" + return [float(1.0)] * (self.dimension - 1) + [float(0.0)] + + +DEFAULT_INDEX_NAME = "langchain-index" +DEFAULT_ENDPOINT = "https://my-search-service.search.windows.net" +DEFAULT_KEY = "mykey" +DEFAULT_EMBEDDING_MODEL = FakeEmbeddingsWithDimension() + + +def mock_default_index(*args, **kwargs): # type: ignore[no-untyped-def] + from azure.search.documents.indexes.models import ( + ExhaustiveKnnAlgorithmConfiguration, + ExhaustiveKnnParameters, + HnswAlgorithmConfiguration, + HnswParameters, + SearchField, + SearchFieldDataType, + SearchIndex, + VectorSearch, + VectorSearchAlgorithmMetric, + VectorSearchProfile, + ) + + return SearchIndex( + name=DEFAULT_INDEX_NAME, + fields=[ + SearchField( + name="id", + type=SearchFieldDataType.String, + key=True, + hidden=False, + searchable=False, + filterable=True, + sortable=False, + facetable=False, + ), + SearchField( + name="content", + type=SearchFieldDataType.String, + key=False, + hidden=False, + searchable=True, + filterable=False, + sortable=False, + facetable=False, + ), + SearchField( + name="content_vector", + type=SearchFieldDataType.Collection(SearchFieldDataType.Single), + searchable=True, + vector_search_dimensions=4, + vector_search_profile_name="myHnswProfile", + ), + SearchField( + name="metadata", + type="Edm.String", + key=False, + hidden=False, + searchable=True, + filterable=False, + sortable=False, + facetable=False, + ), + ], + vector_search=VectorSearch( + profiles=[ + VectorSearchProfile( + name="myHnswProfile", algorithm_configuration_name="default" + ), + VectorSearchProfile( + name="myExhaustiveKnnProfile", + algorithm_configuration_name="default_exhaustive_knn", + ), + ], + algorithms=[ + HnswAlgorithmConfiguration( + name="default", + parameters=HnswParameters( + m=4, + ef_construction=400, + ef_search=500, + metric=VectorSearchAlgorithmMetric.COSINE, + ), + ), + ExhaustiveKnnAlgorithmConfiguration( + name="default_exhaustive_knn", + parameters=ExhaustiveKnnParameters( + metric=VectorSearchAlgorithmMetric.COSINE + ), + ), + ], + ), + ) + + +def create_vector_store() -> AzureSearch: + return AzureSearch( + azure_search_endpoint=DEFAULT_ENDPOINT, + azure_search_key=DEFAULT_KEY, + index_name=DEFAULT_INDEX_NAME, + embedding_function=DEFAULT_EMBEDDING_MODEL, + ) + + +@pytest.mark.requires("azure.search.documents") +def test_init_existing_index() -> None: + from azure.search.documents.indexes import SearchIndexClient + + def mock_create_index() -> None: + pytest.fail("Should not create index in this test") + + with patch.multiple( + SearchIndexClient, get_index=mock_default_index, create_index=mock_create_index + ): + vector_store = create_vector_store() + assert vector_store.client is not None + + +@pytest.mark.requires("azure.search.documents") +def test_init_new_index() -> None: + from azure.core.exceptions import ResourceNotFoundError + from azure.search.documents.indexes import SearchIndexClient + from azure.search.documents.indexes.models import SearchIndex + + def no_index(self, name: str): # type: ignore[no-untyped-def] + raise ResourceNotFoundError + + created_index: Optional[SearchIndex] = None + + def mock_create_index(self, index): # type: ignore[no-untyped-def] + nonlocal created_index + created_index = index + + with patch.multiple( + SearchIndexClient, get_index=no_index, create_index=mock_create_index + ): + vector_store = create_vector_store() + assert vector_store.client is not None + assert created_index is not None + assert json.dumps(created_index.as_dict()) == json.dumps( + mock_default_index().as_dict() + ) From 147a43cfdf42b4c00a197273ea7d8a27d77c2c99 Mon Sep 17 00:00:00 2001 From: ccurme Date: Thu, 25 Apr 2024 16:54:58 -0400 Subject: [PATCH 32/36] langchain: support PineconeVectorStore in self query retriever (#20905) `langchain_pinecone.Pinecone` is deprecated in favor of `PineconeVectorStore`, and is currently a subclass of `PineconeVectorStore`. ```python @deprecated(since="0.0.3", removal="0.2.0", alternative="PineconeVectorStore") class Pinecone(PineconeVectorStore): """Deprecated. Use PineconeVectorStore instead.""" pass ``` --- libs/langchain/langchain/retrievers/self_query/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/langchain/langchain/retrievers/self_query/base.py b/libs/langchain/langchain/retrievers/self_query/base.py index 3295a593b8efa..9d1e79eb61cd4 100644 --- a/libs/langchain/langchain/retrievers/self_query/base.py +++ b/libs/langchain/langchain/retrievers/self_query/base.py @@ -124,11 +124,11 @@ def _get_builtin_translator(vectorstore: VectorStore) -> Visitor: return ElasticsearchTranslator() try: - from langchain_pinecone import Pinecone + from langchain_pinecone import PineconeVectorStore except ImportError: pass else: - if isinstance(vectorstore, Pinecone): + if isinstance(vectorstore, PineconeVectorStore): return PineconeTranslator() raise ValueError( From 474508e825422e33b4f058eb3f30df4b168b9891 Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:51:42 -0700 Subject: [PATCH 33/36] Use lstv2 (#20747) --- libs/core/langchain_core/tracers/context.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/core/langchain_core/tracers/context.py b/libs/core/langchain_core/tracers/context.py index 990049ee36934..1db9530016f72 100644 --- a/libs/core/langchain_core/tracers/context.py +++ b/libs/core/langchain_core/tracers/context.py @@ -147,6 +147,8 @@ def _get_trace_callbacks( def _tracing_v2_is_enabled() -> bool: return ( env_var_is_set("LANGCHAIN_TRACING_V2") + or env_var_is_set("LANGSMITH_TRACING") + or env_var_is_set("LANGSMITH_TRACING_V2") or tracing_v2_callback_var.get() is not None or get_run_tree_context() is not None ) From 170888b0903cfadaa08cc5c3f7626c310441e67e Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Fri, 26 Apr 2024 09:43:41 +0200 Subject: [PATCH 34/36] FIX --- libs/core/langchain_core/output_parsers/pydantic.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/core/langchain_core/output_parsers/pydantic.py b/libs/core/langchain_core/output_parsers/pydantic.py index 4fa525f8360bd..73444d45af283 100644 --- a/libs/core/langchain_core/output_parsers/pydantic.py +++ b/libs/core/langchain_core/output_parsers/pydantic.py @@ -60,10 +60,6 @@ def parse_result( json_object = super().parse_result(result) return self._parse_obj(json_object) - def parse_result( - self, result: List[Generation], *, partial: bool = False - ) -> TBaseModel: - pass def parse(self, text: str) -> TBaseModel: return super().parse(text) From 8f2d86667e653f7e4bfde92d1d40c102878d79bb Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Mon, 6 May 2024 13:30:28 +0200 Subject: [PATCH 35/36] Fix lint --- .../langchain_community/indexes/_sql_record_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/indexes/_sql_record_manager.py b/libs/community/langchain_community/indexes/_sql_record_manager.py index c326998f4cbd4..3fd2ce82f81a3 100644 --- a/libs/community/langchain_community/indexes/_sql_record_manager.py +++ b/libs/community/langchain_community/indexes/_sql_record_manager.py @@ -16,7 +16,8 @@ import contextlib import decimal import uuid -from typing import Any, AsyncGenerator, Dict, Generator, List, Optional, Sequence, Union +from typing import Any, AsyncGenerator, Dict, Generator, List, Optional, Sequence, \ + Union, cast from sqlalchemy import ( URL, @@ -178,7 +179,7 @@ async def _amake_session(self) -> AsyncGenerator[AsyncSession, None]: if not isinstance(self.engine, AsyncEngine): raise AssertionError("This method is not supported for sync engines.") - async with self.session_factory() as session: + async with cast(AsyncSession,self.session_factory()) as session: yield session def get_time(self) -> float: From cca03bdfd8f1d7bae0024894df8a1c885d0360e7 Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Mon, 6 May 2024 13:34:46 +0200 Subject: [PATCH 36/36] Fix lint --- .../indexes/_sql_record_manager.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libs/community/langchain_community/indexes/_sql_record_manager.py b/libs/community/langchain_community/indexes/_sql_record_manager.py index 3fd2ce82f81a3..f70a1dc4f4864 100644 --- a/libs/community/langchain_community/indexes/_sql_record_manager.py +++ b/libs/community/langchain_community/indexes/_sql_record_manager.py @@ -16,8 +16,17 @@ import contextlib import decimal import uuid -from typing import Any, AsyncGenerator, Dict, Generator, List, Optional, Sequence, \ - Union, cast +from typing import ( + Any, + AsyncGenerator, + Dict, + Generator, + List, + Optional, + Sequence, + Union, + cast, +) from sqlalchemy import ( URL, @@ -179,7 +188,7 @@ async def _amake_session(self) -> AsyncGenerator[AsyncSession, None]: if not isinstance(self.engine, AsyncEngine): raise AssertionError("This method is not supported for sync engines.") - async with cast(AsyncSession,self.session_factory()) as session: + async with cast(AsyncSession, self.session_factory()) as session: yield session def get_time(self) -> float: