diff --git a/README.md b/README.md index 273c67d..3f7762b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MongoDB Documentation Notebooks This repository contains Jupyter Notebooks that follow -tutorials and code examples in the official [MongoDB Vector Search documentation](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-overview/) and [Voyage AI documentation](https://www.mongodb.com/docs/voyageai/). You can run, download, and modify these notebooks as you learn how to use MongoDB Vector Search and Voyage AI models for your use case. +tutorials and code examples in the official [MongoDB Vector Search documentation](https://www.mongodb.com/docs/vector-search/) and [Voyage AI documentation](https://www.mongodb.com/docs/voyageai/). You can run, download, and modify these notebooks as you learn how to use MongoDB Vector Search and Voyage AI models for your use case. ## Overview @@ -23,7 +23,7 @@ The following table summarizes the contents of the notebooks in each directory: ## Other Resources -- [MongoDB Vector Search Documentation](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-overview/) +- [MongoDB Vector Search Documentation](https://www.mongodb.com/docs/vector-search/) - [Voyage AI Documentation](https://www.mongodb.com/docs/voyageai/) - [Generative AI Use Cases Repository](https://github.com/mongodb-developer/GenAI-Showcase/tree/main) diff --git a/ai-integrations/langchain-local-rag.ipynb b/ai-integrations/langchain-local-rag.ipynb index 9a25ea2..149f483 100644 --- a/ai-integrations/langchain-local-rag.ipynb +++ b/ai-integrations/langchain-local-rag.ipynb @@ -20,43 +20,29 @@ }, { "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, + "metadata": {}, "source": [ - "## Create a local Atlas deployment\n", - "\n", - "Run the following command in your terminal to set up your local Atlas deployment. \n", - "\n", - "```\n", - "atlas deployments setup\n", - "```" + "## Set up the environment\n", + "> **Prerequisite:** This notebook uses the `atlas-local-lib-py` library, which requires **Python 3.10 or later**. If you run this notebook locally, ensure your Python version meets this requirement." ] }, { - "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "## Set up the environment" + "%pip install --quiet uv" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, + "metadata": {}, "outputs": [], "source": [ - "pip install --quiet --upgrade pymongo langchain langchain-community langchain-huggingface langchain-text-splitters gpt4all pypdf" + "import sys\n", + "!uv pip install --system --python \"{sys.executable}\" atlas-local-lib-py pymongo langchain langchain-classic langchain-mongodb langchain-pymupdf4llm langchain-text-splitters gpt4all langchain-huggingface==1.2.2 sentence-transformers==5.7.0 transformers==4.57.6" ] }, { @@ -65,7 +51,13 @@ "metadata": {}, "outputs": [], "source": [ - "MONGODB_URI = (\"mongodb://localhost:/?directConnection=true\")" + "from atlas_local import LocalDeployment\n", + "\n", + "# Create or reuse a local Atlas deployment\n", + "deployment = LocalDeployment.get_or_create(name=\"local-atlas-deployment\")\n", + "deployment.start()\n", + "MONGODB_URI = deployment.connection_string()\n", + "print(\"Local deployment ready:\", MONGODB_URI)" ] }, { @@ -81,11 +73,24 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_mongodb import MongoDBAtlasVectorSearch\n", "from langchain_huggingface import HuggingFaceEmbeddings\n", "\n", - "# Load the embedding model (https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1)\n", - "embedding_model = HuggingFaceEmbeddings(model_name=\"mixedbread-ai/mxbai-embed-large-v1\")\n", + "# Load the embedding model (https://huggingface.co/voyageai/voyage-4-nano)\n", + "embedding_model = HuggingFaceEmbeddings(\n", + " model_name=\"voyageai/voyage-4-nano\",\n", + " model_kwargs={\"trust_remote_code\": True, \"truncate_dim\": 2048},\n", + " encode_kwargs={\"prompt_name\": \"document\"},\n", + " query_encode_kwargs={\"prompt_name\": \"query\"},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_mongodb import MongoDBAtlasVectorSearch\n", "\n", "# Instantiate vector store\n", "vector_store = MongoDBAtlasVectorSearch.from_connection_string(\n", @@ -102,12 +107,24 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_community.document_loaders import PyPDFLoader\n", + "from pymongo import MongoClient\n", + "\n", + "# Start from a clean collection, in case of previous runs of this tutorial\n", + "MongoClient(MONGODB_URI)[\"langchain_db\"][\"local_rag\"].drop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", + "from langchain_pymupdf4llm import PyMuPDF4LLMLoader \n", "\n", "# Load the PDF\n", - "loader = PyPDFLoader(\"https://investors.mongodb.com/node/13176/pdf\")\n", - "data = loader.load()\n", + "loader = PyMuPDF4LLMLoader(\"https://investors.mongodb.com/node/13176/pdf\") \n", + "data = loader.load() \n", "\n", "# Split PDF into documents\n", "text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=20)\n", @@ -124,7 +141,7 @@ "outputs": [], "source": [ "vector_store.create_vector_search_index(\n", - " dimensions = 1024, # The dimensions of the vector embeddings to be indexed\n", + " dimensions = 2048, # The dimensions of the vector embeddings to be indexed\n", " wait_until_complete = 60 # Number of seconds to wait for the index to build (can take around a minute)\n", ")" ] @@ -143,8 +160,8 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n", - "from langchain_community.llms import GPT4All\n", + "from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n", + "from langchain_classic.llms import GPT4All \n", "\n", "# Configure the LLM\n", "local_path = \"\"\n", diff --git a/ai-integrations/langchain-memory-semantic-cache.ipynb b/ai-integrations/langchain-memory-semantic-cache.ipynb index 7c76090..21a8254 100644 --- a/ai-integrations/langchain-memory-semantic-cache.ipynb +++ b/ai-integrations/langchain-memory-semantic-cache.ipynb @@ -31,7 +31,7 @@ }, "outputs": [], "source": [ - "pip install --quiet --upgrade langchain langchain-community langchain-core langchain-mongodb langchain-voyageai langchain-openai langchain-text-splitters pypdf" + "pip install --quiet --upgrade langchain langchain-community langchain-core langchain-mongodb langchain-voyageai langchain-openai langchain-text-splitters langgraph langgraph-checkpoint-mongodb pypdf" ] }, { @@ -141,18 +141,15 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_mongodb.chat_message_histories import MongoDBChatMessageHistory\n", - "from langchain_core.runnables.history import RunnableWithMessageHistory\n", + "from langgraph.checkpoint.mongodb import MongoDBSaver\n", "from langchain_core.prompts import MessagesPlaceholder\n", - " \n", - "# Define a function that gets the chat message history \n", - "def get_session_history(session_id: str) -> MongoDBChatMessageHistory:\n", - " return MongoDBChatMessageHistory(\n", - " connection_string=MONGODB_URI,\n", - " session_id=session_id,\n", - " database_name=\"langchain_db\",\n", - " collection_name=\"rag_with_memory\"\n", - " )" + "from pymongo import MongoClient\n", + "\n", + "# Connect to your MongoDB cluster\n", + "mongo_client = MongoClient(MONGODB_URI)\n", + "\n", + "# Create the checkpointer to persist conversation state\n", + "checkpointer = MongoDBSaver(mongo_client, db_name=\"langchain_db\")" ] }, { @@ -229,6 +226,9 @@ "metadata": {}, "outputs": [], "source": [ + "from langgraph.graph import StateGraph, START, END, MessagesState\n", + "from langchain_core.messages import AIMessage\n", + "\n", "# Build the RAG chain\n", "rag_chain = (\n", " retriever_chain\n", @@ -237,13 +237,23 @@ " | parse_output\n", ")\n", "\n", - "# Wrap the chain with message history\n", - "rag_with_memory = RunnableWithMessageHistory(\n", - " rag_chain,\n", - " get_session_history,\n", - " input_messages_key=\"question\",\n", - " history_messages_key=\"history\",\n", - ")" + "# Define the node that runs the RAG chain\n", + "def call_rag_chain(state: MessagesState):\n", + " question = state[\"messages\"][-1].content\n", + " history = state[\"messages\"][:-1]\n", + " answer = rag_chain.invoke(\n", + " {\"question\": question, \"history\": history}\n", + " )\n", + " return {\"messages\": [AIMessage(content=answer)]}\n", + "\n", + "# Build the graph\n", + "workflow = StateGraph(MessagesState)\n", + "workflow.add_node(\"rag\", call_rag_chain)\n", + "workflow.add_edge(START, \"rag\")\n", + "workflow.add_edge(\"rag\", END)\n", + "\n", + "# Compile the graph with the checkpointer\n", + "rag_with_memory = workflow.compile(checkpointer=checkpointer)" ] }, { @@ -255,10 +265,10 @@ "source": [ "# First question\n", "response_1 = rag_with_memory.invoke(\n", - " {\"question\": \"What was MongoDB's latest acquisition?\"},\n", - " {\"configurable\": {\"session_id\": \"user_1\"}}\n", + " {\"messages\": [(\"human\", \"What was MongoDB's latest acquisition?\")]},\n", + " {\"configurable\": {\"thread_id\": \"user_1\"}}\n", ")\n", - "print(response_1)" + "print(response_1[\"messages\"][-1].content)" ] }, { @@ -270,10 +280,10 @@ "source": [ "# Follow-up question that references the previous question\n", "response_2 = rag_with_memory.invoke(\n", - " {\"question\": \"Why did they do it?\"},\n", - " {\"configurable\": {\"session_id\": \"user_1\"}}\n", + " {\"messages\": [(\"human\", \"Why did they do it?\")]},\n", + " {\"configurable\": {\"thread_id\": \"user_1\"}}\n", ")\n", - "print(response_2)" + "print(response_2[\"messages\"][-1].content)" ] }, { @@ -320,8 +330,8 @@ "\n", "# First query (not cached)\n", "rag_with_memory.invoke(\n", - " {\"question\": \"What was MongoDB's latest acquisition?\"},\n", - " {\"configurable\": {\"session_id\": \"user_2\"}}\n", + " {\"messages\": [(\"human\", \"What was MongoDB's latest acquisition?\")]},\n", + " {\"configurable\": {\"thread_id\": \"user_2\"}}\n", ")" ] }, @@ -336,8 +346,8 @@ "\n", "# Second query (cached)\n", "rag_with_memory.invoke(\n", - " {\"question\": \"What company did MongoDB acquire recently?\"},\n", - " {\"configurable\": {\"session_id\": \"user_2\"}}\n", + " {\"messages\": [(\"human\", \"What company did MongoDB acquire recently?\")]},\n", + " {\"configurable\": {\"thread_id\": \"user_2\"}}\n", ")" ] } diff --git a/ai-integrations/llamaindex.ipynb b/ai-integrations/llamaindex.ipynb index 404cbff..0ac241e 100644 --- a/ai-integrations/llamaindex.ipynb +++ b/ai-integrations/llamaindex.ipynb @@ -18,6 +18,13 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Prerequisite:** This notebook uses the `atlas-local-lib-py` library, which requires **Python 3.10 or later**. If you run this notebook locally, ensure your Python version meets this requirement." + ] + }, { "cell_type": "code", "execution_count": null, @@ -28,7 +35,7 @@ }, "outputs": [], "source": [ - "pip install --quiet --upgrade llama-index llama-index-vector-stores-mongodb llama-index-llms-openai llama-index-embeddings-voyageai pymongo" + "pip install --quiet --upgrade llama-index llama-index-vector-stores-mongodb llama-index-llms-openai llama-index-embeddings-voyageai pymongo atlas-local-lib-py" ] }, { @@ -39,9 +46,14 @@ "source": [ "import os\n", "\n", - "os.environ[\"VOYAGEAI_API_KEY\"] = \"\"\n", + "from atlas_local import LocalDeployment\n", + "\n", "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "MONGODB_URI = \"\"" + "os.environ[\"VOYAGEAI_API_KEY\"] = \"\"\n", + "\n", + "# Create or reuse a local Atlas deployment\n", + "deployment = LocalDeployment.get_or_create(name=\"llamaindex-local-deployment\")\n", + "MONGODB_URI = deployment.connection_string()" ] }, { diff --git a/create-embeddings/open-source-existing-data.ipynb b/create-embeddings/open-source-existing-data.ipynb index 4b39b1e..9d5156e 100644 --- a/create-embeddings/open-source-existing-data.ipynb +++ b/create-embeddings/open-source-existing-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **existing data in MongoDB** by using the open-source ``nomic-embed-text-v1`` model.\n", "\n", @@ -70,7 +70,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/create-embeddings/open-source-new-data.ipynb b/create-embeddings/open-source-new-data.ipynb index 741635c..2fd4c69 100644 --- a/create-embeddings/open-source-new-data.ipynb +++ b/create-embeddings/open-source-new-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **new data** by using the open-source ``nomic-embed-text-v1`` model.\n", "\n", @@ -70,7 +70,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/create-embeddings/openai-existing-data.ipynb b/create-embeddings/openai-existing-data.ipynb index 3fd4e1c..1835dff 100644 --- a/create-embeddings/openai-existing-data.ipynb +++ b/create-embeddings/openai-existing-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **existing data in MongoDB** by using OpenAI's ``text-embedding-3-small`` model.\n", "\n", @@ -76,7 +76,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/create-embeddings/openai-new-data.ipynb b/create-embeddings/openai-new-data.ipynb index 0682bb6..64e9afe 100644 --- a/create-embeddings/openai-new-data.ipynb +++ b/create-embeddings/openai-new-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **new data** by using OpenAI's ``text-embedding-3-small`` model.\n", "\n", @@ -76,7 +76,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/create-embeddings/voyage-existing-data.ipynb b/create-embeddings/voyage-existing-data.ipynb index a532e5f..36fe740 100644 --- a/create-embeddings/voyage-existing-data.ipynb +++ b/create-embeddings/voyage-existing-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **existing data in MongoDB** by using the ``voyage-3-large`` model from Voyage AI.\n", "\n", @@ -76,7 +76,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/create-embeddings/voyage-new-data.ipynb b/create-embeddings/voyage-new-data.ipynb index 80d7f00..06245bd 100644 --- a/create-embeddings/voyage-new-data.ipynb +++ b/create-embeddings/voyage-new-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create Embeddings](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to generate embeddings from **new data** by using the ``voyage-4-large`` model from Voyage AI.\n", "\n", @@ -76,7 +76,7 @@ "source": [ "### (Optional) Compress your embeddings\n", "\n", - "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-embeddings/#vector-compression)." + "Optionally, run the following code to define a function that converts your embeddings into BSON `binData` vectors for [efficient storage and retrieval](https://www.mongodb.com/docs/vector-search/crud-embeddings/create-embeddings-manual/#vector-compression)." ] }, { diff --git a/get-started/quick-start.ipynb b/get-started/quick-start.ipynb index 04d4e07..a8cb375 100644 --- a/get-started/quick-start.ipynb +++ b/get-started/quick-start.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Quick Start](https://www.mongodb.com/docs/atlas/atlas-vector-search/tutorials/vector-search-quick-start/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Quick Start](https://www.mongodb.com/docs/vector-search/tutorials/vector-search-quick-start/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/manage-indexes/create-indexes-basic.ipynb b/manage-indexes/create-indexes-basic.ipynb index 84727c3..de0d92e 100644 --- a/manage-indexes/create-indexes-basic.ipynb +++ b/manage-indexes/create-indexes-basic.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion for the [Create and Manage Indexes](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion for the [Create and Manage Indexes](https://www.mongodb.com/docs/vector-search/indexes/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/manage-indexes/create-indexes-filter.ipynb b/manage-indexes/create-indexes-filter.ipynb index 35fdbd4..5278c6b 100644 --- a/manage-indexes/create-indexes-filter.ipynb +++ b/manage-indexes/create-indexes-filter.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/vector-search/indexes/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/manage-indexes/delete-indexes.ipynb b/manage-indexes/delete-indexes.ipynb index 66f6389..93b1ba3 100644 --- a/manage-indexes/delete-indexes.ipynb +++ b/manage-indexes/delete-indexes.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/vector-search/indexes/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/manage-indexes/edit-indexes.ipynb b/manage-indexes/edit-indexes.ipynb index 8af4647..bd2d137 100644 --- a/manage-indexes/edit-indexes.ipynb +++ b/manage-indexes/edit-indexes.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/vector-search/indexes/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/manage-indexes/view-indexes.ipynb b/manage-indexes/view-indexes.ipynb index c45b833..9587f36 100644 --- a/manage-indexes/view-indexes.ipynb +++ b/manage-indexes/view-indexes.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Manage Indexes](https://www.mongodb.com/docs/vector-search/indexes/vector-search-type/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/quantization/AutomaticQuantizationWithVoyageAI.ipynb b/quantization/AutomaticQuantizationWithVoyageAI.ipynb new file mode 100644 index 0000000..766bc10 --- /dev/null +++ b/quantization/AutomaticQuantizationWithVoyageAI.ipynb @@ -0,0 +1,933 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Atlas Vector Search - Vector Quantization - Automatic Quantization with Voyage AI" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook is a companion to the [Automatic Quantization with Voyage AI](https://www.mongodb.com/docs/atlas/atlas-vector-search/tutorials/auto-quantize-with-voyage-ai/) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", + "\n", + "This tutorial details techniques needed to design, deploy, and manage advanced AI workloads at scale, ensuring optimal performance and cost efficiency.\n", + "\n", + "\n", + " \"Open\n", + "\n", + "\n", + "## Prerequisites\n", + "\n", + "To run this notebook, you must have the following:\n", + "\n", + "- A MongoDB Atlas cluster at **M20** or higher with 2 or more search nodes using the **High-CPU S20** or higher search tier. To learn more, see [Create a Cluster](https://www.mongodb.com/docs/atlas/create-new-cluster/).\n", + "- A **MongoDB connection URI** for your Atlas cluster with read and write permissions.\n", + "- A **paid-tier Voyage AI API key**. A free-tier key does not support the request volume required for this tutorial. To learn more, see [Voyage AI API Key](https://docs.voyageai.com/docs/api-key-and-installation).\n", + "- A **Hugging Face access token (optional)**. Although not a hard requirement, using a Hugging Face token increases the download rate for the Wikipedia datasets this notebook uses. To learn more, see [User access tokens](https://huggingface.co/docs/hub/security-tokens)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install --quiet datasets==3.6.0 gcsfs==2025.3.0 fsspec==2025.3.0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install --quiet pymongo voyageai pandas==2.2.2 matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hugging Face token set.\n" + ] + } + ], + "source": [ + "import getpass\n", + "import os\n", + "import voyageai\n", + "import huggingface_hub\n", + "\n", + "# Function to securely get and set environment variables\n", + "def set_env_securely(var_name, prompt):\n", + " value = getpass.getpass(prompt)\n", + " os.environ[var_name] = value\n", + "\n", + "# Environment Variables\n", + "set_env_securely(\"VOYAGE_API_KEY\", \"Enter your Voyage API Key: \")\n", + "set_env_securely(\"MONGO_URI\", \"Enter your MongoDB URI: \")\n", + "MONGO_URI = os.environ.get(\"MONGO_URI\")\n", + "if not MONGO_URI:\n", + " raise ValueError(\"MONGO_URI not set in environment variables.\")\n", + "\n", + "# Voyage Client\n", + "voyage_client = voyageai.Client()\n", + "# Hugging Face Token (optional)\n", + "# Providing a token increases rate limits and speeds up dataset downloads.\n", + "# Press Enter to skip.\n", + "hf_token = getpass.getpass(\"Enter your Hugging Face token (optional, press Enter to skip): \")\n", + "if hf_token:\n", + " os.environ[\"HF_TOKEN\"] = hf_token\n", + " huggingface_hub.login(hf_token)\n", + " print(\"Hugging Face token set.\")\n", + "else:\n", + " print(\"No Hugging Face token provided. You may experience slower download speeds.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from datasets import load_dataset\n", + "from bson.binary import Binary, BinaryVectorDtype\n", + "import pymongo\n", + "\n", + "# Connect to Cluster\n", + "def get_mongo_client(uri):\n", + " \"\"\"Connect to MongoDB and confirm the connection.\"\"\"\n", + " client = pymongo.MongoClient(uri)\n", + " if client.admin.command(\"ping\").get(\"ok\") == 1.0:\n", + " print(\"Connected to MongoDB successfully.\")\n", + " return client\n", + " print(\"Failed to connect to MongoDB.\")\n", + " return None\n", + "\n", + "# Generate BSON Vector\n", + "def generate_bson_vector(array, data_type):\n", + " \"\"\"Convert an array to BSON vector format.\"\"\"\n", + " array = [float(val) for val in eval(array)]\n", + " return Binary.from_vector(array, BinaryVectorDtype(data_type))\n", + "\n", + "# Load Datasets\n", + "def load_and_prepare_data(dataset_name, amount):\n", + " \"\"\"Load and prepare streaming datasets for DataFrame.\"\"\"\n", + " print(f\"Loading {amount} records from {dataset_name}...\")\n", + " data = load_dataset(dataset_name, streaming=True, split=\"train\").take(amount)\n", + " return pd.DataFrame(data)\n", + "\n", + "# Insert datasets into MongoDB Collection\n", + "def insert_dataframe_into_collection(df, collection):\n", + " \"\"\"Insert Dataset records into MongoDB collection.\"\"\"\n", + " collection.insert_many(df.to_dict(\"records\"))\n", + " print(f\"Inserted {len(df)} records into '{collection.name}' collection.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading 250000 records from MongoDB/wikipedia-22-12-en-voyage-embed...\n" + ] + } + ], + "source": [ + "# NOTE: Loading 250,000 documents may take around 40 minutes.\n", + "# A higher-tier Voyage AI API key is required to handle this volume.\n", + "# To test the notebook logic quickly, reduce the amount to a smaller value.\n", + "import pandas as pd\n", + "from bson.binary import Binary, BinaryVectorDtype\n", + "from pymongo.errors import CollectionInvalid\n", + "\n", + "wikipedia_data_df = load_and_prepare_data(\"MongoDB/wikipedia-22-12-en-voyage-embed\", amount=250000)\n", + "wikipedia_annotation_data_df = load_and_prepare_data(\"MongoDB/wikipedia-22-12-en-annotation\", amount=250000)\n", + "wikipedia_annotation_data_df.drop(columns=[\"_id\"], inplace=True)\n", + "\n", + "# Convert embeddings to BSON format\n", + "wikipedia_data_df[\"embedding\"] = wikipedia_data_df[\"embedding\"].apply(\n", + " lambda x: generate_bson_vector(x, BinaryVectorDtype.FLOAT32)\n", + ")\n", + "\n", + "# MongoDB Setup\n", + "mongo_client = get_mongo_client(MONGO_URI)\n", + "DB_NAME = \"testing_datasets\"\n", + "db = mongo_client[DB_NAME]\n", + "\n", + "collections = {\n", + " \"wikipedia-22-12-en\": wikipedia_data_df,\n", + " \"wikipedia-22-12-en-annotation\": wikipedia_annotation_data_df,\n", + "}\n", + "\n", + "# Create Collections and Insert Data\n", + "for collection_name, df in collections.items():\n", + " if collection_name not in db.list_collection_names():\n", + " try:\n", + " db.create_collection(collection_name)\n", + " print(f\"Collection '{collection_name}' created successfully.\")\n", + " except CollectionInvalid:\n", + " print(f\"Error creating collection '{collection_name}'.\")\n", + " else:\n", + " print(f\"Collection '{collection_name}' already exists.\")\n", + "\n", + " # Clear collection and insert fresh data\n", + " collection = db[collection_name]\n", + " collection.delete_many({})\n", + " insert_dataframe_into_collection(df, collection)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "from pymongo.operations import SearchIndexModel\n", + "\n", + "def setup_vector_search_index(collection, index_definition, index_name=\"vector_index\"):\n", + " new_vector_search_index_model = SearchIndexModel(\n", + " definition=index_definition, name=index_name, type=\"vectorSearch\"\n", + " )\n", + "\n", + " # Create the new index\n", + " try:\n", + " result = collection.create_search_index(model=new_vector_search_index_model)\n", + " print(f\"Creating index '{index_name}'...\")\n", + "\n", + " # Wait for initial sync to complete\n", + " print(\"Polling to check if the index is ready. This might take a few minutes.\")\n", + " predicate=None\n", + " if predicate is None:\n", + " predicate = lambda index: index.get(\"queryable\") is True\n", + " while True:\n", + " indices = list(collection.list_search_indexes(result))\n", + " if len(indices) and predicate(indices[0]):\n", + " break\n", + " time.sleep(5)\n", + " print(f\"Index '{index_name}' is ready for querying.\")\n", + " return result\n", + "\n", + " except Exception as e:\n", + " print(f\"Error creating new vector search index '{index_name}': {e!s}\")\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vector_index_definition_scalar_quantized = {\n", + " \"fields\": [\n", + " {\n", + " \"type\": \"vector\",\n", + " \"path\": \"embedding\",\n", + " \"quantization\": \"scalar\", # Added quantization for scalar vector quantization\n", + " \"numDimensions\": 1024,\n", + " \"similarity\": \"cosine\",\n", + " }\n", + " ]\n", + "}\n", + "\n", + "vector_index_definition_binary_quantized = {\n", + " \"fields\": [\n", + " {\n", + " \"type\": \"vector\",\n", + " \"path\": \"embedding\",\n", + " \"quantization\": \"binary\", # Changed quantization to binary for binary vector quantization\n", + " \"numDimensions\": 1024,\n", + " \"similarity\": \"cosine\",\n", + " }\n", + " ]\n", + "}\n", + "\n", + "vector_index_definition_float32_ann = {\n", + " \"fields\": [\n", + " {\n", + " \"type\": \"vector\",\n", + " \"path\": \"embedding\",\n", + " \"numDimensions\": 1024,\n", + " \"similarity\": \"cosine\",\n", + " }\n", + " ]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pymongo\n", + "vector_search_scalar_quantized_index_name = \"vector_index_scalar_quantized\"\n", + "vector_search_binary_quantized_index_name = \"vector_index_binary_quantized\"\n", + "vector_search_float32_ann_index_name = \"vector_index_float32_ann\"\n", + "db = mongo_client[DB_NAME]\n", + "wiki_data_collection = db[\"wikipedia-22-12-en\"]\n", + "wiki_annotation_data_collection = db[\"wikipedia-22-12-en-annotation\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pymongo.operations import SearchIndexModel\n", + "\n", + "setup_vector_search_index(\n", + " wiki_data_collection,\n", + " vector_index_definition_scalar_quantized,\n", + " vector_search_scalar_quantized_index_name,\n", + ")\n", + "setup_vector_search_index(\n", + " wiki_data_collection,\n", + " vector_index_definition_binary_quantized,\n", + " vector_search_binary_quantized_index_name,\n", + ")\n", + "setup_vector_search_index(\n", + " wiki_data_collection,\n", + " vector_index_definition_float32_ann,\n", + " vector_search_float32_ann_index_name,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_embedding(text, task_prefix=\"document\"):\n", + " \"\"\"Fetch embedding for a given text using Voyage AI.\"\"\"\n", + " if not text.strip():\n", + " print(\"Empty text provided for embedding.\")\n", + " return []\n", + " result = voyage_client.embed([text], model=\"voyage-3-large\", input_type=task_prefix)\n", + " return result.embeddings[0]\n", + "\n", + "def custom_vector_search(\n", + " user_query,\n", + " collection,\n", + " embedding_path,\n", + " vector_search_index_name=\"vector_index\",\n", + " top_k=5,\n", + " num_candidates=25,\n", + " use_full_precision=False,\n", + "):\n", + " \"\"\"Perform vector search on a MongoDB collection using specified index.\"\"\"\n", + " # Generate embedding for the user query\n", + " query_embedding = get_embedding(user_query, task_prefix=\"query\")\n", + "\n", + " if query_embedding is None or not query_embedding:\n", + " return {\n", + " \"error\": \"Invalid query or embedding generation failed.\",\n", + " \"execution_time_ms\": None,\n", + " \"results\": [],\n", + " }\n", + "\n", + " # Define the vector search stage\n", + " vector_search_stage = {\n", + " \"$vectorSearch\": {\n", + " \"index\": vector_search_index_name,\n", + " \"queryVector\": query_embedding,\n", + " \"path\": embedding_path,\n", + " \"limit\": top_k,\n", + " }\n", + " }\n", + "\n", + " # Configure search precision approach\n", + " if not use_full_precision:\n", + " # For approximate nearest neighbor (ANN) search\n", + " vector_search_stage[\"$vectorSearch\"][\"numCandidates\"] = num_candidates\n", + " else:\n", + " # For exact nearest neighbor (ENN) search\n", + " vector_search_stage[\"$vectorSearch\"][\"exact\"] = True\n", + "\n", + " # Project stage to fetch desired fields and vector search score\n", + " project_stage = {\n", + " \"$project\": {\n", + " \"_id\": 0,\n", + " \"title\": 1,\n", + " \"text\": 1,\n", + " \"wiki_id\": 1,\n", + " \"url\": 1,\n", + " \"score\": {\"$meta\": \"vectorSearchScore\"},\n", + " }\n", + " }\n", + "\n", + " # Define the aggregate pipeline\n", + " pipeline = [vector_search_stage, project_stage]\n", + "\n", + " try:\n", + " # Execute the explain command to measure latency\n", + " explain_result = collection.database.command(\n", + " \"explain\",\n", + " {\"aggregate\": collection.name, \"pipeline\": pipeline, \"cursor\": {}},\n", + " verbosity=\"executionStats\",\n", + " )\n", + "\n", + " # Extract the execution time\n", + " vector_search_explain = explain_result[\"stages\"][0][\"$vectorSearch\"]\n", + " execution_time_ms = vector_search_explain[\"explain\"][\"query\"][\"stats\"][\"context\"][\"millisElapsed\"]\n", + "\n", + " # Execute the actual aggregate query\n", + " results = list(collection.aggregate(pipeline))\n", + "\n", + " return {\n", + " \"results\": results,\n", + " \"execution_time_ms\": execution_time_ms,\n", + " }\n", + " except Exception as e:\n", + " print(f\"Error during vector search: {e}\")\n", + " return {\n", + " \"error\": str(e),\n", + " \"execution_time_ms\": None,\n", + " \"results\": [],\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vector_search_indices = [\n", + " vector_search_float32_ann_index_name,\n", + " vector_search_scalar_quantized_index_name,\n", + " vector_search_binary_quantized_index_name,\n", + "]\n", + "\n", + "# Random query\n", + "user_query = \"How do I increase my productivity for maximum output\"\n", + "test_top_k = 5\n", + "test_num_candidates = 25\n", + "\n", + "# Result is a list of dictionaries with the following headings: precision, top_k, latency_ms, results\n", + "results = []\n", + "\n", + "for vector_search_index in vector_search_indices:\n", + "# Conduct a vector search operation using scalar quantized\n", + " vector_search_results = custom_vector_search(\n", + " user_query,\n", + " wiki_data_collection,\n", + " embedding_path=\"embedding\",\n", + " vector_search_index_name=vector_search_index,\n", + " top_k=test_top_k,\n", + " num_candidates=test_num_candidates,\n", + " use_full_precision=False,\n", + " )\n", + " # Include the precision in the results\n", + " precision = vector_search_index.split(\"vector_index\")[1]\n", + " precision = precision.replace(\"quantized\", \"\").capitalize()\n", + "\n", + " results.append(\n", + " {\n", + " \"precision\": precision,\n", + " \"top_k\": test_top_k,\n", + " \"num_candidates\": test_num_candidates,\n", + " \"latency_ms\": vector_search_results[\"execution_time_ms\"],\n", + " \"results\": vector_search_results[\"results\"][\n", + " 0\n", + " ], # Just taking the first result\n", + " }\n", + " )\n", + "\n", + "# Conduct a vector search operation using full precision\n", + "precision = \"Float32_ENN\"\n", + "vector_search_results = custom_vector_search(\n", + " user_query,\n", + " wiki_data_collection,\n", + " embedding_path=\"embedding\",\n", + " vector_search_index_name=\"vector_index_scalar_quantized\",\n", + " top_k=test_top_k,\n", + " num_candidates=test_num_candidates,\n", + " use_full_precision=True,\n", + ")\n", + "\n", + "results.append(\n", + " {\n", + " \"precision\": precision,\n", + " \"top_k\": test_top_k,\n", + " \"num_candidates\": test_num_candidates,\n", + " \"latency_ms\": vector_search_results[\"execution_time_ms\"],\n", + " \"results\": vector_search_results[\"results\"][0], # Just taking the first result\n", + " }\n", + ")\n", + "\n", + "# Convert the results to a pandas DataFrame with the headings: precision, top_k, latency_ms\n", + "results_df = pd.DataFrame(results)\n", + "results_df.columns = [\"precision\", \"top_k\", \"num_candidates\", \"latency_ms\", \"results\"]\n", + "\n", + "# To display the results:\n", + "results_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "\n", + "def format_time(ms):\n", + " \"\"\"Convert milliseconds to a human-readable format\"\"\"\n", + " delta = timedelta(milliseconds=ms)\n", + "\n", + " # Extract minutes, seconds, and milliseconds with more precision\n", + " minutes = delta.seconds // 60\n", + " seconds = delta.seconds % 60\n", + " milliseconds = round(ms % 1000, 3) # Keep 3 decimal places for milliseconds\n", + "\n", + " # Format based on duration\n", + " if minutes > 0:\n", + " return f\"{minutes}m {seconds}.{milliseconds:03.0f}s\"\n", + " elif seconds > 0:\n", + " return f\"{seconds}.{milliseconds:03.0f}s\"\n", + " else:\n", + " return f\"{milliseconds:.3f}ms\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def measure_latency_with_varying_topk(\n", + " user_query,\n", + " collection,\n", + " vector_search_index_name=\"vector_index_scalar_quantized\",\n", + " use_full_precision=False,\n", + " top_k_values=[5, 10, 100],\n", + " num_candidates_values=[25, 50, 100, 200, 500, 1000, 2000, 5000, 10000],\n", + "):\n", + " results_data = []\n", + "\n", + " # Conduct vector search operation for each (top_k, num_candidates) combination\n", + " for top_k in top_k_values:\n", + " for num_candidates in num_candidates_values:\n", + " # Skip scenarios where num_candidates < top_k\n", + " if num_candidates < top_k:\n", + " continue\n", + "\n", + " # Construct the precision name\n", + " precision_name = vector_search_index_name.split(\"vector_index\")[1]\n", + " precision_name = precision_name.replace(\"quantized\", \"\").capitalize()\n", + "\n", + " # If use_full_precision is true, then the precision name is \"_float32_\"\n", + " if use_full_precision:\n", + " precision_name = \"_float32_ENN\"\n", + "\n", + " # Perform the vector search\n", + " vector_search_results = custom_vector_search(\n", + " user_query=user_query,\n", + " collection=collection,\n", + " embedding_path=\"embedding\",\n", + " vector_search_index_name=vector_search_index_name,\n", + " top_k=top_k,\n", + " num_candidates=num_candidates,\n", + " use_full_precision=use_full_precision,\n", + " )\n", + "\n", + " # Extract the execution time (latency)\n", + " latency_ms = vector_search_results[\"execution_time_ms\"]\n", + "\n", + " # Append the results to the list\n", + " results_data.append(\n", + " {\n", + " \"precision\": precision_name,\n", + " \"top_k\": top_k,\n", + " \"num_candidates\": num_candidates,\n", + " \"latency_ms\": latency_ms,\n", + " }\n", + " )\n", + "\n", + " print(f\"Top-K: {top_k}, NumCandidates: {num_candidates}, \"\n", + " f\"Latency: {format_time(latency_ms)}, Precision: {precision_name}\")\n", + "\n", + " return results_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define vector search indices\n", + "vector_search_float32_ann_index_name = \"vector_index_float32_ann\"\n", + "vector_search_scalar_quantized_index_name = \"vector_index_scalar_quantized\"\n", + "vector_search_binary_quantized_index_name = \"vector_index_binary_quantized\"\n", + "\n", + "vector_search_indices = [\n", + " vector_search_float32_ann_index_name,\n", + " vector_search_scalar_quantized_index_name,\n", + " vector_search_binary_quantized_index_name,\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run the measurements\n", + "user_query = \"How do I increase my productivity for maximum output\"\n", + "top_k_values = [5, 10, 50, 100]\n", + "num_candidates_values = [25, 50, 100, 200, 500, 1000, 2000, 5000, 10000]\n", + "\n", + "latency_results = []\n", + "\n", + "for vector_search_index in vector_search_indices:\n", + " latency_results.append(\n", + " measure_latency_with_varying_topk(\n", + " user_query,\n", + " wiki_data_collection,\n", + " vector_search_index_name=vector_search_index,\n", + " use_full_precision=False,\n", + " top_k_values=top_k_values,\n", + " num_candidates_values=num_candidates_values,\n", + " )\n", + " )\n", + "\n", + "# Conduct vector search operation using full precision\n", + "latency_results.append(\n", + " measure_latency_with_varying_topk(\n", + " user_query,\n", + " wiki_data_collection,\n", + " vector_search_index_name=\"vector_index_scalar_quantized\",\n", + " use_full_precision=True,\n", + " top_k_values=top_k_values,\n", + " num_candidates_values=num_candidates_values,\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# Map your precision field to the labels and colors you want in the legend\n", + "precision_label_map = {\n", + " \"_scalar_\": \"scalar\",\n", + " \"_binary_\": \"binary\",\n", + " \"_float32_ann\": \"float32_ann\",\n", + " \"_float32_ENN\": \"float32_ENN\",\n", + "}\n", + "\n", + "precision_color_map = {\n", + " \"_scalar_\": \"orange\",\n", + " \"_binary_\": \"red\",\n", + " \"_float32_ann\": \"blue\",\n", + " \"_float32_ENN\": \"purple\",\n", + "}\n", + "\n", + "# Assume latency_results is a list of lists. Each inner list corresponds to one precision type.\n", + "# Each dictionary in an inner list has keys: 'precision', 'top_k', 'num_candidates', 'latency_ms'\n", + "\n", + "# Flatten all measurements and find the unique top_k values\n", + "all_measurements = [m for precision_list in latency_results for m in precision_list]\n", + "unique_topk = sorted(set(m[\"top_k\"] for m in all_measurements))\n", + "\n", + "# For each top_k, create a separate plot\n", + "for k in unique_topk:\n", + " plt.figure(figsize=(10, 6))\n", + "\n", + " # For each precision type, filter out measurements for the current top_k value\n", + " for measurements in latency_results:\n", + " # Filter measurements with top_k equal to the current k\n", + " filtered = [m for m in measurements if m[\"top_k\"] == k]\n", + " if not filtered:\n", + " continue\n", + "\n", + " # Extract x (num_candidates) and y (latency) values\n", + " x = [m[\"num_candidates\"] for m in filtered]\n", + " y = [m[\"latency_ms\"] for m in filtered]\n", + "\n", + " # Determine the precision, label, and color from the first measurement in this filtered list\n", + " precision = filtered[0][\"precision\"]\n", + " label = precision_label_map.get(precision, precision)\n", + " color = precision_color_map.get(precision, \"blue\")\n", + "\n", + " # Plot the line for this precision type\n", + " plt.plot(x, y, marker=\"o\", color=color, label=label)\n", + "\n", + " # Label axes and add title including the top_k value\n", + " plt.xlabel(\"Number of Candidates\")\n", + " plt.ylabel(\"Latency (ms)\")\n", + " plt.title(f\"Search Latency vs Num Candidates for Top-K = {k}\")\n", + "\n", + " # Add a legend and grid, then show the plot\n", + " plt.legend()\n", + " plt.grid(True)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def measure_representational_capacity_retention_against_float_enn(\n", + " ground_truth_collection,\n", + " collection,\n", + " quantized_index_name, # This is used for both the quantized search and (with use_full_precision=True) for the baseline.\n", + " top_k_values, # List/array of top-k values to test.\n", + " num_candidates_values, # List/array of num_candidates values to test.\n", + " num_queries_to_test=1,\n", + "):\n", + " retention_results = {\"per_query_retention\": {}}\n", + " overall_retention = {} # overall_retention[top_k][num_candidates] = [list of retention values]\n", + "\n", + " # Initialize overall retention structure\n", + " for top_k in top_k_values:\n", + " overall_retention[top_k] = {}\n", + " for num_candidates in num_candidates_values:\n", + " if num_candidates < top_k:\n", + " continue\n", + " overall_retention[top_k][num_candidates] = []\n", + "\n", + " # Extract and store the precision name from the quantized index name.\n", + " precision_name = quantized_index_name.split(\"vector_index\")[1]\n", + " precision_name = precision_name.replace(\"quantized\", \"\").capitalize()\n", + " retention_results[\"precision_name\"] = precision_name\n", + " retention_results[\"top_k_values\"] = top_k_values\n", + " retention_results[\"num_candidates_values\"] = num_candidates_values\n", + "\n", + " # Load ground truth annotations\n", + " ground_truth_annotations = list(\n", + " ground_truth_collection.find().limit(num_queries_to_test)\n", + " )\n", + " print(f\"Loaded {len(ground_truth_annotations)} ground truth annotations\")\n", + "\n", + " # Process each ground truth annotation\n", + " for annotation in ground_truth_annotations:\n", + " # Use the ground truth wiki_id from the annotation.\n", + " ground_truth_wiki_id = annotation[\"wiki_id\"]\n", + "\n", + " # Process only queries that are questions.\n", + " for query_type, queries in annotation[\"queries\"].items():\n", + " if query_type.lower() not in [\"question\", \"questions\"]:\n", + " continue\n", + "\n", + " for query in queries:\n", + " # Prepare nested dict for this query\n", + " if query not in retention_results[\"per_query_retention\"]:\n", + " retention_results[\"per_query_retention\"][query] = {}\n", + "\n", + " # For each valid combination of top_k and num_candidates\n", + " for top_k in top_k_values:\n", + " if top_k not in retention_results[\"per_query_retention\"][query]:\n", + " retention_results[\"per_query_retention\"][query][top_k] = {}\n", + " for num_candidates in num_candidates_values:\n", + " if num_candidates < top_k:\n", + " continue\n", + "\n", + " # Baseline search: full precision using ENN (Float32)\n", + " baseline_result = custom_vector_search(\n", + " user_query=query,\n", + " collection=collection,\n", + " embedding_path=\"embedding\",\n", + " vector_search_index_name=quantized_index_name,\n", + " top_k=top_k,\n", + " num_candidates=num_candidates,\n", + " use_full_precision=True,\n", + " )\n", + " baseline_ids = {\n", + " res[\"wiki_id\"] for res in baseline_result[\"results\"]\n", + " }\n", + "\n", + " # Quantized search:\n", + " quantized_result = custom_vector_search(\n", + " user_query=query,\n", + " collection=collection,\n", + " embedding_path=\"embedding\",\n", + " vector_search_index_name=quantized_index_name,\n", + " top_k=top_k,\n", + " num_candidates=num_candidates,\n", + " use_full_precision=False,\n", + " )\n", + " quantized_ids = {\n", + " res[\"wiki_id\"] for res in quantized_result[\"results\"]\n", + " }\n", + "\n", + " # Compute retention for this combination\n", + " if baseline_ids:\n", + " retention = len(\n", + " baseline_ids.intersection(quantized_ids)\n", + " ) / len(baseline_ids)\n", + " else:\n", + " retention = 0\n", + "\n", + " # Store the results per query\n", + " retention_results[\"per_query_retention\"][query].setdefault(\n", + " top_k, {}\n", + " )[num_candidates] = {\n", + " \"ground_truth_wiki_id\": ground_truth_wiki_id,\n", + " \"baseline_ids\": sorted(baseline_ids),\n", + " \"quantized_ids\": sorted(quantized_ids),\n", + " \"retention\": retention,\n", + " }\n", + " overall_retention[top_k][num_candidates].append(retention)\n", + "\n", + " print(\n", + " f\"Query: '{query}' | top_k: {top_k}, num_candidates: {num_candidates}\"\n", + " )\n", + " print(f\" Ground Truth wiki_id: {ground_truth_wiki_id}\")\n", + " print(f\" Baseline IDs (Float32): {sorted(baseline_ids)}\")\n", + " print(\n", + " f\" Quantized IDs: {precision_name}: {sorted(quantized_ids)}\"\n", + " )\n", + " print(f\" Retention: {retention:.4f}\\n\")\n", + "\n", + " # Compute overall average retention per combination\n", + " avg_overall_retention = {}\n", + " for top_k, cand_dict in overall_retention.items():\n", + " avg_overall_retention[top_k] = {}\n", + " for num_candidates, retentions in cand_dict.items():\n", + " if retentions:\n", + " avg = sum(retentions) / len(retentions)\n", + " else:\n", + " avg = 0\n", + " avg_overall_retention[top_k][num_candidates] = avg\n", + " print(\n", + " f\"Overall Average Retention for top_k {top_k}, num_candidates {num_candidates}: {avg:.4f}\"\n", + " )\n", + "\n", + " retention_results[\"average_retention\"] = avg_overall_retention\n", + " return retention_results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Access the database\n", + "DB_NAME = \"testing_datasets\"\n", + "db = mongo_client[DB_NAME]\n", + "\n", + "# Access collections\n", + "wiki_data_collection = db[\"wikipedia-22-12-en\"]\n", + "ground_truth_collection = db[\"wikipedia-22-12-en-annotation\"]\n", + "\n", + "overall_recall_results = []\n", + "top_k_values = [5, 10, 50, 100]\n", + "num_candidates_values = [25, 50, 100, 200, 500, 1000, 5000]\n", + "num_queries_to_test = 1\n", + "\n", + "for vector_search_index in vector_search_indices:\n", + " overall_recall_results.append(\n", + " measure_representational_capacity_retention_against_float_enn(\n", + " ground_truth_collection=ground_truth_collection,\n", + " collection=wiki_data_collection,\n", + " quantized_index_name=vector_search_index,\n", + " top_k_values=top_k_values,\n", + " num_candidates_values=num_candidates_values,\n", + " num_queries_to_test=num_queries_to_test,\n", + " )\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# Define colors and labels for each precision type\n", + "precision_colors = {\"_scalar_\": \"orange\", \"_binary_\": \"red\", \"_float32_\": \"green\"}\n", + "\n", + "# Assume overall_retention_results is a list of dictionaries returned by your\n", + "# measure_representational_capacity_retention_against_float_enn function.\n", + "# Each dictionary should contain:\n", + "# - 'precision_name': the precision type (e.g., '_scalar_')\n", + "# - 'average_retention': a dict mapping each top_k to a dict mapping num_candidates\n", + "# to the average retention, e.g.,\n", + "# average_retention[top_k][num_candidates] = retention_value\n", + "\n", + "if overall_recall_results:\n", + " # Determine unique top_k values from the first result's average_retention keys\n", + " unique_topk = sorted(list(overall_recall_results[0][\"average_retention\"].keys()))\n", + "\n", + " for k in unique_topk:\n", + " plt.figure(figsize=(10, 6))\n", + " # For each precision type, plot retention vs. number of candidates at this top_k\n", + " for result in overall_recall_results:\n", + " precision_name = result.get(\"precision_name\", \"unknown\")\n", + " color = precision_colors.get(precision_name, \"blue\")\n", + " # Get candidate values from the average_retention dictionary for top_k k\n", + " candidate_values = sorted(result[\"average_retention\"][k].keys())\n", + " retention_values = [\n", + " result[\"average_retention\"][k][nc] for nc in candidate_values\n", + " ]\n", + "\n", + " plt.plot(\n", + " candidate_values,\n", + " retention_values,\n", + " marker=\"o\",\n", + " label=precision_name.strip(\"_\"),\n", + " color=color,\n", + " )\n", + "\n", + " plt.xlabel(\"Number of Candidates\")\n", + " plt.ylabel(\"Retention Score\")\n", + " plt.title(f\"Retention vs Number of Candidates for Top-K = {k}\")\n", + " plt.legend()\n", + " plt.grid(True)\n", + " plt.show()\n", + "\n", + " # Print detailed average retention results\n", + " print(\"\\nDetailed Average Retention Results:\")\n", + " for result in overall_recall_results:\n", + " precision_name = result.get(\"precision_name\", \"unknown\")\n", + " print(f\"\\n{precision_name} Embedding:\")\n", + " for k in sorted(result[\"average_retention\"].keys()):\n", + " print(f\"\\nTop-K: {k}\")\n", + " for nc in sorted(result[\"average_retention\"][k].keys()):\n", + " ret = result[\"average_retention\"][k][nc]\n", + " print(f\" NumCandidates: {nc}, Retention: {ret:.4f}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/quantization/existing-data.ipynb b/quantization/existing-data.ipynb index fed8c1e..f94638b 100644 --- a/quantization/existing-data.ipynb +++ b/quantization/existing-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Vector Quantization](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-quantization/#how-to-ingest-pre-quantized-vectors) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Vector Quantization](https://www.mongodb.com/docs/vector-search/about/vector-quantization/#how-to-ingest-pre-quantized-vectors) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to pre-quantize and ingest your vectors for vector search from **existing data in Atlas** by using the `voyage-4-large` model from [Voyage AI](https://www.voyageai.com).\n", "\n", diff --git a/quantization/new-data.ipynb b/quantization/new-data.ipynb index 1822fb3..036bc6c 100644 --- a/quantization/new-data.ipynb +++ b/quantization/new-data.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Vector Quantization](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-quantization/#how-to-ingest-pre-quantized-vectors) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Vector Quantization](https://www.mongodb.com/docs/vector-search/about/vector-quantization/#how-to-ingest-pre-quantized-vectors) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to pre-quantize and ingest your vectors for vector search from **existing data in Atlas** by using the `voyage-4-large` model from [Voyage AI](https://www.voyageai.com).\n", "\n", diff --git a/run-queries/ann-basic.ipynb b/run-queries/ann-basic.ipynb index 878c4f2..ae26d00 100644 --- a/run-queries/ann-basic.ipynb +++ b/run-queries/ann-basic.ipynb @@ -19,7 +19,7 @@ } }, "source": [ - "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/vector-search/query/aggregation-stages/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/run-queries/ann-filter.ipynb b/run-queries/ann-filter.ipynb index a8fb720..c4de46a 100644 --- a/run-queries/ann-filter.ipynb +++ b/run-queries/ann-filter.ipynb @@ -19,7 +19,7 @@ } }, "source": [ - "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/vector-search/query/aggregation-stages/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/run-queries/enn.ipynb b/run-queries/enn.ipynb index a961d4c..1e72e07 100644 --- a/run-queries/enn.ipynb +++ b/run-queries/enn.ipynb @@ -19,7 +19,7 @@ } }, "source": [ - "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Create and Run Queries](https://www.mongodb.com/docs/vector-search/query/aggregation-stages/vector-search-stage/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/run-queries/semantic-search.ipynb b/run-queries/semantic-search.ipynb index f3ff553..b450f25 100644 --- a/run-queries/semantic-search.ipynb +++ b/run-queries/semantic-search.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Semantic Search for Text](https://www.mongodb.com/docs/atlas/atlas-vector-search/tutorials/vector-search-tutorial/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Semantic Search for Text](https://www.mongodb.com/docs/vector-search/tutorials/vector-search-tutorial/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "\n", " \"Open\n", diff --git a/use-cases/ai-agent.ipynb b/use-cases/ai-agent.ipynb index 1b5d151..cb13ff6 100644 --- a/use-cases/ai-agent.ipynb +++ b/use-cases/ai-agent.ipynb @@ -13,7 +13,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Build AI Agents with MongoDB](https://www.mongodb.com/docs/atlas/atlas-vector-search/ai-agents) page. For a more traditional Python development example and detailed explanations of the code, refer to the tutorial on the page.\n", + "This notebook is a companion to the [Build AI Agents with MongoDB](https://www.mongodb.com/docs/vector-search/about/ai-agents) page. For a more traditional Python development example and detailed explanations of the code, refer to the tutorial on the page.\n", "\n", "This notebook demonstrates an AI agent that uses MongoDB as the database for both agentic RAG and agent memory.\n", "\n", diff --git a/use-cases/local-rag.ipynb b/use-cases/local-rag.ipynb index 580db8e..e07852a 100644 --- a/use-cases/local-rag.ipynb +++ b/use-cases/local-rag.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Local Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/atlas/atlas-vector-search/tutorials/local-rag/) page. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Local Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/vector-search/tutorials/local-rag/) page. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you a RAG implementation with MongoDB Vector Search that you can run **completely locally** by using models from Hugging Face and GPT4All.\n", "\n", @@ -24,15 +24,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a local Atlas deployment\n", - "\n", - "Run the following commands in your terminal to set up your local Atlas deployment. \n", - "\n", - "```\n", - "atlas deployments setup\n", - "curl https://atlas-education.s3.amazonaws.com/sampledata.archive -o sampledata.archive\n", - "mongorestore --archive=sampledata.archive --port=\n", - "```" + "## Set up the environment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Prerequisite:** This notebook uses the `atlas-local-lib-py` library, which requires **Python 3.10 or later**. If you run this notebook locally, ensure your Python version meets this requirement." ] }, { @@ -45,7 +44,7 @@ }, "outputs": [], "source": [ - "pip install --quiet --upgrade pymongo gpt4all sentence_transformers" + "pip install --quiet --upgrade pymongo gpt4all sentence_transformers atlas-local-lib-py" ] }, { @@ -54,8 +53,11 @@ "metadata": {}, "outputs": [], "source": [ - "MONGODB_URI = (\"\")\n", - "# Use \"mongodb://localhost:/?directConnection=true\" for local Atlas deployments" + "from atlas_local import LocalDeployment\n", + "\n", + "# Create or reuse a local Atlas deployment\n", + "deployment = LocalDeployment.get_or_create(name=\"local-atlas-deployment\")\n", + "MONGODB_URI = deployment.connection_string()" ] }, { diff --git a/use-cases/rag-with-voyage.ipynb b/use-cases/rag-with-voyage.ipynb index 13cee7d..7da0c26 100644 --- a/use-cases/rag-with-voyage.ipynb +++ b/use-cases/rag-with-voyage.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/atlas/atlas-vector-search/rag/#get-started) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/vector-search/tutorials/rag/#get-started) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to implement RAG with MongoDB Vector Search by using the ``voyage-4-large`` embedding model from Voyage AI and an open-source generative model from Hugging Face.\n", "\n", diff --git a/use-cases/rag.ipynb b/use-cases/rag.ipynb index 58413dd..f99da9a 100644 --- a/use-cases/rag.ipynb +++ b/use-cases/rag.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook is a companion to the [Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/atlas/atlas-vector-search/rag/#get-started) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", + "This notebook is a companion to the [Retrieval-Augmented Generation (RAG)](https://www.mongodb.com/docs/vector-search/tutorials/rag/#get-started) tutorial. Refer to the page for set-up instructions and detailed explanations.\n", "\n", "This notebook takes you through how to implement RAG with MongoDB Vector Search by using open-source models from Hugging Face.\n", "\n", @@ -103,7 +103,7 @@ "source": [ "from pymongo import MongoClient\n", "\n", - "# Connect to your MongoDB cluster\n", + "# Connect to your MongoDB deployment\n", "client = MongoClient(\"\")\n", "collection = client[\"rag_db\"][\"test\"]\n", "\n",