notebooks/rag-from-scratch/12-03-multi-vector-hypothetical-questions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "580c951b-6b28-47a8-9708-2efc03238f25",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import re\n",
    "\n",
    "from dotenv import find_dotenv, load_dotenv\n",
    "from IPython.display import display\n",
    "from rich import print as rprint\n",
    "from rich.markdown import Markdown\n",
    "from rich.pretty import Pretty\n",
    "from rich.text import Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db0e587b-bdfa-4552-8220-64144917eb2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv(find_dotenv('.env'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13bcb6ad-e1b8-4729-8bf3-1d871f991747",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"LANGCHAIN_PROJECT\"] = \"RAG From Scratch: Part 12-3 (Indexing - Multi-Vector (Hypothetical Questions))\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e806a94a-c35e-4acb-a0b3-c67091dfcce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path('../../data')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63aef84c-efb9-4612-956f-e0346bcfac2b",
   "metadata": {},
   "source": [
    "# Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "676051da-a5ae-4348-abc7-8364e400396c",
   "metadata": {},
   "source": [
    "![](images/indexing-01.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ed51e36-ecac-4598-9fe6-5ba15abcbf39",
   "metadata": {},
   "source": [
    "# Part 12-3: Multi-Vector (Hypothetical Questions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13a04be8-4b40-4532-8183-67141487e2a5",
   "metadata": {},
   "source": [
    "![](images/12-03-multi-vector-hypothetical-questions.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6f21622-04a0-45ae-aa28-71ac8cba6fbf",
   "metadata": {},
   "source": [
    "## Configure components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "451cf41e-0256-491f-b4f2-4ca0e8b82b61",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1e39e7ab-64a8-43c7-a1d9-5d9267eb4bd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='Hello! How can I assist you today?', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 10, 'prompt_tokens': 8, 'total_tokens': 18, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_b376dfbbd5', 'id': 'chatcmpl-BKP8jOjsliSWzbirJjgz4RD00oPcX', 'finish_reason': 'stop', 'logprobs': None}, id='run-fda0a3a8-b937-4dd6-821a-48913765747d-0', usage_metadata={'input_tokens': 8, 'output_tokens': 10, 'total_tokens': 18, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm = ChatOpenAI(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    temperature=1\n",
    ")\n",
    "llm.invoke(\"Hello\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9d5353b6-5ed8-4b66-bee5-a01fbba988e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1536"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
    "len(embeddings.embed_query(\"Hello\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac755af1-1f44-4352-b360-cbb856aeb95b",
   "metadata": {},
   "source": [
    "## Load documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a129af0c-9980-4f69-921b-f292892f0d53",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
     ]
    }
   ],
   "source": [
    "import bs4\n",
    "from langchain_community.document_loaders import WebBaseLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "58745065-99f8-41e9-a045-26cbfa3f32e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles = [\n",
    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
    "    \"https://lilianweng.github.io/posts/2024-02-05-human-data-quality/\",\n",
    "]\n",
    "\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=articles,\n",
    "    bs_kwargs=dict(\n",
    "        parse_only=bs4.SoupStrainer(\n",
    "            class_=(\"post-content\", \"post-title\", \"post-header\")\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "docs = loader.load()\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b4280f16-6d09-4c71-934f-e5fb029c37ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "      LLM Powered Autonomous Agents\n",
      "    \n",
      "Date: June 23, 2023  |  Estimated Reading Time: 31 min  |  Author: Lilian Weng\n",
      "\n",
      "\n",
      "Building agents with LLM (large language model) as its core controller is a cool concept. Several proof-of-concepts demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of LLM extends beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem solver.\n",
      "Agent System Overview#\n",
      "In a LLM-powered autonomous agent system, LLM functions as the agent’s brain, complemented by several key components:\n",
      "\n",
      "Planning\n",
      "\n",
      "Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable subgoals, enabling efficient handling of complex tasks.\n",
      "Reflection and refinement: The agent can do self-criticism and self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the quality of final results.\n",
      "\n",
      "\n",
      "Memory\n",
      "\n",
      "Short-term memory: I \n"
     ]
    }
   ],
   "source": [
    "print(docs[0].page_content[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e5042bf-4d51-47ac-b690-bc03ab6aee61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "      Thinking about High-Quality Human Data\n",
      "    \n",
      "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n",
      "\n",
      "\n",
      "[Special thank you to Ian Kivlichan for many useful pointers (E.g. the 100+ year old Nature paper “Vox populi”) and nice feedback. 🙏 ]\n",
      "High-quality data is the fuel for modern data deep learning model training. Most of the task-specific labeled data comes from human annotation, such as classification task or RLHF labeling (which can be constructed as classification format) for LLM alignment training. Lots of ML techniques in the post can help with data quality, but fundamentally human data collection involves attention to details and careful execution. The community knows the value of high quality data, but somehow we have this subtle impression that “Everyone wants to do the model work, not the data work” (Sambasivan et al. 2021).\n",
      "\n",
      "Fig. 1. Two directions to approach high data quality.\n",
      "Human Raters ↔ Data Quality#\n",
      "Collecting human data involve a set of\n"
     ]
    }
   ],
   "source": [
    "print(docs[1].page_content[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8cff0076-395a-46e7-b1d2-94cecb992225",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[43130, 29018]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(doc.page_content) for doc in docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc8a5249-d24c-4fb8-8ad2-39d396e42a39",
   "metadata": {},
   "source": [
    "## Generate hypothetical questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dafaecb3-e18f-4fe3-83dc-075868d06a65",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.messages import HumanMessage\n",
    "from langchain_core.runnables import chain\n",
    "from pydantic import BaseModel, Field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "39fe49e3-7c96-44df-8128-f09a4a1d91ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generate a list of exactly {hypothetical_questions_count} hypothetical questions that the below document could be used to answer:\n",
      "\n",
      "{doc}\n"
     ]
    }
   ],
   "source": [
    "hypothetical_questions_prompt_template = \"Generate a list of exactly {hypothetical_questions_count} hypothetical questions that the below document could be used to answer:\\n\\n{doc}\"\n",
    "print(hypothetical_questions_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c9b90e1a-261b-48ec-a3d0-e23431a403ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HypotheticalQuestions(BaseModel):\n",
    "    \"\"\"Generate hypothetical questions.\"\"\"\n",
    "\n",
    "    questions: list[str] = Field(..., description=\"List of questions\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "451b5e42-a5c4-4a4f-9f04-4b475b0b996f",
   "metadata": {},
   "outputs": [],
   "source": [
    "@chain\n",
    "def generate_hypothetical_questions(doc, hypothetical_questions_count=3):\n",
    "    hypothetical_questions_prompt = hypothetical_questions_prompt_template.format(\n",
    "        hypothetical_questions_count=hypothetical_questions_count,\n",
    "        doc=doc.page_content\n",
    "    )\n",
    "    structured_llm = llm.with_structured_output(HypotheticalQuestions)\n",
    "    response = structured_llm.invoke([\n",
    "        HumanMessage(content=hypothetical_questions_prompt)\n",
    "    ])\n",
    "    return response.questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d0287c7c-a794-4484-9011-e638af2c3612",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What capabilities would an LLM-powered autonomous agent need to effectively perform complex scientific </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">discovery tasks?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How could the integration of self-reflection improve the decision-making process for a large language </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">model in autonomous agents?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'In what ways can the challenges posed by finite context length impact the performance of LLMs in </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">real-world applications?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What changes might occur in data quality if the majority voting approach is replaced with probabilistic </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">graph modeling in human annotation tasks?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How would the performance of machine learning models be affected if diverse perspectives from annotators </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">were systematically incorporated during data annotation?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What could be the implications for bias in model training if non-experts were relied upon exclusively to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">annotate complex datasets without expert validation?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'What capabilities would an LLM-powered autonomous agent need to effectively perform complex scientific \u001b[0m\n",
       "\u001b[32mdiscovery tasks?'\u001b[0m,\n",
       "        \u001b[32m'How could the integration of self-reflection improve the decision-making process for a large language \u001b[0m\n",
       "\u001b[32mmodel in autonomous agents?'\u001b[0m,\n",
       "        \u001b[32m'In what ways can the challenges posed by finite context length impact the performance of LLMs in \u001b[0m\n",
       "\u001b[32mreal-world applications?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'What changes might occur in data quality if the majority voting approach is replaced with probabilistic \u001b[0m\n",
       "\u001b[32mgraph modeling in human annotation tasks?'\u001b[0m,\n",
       "        \u001b[32m'How would the performance of machine learning models be affected if diverse perspectives from annotators \u001b[0m\n",
       "\u001b[32mwere systematically incorporated during data annotation?'\u001b[0m,\n",
       "        \u001b[32m'What could be the implications for bias in model training if non-experts were relied upon exclusively to \u001b[0m\n",
       "\u001b[32mannotate complex datasets without expert validation?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rprint(generate_hypothetical_questions.batch(docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0444f1f9-2da1-48db-baaa-621a563404bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What would happen if an LLM-powered autonomous agent faced conflicting information when trying to make a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">decision?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How would the performance of an LLM-based agent change if it had infinite long-term memory compared to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">finite memory?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What challenges might arise if an LLM-powered agent were tasked with emotional support rather than </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">technical problem-solving?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How would the integration of additional external tools impact the effectiveness of an LLM-powered agent in</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">a complex task?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'If a team of LLM-powered agents were deployed to collaborate on a multifaceted project, what potential </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">benefits and drawbacks could arise from their interactions?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How would the research in this document affect the future of high-stakes AI applications that rely on </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">human annotations?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What would happen if the Community prioritized model development over data quality improvements as </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">suggested in the document?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What implications could arise if the discrepancies in annotations were acknowledged as valid rather than </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">errors, according to the findings mentioned?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How might the techniques for improving data quality impact the reliability of machine learning models </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">trained on human-generated data?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What changes could occur in data annotation practices if the contradictory paradigms of prescriptive and </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">descriptive annotation were widely adopted?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'What would happen if an LLM-powered autonomous agent faced conflicting information when trying to make a \u001b[0m\n",
       "\u001b[32mdecision?'\u001b[0m,\n",
       "        \u001b[32m'How would the performance of an LLM-based agent change if it had infinite long-term memory compared to \u001b[0m\n",
       "\u001b[32mfinite memory?'\u001b[0m,\n",
       "        \u001b[32m'What challenges might arise if an LLM-powered agent were tasked with emotional support rather than \u001b[0m\n",
       "\u001b[32mtechnical problem-solving?'\u001b[0m,\n",
       "        \u001b[32m'How would the integration of additional external tools impact the effectiveness of an LLM-powered agent in\u001b[0m\n",
       "\u001b[32ma complex task?'\u001b[0m,\n",
       "        \u001b[32m'If a team of LLM-powered agents were deployed to collaborate on a multifaceted project, what potential \u001b[0m\n",
       "\u001b[32mbenefits and drawbacks could arise from their interactions?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'How would the research in this document affect the future of high-stakes AI applications that rely on \u001b[0m\n",
       "\u001b[32mhuman annotations?'\u001b[0m,\n",
       "        \u001b[32m'What would happen if the Community prioritized model development over data quality improvements as \u001b[0m\n",
       "\u001b[32msuggested in the document?'\u001b[0m,\n",
       "        \u001b[32m'What implications could arise if the discrepancies in annotations were acknowledged as valid rather than \u001b[0m\n",
       "\u001b[32merrors, according to the findings mentioned?'\u001b[0m,\n",
       "        \u001b[32m'How might the techniques for improving data quality impact the reliability of machine learning models \u001b[0m\n",
       "\u001b[32mtrained on human-generated data?'\u001b[0m,\n",
       "        \u001b[32m'What changes could occur in data annotation practices if the contradictory paradigms of prescriptive and \u001b[0m\n",
       "\u001b[32mdescriptive annotation were widely adopted?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rprint(generate_hypothetical_questions.batch(docs, hypothetical_questions_count=5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "98026683-7cfa-4de4-bf8c-94f8bd1573f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How might LLM-powered autonomous agents address the challenge of finite context length in real-time </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">applications?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What implications could the integration of tool use capabilities in LLM agents have on their performance </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">in complex task execution?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from past mistakes </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">during autonomous operations?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How might the understanding of human data collection methods change if a new, more efficient technique </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">were discovered?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'What would be the implications for machine learning models if the significance of human subjectivity in </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">annotations is further established by future research?'</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'How could the quality control measures for human annotations evolve if AI tools were developed to assist </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">in real-time data validation during the annotation process?'</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'How might LLM-powered autonomous agents address the challenge of finite context length in real-time \u001b[0m\n",
       "\u001b[32mapplications?'\u001b[0m,\n",
       "        \u001b[32m'What implications could the integration of tool use capabilities in LLM agents have on their performance \u001b[0m\n",
       "\u001b[32min complex task execution?'\u001b[0m,\n",
       "        \u001b[32m'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from past mistakes \u001b[0m\n",
       "\u001b[32mduring autonomous operations?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[1m[\u001b[0m\n",
       "        \u001b[32m'How might the understanding of human data collection methods change if a new, more efficient technique \u001b[0m\n",
       "\u001b[32mwere discovered?'\u001b[0m,\n",
       "        \u001b[32m'What would be the implications for machine learning models if the significance of human subjectivity in \u001b[0m\n",
       "\u001b[32mannotations is further established by future research?'\u001b[0m,\n",
       "        \u001b[32m'How could the quality control measures for human annotations evolve if AI tools were developed to assist \u001b[0m\n",
       "\u001b[32min real-time data validation during the annotation process?'\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hypothetical_questions = generate_hypothetical_questions.batch(docs, {\"max_concurrency\": len(docs)})\n",
    "rprint(hypothetical_questions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e21fd3d-c42f-4d56-8793-a77a8b0ee8ac",
   "metadata": {},
   "source": [
    "## Store documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f5d0342f-c075-42da-8133-3f651cb10272",
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import attrgetter\n",
    "import uuid\n",
    "\n",
    "from langchain_core.documents import Document\n",
    "from langchain_core.stores import InMemoryByteStore\n",
    "from langchain_core.vectorstores import InMemoryVectorStore\n",
    "from langchain.retrievers.multi_vector import MultiVectorRetriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f69d0587-eb99-47c1-9977-28e90f653bfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 6\n",
      "2 2\n"
     ]
    }
   ],
   "source": [
    "vectorstore = InMemoryVectorStore(embeddings)\n",
    "store = InMemoryByteStore()\n",
    "id_key = \"doc_id\"\n",
    "\n",
    "retriever = MultiVectorRetriever(\n",
    "    vectorstore=vectorstore,\n",
    "    byte_store=store,\n",
    "    id_key=id_key,\n",
    ")\n",
    "\n",
    "doc_ids = [str(uuid.uuid4()) for _ in docs]\n",
    "\n",
    "question_docs = []\n",
    "\n",
    "for i, questions in enumerate(hypothetical_questions):\n",
    "    question_docs.extend(\n",
    "        [Document(page_content=question, metadata={id_key: doc_ids[i]}) for question in questions]\n",
    "    )\n",
    "\n",
    "retriever.vectorstore.add_documents(question_docs)\n",
    "print(len(question_docs), len(retriever.vectorstore.store))\n",
    "\n",
    "retriever.docstore.mset(list(zip(doc_ids, docs)))\n",
    "print(len(doc_ids), len(list(retriever.docstore.store.yield_keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6c8bfc48-bcdf-425c-b5a2-e06061d380fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_query = \"What is task decomposition for LLM agents?\"\n",
    "human_data_query = \"What are main steps for collecting human data?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "07826f12-eca2-42e6-a5bb-ccaa87914a3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'df048473-f854-4e20-9bbc-d1c8f8677ea9'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What implications could the integration of tool use capabilities in LLM agents have on their </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">perform'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'268d0df0-5cc8-48d7-8aa1-2090f01ac10a'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How might LLM-powered autonomous agents address the challenge of finite context length in </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">real-time '</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'7c29ec00-e7de-4609-8b69-10f3949718e0'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">past mista'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'0c33b88f-0594-48d2-8912-19ea2c2ceb88'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What would be the implications for machine learning models if the significance of human </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subjectivity'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">58</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'df048473-f854-4e20-9bbc-d1c8f8677ea9'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'What implications could the integration of tool use capabilities in LLM agents have on their \u001b[0m\n",
       "\u001b[32mperform'\u001b[0m+\u001b[1;36m31\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'268d0df0-5cc8-48d7-8aa1-2090f01ac10a'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'How might LLM-powered autonomous agents address the challenge of finite context length in \u001b[0m\n",
       "\u001b[32mreal-time '\u001b[0m+\u001b[1;36m13\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'7c29ec00-e7de-4609-8b69-10f3949718e0'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from \u001b[0m\n",
       "\u001b[32mpast mista'\u001b[0m+\u001b[1;36m33\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'0c33b88f-0594-48d2-8912-19ea2c2ceb88'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'What would be the implications for machine learning models if the significance of human \u001b[0m\n",
       "\u001b[32msubjectivity'\u001b[0m+\u001b[1;36m58\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Time: 31 min  |'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">43030</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  Estimated </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Reading Tim'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28918</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading \u001b[0m\n",
       "\u001b[32mTime: 31 min  |'\u001b[0m+\u001b[1;36m43030\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  Estimated \u001b[0m\n",
       "\u001b[32mReading Tim'\u001b[0m+\u001b[1;36m28918\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "response = retriever.vectorstore.similarity_search(agent_query)\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))\n",
    "\n",
    "response = retriever.invoke(agent_query)\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d3a83345-4997-4e9b-b255-17696e7947e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'a62993bc-8939-4c59-b61c-3487f259d5da'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How might the understanding of human data collection methods change if a new, more efficient </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">techniq'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'d6eddc3b-5c06-4adf-80fa-3754c04c2ea7'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How could the quality control measures for human annotations evolve if AI tools were </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">developed to as'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">64</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'0c33b88f-0594-48d2-8912-19ea2c2ceb88'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What would be the implications for machine learning models if the significance of human </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subjectivity'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">58</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'df048473-f854-4e20-9bbc-d1c8f8677ea9'</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What implications could the integration of tool use capabilities in LLM agents have on their </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">perform'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'a62993bc-8939-4c59-b61c-3487f259d5da'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'How might the understanding of human data collection methods change if a new, more efficient \u001b[0m\n",
       "\u001b[32mtechniq'\u001b[0m+\u001b[1;36m19\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'd6eddc3b-5c06-4adf-80fa-3754c04c2ea7'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'How could the quality control measures for human annotations evolve if AI tools were \u001b[0m\n",
       "\u001b[32mdeveloped to as'\u001b[0m+\u001b[1;36m64\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'0c33b88f-0594-48d2-8912-19ea2c2ceb88'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'What would be the implications for machine learning models if the significance of human \u001b[0m\n",
       "\u001b[32msubjectivity'\u001b[0m+\u001b[1;36m58\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mid\u001b[0m=\u001b[32m'df048473-f854-4e20-9bbc-d1c8f8677ea9'\u001b[0m,\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'What implications could the integration of tool use capabilities in LLM agents have on their \u001b[0m\n",
       "\u001b[32mperform'\u001b[0m+\u001b[1;36m31\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  Estimated </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Reading Tim'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28918</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Time: 31 min  |'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">43030</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  Estimated \u001b[0m\n",
       "\u001b[32mReading Tim'\u001b[0m+\u001b[1;36m28918\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading \u001b[0m\n",
       "\u001b[32mTime: 31 min  |'\u001b[0m+\u001b[1;36m43030\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "response = retriever.vectorstore.similarity_search(human_data_query)\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))\n",
    "\n",
    "response = retriever.invoke(human_data_query)\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d2ceae-3823-46c3-9246-465b7655af7d",
   "metadata": {},
   "source": [
    "## RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b4a5591c-8255-4d49-9a05-54700ac46b6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import TypedDict\n",
    "\n",
    "from langchain_core.documents import Document\n",
    "from langgraph.graph import END, START, StateGraph\n",
    "from langchain_core.messages import HumanMessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "13eca5a4-4772-4d17-a300-550ff64e17e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answer the following question based on this context:\n",
      "\n",
      "{context}\n",
      "\n",
      "Question: {question}\n"
     ]
    }
   ],
   "source": [
    "rag_prompt_template = \"\"\"Answer the following question based on this context:\n",
    "\n",
    "{context}\n",
    "\n",
    "Question: {question}\"\"\"\n",
    "print(rag_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7c01f711-b56e-4398-8b88-20a456d7b7fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "07d68e91-f6b2-442a-b516-08bedaf0dd0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class State(TypedDict):\n",
    "    question: str\n",
    "    search_results: list[Document]\n",
    "    context: list[Document]\n",
    "    answer: str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f9fc6edf-1a64-4a95-b4cd-435619b0ba18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAG0AAAFNCAIAAACFQXaDAAAAAXNSR0IArs4c6QAAHw1JREFUeJztnXdYFGf+wN/tfYFdegcp0lUsRL0oUbHExAbRQz29JJfEckZjjZrTFONdLmp+XkxiNKfYY0NjOfUssSQxRkUR0AUEKStL2WV7L78/xuM42d2ZhXfZHZjPc8897s47M18+eafs274km80GCLoM2dMB9BAIj3AgPMKB8AgHwiMcCI9woEI5iqRGr1WatWqLxWQz6KxQjulWaEwylUJi8ylsHiUokkmmkLp4QFJX3h9Fd1RVD9TVJZroZI7NBthcil8Q3ajHgUc6iyxvNmqVFoPW8rRaH5HAjk3j9B3Mo1I7eYF20mPJL4qff5BGJ7Nj07gxqRwKtav/PT1LzUNN1QNNfYWu7yDeoBxBJ47gskfpU8P5PZKQGNbQV4UMFqUTp/Rmbp6V3r8qz5kdFJPKdWlH1zyK7qjuXGqd+GYIX0BzPUh8YDRYfzzS5BdId6liuuDxSZmm/I4qZ3ZwZyPEEzfPSmkMcuYoP4zlsXosutLaWGsYN6dXSET4+XSLTm0ZNSMIS2FMj6faR9pakbZXSQQADJ3oT6OT71+TYymM7lGtMN+/Lp/0ThiM2HDGi1MDpA1GcaUWtSS6x59OtiRm8iAFhj/ShvtcL2xBLYbisVlsaG00JgzovR4Dwhh+QfTyuyrnxVA8lvykGD7FH2pg+GPYq8KKoi54NBmtotuq8Dg27MBwBteXpm61NNXrnZRx5rG6RBOTynFDYM44fPjw+vXrO7HjypUrT5065YaIAAAgJo1T/UDjpIAzjw1Vuvj+rv086joPHz7s5h2xEJfBbRYbnBRw9h7+/ed12TMCAsOZ7oisqKho27ZtlZWVFoslISFhwYIFAwYMeOutt+7evYsU2L9/f2Ji4rlz5/bu3VtbW0un09PT05cuXRoeHo7UPhKJFB0dvW/fvo0bNy5ZsgTZi8vl/vjjj9CjNRutO9ZWz/usj6MCzuqjRmXm8OA0UD6HTqdbvHhxbGzsrl27CgoK4uPjFy1apFQqN2/e3Ldv35ycnIsXL8bFxZWWlq5du3bYsGF79+7dunWrTqdbvnw5cgQajVZZWfno0aOtW7empaWdPXsWALB8+fKTJ0+6I2AqnUyhkAw6i8MCTnbWqixsnltadCQSiUajmTBhQkxMDABg2bJlY8aModPpTCaTSqXS6XRfX18AQFRU1N69e+Pj46lUKgAgPz//vffek8lkAoEAAFBfX//dd9/5+PgAAAwGAwCAzWYjH90Bh0/RKC2OmrgcerRarSwOmUR2S8NiZGRkVFTU2rVrc3Nzs7KyEhMTMzMzOxbjcrlisfjLL7+sq6vT6/UmkwkAoFQqEY9RUVHus9YRJoditTi8Bzq8rslkss0GdGqHNbkrUCiUnTt3jh49urCwcNasWa+88sqZM2c6Frtw4cKqVatSU1O3bt164MCBNWvWtN/K5XbrM7C1ycjhO6x2zu6PbD5VqzS7Jyrg5+e3ePHikydPHj58ePDgwevWrev4wC0sLBw4cOC8efOio6P9/f31emdvcG7FarEZdFYW1+FdzpnH0Bim1j31USwWtz1VY2NjV69eTSaTHz9+jHzT9gphNBqRGyXCuXPn2m/tiPvGKqkV5uhkZ6/Szjz6hzEq76ndEBWQSCQrVqzYt2/fkydPampqdu7cSSaT09LSAAA8Hk8kEolEIrlcnpqaevPmzZKSkoaGho0bN/r7+wMAysrKOlZMBoPBYDDu3r0rEonMZvjXUNUDDV/g7JlMcfLjgeNDvXGipX821jZh7ISGhoaGhh47dmz37t0nT57UarWrVq1KT08HAPj4+Jw5c+b48eP9+/fPycmpqKj49ttvz549m5mZuWTJkuLi4u+//z46Orq2tlatVk+aNKntmFartbCw8Pz587m5uQwGA27Av5yWpg7zcdabYnPK+T0NTXU652V6PEa9ufDLOudlUNp7Egfyfjkjg/vfFnfcPCuLRus+RPm5EpXEuXtJLq7UhcWx7BZYuHBhSUmJ3U0Wi4VCsf+A+/DDD0eMGOH81J1m5MiRjuJBXrnsbr148SLytv8cGqW5okj9+kcxzk+K3s/VWKsvvqEYk2+/u0er1SLxdcRsNtuNDADAYrEcbeo6KpX9tkLk+ePovDye/bbqn0+3BIQy4tFasjH1Fz64oZBKDCNzA1FL9jCKr8tbm0wjpgWglsTUX5g23MdmBbfOSWHEhhsq76kr76uxSHRtHMCdS60Ws23w2M4Mf8Ed5XdVVSWacX/A2tXswvCqzFF+ZpP1/B5JZ2PDDb9dkFU9cEFiZ8ZJld9VXT3WNGScMP13vhiK44yKItXPp6Rpw/gDRrl22XVm3J7JYPn5tKzqgTp9uG9MGkcQRHf1CN6GqtVUXaJ5UqqhsyhDXxF2YhRY58eRquXm4hvy6gcaqxXEpHGoVBKHT+ULqBYcDCMFFApJJTdplRad2tJQpdNrrTGpnOQhvIDOdqJ0aTwugrzZKHmiV7WaNUozmUJSySA3E9y/fz8lJQXu+ybXl2o129h8CseXGhTJDAjr6u9xCB7dzejRo48ePdq+Ac0LIeYrwIHwCAcceExMTPR0COjgwKNIJPJ0COjgwGN3dq52Ghx4VCgUng4BHRx4DAkJ8XQI6ODAY0NDg6dDQAcHHlNSUjwdAjo48FhaWurpENDBgUdcgAOPyDAKLwcHHlta0KeveBwceCTqIxyI+tiLwIHHPn0czhLwHnDgsW18qTeDA4+4AAcek5KSPB0COjjw6NYJb7DAgUdcgAOPRHsPHIj2nl4EDjwS/a5wIPpdexE48Ej0X8OB6L+GA9HeAweivacXgQOPQUGYVmD0LDjw2NjY6OkQ0MGBx+TkZE+HgA4OPJaVlXk6BHRw4JGoj3Ag6iMckIXhvBzvnYc0YcIEZA5XS0uLQCAgk8k2m83f33/Xrl2eDs0O7lrcoOuQSKSnT58i/5ZIJMgycIsXL/Z0XPbx3uu6f//+z10rMTExo0aN8lxEzvBej7Nnzw4O/u9MchaLNXPmTI9G5Azv9ZiYmNivX7+2j3369MnJyfFoRM7wXo8AgFmzZiE/rtlsdn5+vqfDcYZXe0xKSsrIyLDZbDExMd5cGTvzvDYarC1ig17bTbP+x704p77cNDlnSlWJs2WnIUJnkIQhDCdLPdrFtffHf++XPC7WBEezyO5Z79UboLPIdSJNeBxrdH4QjYH1esXq0Wq1FX4l7pPO75PB71qc+KCxVvfr2eZpC8OYHEwVE6vHk1+L4zN9IxK7e3l7D6KWm87vFs9dF42lMKZ6W1OmYfKovUoiklYhfgC/+AakPD4AgJanRgazp6WGwwLHh9r4xFk6gDYwedRpLD4BuF/sqBP4+NONBkxvJpg8mo02i8lLm4XcitUC9NhWrPbq93AcQXiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeIQD4REO3u5x0pRRe/bu9HQU6HjeY3X14xn5Ex1tnf/Okqys4d0bUWfw/LiU8nJn06vHjnWo2KtwV32cPHX00WMHVr6/KGfcC2q1GgBw6fL5d+bNHv/y8Km5OV9u24Tk2NpdsP2vn61vbJRkjxp49NiBwhOHp0wb89NPV6dMG/P1N188d12XVzxasXLhpCmjXn7lxQ/+skwiaQAA7Pxu28RXRyCpDREOHipwflJ34C6PVCr11OnjsTFxWzZtZzKZN278+MmGNZmZQ3Z8e3DF8nXXrl/atGUDAGDG9DlTp84IDAw6cfziKxOn0Wg0vV53vPDQyhXrJ03Ka3/AxkbJe0vfJpHJWzZt3/T5N0qVYunyeUaj8aXssRqN5s7dW20lr127lDVkOJfLdXRSd+AujyQSiclgvv3WopSUdCqVeuDQ7oyMAX96c2F4WETWkGF/evPPFy/+q6mpkclkMugMEonk4+PLYDBIJJJer8+dlp81ZFhoSFj7A/5w6iiJRFq7ZkNsbFzfxOTVqz5uaBBfvXYpNjYuMjL6xo0rSLHGRskjUdmoUeMAAHZPKpW6ZVUlNz5nUlLSkX9Yrdby8ocDM7PaNvXLyAQAVFVV2N0xOTmt45cPH5b0TUzhcZ/lJQoKCg4JCausFAEAskfm/PTzVavVCgC4dv0Sh8PJGjLc0Umrn7hlVpMbnzMczrPcYHq93mKx7C7YvmfvjvYFpDL7VaNtx/ZoNOqKSlHOuBfavjGZTMgRXsrOKdjzbUnJ/fT0/levXRo+LJvBYCAJrzqetLXVLWnbuuN5jWQRnjplxssTJrf/3tfPhdwkHA43La3f0iX/k+KVxWIDACIjo2Nj467fuBIaGl5aWjznD285OalA4JZV57rDI5lMjo/v29jYEBn5rE/dZDI1NTfyeS4MzUhKSj1/4XRoaHhbwoq6uhqh8JmU7JE55y+cDg+P9PMTDOg/yMlJ3ZRdt5vew2dM/8O165cPHNxdV1dTUSn6dOMHi959Q6PRAAC4XJ5U2lJcXIS8xzjilYnTdDrt3z5bX1Epqq+v3bN35x/feO3Ro2dLgGRn59TX1546fWzkyDFtmfXsnlSr1brjD+wmjy/+7qXV73986fK519+cvnzFApPZtGXTdg6HAwAY9dK40NDwpcvn/eucs1TqwcEhmzdtl8mki9594535s2/99vMnH29ueyKFhYYnxPd9/Lhi9EvjnJ+UzWa74w/ENL7nxyPNXD964iAczMuHS1Ot/t7llmnvok888fzv654B4REOhEc4EB7hQHiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeIQDJo8sHoVM7bETCp1iwzjhBZNHvh+1qUbX5ZjwR1O9nsnBpAhTofBEllYJOas1LlA0GaOTMbX7YvLI86X1Hcy78j0O8ghC5NezzXwhNTwek0cX5l9X3lPfOi9LHOQjDGUy2T12uqHFZG0W6xuqtMIQ+uCxWHs0XZvHLm0w3L+mkDeblFIThuJwMBgMdDqdROqmB50ghMFkkxMGcKKTXehZ9N71pNog8tr3IgiPcMCBRyJvChyIvClwINZhhwOxDjsc+vbt6+kQ0MGBx0ePHnk6BHRw4JG4P8KBuD/2InDgMT4+3tMhoIMDjxUV9qeHeBU48IgLcOCRyWR6OgR0cODRfZMrIYIDj3w+DlZAxYFHpVLp6RDQwYFHXIADj2FhYRhKeRgceBSLxZ4OAR0ceMQFOPBItPfAgWjv6UXgwCPR7woHot+1F4EDj8TzGg7E8xoOXj5iDwEHHuVyTJlLPAsOPOICHHhMTEz0dAjo4MCjSCTydAjo4MBjUlKSp0NABwceHz50tvCrl4ADj8S4PTgQ4/bggIv7o/fOQ8rLy2MymWQyuby8PDw8HPk3k8ncvn27p0Ozg+fXD3fE48ePyeRnl0t1dTUAgEKhEHntXWbw4MHPfRMRETFjxgwPhYOC93qcO3du+xEpZDJ56tSp3TZb01W812NWVlZCQkLb7Ts8PHz69OmeDsoh3usRqZI+Pj7InTEvL69t4VsvxKs9ZmVlJSYm2my20NBQb66MWJ/XZpNVp+6mRPbPMSP3jzWPm/KmzNIorAB4IAYanYxlqQ+U98eHt5TF1xUyiZHF9d5ryq0w2BSjzpLyAn/gGGdrLDjzeOuCrOWpqd8IAU9Ac0+Q+EAtN1XdV6lajePmBDsq49Djr+dkSqk5a2KgOyPEE2U35bIG/fi59lXav/Jbm4wtYgMhsT3JWb50FuVJmcbuVvseW8QGm81L33g9CJ1JaayxP+jfvke1whIQgYPZFt2MMJSh19p/Z7D/3mMyWE04mGzR3VjNNkfrk3n1eziOIDzCgfAIB8IjHAiPcCA8woHwCAfCIxwIj3AgPMKB8AgHwiMcerjH9R+uPHf+VDecqId7LC/vprGT9vsVbp2XGfUgY6QL+YBbWpo3bdlQVPQbl8vLnZav0aivXb9csOsoAMBsNu/b/93lKxcaGxsCAoLycmdOejUXAFBTUz339bzNm745dvzggwf3yGRy9sgxC+YvRfqp5fLWr77Zcv/+HYVCHhsb/6c3F/bvNxAAUHji8J69O5a9t/bzzZ/kjHl53juLW1tlX2//4u7dWyqVMiAgaOrk6VOnzgAAZI8aiMTG5XJPnfwRSXN/5Mi+mtpqFov9UvbYN99Y4NKiNjVl6rpHqvF/DOm4Cdo4qc83f1JZKfr4o00CP+HOf26rrX1Cpz/L8PDN9v87c7Zw8aJVKakZd+78+uW2z6lU6ssTJlOoVADAtq82LXn3/U8+2nTn7q1ly+enpfXPHjnGarWuXPVntUa9csV6ocD/5A9HVr2/6Otte2Jj42g0ml6vO154aOWK9Ugu4c8+/6iu9skHaz4VCIQPSu5t2rwhMCh4+LCRhw+dfW3GhD8vXI6kZ0fS3Of/fu7atZ/W19du3rJBoZSvef9jKH8+nOtaJpPeuvXzrJlvDBqY1adP/NrVG5SKZ5Ne1Gr1yR+OTH9t9tixE8PDIia9mjs2Z+KBg7vb9h3x4mgkc3vmgMGhIWEiURkA4PadX8srHi1bunZA/0FRUTELFywLCgo5XngIAEAikfR6fe60/Kwhw0JDwgAAC+Yv/eyzbRkZAyIioiaMnxTXJ+H27ZsAAD7fBwDAZrN9+D6O0tw3NTVCMQCnPorFdTabLTUlA/nI4XAyM4fU1FYDAB4/Ljebze3zy2dkZJ45e6Itf3Kf2P8uA8fl8tRqFZLFnkajIZnokUFS6Wn9kSz2CG2ZhgEALCbrwKHd9+7dVijkVqtVpVKGhUU8FyGS5n7unLfbvkEOXlVVERgY1HUDcDwqFHIAAKtdSmSkLgAAtFoNAGDJ0rfbhoohd2RZqxT5SGcw2h8K2arVakwm09jxQ9u+t1gsAoGw7SOH82zRfrPZvGLVQovFsnDBssiIaAqFsvYvSztGqNfr7aa5l8paYAiA5BFxYWi3gJZK9WzxIuQPXrP6k9iYuPa7BAYENTU7vKY4HC6dTt+x/UD7L9uGlbbn4cOSqqrK/9uyIz29P/KNQt4aEhz6XDFHae59/Vx4ljoBjkfkOnokKo2NjQMAaDSaO3d+FfoHAABiY+NpNFprqyxyxLP88nJ5K4lEansK2aVv3xSj0WixWGJink0alkgafH39OpY0GA3tq39paXGD5GliYnJbAaSCO0pzz+fBWfQLznMGSYe+f/8/S0uLa2ufbPzbX/z+cw1yudyJE6fuLth++cqFpw3ionu3l62Y/9fP1js/YOaAwfFxiZ9u/ODevTsNkqcXL5176+38kz8c6Vgyrk8CnU4/XnhIKm357fbNrf/4bNDArLr6mtZWGYPBYDAY94vvVlSKzGaz3TT3Go39fn1Xgfbes3bNhr9v+njJ0rf9hQEzZ74uFPg/evRsPYT57yzhcXnf7tgqlbYIBMKhL7z4xusLnB+NQqH87a//+Hr7F+s+XKHX64KDQ2fPfjMvd2bHkr6+fiuWr9u588sL/z6TkJC0csX65pamjz95/71l7+z67vDvZ8w99H3BL79c37f3BJLm/uCh3bt2f8PhcFNTM7Zs2s7hcKD8+dDew/V6vcls4nF5yMf3lr7D5/usX/c3KFF6Cd3xHr56zWJZq3TpkjV+foJfbl4vund744YvYB3c+4F5XX/19eYP1i0zGPShoeGrVqzPyhoO6+DeDzSPAoFw7ZoNsI6GO3p4e0+3QXiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeISD/d+FdCbJCoj5M89DppA4PvaN2a+PPD9ac69MZO+cFrHe0XxV+x4DIxjeuoCBJzHqLcEx9scNOKyPYXHMa8ckbg4MTxRdlpJIIMJBmntn84ZLf1FU3FNnjBD6BdEp1N77RJI26B/fV9JopBenBjgqgzKPvbpUc++qXFKtp1A9dp1brBYymeKp07M4FBqTnDqUlzrU2fKyWNeTMug8s64CAGDy5MkFBQXIgh/dD51JxvKowNoezmB57Lo2WbR0JsmDAWDBq4PDETjwSKzDDgdiHXY4EPk+4EDk+4ADUR/hQNRHOBB5SeFA5CXtReDAI/GcgQPxnOlF4MBjVFSUp0NABwcea2pqPB0COjjwiAtw4NFTLeEugQOPCoXC0yGggwOPdqcVehs4CNFq9VgXG3Zw4BEX4MAjkZcUDkRe0l4EDjwS/a5wIPpdexE48Ei048KBaMftReDAI4/H83QI6ODAo0ql8nQI6ODAI/GcgQPxnIFDWFiYp0NABwcexWKxp0NABwceQ0OfXzzPC8GBx6dPn3o6BHRw4DE5ORlDKQ+DA49lZWWeDgEdrPO5up/MzEybzUYmk61WK/L/FAplzpw5Cxcu9HRodvDe+hgXF4csqYv0u5LJ5PDw8Pz8fE/HZR/v9Th79uznFkkfN26cQABnOVvoeK/HiRMnxsTEtH2MiIjIy8vzaETO8F6PAICZM2ey/7OW9tixY722Mnq7x/HjxyNVMjo6+rXXXvN0OM7wao8AgOnTpzOZzPHjx3tzZYT23mM2WqtLNXUVBmmDQae2UOlkpdQIIzwAADCbTFQqFUBaeMQvkKHXmFlcqm8QLSSaEZfOdbQEikt01WNdubboirK+QsMLZPMDOGQqicagUhkUEtlL11shAZtRbzEbLBazVd2iVbdoffzp/Ub69B3YpVb3znuU1OivFUp1Gpt/tC9HwOpKEJ5FI9fL65UWo+l3U/xjku0vh4JKZzzabOD6D611Ip1PKJ8rxLHB9uhUBmm13C+QOn5OYCcGXHbG49ldEqWSHJwgxFAWZ8jqlEalZsaycFd3dNnjvw82K5UUYSQOxmx3DrVUp5Mp8xa51ujpWg0+v6dRperJEgEAXCGLJeAd/LzOpb1c8HjnUqtcThJE9GSJCFwhm+nDvbC/CfsuWD3KGg1lv6mD4nvgPdEufuF8WZO16gHWrnOsHq8XSn1Cen5NbI9fhM/1QhnGwpg8Sp7oW5st/EA4qVrwAoNDp3MZZTcxzd7B5LHoR7k33xaPn/r73//xe3cc2S/Cp/gnTJc2Jo/VJRqufw9533YJJpeuajUrZSbUkugea0VanpBBpnh7y5Cb4Pmzqx6g5/BCX2+vqUbPEbrxzlhUfOHqTwcam6sZDHb/tJzxo+fR6UwAwJ5Dq0kkkBj/wpVrexSq5kD/qCkTl0VFpAEAFMrmIyc2VFbfYTK5Lwya6r7YAAAcIatZjL5UMHota5GYSG5bxbKk7Or+Ix8kxA1eumDf9CkfFJdePvrDRmQThUKtrrlfW1e6eP6e9SvPsdk+3x//BNl08Nh6SVPVG7O3zPvjVxqN/EHZFTeFBwCg0CgtYgNqMXSPGoWZxoCWNuk5Ll/fExs9YMKY+f7CiKSEoS/nLLh7/5xc8Sw/pNGoe3X8YgadRaczB6SPa2p5YjTq5Yqmyqrb2b/7Q3zswKDAmCkTlzEZbrxcaAyKVmVGLYbukUIlURkQWjo7YrVa658+TIgb3PZNbPQAAECDpBL56C+MQK5xAACbxQcAaHXKpuYnAIDI8GeDLEgkUkS4GwdcUBkUJpeK2gqBXtEMOivN5JYZpyaT3mq1XLi8499Xvmv/vVL1LOcqlcrosJPNYNQ+t4lB72SjIRYsJqtGbiKhtcaje+T4UM0G9IrdCWg0JoVCHZ41fUjmq+2/53KcdcXQ6SwAgF6vbvtGp3fjwGezwcLioltCv655fhSTwQIpqv89N5kcFtK3Vd4QGBCN/E/gF0YmU9lsZwuaBQgjAQBPJRXIR4vF/Lj6rjvCQzAbLWw++m0N3WNwJNOkRX9gdY6Rw2c9KLty+VpBU3ON+KnowNF123a+pdc7e18T+IVERaRdvlYgqvxV/FR05MSnVCrNTeEBAHQKQ3BUx9vL86B7jEnlyCVaSFE9T3pK9u+nfVhUfGHTl/nfFiyyWEzzXv+KyUR5/s7M+yjAP/Kf+5bu2POur2/wgIzxNretGaCVaePSuajFMLWHf7+5nhvsx/Gzn1qgB2M2WKpvif/0aQxqSUy/9tKH81XNcPIb4wtFozplKKbVJzG9YCcN5v96rtWgMTI49pN/37x94vT5f9jdZDYZqDT795cZU9elJr2IJQAsVNfc+26fnYz2AACz2Uil0OyOJMh9dVW/tDGOjtnwSDbt7ThHW9uDtZ/r8QP1L/9ShqcF2d2q12u0OvvtdFqdis2y38XO5QjaXrO7jslkUKmlDsJT0+lsu+vXcDh+DLr9pqzGCllsEmXQGEzjYVzoL/xXgcRMZvP8e0Vrrl5jVNRKp7+HtQPWhdaw8XOCpVUygwa9Ma4HUPmT+LXFLsx/cq1VcfaaqMbyJrPRLa/l3kPtvYZZqyNdGqLkmkcKlZS/LLzqZr1a1jOzdxl1poeXn0x+O9A3wP4T1RGdHCd15It6MosljMTBCkXYkdUr5fWKWe9H0pkuN/53frzZbxdkt87LguMFwijv7QLDiLxB3fxYFtePm53nMHOUc7o0/tFitl073lIj0lIZNK6QwwtgUWhuaal0B1aLVS3VqZq1WrkuNJY1Yqo/17fzzdUQxuOajNaaMm15kVrVamkR6xgsKlfINOnd0tTWdZg8mrJJZ9RZeP50Lo+SmMmNTmFjaRlzDuT5XBazTaM0a1UWi8lLp4mRySQWj8zhU2kMmD2g3jsvDl/00l5p6BAe4UB4hAPhEQ6ERzgQHuHw/6dv5ct5mp8JAAAAAElFTkSuQmCC",
      "text/plain": [
       "<langgraph.graph.state.CompiledStateGraph object at 0x7f6e9e4992d0>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def retrieve(state: State):\n",
    "    search_results = retriever.vectorstore.similarity_search(state[\"question\"])\n",
    "    retrieved_docs = retriever.invoke(state[\"question\"])\n",
    "    return {\n",
    "        \"search_results\": search_results,\n",
    "        \"context\": retrieved_docs\n",
    "    }\n",
    "\n",
    "def generate(state: State):\n",
    "    docs_content = format_docs(state[\"context\"])\n",
    "    rag_prompt = rag_prompt_template.format(\n",
    "        question=state[\"question\"],\n",
    "        context=docs_content\n",
    "    )\n",
    "    response = llm.invoke([\n",
    "        HumanMessage(content=rag_prompt)\n",
    "    ])\n",
    "    return {\"answer\": response.content}\n",
    "\n",
    "\n",
    "graph_builder = StateGraph(State)\n",
    "\n",
    "graph_builder.add_node(\"retrieve\", retrieve)\n",
    "graph_builder.add_node(\"generate\", generate)\n",
    "\n",
    "graph_builder.add_edge(START, \"retrieve\")\n",
    "graph_builder.add_edge(\"retrieve\", \"generate\")\n",
    "graph_builder.add_edge(\"generate\", END)\n",
    "\n",
    "graph = graph_builder.compile()\n",
    "graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d07cd5e8-dd99-4539-b1f6-1beeefb656ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is task decomposition for LLM agents?\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'What is task decomposition for LLM agents?'</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'search_results'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'df048473-f854-4e20-9bbc-d1c8f8677ea9'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What implications could the integration of tool use capabilities in LLM agents have on </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">their perform'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'268d0df0-5cc8-48d7-8aa1-2090f01ac10a'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How might LLM-powered autonomous agents address the challenge of finite context length in</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">real-time '</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'7c29ec00-e7de-4609-8b69-10f3949718e0'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">past mista'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'0c33b88f-0594-48d2-8912-19ea2c2ceb88'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What would be the implications for machine learning models if the significance of human </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subjectivity'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">58</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'context'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Time: 31 min  |'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">43030</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Estimated Reading Tim'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28918</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'answer'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition for LLM (large language model) agents is the process of breaking down complex </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">tas'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1758</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'question'\u001b[0m: \u001b[32m'What is task decomposition for LLM agents?'\u001b[0m,\n",
       "    \u001b[32m'search_results'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'df048473-f854-4e20-9bbc-d1c8f8677ea9'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'What implications could the integration of tool use capabilities in LLM agents have on \u001b[0m\n",
       "\u001b[32mtheir perform'\u001b[0m+\u001b[1;36m31\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'268d0df0-5cc8-48d7-8aa1-2090f01ac10a'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'How might LLM-powered autonomous agents address the challenge of finite context length in\u001b[0m\n",
       "\u001b[32mreal-time '\u001b[0m+\u001b[1;36m13\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'7c29ec00-e7de-4609-8b69-10f3949718e0'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'In what ways could self-reflection mechanisms in LLMs enhance their ability to learn from\u001b[0m\n",
       "\u001b[32mpast mista'\u001b[0m+\u001b[1;36m33\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'0c33b88f-0594-48d2-8912-19ea2c2ceb88'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'What would be the implications for machine learning models if the significance of human \u001b[0m\n",
       "\u001b[32msubjectivity'\u001b[0m+\u001b[1;36m58\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'context'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading \u001b[0m\n",
       "\u001b[32mTime: 31 min  |'\u001b[0m+\u001b[1;36m43030\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  \u001b[0m\n",
       "\u001b[32mEstimated Reading Tim'\u001b[0m+\u001b[1;36m28918\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'answer'\u001b[0m: \u001b[32m'Task decomposition for LLM \u001b[0m\u001b[32m(\u001b[0m\u001b[32mlarge language model\u001b[0m\u001b[32m)\u001b[0m\u001b[32m agents is the process of breaking down complex \u001b[0m\n",
       "\u001b[32mtas'\u001b[0m+\u001b[1;36m1758\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Task decomposition for LLM (large language model) agents is the process of breaking down complex tasks into        \n",
       "smaller, manageable subgoals or steps. This is important for LLM-powered autonomous agents as it enables them to   \n",
       "handle complicated tasks more efficiently and effectively. The key concepts involved in task decomposition include:\n",
       "\n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 1 </span><span style=\"font-weight: bold\">Subgoal Identification</span>: The agent identifies specific subgoals that must be achieved to complete the larger     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>task. This often involves prompting the model with questions like \"What are the subgoals for achieving XYZ?\" or \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>providing task-specific instructions.                                                                           \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 2 </span><span style=\"font-weight: bold\">Chain of Thought (CoT)</span>: A prompting technique that encourages the model to think step by step, making its       \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>thought process more transparent and organized. By utilizing CoT, the model can decompose difficult tasks into  \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>simpler ones and articulate the reasoning behind its decisions.                                                 \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 3 </span><span style=\"font-weight: bold\">Tree of Thoughts</span>: An extension of CoT that allows the model to explore multiple reasoning possibilities at each \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>step. It organizes the reasoning process into a tree structure where each thought can branch out to explore     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>different paths or actions.                                                                                     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 4 </span><span style=\"font-weight: bold\">LLM+P Approach</span>: This approach involves the use of an external classical planner to perform long-horizon         \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>planning. The LLM translates the problem into a format known as PDDL (Planning Domain Definition Language),     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>allows the planner to create a plan, and then translates that plan back into natural language.                  \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 5 </span><span style=\"font-weight: bold\">Human Inputs</span>: Human intervention can also assist in task decomposition by providing clarity or additional       \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>context for complex tasks, which may not be easily broken down by the model alone.                              \n",
       "\n",
       "By effectively utilizing task decomposition, LLM agents can enhance their performance on complex problems, improve \n",
       "reliability in executing tasks, and obtain better results in various applications.                                 \n",
       "</pre>\n"
      ],
      "text/plain": [
       "Task decomposition for LLM (large language model) agents is the process of breaking down complex tasks into        \n",
       "smaller, manageable subgoals or steps. This is important for LLM-powered autonomous agents as it enables them to   \n",
       "handle complicated tasks more efficiently and effectively. The key concepts involved in task decomposition include:\n",
       "\n",
       "\u001b[1;33m 1 \u001b[0m\u001b[1mSubgoal Identification\u001b[0m: The agent identifies specific subgoals that must be achieved to complete the larger     \n",
       "\u001b[1;33m   \u001b[0mtask. This often involves prompting the model with questions like \"What are the subgoals for achieving XYZ?\" or \n",
       "\u001b[1;33m   \u001b[0mproviding task-specific instructions.                                                                           \n",
       "\u001b[1;33m 2 \u001b[0m\u001b[1mChain of Thought (CoT)\u001b[0m: A prompting technique that encourages the model to think step by step, making its       \n",
       "\u001b[1;33m   \u001b[0mthought process more transparent and organized. By utilizing CoT, the model can decompose difficult tasks into  \n",
       "\u001b[1;33m   \u001b[0msimpler ones and articulate the reasoning behind its decisions.                                                 \n",
       "\u001b[1;33m 3 \u001b[0m\u001b[1mTree of Thoughts\u001b[0m: An extension of CoT that allows the model to explore multiple reasoning possibilities at each \n",
       "\u001b[1;33m   \u001b[0mstep. It organizes the reasoning process into a tree structure where each thought can branch out to explore     \n",
       "\u001b[1;33m   \u001b[0mdifferent paths or actions.                                                                                     \n",
       "\u001b[1;33m 4 \u001b[0m\u001b[1mLLM+P Approach\u001b[0m: This approach involves the use of an external classical planner to perform long-horizon         \n",
       "\u001b[1;33m   \u001b[0mplanning. The LLM translates the problem into a format known as PDDL (Planning Domain Definition Language),     \n",
       "\u001b[1;33m   \u001b[0mallows the planner to create a plan, and then translates that plan back into natural language.                  \n",
       "\u001b[1;33m 5 \u001b[0m\u001b[1mHuman Inputs\u001b[0m: Human intervention can also assist in task decomposition by providing clarity or additional       \n",
       "\u001b[1;33m   \u001b[0mcontext for complex tasks, which may not be easily broken down by the model alone.                              \n",
       "\n",
       "By effectively utilizing task decomposition, LLM agents can enhance their performance on complex problems, improve \n",
       "reliability in executing tasks, and obtain better results in various applications.                                 \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(agent_query)\n",
    "response = graph.invoke({\"question\": agent_query})\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))\n",
    "rprint(Markdown(response[\"answer\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b8c7b0b5-8c2c-4432-8b6e-70939306b57c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are main steps for collecting human data?\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'What are main steps for collecting human data?'</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'search_results'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'a62993bc-8939-4c59-b61c-3487f259d5da'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How might the understanding of human data collection methods change if a new, more </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">efficient techniq'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'d6eddc3b-5c06-4adf-80fa-3754c04c2ea7'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'How could the quality control measures for human annotations evolve if AI tools were </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">developed to as'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">64</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'0c33b88f-0594-48d2-8912-19ea2c2ceb88'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What would be the implications for machine learning models if the significance of human </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subjectivity'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">58</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'df048473-f854-4e20-9bbc-d1c8f8677ea9'</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'doc_id'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'530a8eea-8510-4afd-8582-93ad3a5cbd6d'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'What implications could the integration of tool use capabilities in LLM agents have on </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">their perform'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'context'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Estimated Reading Tim'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28918</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Time: 31 min  |'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">43030</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'answer'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'The main steps for collecting human data, as outlined in the provided context, are:\\n\\n1. **Task </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Desig'</span>+<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">900</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'question'\u001b[0m: \u001b[32m'What are main steps for collecting human data?'\u001b[0m,\n",
       "    \u001b[32m'search_results'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'a62993bc-8939-4c59-b61c-3487f259d5da'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'How might the understanding of human data collection methods change if a new, more \u001b[0m\n",
       "\u001b[32mefficient techniq'\u001b[0m+\u001b[1;36m19\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'd6eddc3b-5c06-4adf-80fa-3754c04c2ea7'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'How could the quality control measures for human annotations evolve if AI tools were \u001b[0m\n",
       "\u001b[32mdeveloped to as'\u001b[0m+\u001b[1;36m64\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'0c33b88f-0594-48d2-8912-19ea2c2ceb88'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'0d92fccc-9d76-4edb-9067-ef3fbe824ec4'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'What would be the implications for machine learning models if the significance of human \u001b[0m\n",
       "\u001b[32msubjectivity'\u001b[0m+\u001b[1;36m58\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mid\u001b[0m=\u001b[32m'df048473-f854-4e20-9bbc-d1c8f8677ea9'\u001b[0m,\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'doc_id'\u001b[0m: \u001b[32m'530a8eea-8510-4afd-8582-93ad3a5cbd6d'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'What implications could the integration of tool use capabilities in LLM agents have on \u001b[0m\n",
       "\u001b[32mtheir perform'\u001b[0m+\u001b[1;36m31\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'context'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      Thinking about High-Quality Human Data\\n    \\nDate: February 5, 2024  |  \u001b[0m\n",
       "\u001b[32mEstimated Reading Tim'\u001b[0m+\u001b[1;36m28918\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'\\n\\n      LLM Powered Autonomous Agents\\n    \\nDate: June 23, 2023  |  Estimated Reading \u001b[0m\n",
       "\u001b[32mTime: 31 min  |'\u001b[0m+\u001b[1;36m43030\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'answer'\u001b[0m: \u001b[32m'The main steps for collecting human data, as outlined in the provided context, are:\\n\\n1. **Task \u001b[0m\n",
       "\u001b[32mDesig'\u001b[0m+\u001b[1;36m900\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">The main steps for collecting human data, as outlined in the provided context, are:                                \n",
       "\n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 1 </span><span style=\"font-weight: bold\">Task Design</span>: Create a clear task workflow to enhance clarity and reduce complexity. This involves providing     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>detailed guidelines that are straightforward enough to be useful without being overly complicated.              \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 2 </span><span style=\"font-weight: bold\">Select and Train a Pool of Raters</span>: Identify annotators with the appropriate skill set and ensure consistency in \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>their annotations. This requires conducting training sessions to educate raters about the task. After           \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>onboarding, regular feedback and calibration sessions should be implemented to maintain quality.                \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 3 </span><span style=\"font-weight: bold\">Collect and Aggregate Data</span>: This step involves the collection of annotations from the human raters, followed by \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>the application of machine learning techniques to clean, filter, and intelligently aggregate the data to        \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>determine the true labels.                                                                                      \n",
       "\n",
       "These steps emphasize the importance of careful execution throughout the data collection process to ensure         \n",
       "high-quality outcomes.                                                                                             \n",
       "</pre>\n"
      ],
      "text/plain": [
       "The main steps for collecting human data, as outlined in the provided context, are:                                \n",
       "\n",
       "\u001b[1;33m 1 \u001b[0m\u001b[1mTask Design\u001b[0m: Create a clear task workflow to enhance clarity and reduce complexity. This involves providing     \n",
       "\u001b[1;33m   \u001b[0mdetailed guidelines that are straightforward enough to be useful without being overly complicated.              \n",
       "\u001b[1;33m 2 \u001b[0m\u001b[1mSelect and Train a Pool of Raters\u001b[0m: Identify annotators with the appropriate skill set and ensure consistency in \n",
       "\u001b[1;33m   \u001b[0mtheir annotations. This requires conducting training sessions to educate raters about the task. After           \n",
       "\u001b[1;33m   \u001b[0monboarding, regular feedback and calibration sessions should be implemented to maintain quality.                \n",
       "\u001b[1;33m 3 \u001b[0m\u001b[1mCollect and Aggregate Data\u001b[0m: This step involves the collection of annotations from the human raters, followed by \n",
       "\u001b[1;33m   \u001b[0mthe application of machine learning techniques to clean, filter, and intelligently aggregate the data to        \n",
       "\u001b[1;33m   \u001b[0mdetermine the true labels.                                                                                      \n",
       "\n",
       "These steps emphasize the importance of careful execution throughout the data collection process to ensure         \n",
       "high-quality outcomes.                                                                                             \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(human_data_query)\n",
    "response = graph.invoke({\"question\": human_data_query})\n",
    "rprint(Pretty(response, max_string=100, no_wrap=False))\n",
    "rprint(Markdown(response[\"answer\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}