notebooks/rag-from-scratch/14-colbert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "480d3c90-ff0e-4257-bb4e-4883a5926c23",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import re\n",
    "import warnings\n",
    "\n",
    "from dotenv import find_dotenv, load_dotenv\n",
    "from IPython.display import display\n",
    "from rich import print as rprint\n",
    "from rich.markdown import Markdown\n",
    "from rich.pretty import Pretty\n",
    "from rich.text import Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d47f7015-0428-4388-a0bf-36413850ddea",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f8f0069c-442e-44aa-a708-2d6eff0cc8d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv(find_dotenv('.env'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62aaacc4-a1f1-406b-9598-6c8cdbea6dae",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"LANGCHAIN_PROJECT\"] = \"RAG From Scratch: Part 14 (Indexing - ColBERT)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "77dc7c4c-f767-49d4-aadd-22f140bd81a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path('../../data')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67487dee-134e-472e-9236-f930973acc3a",
   "metadata": {},
   "source": [
    "# Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7d07f7c-37c6-4a3d-a5b9-15873c378028",
   "metadata": {},
   "source": [
    "![](images/indexing-01.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12b7b630-57a5-41d9-8eb7-c80eb7a9de03",
   "metadata": {},
   "source": [
    "# Part 14: ColBERT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fa93db7-44b1-491f-ab5c-fe9546d917c1",
   "metadata": {},
   "source": [
    "![](images/14-01-colbert.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "431ec57b-74cc-4b10-94b7-48099af92ba3",
   "metadata": {},
   "source": [
    "![](images/14-02-colbert.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d6b97d9-6bec-4833-a5c9-a7fae1c6c105",
   "metadata": {},
   "source": [
    "## Configure components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3da3a72c-bcff-4686-9656-33739427ceba",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3bd44cf4-763b-41c7-892e-d7820f00c5b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='Hello! How can I assist you today?', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 10, 'prompt_tokens': 8, 'total_tokens': 18, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_b376dfbbd5', 'id': 'chatcmpl-BKPueXufJHLtjqQ7UK2irCMmUtSGA', 'finish_reason': 'stop', 'logprobs': None}, id='run-19a8c321-b21d-4606-8a03-17487eedc7aa-0', usage_metadata={'input_tokens': 8, 'output_tokens': 10, 'total_tokens': 18, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm = ChatOpenAI(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    temperature=1\n",
    ")\n",
    "llm.invoke(\"Hello\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6934f30a-24a4-484e-adb9-8d22f88e581b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1536"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
    "len(embeddings.embed_query(\"Hello\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9183418-6f87-458b-845c-1a4fb796c58b",
   "metadata": {},
   "source": [
    "## Load documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2fece486-596d-4b83-b6a4-52a9af384f36",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
     ]
    }
   ],
   "source": [
    "import bs4\n",
    "from langchain_community.document_loaders import WebBaseLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cd5f6654-d115-4a96-8d3d-ccb902d9def8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles = [\n",
    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
    "    \"https://lilianweng.github.io/posts/2024-02-05-human-data-quality/\",\n",
    "]\n",
    "\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=articles,\n",
    "    bs_kwargs=dict(\n",
    "        parse_only=bs4.SoupStrainer(\n",
    "            class_=(\"post-content\", \"post-title\", \"post-header\")\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "docs = loader.load()\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c42f8c00-12c7-4a31-b054-ff6fc8fd2f4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "      LLM Powered Autonomous Agents\n",
      "    \n",
      "Date: June 23, 2023  |  Estimated Reading Time: 31 min  |  Author: Lilian Weng\n",
      "\n",
      "\n",
      "Building agents with LLM (large language model) as its core controller is a cool concept. Several proof-of-concepts demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of LLM extends beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem solver.\n",
      "Agent System Overview#\n",
      "In a LLM-powered autonomous agent system, LLM functions as the agent’s brain, complemented by several key components:\n",
      "\n",
      "Planning\n",
      "\n",
      "Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable subgoals, enabling efficient handling of complex tasks.\n",
      "Reflection and refinement: The agent can do self-criticism and self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the quality of final results.\n",
      "\n",
      "\n",
      "Memory\n",
      "\n",
      "Short-term memory: I \n"
     ]
    }
   ],
   "source": [
    "print(docs[0].page_content[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2fa8dd8d-3591-4754-a61c-b71bed3981f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "      Thinking about High-Quality Human Data\n",
      "    \n",
      "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n",
      "\n",
      "\n",
      "[Special thank you to Ian Kivlichan for many useful pointers (E.g. the 100+ year old Nature paper “Vox populi”) and nice feedback. 🙏 ]\n",
      "High-quality data is the fuel for modern data deep learning model training. Most of the task-specific labeled data comes from human annotation, such as classification task or RLHF labeling (which can be constructed as classification format) for LLM alignment training. Lots of ML techniques in the post can help with data quality, but fundamentally human data collection involves attention to details and careful execution. The community knows the value of high quality data, but somehow we have this subtle impression that “Everyone wants to do the model work, not the data work” (Sambasivan et al. 2021).\n",
      "\n",
      "Fig. 1. Two directions to approach high data quality.\n",
      "Human Raters ↔ Data Quality#\n",
      "Collecting human data involve a set of\n"
     ]
    }
   ],
   "source": [
    "print(docs[1].page_content[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5bad43f7-3395-43e6-a6d4-ed5782fd477d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[43130, 29018]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(doc.page_content) for doc in docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fb30bd8-8eae-416f-9ae6-6d49b6261f58",
   "metadata": {},
   "source": [
    "## Split documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "daec3f6c-02a6-4f1e-9a89-b918526332ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e2e76069-fc7a-4d41-93f1-51f5ea99ad4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=0)\n",
    "splits = text_splitter.split_documents(docs)\n",
    "len(splits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbb41010-959e-469e-8b81-57f099d4b685",
   "metadata": {},
   "source": [
    "## Store documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e2eddba9-70fe-45e0-82ef-ca27c86748cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use transformers==4.49.0\n",
    "# https://github.com/huggingface/transformers/issues/36954"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "144f7190-ef47-4026-bb25-7c15e1ddab64",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.vectorstores import InMemoryVectorStore\n",
    "from ragatouille import RAGPretrainedModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4a37177a-6225-43d2-9414-af63afe27d0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99, 99)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorstore = InMemoryVectorStore(embeddings)\n",
    "doc_ids = vectorstore.add_documents(documents=splits)\n",
    "len(doc_ids), len(vectorstore.store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "477d1a0d-c9cb-446e-b904-b672e78621e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Apr 09, 16:38:11] Loading segmented_maxsim_cpp extension (set COLBERT_LOAD_TORCH_EXTENSION_VERBOSE=True for more info)...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/llm-rag/.venv/lib/python3.11/site-packages/torch/amp/grad_scaler.py:132: UserWarning: torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# https://github.com/AnswerDotAI/RAGatouille/issues/213#issuecomment-2285897809\n",
    "\n",
    "RAG = RAGPretrainedModel.from_pretrained(\"colbert-ir/colbertv2.0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "28023b0c-cf55-4a43-8d92-9cea191dbd41",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_texts = [doc.page_content for doc in splits]\n",
    "docs_metadatas = [doc.metadata for doc in splits]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa8d5a2a-c272-4898-b242-b7bcc5212431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---- WARNING! You are using PLAID with an experimental replacement for FAISS for greater compatibility ----\n",
      "This is a behaviour change from RAGatouille 0.8.0 onwards.\n",
      "This works fine for most users and smallish datasets, but can be considerably slower than FAISS and could cause worse results in some situations.\n",
      "If you're confident with FAISS working on your machine, pass use_faiss=True to revert to the FAISS-using behaviour.\n",
      "--------------------\n",
      "\n",
      "\n",
      "[Apr 09, 16:38:11] #> Note: Output directory .ragatouille/colbert/indexes/blog already exists\n",
      "\n",
      "\n",
      "[Apr 09, 16:38:11] #> Will delete 11 files already at .ragatouille/colbert/indexes/blog in 20 seconds...\n",
      "[Apr 09, 16:38:34] [0] \t\t #> Encoding 99 passages..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                                           | 0/4 [00:00<?, ?it/s]/home/dlabazkin/projects/gpt/llm-rag/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n",
      " 50%|█████████████████████████████████████████████████████████▌                                                         | 2/4 [00:22<00:22, 11.34s/it]"
     ]
    }
   ],
   "source": [
    "RAG.index(\n",
    "    collection=docs_texts,\n",
    "    document_metadatas=docs_metadatas,\n",
    "    index_name=\"blog\",\n",
    "    split_documents=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd6224e5-94ac-4df9-a5b4-6781fe46d7b6",
   "metadata": {},
   "source": [
    "## RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c8c6f142-654e-492d-bf25-519d8b07a50b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answer the following question based on this context:\n",
      "\n",
      "{context}\n",
      "\n",
      "Question: {question}\n"
     ]
    }
   ],
   "source": [
    "rag_prompt_template = \"\"\"Answer the following question based on this context:\n",
    "\n",
    "{context}\n",
    "\n",
    "Question: {question}\"\"\"\n",
    "print(rag_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "68229030-18fd-4cab-8456-6a5703e2d565",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0342442e-b7b9-4896-a977-b1217d4eab65",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_kwargs = {\n",
    "    \"k\": 10\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e0e6a007-8aac-4e88-909c-ce7e8279402e",
   "metadata": {},
   "outputs": [],
   "source": [
    "queries = [\n",
    "    \"What is task decomposition for LLM agents?\",\n",
    "    \"What are main steps for collecting human data?\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80736739-e0ac-43aa-a617-5ea63de33456",
   "metadata": {},
   "source": [
    "### Reranker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4053fc87-4c45-4bb9-b0cd-01b26543b22d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.retrievers import ContextualCompressionRetriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "31475e61-7fe8-4d5f-b100-c302dfa1a9b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is task decomposition for LLM agents?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.78s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1:</span>\n",
       "Fig. 1. Overview of a <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> system.\n",
       "Component One: Planning#\n",
       "A complicated <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> usually involves many steps. An <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> needs to know <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">what</span> they are and plan ahead.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Decomposition</span>#\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1:\u001b[0m\n",
       "Fig. 1. Overview of a \u001b[4;31mLLM\u001b[0m-powered autonomous \u001b[4;31magent\u001b[0m system.\n",
       "Component One: Planning#\n",
       "A complicated \u001b[4;31mtask\u001b[0m usually involves many steps. An \u001b[4;31magent\u001b[0m needs to know \u001b[4;31mwhat\u001b[0m they are and plan ahead.\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mDecomposition\u001b[0m#\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1 (compressed):</span>\n",
       "Fig. 1. Overview of a <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> system.\n",
       "Component One: Planning#\n",
       "A complicated <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> usually involves many steps. An <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> needs to know <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">what</span> they are and plan ahead.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Decomposition</span>#\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1 (compressed):\u001b[0m\n",
       "Fig. 1. Overview of a \u001b[4;31mLLM\u001b[0m-powered autonomous \u001b[4;31magent\u001b[0m system.\n",
       "Component One: Planning#\n",
       "A complicated \u001b[4;31mtask\u001b[0m usually involves many steps. An \u001b[4;31magent\u001b[0m needs to know \u001b[4;31mwhat\u001b[0m they are and plan ahead.\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mDecomposition\u001b[0m#\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2:</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span> can be done (1) by <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> with simple prompting like \"Steps <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> are the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with human inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2:\u001b[0m\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mdecomposition\u001b[0m can be done (1) by \u001b[4;31mLLM\u001b[0m with simple prompting like \"Steps \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m are the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using \u001b[4;31mtask\u001b[0m-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with human inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2 (compressed):</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span> can be done (1) by <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> with simple prompting like \"Steps <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> are the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with human inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2 (compressed):\u001b[0m\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mdecomposition\u001b[0m can be done (1) by \u001b[4;31mLLM\u001b[0m with simple prompting like \"Steps \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m are the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using \u001b[4;31mtask\u001b[0m-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with human inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3:</span>\n",
       "Building <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> (large language model) as its core controller <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> a cool concept. Several proof-of-concepts\n",
       "demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> extends \n",
       "beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem\n",
       "solver.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Agent</span> System Overview#\n",
       "In a <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> system, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> functions as the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span>’s brain, complemented by several key \n",
       "components:\n",
       "\n",
       "Planning\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3:\u001b[0m\n",
       "Building \u001b[4;31magents\u001b[0m with \u001b[4;31mLLM\u001b[0m (large language model) as its core controller \u001b[4;31mis\u001b[0m a cool concept. Several proof-of-concepts\n",
       "demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of \u001b[4;31mLLM\u001b[0m extends \n",
       "beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem\n",
       "solver.\n",
       "\u001b[4;31mAgent\u001b[0m System Overview#\n",
       "In a \u001b[4;31mLLM\u001b[0m-powered autonomous \u001b[4;31magent\u001b[0m system, \u001b[4;31mLLM\u001b[0m functions as the \u001b[4;31magent\u001b[0m’s brain, complemented by several key \n",
       "components:\n",
       "\n",
       "Planning\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3 (compressed):</span>\n",
       "Challenges in long-term planning and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3 (compressed):\u001b[0m\n",
       "Challenges in long-term planning and \u001b[4;31mtask\u001b[0m \u001b[4;31mdecomposition\u001b[0m: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4:</span>\n",
       "Challenges in long-term planning and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4:\u001b[0m\n",
       "Challenges in long-term planning and \u001b[4;31mtask\u001b[0m \u001b[4;31mdecomposition\u001b[0m: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4 (compressed):</span>\n",
       "Subgoal and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4 (compressed):\u001b[0m\n",
       "Subgoal and \u001b[4;31mdecomposition\u001b[0m: The \u001b[4;31magent\u001b[0m breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The \u001b[4;31magent\u001b[0m can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them \u001b[4;31mfor\u001b[0m future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5:</span>\n",
       "(2) Model selection: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> distributes the tasks to expert models, where the request <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> framed as a multiple-choice \n",
       "question. <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> presented with a list of models to choose from. Due to the limited context length, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type based\n",
       "filtration <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> needed.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5:\u001b[0m\n",
       "(2) Model selection: \u001b[4;31mLLM\u001b[0m distributes the tasks to expert models, where the request \u001b[4;31mis\u001b[0m framed as a multiple-choice \n",
       "question. \u001b[4;31mLLM\u001b[0m \u001b[4;31mis\u001b[0m presented with a list of models to choose from. Due to the limited context length, \u001b[4;31mtask\u001b[0m type based\n",
       "filtration \u001b[4;31mis\u001b[0m needed.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5 (compressed):</span>\n",
       "(2) Model selection: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> distributes the tasks to expert models, where the request <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> framed as a multiple-choice \n",
       "question. <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> presented with a list of models to choose from. Due to the limited context length, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type based\n",
       "filtration <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> needed.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5 (compressed):\u001b[0m\n",
       "(2) Model selection: \u001b[4;31mLLM\u001b[0m distributes the tasks to expert models, where the request \u001b[4;31mis\u001b[0m framed as a multiple-choice \n",
       "question. \u001b[4;31mLLM\u001b[0m \u001b[4;31mis\u001b[0m presented with a list of models to choose from. Due to the limited context length, \u001b[4;31mtask\u001b[0m type based\n",
       "filtration \u001b[4;31mis\u001b[0m needed.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6:</span>\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> planning: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to do <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> parsing and planning.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6:\u001b[0m\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) \u001b[4;31mTask\u001b[0m planning: \u001b[4;31mLLM\u001b[0m works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each \u001b[4;31mtask\u001b[0m: \u001b[4;31mtask\u001b[0m type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide \u001b[4;31mLLM\u001b[0m to do \u001b[4;31mtask\u001b[0m parsing and planning.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6 (compressed):</span>\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> planning: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to do <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> parsing and planning.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6 (compressed):\u001b[0m\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) \u001b[4;31mTask\u001b[0m planning: \u001b[4;31mLLM\u001b[0m works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each \u001b[4;31mtask\u001b[0m: \u001b[4;31mtask\u001b[0m type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide \u001b[4;31mLLM\u001b[0m to do \u001b[4;31mtask\u001b[0m parsing and planning.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7:</span>\n",
       "Boiko et al. (2023) also looked into <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-empowered <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> scientific discovery, to handle autonomous design, \n",
       "planning, and performance of complex scientific experiments. This <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> can use tools to browse the Internet, read \n",
       "documentation, execute code, call robotics experimentation APIs and leverage other LLMs.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">For</span> example, when requested to \"develop a novel anticancer drug\", the model came up with the following reasoning \n",
       "steps:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7:\u001b[0m\n",
       "Boiko et al. (2023) also looked into \u001b[4;31mLLM\u001b[0m-empowered \u001b[4;31magents\u001b[0m \u001b[4;31mfor\u001b[0m scientific discovery, to handle autonomous design, \n",
       "planning, and performance of complex scientific experiments. This \u001b[4;31magent\u001b[0m can use tools to browse the Internet, read \n",
       "documentation, execute code, call robotics experimentation APIs and leverage other LLMs.\n",
       "\u001b[4;31mFor\u001b[0m example, when requested to \"develop a novel anticancer drug\", the model came up with the following reasoning \n",
       "steps:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7 (compressed):</span>\n",
       "Boiko et al. (2023) also looked into <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-empowered <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> scientific discovery, to handle autonomous design, \n",
       "planning, and performance of complex scientific experiments. This <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> can use tools to browse the Internet, read \n",
       "documentation, execute code, call robotics experimentation APIs and leverage other LLMs.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">For</span> example, when requested to \"develop a novel anticancer drug\", the model came up with the following reasoning \n",
       "steps:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7 (compressed):\u001b[0m\n",
       "Boiko et al. (2023) also looked into \u001b[4;31mLLM\u001b[0m-empowered \u001b[4;31magents\u001b[0m \u001b[4;31mfor\u001b[0m scientific discovery, to handle autonomous design, \n",
       "planning, and performance of complex scientific experiments. This \u001b[4;31magent\u001b[0m can use tools to browse the Internet, read \n",
       "documentation, execute code, call robotics experimentation APIs and leverage other LLMs.\n",
       "\u001b[4;31mFor\u001b[0m example, when requested to \"develop a novel anticancer drug\", the model came up with the following reasoning \n",
       "steps:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8:</span>\n",
       "Subgoal and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8:\u001b[0m\n",
       "Subgoal and \u001b[4;31mdecomposition\u001b[0m: The \u001b[4;31magent\u001b[0m breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The \u001b[4;31magent\u001b[0m can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them \u001b[4;31mfor\u001b[0m future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8 (compressed):</span>\n",
       "Building <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> (large language model) as its core controller <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> a cool concept. Several proof-of-concepts\n",
       "demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> extends \n",
       "beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem\n",
       "solver.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Agent</span> System Overview#\n",
       "In a <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> system, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> functions as the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span>’s brain, complemented by several key \n",
       "components:\n",
       "\n",
       "Planning\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8 (compressed):\u001b[0m\n",
       "Building \u001b[4;31magents\u001b[0m with \u001b[4;31mLLM\u001b[0m (large language model) as its core controller \u001b[4;31mis\u001b[0m a cool concept. Several proof-of-concepts\n",
       "demos, such as AutoGPT, GPT-Engineer and BabyAGI, serve as inspiring examples. The potentiality of \u001b[4;31mLLM\u001b[0m extends \n",
       "beyond generating well-written copies, stories, essays and programs; it can be framed as a powerful general problem\n",
       "solver.\n",
       "\u001b[4;31mAgent\u001b[0m System Overview#\n",
       "In a \u001b[4;31mLLM\u001b[0m-powered autonomous \u001b[4;31magent\u001b[0m system, \u001b[4;31mLLM\u001b[0m functions as the \u001b[4;31magent\u001b[0m’s brain, complemented by several key \n",
       "components:\n",
       "\n",
       "Planning\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9:</span>\n",
       "}\n",
       "]\n",
       "Challenges#\n",
       "After going through key ideas and demos of building <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-centered <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span>, I start to see a couple common \n",
       "limitations:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9:\u001b[0m\n",
       "}\n",
       "]\n",
       "Challenges#\n",
       "After going through key ideas and demos of building \u001b[4;31mLLM\u001b[0m-centered \u001b[4;31magents\u001b[0m, I start to see a couple common \n",
       "limitations:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9 (compressed):</span>\n",
       "(4) Response generation: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> receives the execution results and provides summarized results to users.\n",
       "To put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> needed as\n",
       "both <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> inference rounds and interactions with other models slow down the process; (2) It relies on a long context\n",
       "window to communicate over complicated <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> content; (3) Stability improvement of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> outputs and external model \n",
       "services.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9 (compressed):\u001b[0m\n",
       "(4) Response generation: \u001b[4;31mLLM\u001b[0m receives the execution results and provides summarized results to users.\n",
       "To put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement \u001b[4;31mis\u001b[0m needed as\n",
       "both \u001b[4;31mLLM\u001b[0m inference rounds and interactions with other models slow down the process; (2) It relies on a long context\n",
       "window to communicate over complicated \u001b[4;31mtask\u001b[0m content; (3) Stability improvement of \u001b[4;31mLLM\u001b[0m outputs and external model \n",
       "services.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10:</span>\n",
       "(4) Response generation: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> receives the execution results and provides summarized results to users.\n",
       "To put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> needed as\n",
       "both <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> inference rounds and interactions with other models slow down the process; (2) It relies on a long context\n",
       "window to communicate over complicated <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> content; (3) Stability improvement of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> outputs and external model \n",
       "services.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10:\u001b[0m\n",
       "(4) Response generation: \u001b[4;31mLLM\u001b[0m receives the execution results and provides summarized results to users.\n",
       "To put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement \u001b[4;31mis\u001b[0m needed as\n",
       "both \u001b[4;31mLLM\u001b[0m inference rounds and interactions with other models slow down the process; (2) It relies on a long context\n",
       "window to communicate over complicated \u001b[4;31mtask\u001b[0m content; (3) Stability improvement of \u001b[4;31mLLM\u001b[0m outputs and external model \n",
       "services.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10 (compressed):</span>\n",
       "}\n",
       "]\n",
       "Challenges#\n",
       "After going through key ideas and demos of building <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-centered <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span>, I start to see a couple common \n",
       "limitations:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10 (compressed):\u001b[0m\n",
       "}\n",
       "]\n",
       "Challenges#\n",
       "After going through key ideas and demos of building \u001b[4;31mLLM\u001b[0m-centered \u001b[4;31magents\u001b[0m, I start to see a couple common \n",
       "limitations:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are main steps for collecting human data?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.57s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1:</span>\n",
       "Fig. 1. Two directions to approach high <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> Raters ↔ <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span> Quality#\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Collecting</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> involve a set of operation <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">steps</span> and every step contributes to the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1:\u001b[0m\n",
       "Fig. 1. Two directions to approach high \u001b[4;31mdata\u001b[0m quality.\n",
       "\u001b[4;31mHuman\u001b[0m Raters ↔ \u001b[4;31mData\u001b[0m Quality#\n",
       "\u001b[4;31mCollecting\u001b[0m \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m involve a set of operation \u001b[4;31msteps\u001b[0m and every step contributes to the \u001b[4;31mdata\u001b[0m quality:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1 (compressed):</span>\n",
       "Fig. 1. Two directions to approach high <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> Raters ↔ <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span> Quality#\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Collecting</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> involve a set of operation <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">steps</span> and every step contributes to the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1 (compressed):\u001b[0m\n",
       "Fig. 1. Two directions to approach high \u001b[4;31mdata\u001b[0m quality.\n",
       "\u001b[4;31mHuman\u001b[0m Raters ↔ \u001b[4;31mData\u001b[0m Quality#\n",
       "\u001b[4;31mCollecting\u001b[0m \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m involve a set of operation \u001b[4;31msteps\u001b[0m and every step contributes to the \u001b[4;31mdata\u001b[0m quality:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2:</span>\n",
       "Or\n",
       "@article{weng2024humandata,\n",
       "  title   = \"Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>\",\n",
       "  author  = \"Weng, Lilian\",\n",
       "  journal = \"lilianweng.github.io\",\n",
       "  year    = \"2024\",\n",
       "  month   = \"Feb\",\n",
       "  url     = \"https://lilianweng.github.io/posts/2024-02-05-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span>-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>-quality/\"\n",
       "}\n",
       "References#\n",
       "[1] Francis Galton “Vox populi”  Nature 75, 450-451 (1907).\n",
       "[2] Sambasivan et al. “Everyone wants to do the model work, not the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> work”: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span> Cascades in High-Stakes AI\" \n",
       "CHI 2021\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2:\u001b[0m\n",
       "Or\n",
       "@article{weng2024humandata,\n",
       "  title   = \"Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m\",\n",
       "  author  = \"Weng, Lilian\",\n",
       "  journal = \"lilianweng.github.io\",\n",
       "  year    = \"2024\",\n",
       "  month   = \"Feb\",\n",
       "  url     = \"https://lilianweng.github.io/posts/2024-02-05-\u001b[4;31mhuman\u001b[0m-\u001b[4;31mdata\u001b[0m-quality/\"\n",
       "}\n",
       "References#\n",
       "[1] Francis Galton “Vox populi”  Nature 75, 450-451 (1907).\n",
       "[2] Sambasivan et al. “Everyone wants to do the model work, not the \u001b[4;31mdata\u001b[0m work”: \u001b[4;31mData\u001b[0m Cascades in High-Stakes AI\" \n",
       "CHI 2021\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2 (compressed):</span>\n",
       "High-quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> is the fuel <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> modern <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> deep learning model training. Most of the task-specific labeled <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> \n",
       "comes from <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> LLM alignment training. Lots of ML techniques in the post can help with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality, \n",
       "but fundamentally <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>, but somehow we have this subtle\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2 (compressed):\u001b[0m\n",
       "High-quality \u001b[4;31mdata\u001b[0m is the fuel \u001b[4;31mfor\u001b[0m modern \u001b[4;31mdata\u001b[0m deep learning model training. Most of the task-specific labeled \u001b[4;31mdata\u001b[0m \n",
       "comes from \u001b[4;31mhuman\u001b[0m annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) \u001b[4;31mfor\u001b[0m LLM alignment training. Lots of ML techniques in the post can help with \u001b[4;31mdata\u001b[0m quality, \n",
       "but fundamentally \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality \u001b[4;31mdata\u001b[0m, but somehow we have this subtle\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3:</span>\n",
       "Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>\n",
       "    \n",
       "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3:\u001b[0m\n",
       "Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m\n",
       "    \n",
       "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3 (compressed):</span>\n",
       "Collect and aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> to identify the true labels.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3 (compressed):\u001b[0m\n",
       "Collect and aggregate \u001b[4;31mdata\u001b[0m. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate \u001b[4;31mdata\u001b[0m to identify the true labels.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4:</span>\n",
       "Collect and aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> to identify the true labels.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4:\u001b[0m\n",
       "Collect and aggregate \u001b[4;31mdata\u001b[0m. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate \u001b[4;31mdata\u001b[0m to identify the true labels.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4 (compressed):</span>\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Steps</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4 (compressed):\u001b[0m\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"\u001b[4;31mSteps\u001b[0m \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m \u001b[4;31mare\u001b[0m the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with \u001b[4;31mhuman\u001b[0m inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5:</span>\n",
       "High-quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> is the fuel <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> modern <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> deep learning model training. Most of the task-specific labeled <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> \n",
       "comes from <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> LLM alignment training. Lots of ML techniques in the post can help with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality, \n",
       "but fundamentally <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>, but somehow we have this subtle\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5:\u001b[0m\n",
       "High-quality \u001b[4;31mdata\u001b[0m is the fuel \u001b[4;31mfor\u001b[0m modern \u001b[4;31mdata\u001b[0m deep learning model training. Most of the task-specific labeled \u001b[4;31mdata\u001b[0m \n",
       "comes from \u001b[4;31mhuman\u001b[0m annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) \u001b[4;31mfor\u001b[0m LLM alignment training. Lots of ML techniques in the post can help with \u001b[4;31mdata\u001b[0m quality, \n",
       "but fundamentally \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality \u001b[4;31mdata\u001b[0m, but somehow we have this subtle\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5 (compressed):</span>\n",
       "Fig. 15. Algorithm of INCV (iterative noisy cross-validation). (Image source: Chen et al. 2019)\n",
       "Citation#\n",
       "Cited as:\n",
       "\n",
       "Weng, Lilian. (Feb 2024). “Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>”. Lil’Log. \n",
       "https://lilianweng.github.io/posts/2024-02-05-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span>-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>-quality/.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5 (compressed):\u001b[0m\n",
       "Fig. 15. Algorithm of INCV (iterative noisy cross-validation). (Image source: Chen et al. 2019)\n",
       "Citation#\n",
       "Cited as:\n",
       "\n",
       "Weng, Lilian. (Feb 2024). “Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m”. Lil’Log. \n",
       "https://lilianweng.github.io/posts/2024-02-05-\u001b[4;31mhuman\u001b[0m-\u001b[4;31mdata\u001b[0m-quality/.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6:</span>\n",
       "Prompt LM with 100 most recent observations and to generate 3 most salient high-level questions given a set of \n",
       "observations/statements. Then ask LM to answer those questions.\n",
       "\n",
       "\n",
       "Planning &amp; Reacting: translate the reflections and the environment information into actions\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6:\u001b[0m\n",
       "Prompt LM with 100 most recent observations and to generate 3 most salient high-level questions given a set of \n",
       "observations/statements. Then ask LM to answer those questions.\n",
       "\n",
       "\n",
       "Planning & Reacting: translate the reflections and the environment information into actions\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6 (compressed):</span>\n",
       "Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>\n",
       "    \n",
       "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6 (compressed):\u001b[0m\n",
       "Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m\n",
       "    \n",
       "Date: February 5, 2024  |  Estimated Reading Time: 21 min  |  Author: Lilian Weng\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7:</span>\n",
       "With the input and the inference results, the AI assistant needs to describe the process and results. The previous \n",
       "stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model \n",
       "Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward \n",
       "manner. Then describe the task process and show your analysis and model inference results to the user in the first \n",
       "person. If inference results contain a file path, must tell\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7:\u001b[0m\n",
       "With the input and the inference results, the AI assistant needs to describe the process and results. The previous \n",
       "stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model \n",
       "Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward \n",
       "manner. Then describe the task process and show your analysis and model inference results to the user in the first \n",
       "person. If inference results contain a file path, must tell\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7 (compressed):</span>\n",
       "Or\n",
       "@article{weng2024humandata,\n",
       "  title   = \"Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>\",\n",
       "  author  = \"Weng, Lilian\",\n",
       "  journal = \"lilianweng.github.io\",\n",
       "  year    = \"2024\",\n",
       "  month   = \"Feb\",\n",
       "  url     = \"https://lilianweng.github.io/posts/2024-02-05-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span>-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>-quality/\"\n",
       "}\n",
       "References#\n",
       "[1] Francis Galton “Vox populi”  Nature 75, 450-451 (1907).\n",
       "[2] Sambasivan et al. “Everyone wants to do the model work, not the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> work”: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span> Cascades in High-Stakes AI\" \n",
       "CHI 2021\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7 (compressed):\u001b[0m\n",
       "Or\n",
       "@article{weng2024humandata,\n",
       "  title   = \"Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m\",\n",
       "  author  = \"Weng, Lilian\",\n",
       "  journal = \"lilianweng.github.io\",\n",
       "  year    = \"2024\",\n",
       "  month   = \"Feb\",\n",
       "  url     = \"https://lilianweng.github.io/posts/2024-02-05-\u001b[4;31mhuman\u001b[0m-\u001b[4;31mdata\u001b[0m-quality/\"\n",
       "}\n",
       "References#\n",
       "[1] Francis Galton “Vox populi”  Nature 75, 450-451 (1907).\n",
       "[2] Sambasivan et al. “Everyone wants to do the model work, not the \u001b[4;31mdata\u001b[0m work”: \u001b[4;31mData\u001b[0m Cascades in High-Stakes AI\" \n",
       "CHI 2021\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8:</span>\n",
       "Fig. 15. Algorithm of INCV (iterative noisy cross-validation). (Image source: Chen et al. 2019)\n",
       "Citation#\n",
       "Cited as:\n",
       "\n",
       "Weng, Lilian. (Feb 2024). “Thinking about High-Quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span>”. Lil’Log. \n",
       "https://lilianweng.github.io/posts/2024-02-05-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span>-<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>-quality/.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8:\u001b[0m\n",
       "Fig. 15. Algorithm of INCV (iterative noisy cross-validation). (Image source: Chen et al. 2019)\n",
       "Citation#\n",
       "Cited as:\n",
       "\n",
       "Weng, Lilian. (Feb 2024). “Thinking about High-Quality \u001b[4;31mHuman\u001b[0m \u001b[4;31mData\u001b[0m”. Lil’Log. \n",
       "https://lilianweng.github.io/posts/2024-02-05-\u001b[4;31mhuman\u001b[0m-\u001b[4;31mdata\u001b[0m-quality/.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8 (compressed):</span>\n",
       "With the input and the inference results, the AI assistant needs to describe the process and results. The previous \n",
       "stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model \n",
       "Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward \n",
       "manner. Then describe the task process and show your analysis and model inference results to the user in the first \n",
       "person. If inference results contain a file path, must tell\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8 (compressed):\u001b[0m\n",
       "With the input and the inference results, the AI assistant needs to describe the process and results. The previous \n",
       "stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model \n",
       "Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward \n",
       "manner. Then describe the task process and show your analysis and model inference results to the user in the first \n",
       "person. If inference results contain a file path, must tell\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9:</span>\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Steps</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9:\u001b[0m\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"\u001b[4;31mSteps\u001b[0m \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m \u001b[4;31mare\u001b[0m the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with \u001b[4;31mhuman\u001b[0m inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9 (compressed):</span>\n",
       "Prompt LM with 100 most recent observations and to generate 3 most salient high-level questions given a set of \n",
       "observations/statements. Then ask LM to answer those questions.\n",
       "\n",
       "\n",
       "Planning &amp; Reacting: translate the reflections and the environment information into actions\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9 (compressed):\u001b[0m\n",
       "Prompt LM with 100 most recent observations and to generate 3 most salient high-level questions given a set of \n",
       "observations/statements. Then ask LM to answer those questions.\n",
       "\n",
       "\n",
       "Planning & Reacting: translate the reflections and the environment information into actions\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10:</span>\n",
       "Task design: Design task workflow to improve clarity and reduce complexity. Detailed guidelines <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> helpful but \n",
       "very long and complicated guidelines demand a decent amount of training to be useful.\n",
       "Select and train a pool of raters: Select annotators with matched skillset and consistency. Training sessions <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> \n",
       "necessary. After onboarding, regular feedback and calibration sessions <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> also needed.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10:\u001b[0m\n",
       "Task design: Design task workflow to improve clarity and reduce complexity. Detailed guidelines \u001b[4;31mare\u001b[0m helpful but \n",
       "very long and complicated guidelines demand a decent amount of training to be useful.\n",
       "Select and train a pool of raters: Select annotators with matched skillset and consistency. Training sessions \u001b[4;31mare\u001b[0m \n",
       "necessary. After onboarding, regular feedback and calibration sessions \u001b[4;31mare\u001b[0m also needed.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10 (compressed):</span>\n",
       "Task design: Design task workflow to improve clarity and reduce complexity. Detailed guidelines <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> helpful but \n",
       "very long and complicated guidelines demand a decent amount of training to be useful.\n",
       "Select and train a pool of raters: Select annotators with matched skillset and consistency. Training sessions <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> \n",
       "necessary. After onboarding, regular feedback and calibration sessions <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> also needed.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10 (compressed):\u001b[0m\n",
       "Task design: Design task workflow to improve clarity and reduce complexity. Detailed guidelines \u001b[4;31mare\u001b[0m helpful but \n",
       "very long and complicated guidelines demand a decent amount of training to be useful.\n",
       "Select and train a pool of raters: Select annotators with matched skillset and consistency. Training sessions \u001b[4;31mare\u001b[0m \n",
       "necessary. After onboarding, regular feedback and calibration sessions \u001b[4;31mare\u001b[0m also needed.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "retriever = vectorstore.as_retriever(\n",
    "    search_kwargs=search_kwargs\n",
    ")\n",
    "\n",
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=RAG.as_langchain_document_compressor(k=search_kwargs[\"k\"]), base_retriever=retriever\n",
    ")\n",
    "\n",
    "for query in queries:\n",
    "    print(query)\n",
    "    docs = retriever.invoke(query)\n",
    "    compressed_docs = compression_retriever.invoke(query)\n",
    "    \n",
    "    chunk_pattern = re.compile(r'^Chunk \\d+.*:$', flags=re.MULTILINE)\n",
    "    terms_pattern = re.compile(rf'\\b({\"|\".join(query.split())})\\b', flags=re.IGNORECASE)\n",
    "    \n",
    "    for chunk_id, (doc, compressed_doc) in enumerate(zip(docs, compressed_docs), start=1):\n",
    "        text = Text(f\"Chunk {chunk_id}:\\n{doc.page_content}\")\n",
    "        text.highlight_regex(chunk_pattern, \"bold green\")\n",
    "        text.highlight_regex(terms_pattern, \"underline red\")\n",
    "        rprint(text)\n",
    "\n",
    "        text = Text(f\"Chunk {chunk_id} (compressed):\\n{compressed_doc.page_content}\")\n",
    "        text.highlight_regex(chunk_pattern, \"bold green\")\n",
    "        text.highlight_regex(terms_pattern, \"underline red\")\n",
    "        rprint(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9300e7f-b6b0-48c9-9689-221e4d7e8193",
   "metadata": {},
   "source": [
    "### Retriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "eb6e7c29-b1f0-4542-b6ab-45656f81043c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import TypedDict\n",
    "\n",
    "from langchain_core.documents import Document\n",
    "from langgraph.graph import END, START, StateGraph\n",
    "from langchain_core.messages import HumanMessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8978ec47-b9a5-44ac-b29a-031cac9e2835",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is task decomposition for LLM agents?\n",
      "Loading searcher for index blog for the first time... This may take a few seconds\n",
      "[Apr 08, 14:53:52] #> Loading codec...\n",
      "[Apr 08, 14:53:52] #> Loading IVF...\n",
      "[Apr 08, 14:53:52] Loading segmented_lookup_cpp extension (set COLBERT_LOAD_TORCH_EXTENSION_VERBOSE=True for more info)...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/grad_scaler.py:132: UserWarning: torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Apr 08, 14:53:52] #> Loading doclens...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 6278.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Apr 08, 14:53:52] #> Loading codes and residuals...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 468.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Apr 08, 14:53:52] Loading filter_pids_cpp extension (set COLBERT_LOAD_TORCH_EXTENSION_VERBOSE=True for more info)...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Apr 08, 14:53:52] Loading decompress_residuals_cpp extension (set COLBERT_LOAD_TORCH_EXTENSION_VERBOSE=True for more info)...\n",
      "Searcher loaded!\n",
      "\n",
      "#> QueryTokenizer.tensorize(batch_text[0], batch_background[0], bsize) ==\n",
      "#> Input: What is task decomposition for LLM agents?, \t\t True, \t\t None\n",
      "#> Output IDs: torch.Size([32]), tensor([  101,     1,  2054,  2003,  4708, 22511,  2005,  2222,  2213,  6074,\n",
      "         1029,   102,   103,   103,   103,   103,   103,   103,   103,   103,\n",
      "          103,   103,   103,   103,   103,   103,   103,   103,   103,   103,\n",
      "          103,   103])\n",
      "#> Output Mask: torch.Size([32]), tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0])\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Decomposition#'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">for writing a novel, or (3) with human inputs.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Challenges in long-term planning and task decomposition: Planning over a lengthy history and </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">effectively exploring the solution space remain challenging. LLMs struggle to adjust plans when faced with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">unexpected errors, making them less robust compared to humans who learn from trial and error.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subgoals, enabling efficient handling of complex tasks.\\nReflection and refinement: The agent can do self-criticism</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">and self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">quality of final results.\\n\\n\\nMemory'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Case Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'(2) Model selection: LLM distributes the tasks to expert models, where the request is framed </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">length, task type based filtration is needed.\\nInstruction:'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Generative Agents (Park, et al. 2023) is super fun experiment where 25 virtual characters, </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">each controlled by a LLM-powered agent, are living and interacting in a sandbox environment, inspired by The Sims. </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Generative agents create believable simulacra of human behavior for interactive applications.\\nThe design of </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">generative agents combines LLM with memory, planning and reflection mechanisms to enable agents to behave </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">conditioned on past experience, as well as to interact with other agents.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'MRKL (Karpas et al. 2022), short for “Modular Reasoning, Knowledge and Language”, is a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of “expert” </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">API).'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'ReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">space to be a combination of task-specific discrete actions and the language space. The former enables LLM to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks.</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">examples to guide LLM to do task parsing and planning.\\nInstruction:'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA \u001b[0m\n",
       "\u001b[32mcomplicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask \u001b[0m\n",
       "\u001b[32mDecomposition#'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Task decomposition can be done \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \u001b[0m\n",
       "\u001b[32m\"What are the subgoals for achieving XYZ?\", \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by using task-specific instructions; e.g. \"Write a story outline.\" \u001b[0m\n",
       "\u001b[32mfor writing a novel, or \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with human inputs.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Challenges in long-term planning and task decomposition: Planning over a lengthy history and \u001b[0m\n",
       "\u001b[32meffectively exploring the solution space remain challenging. LLMs struggle to adjust plans when faced with \u001b[0m\n",
       "\u001b[32munexpected errors, making them less robust compared to humans who learn from trial and error.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable \u001b[0m\n",
       "\u001b[32msubgoals, enabling efficient handling of complex tasks.\\nReflection and refinement: The agent can do self-criticism\u001b[0m\n",
       "\u001b[32mand self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the \u001b[0m\n",
       "\u001b[32mquality of final results.\\n\\n\\nMemory'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Case Studies#\\nScientific Discovery Agent#\\nChemCrow \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBran et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is a domain-specific \u001b[0m\n",
       "\u001b[32mexample in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug \u001b[0m\n",
       "\u001b[32mdiscovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in \u001b[0m\n",
       "\u001b[32mthe ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'\u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Model selection: LLM distributes the tasks to expert models, where the request is framed \u001b[0m\n",
       "\u001b[32mas a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context \u001b[0m\n",
       "\u001b[32mlength, task type based filtration is needed.\\nInstruction:'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Generative Agents \u001b[0m\u001b[32m(\u001b[0m\u001b[32mPark, et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is super fun experiment where 25 virtual characters, \u001b[0m\n",
       "\u001b[32meach controlled by a LLM-powered agent, are living and interacting in a sandbox environment, inspired by The Sims. \u001b[0m\n",
       "\u001b[32mGenerative agents create believable simulacra of human behavior for interactive applications.\\nThe design of \u001b[0m\n",
       "\u001b[32mgenerative agents combines LLM with memory, planning and reflection mechanisms to enable agents to behave \u001b[0m\n",
       "\u001b[32mconditioned on past experience, as well as to interact with other agents.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'MRKL \u001b[0m\u001b[32m(\u001b[0m\u001b[32mKarpas et al. 2022\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, short for “Modular Reasoning, Knowledge and Language”, is a \u001b[0m\n",
       "\u001b[32mneuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of “expert” \u001b[0m\n",
       "\u001b[32mmodules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These \u001b[0m\n",
       "\u001b[32mmodules can be neural \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. deep learning models\u001b[0m\u001b[32m)\u001b[0m\u001b[32m or symbolic \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. math calculator, currency converter, weather \u001b[0m\n",
       "\u001b[32mAPI\u001b[0m\u001b[32m)\u001b[0m\u001b[32m.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'ReAct \u001b[0m\u001b[32m(\u001b[0m\u001b[32mYao et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m integrates reasoning and acting within LLM by extending the action \u001b[0m\n",
       "\u001b[32mspace to be a combination of task-specific discrete actions and the language space. The former enables LLM to \u001b[0m\n",
       "\u001b[32minteract with the environment \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. use Wikipedia search API\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, while the latter prompting LLM to generate reasoning\u001b[0m\n",
       "\u001b[32mtraces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly \u001b[0m\n",
       "\u001b[32mformatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... \u001b[0m\u001b[32m(\u001b[0m\u001b[32mRepeated many times\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 11. Illustration of how HuggingGPT works. \u001b[0m\u001b[32m(\u001b[0m\u001b[32mImage source: Shen et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nThe system \u001b[0m\n",
       "\u001b[32mcomprises of 4 stages:\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Task planning: LLM works as the brain and parses the user requests into multiple tasks.\u001b[0m\n",
       "\u001b[32mThere are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot \u001b[0m\n",
       "\u001b[32mexamples to guide LLM to do task parsing and planning.\\nInstruction:'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1:</span>\n",
       "Fig. 1. Overview of a <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> system.\n",
       "Component One: Planning#\n",
       "A complicated <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> usually involves many steps. An <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> needs to know <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">what</span> they are and plan ahead.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Decomposition</span>#\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1:\u001b[0m\n",
       "Fig. 1. Overview of a \u001b[4;31mLLM\u001b[0m-powered autonomous \u001b[4;31magent\u001b[0m system.\n",
       "Component One: Planning#\n",
       "A complicated \u001b[4;31mtask\u001b[0m usually involves many steps. An \u001b[4;31magent\u001b[0m needs to know \u001b[4;31mwhat\u001b[0m they are and plan ahead.\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mDecomposition\u001b[0m#\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2:</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span> can be done (1) by <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> with simple prompting like \"Steps <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> are the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with human inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2:\u001b[0m\n",
       "\u001b[4;31mTask\u001b[0m \u001b[4;31mdecomposition\u001b[0m can be done (1) by \u001b[4;31mLLM\u001b[0m with simple prompting like \"Steps \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m are the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using \u001b[4;31mtask\u001b[0m-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with human inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3:</span>\n",
       "Challenges in long-term planning and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3:\u001b[0m\n",
       "Challenges in long-term planning and \u001b[4;31mtask\u001b[0m \u001b[4;31mdecomposition\u001b[0m: Planning over a lengthy history and effectively exploring \n",
       "the solution space remain challenging. LLMs struggle to adjust plans when faced with unexpected errors, making them\n",
       "less robust compared to humans who learn from trial and error.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4:</span>\n",
       "Subgoal and <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">decomposition</span>: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span> can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4:\u001b[0m\n",
       "Subgoal and \u001b[4;31mdecomposition\u001b[0m: The \u001b[4;31magent\u001b[0m breaks down large tasks into smaller, manageable subgoals, enabling efficient \n",
       "handling of complex tasks.\n",
       "Reflection and refinement: The \u001b[4;31magent\u001b[0m can do self-criticism and self-reflection over past actions, learn from \n",
       "mistakes and refine them \u001b[4;31mfor\u001b[0m future steps, thereby improving the quality of final results.\n",
       "\n",
       "\n",
       "Memory\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5:</span>\n",
       "Case Studies#\n",
       "Scientific Discovery <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Agent</span>#\n",
       "ChemCrow (Bran et al. 2023) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> a domain-specific example in which <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> augmented with 13 expert-designed tools to\n",
       "accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in \n",
       "LangChain, reflects <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">what</span> was previously described in the ReAct and MRKLs and combines CoT reasoning with tools \n",
       "relevant to the tasks:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5:\u001b[0m\n",
       "Case Studies#\n",
       "Scientific Discovery \u001b[4;31mAgent\u001b[0m#\n",
       "ChemCrow (Bran et al. 2023) \u001b[4;31mis\u001b[0m a domain-specific example in which \u001b[4;31mLLM\u001b[0m \u001b[4;31mis\u001b[0m augmented with 13 expert-designed tools to\n",
       "accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in \n",
       "LangChain, reflects \u001b[4;31mwhat\u001b[0m was previously described in the ReAct and MRKLs and combines CoT reasoning with tools \n",
       "relevant to the tasks:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6:</span>\n",
       "(2) Model selection: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> distributes the tasks to expert models, where the request <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> framed as a multiple-choice \n",
       "question. <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> presented with a list of models to choose from. Due to the limited context length, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type based\n",
       "filtration <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> needed.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6:\u001b[0m\n",
       "(2) Model selection: \u001b[4;31mLLM\u001b[0m distributes the tasks to expert models, where the request \u001b[4;31mis\u001b[0m framed as a multiple-choice \n",
       "question. \u001b[4;31mLLM\u001b[0m \u001b[4;31mis\u001b[0m presented with a list of models to choose from. Due to the limited context length, \u001b[4;31mtask\u001b[0m type based\n",
       "filtration \u001b[4;31mis\u001b[0m needed.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7:</span>\n",
       "Generative <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Agents</span> (Park, et al. 2023) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> super fun experiment where 25 virtual characters, each controlled by a \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span>-powered <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agent</span>, are living and interacting in a sandbox environment, inspired by The Sims. Generative <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> \n",
       "create believable simulacra of human behavior <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> interactive applications.\n",
       "The design of generative <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> combines <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> with memory, planning and reflection mechanisms to enable <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span> to \n",
       "behave conditioned on past experience, as well as to interact with other <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span>.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7:\u001b[0m\n",
       "Generative \u001b[4;31mAgents\u001b[0m (Park, et al. 2023) \u001b[4;31mis\u001b[0m super fun experiment where 25 virtual characters, each controlled by a \n",
       "\u001b[4;31mLLM\u001b[0m-powered \u001b[4;31magent\u001b[0m, are living and interacting in a sandbox environment, inspired by The Sims. Generative \u001b[4;31magents\u001b[0m \n",
       "create believable simulacra of human behavior \u001b[4;31mfor\u001b[0m interactive applications.\n",
       "The design of generative \u001b[4;31magents\u001b[0m combines \u001b[4;31mLLM\u001b[0m with memory, planning and reflection mechanisms to enable \u001b[4;31magents\u001b[0m to \n",
       "behave conditioned on past experience, as well as to interact with other \u001b[4;31magents\u001b[0m.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8:</span>\n",
       "MRKL (Karpas et al. 2022), short <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> “Modular Reasoning, Knowledge and Language”, <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> a neuro-symbolic architecture \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> autonomous <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">agents</span>. A MRKL system <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">is</span> proposed to contain a collection of “expert” modules and the \n",
       "general-purpose <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> works as a router to route inquiries to the best suitable expert module. These modules can be \n",
       "neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8:\u001b[0m\n",
       "MRKL (Karpas et al. 2022), short \u001b[4;31mfor\u001b[0m “Modular Reasoning, Knowledge and Language”, \u001b[4;31mis\u001b[0m a neuro-symbolic architecture \n",
       "\u001b[4;31mfor\u001b[0m autonomous \u001b[4;31magents\u001b[0m. A MRKL system \u001b[4;31mis\u001b[0m proposed to contain a collection of “expert” modules and the \n",
       "general-purpose \u001b[4;31mLLM\u001b[0m works as a router to route inquiries to the best suitable expert module. These modules can be \n",
       "neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9:</span>\n",
       "ReAct (Yao et al. 2023) integrates reasoning and acting within <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> by extending the action space to be a \n",
       "combination of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>-specific discrete actions and the language space. The former enables <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to interact with the \n",
       "environment (e.g. use Wikipedia search API), while the latter prompting <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to generate reasoning traces in natural\n",
       "language.\n",
       "The ReAct prompt template incorporates explicit steps <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to think, roughly formatted as:\n",
       "Thought: ...\n",
       "Action: ...\n",
       "Observation: ...\n",
       "... (Repeated many times)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9:\u001b[0m\n",
       "ReAct (Yao et al. 2023) integrates reasoning and acting within \u001b[4;31mLLM\u001b[0m by extending the action space to be a \n",
       "combination of \u001b[4;31mtask\u001b[0m-specific discrete actions and the language space. The former enables \u001b[4;31mLLM\u001b[0m to interact with the \n",
       "environment (e.g. use Wikipedia search API), while the latter prompting \u001b[4;31mLLM\u001b[0m to generate reasoning traces in natural\n",
       "language.\n",
       "The ReAct prompt template incorporates explicit steps \u001b[4;31mfor\u001b[0m \u001b[4;31mLLM\u001b[0m to think, roughly formatted as:\n",
       "Thought: ...\n",
       "Action: ...\n",
       "Observation: ...\n",
       "... (Repeated many times)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10:</span>\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Task</span> planning: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span>: <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">LLM</span> to do <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">task</span> parsing and planning.\n",
       "Instruction:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10:\u001b[0m\n",
       "Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\n",
       "The system comprises of 4 stages:\n",
       "(1) \u001b[4;31mTask\u001b[0m planning: \u001b[4;31mLLM\u001b[0m works as the brain and parses the user requests into multiple tasks. There are four \n",
       "attributes associated with each \u001b[4;31mtask\u001b[0m: \u001b[4;31mtask\u001b[0m type, ID, dependencies, and arguments. They use few-shot examples to \n",
       "guide \u001b[4;31mLLM\u001b[0m to do \u001b[4;31mtask\u001b[0m parsing and planning.\n",
       "Instruction:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are main steps for collecting human data?\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 1. Two directions to approach high data quality.\\nHuman Raters ↔ Data </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Quality#\\nCollecting human data involve a set of operation steps and every step contributes to the data quality:'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'High-quality data is the fuel for modern data deep learning model training. Most of the </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">task-specific labeled data comes from human annotation, such as classification task or RLHF labeling (which can be </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">constructed as classification format) for LLM alignment training. Lots of ML techniques in the post can help with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">data quality, but fundamentally human data collection involves attention to details and careful execution. The </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">community knows the value of high quality data, but somehow we have this subtle'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Collect and aggregate data. This is the stage where more ML techniques can be applied to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">clean, filter and smartly aggregate data to identify the true labels.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Memory can be defined as the processes used to acquire, store, retain, and later retrieve </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">information. There are several types of memory in human brains.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Chain of Hindsight (CoH; Liu et al. 2023) encourages the model to improve on its own outputs </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">by explicitly presenting it with a sequence of past outputs, each annotated with feedback. Human feedback data is a</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">collection of $D_h = \\\\{(x, y_i , r_i , z_i)\\\\}_{i=1}^n$, where $x$ is the prompt, each $y_i$ is a model </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">completion, $r_i$ is the human rating of $y_i$, and $z_i$ is the corresponding human-provided hindsight feedback. </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Assume the feedback tuples are ranked by reward, $r_n \\\\geq r_{n-1} \\\\geq \\\\dots \\\\geq'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Aroyo &amp; Welty (2015) discussed a set of “myths” in the practice of human annotation </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">collection and found all of them somewhat inaccurate, key findings including:'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">for writing a novel, or (3) with human inputs.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'In a harder task, non-expert human annotators were asked to create new gold reference </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">translations. Callison-Burch designed the task in two stages, where the first stage created new translations with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">reference to MT outputs and the second one filtered translations that may seem to be gerated by a MT system. The </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">correlation between experts’ and crowdsourced translations is higher than that between expert and MT system </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">outputs.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Resources:\\n1. Internet access for searches and information gathering.\\n2. Long Term memory </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">management.\\n3. GPT-3.5 powered Agents for delegation of simple tasks.\\n4. File output.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Almost 100 years later, Callison-Burch (2009) did an early study on using Amazon Mechanical </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Turk (AMT) to run non-expert human evaluation on Machine Translation (MT) tasks and even to rely on non-experts to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">create new gold reference translations. The setup for human evaluation was simple: Each turker is shown a source </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">sentence, a reference translation, and 5 translations from 5 MT systems. They are asked to rank 5 translations from</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">best to worst. Each task is completed by 5 turkers.'</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 1. Two directions to approach high data quality.\\nHuman Raters ↔ Data \u001b[0m\n",
       "\u001b[32mQuality#\\nCollecting human data involve a set of operation steps and every step contributes to the data quality:'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'High-quality data is the fuel for modern data deep learning model training. Most of the \u001b[0m\n",
       "\u001b[32mtask-specific labeled data comes from human annotation, such as classification task or RLHF labeling \u001b[0m\u001b[32m(\u001b[0m\u001b[32mwhich can be \u001b[0m\n",
       "\u001b[32mconstructed as classification format\u001b[0m\u001b[32m)\u001b[0m\u001b[32m for LLM alignment training. Lots of ML techniques in the post can help with \u001b[0m\n",
       "\u001b[32mdata quality, but fundamentally human data collection involves attention to details and careful execution. The \u001b[0m\n",
       "\u001b[32mcommunity knows the value of high quality data, but somehow we have this subtle'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Collect and aggregate data. This is the stage where more ML techniques can be applied to \u001b[0m\n",
       "\u001b[32mclean, filter and smartly aggregate data to identify the true labels.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Memory can be defined as the processes used to acquire, store, retain, and later retrieve \u001b[0m\n",
       "\u001b[32minformation. There are several types of memory in human brains.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Chain of Hindsight \u001b[0m\u001b[32m(\u001b[0m\u001b[32mCoH; Liu et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m encourages the model to improve on its own outputs \u001b[0m\n",
       "\u001b[32mby explicitly presenting it with a sequence of past outputs, each annotated with feedback. Human feedback data is a\u001b[0m\n",
       "\u001b[32mcollection of $D_h = \\\\\u001b[0m\u001b[32m{\u001b[0m\u001b[32m(\u001b[0m\u001b[32mx, y_i , r_i , z_i\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\u001b[0m\u001b[32m}\u001b[0m\u001b[32m_\u001b[0m\u001b[32m{\u001b[0m\u001b[32mi\u001b[0m\u001b[32m=\u001b[0m\u001b[32m1\u001b[0m\u001b[32m}\u001b[0m\u001b[32m^n$, where $x$ is the prompt, each $y_i$ is a model \u001b[0m\n",
       "\u001b[32mcompletion, $r_i$ is the human rating of $y_i$, and $z_i$ is the corresponding human-provided hindsight feedback. \u001b[0m\n",
       "\u001b[32mAssume the feedback tuples are ranked by reward, $r_n \\\\geq r_\u001b[0m\u001b[32m{\u001b[0m\u001b[32mn-1\u001b[0m\u001b[32m}\u001b[0m\u001b[32m \\\\geq \\\\dots \\\\geq'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Aroyo & Welty \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2015\u001b[0m\u001b[32m)\u001b[0m\u001b[32m discussed a set of “myths” in the practice of human annotation \u001b[0m\n",
       "\u001b[32mcollection and found all of them somewhat inaccurate, key findings including:'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Task decomposition can be done \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \u001b[0m\n",
       "\u001b[32m\"What are the subgoals for achieving XYZ?\", \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by using task-specific instructions; e.g. \"Write a story outline.\" \u001b[0m\n",
       "\u001b[32mfor writing a novel, or \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with human inputs.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'In a harder task, non-expert human annotators were asked to create new gold reference \u001b[0m\n",
       "\u001b[32mtranslations. Callison-Burch designed the task in two stages, where the first stage created new translations with \u001b[0m\n",
       "\u001b[32mreference to MT outputs and the second one filtered translations that may seem to be gerated by a MT system. The \u001b[0m\n",
       "\u001b[32mcorrelation between experts’ and crowdsourced translations is higher than that between expert and MT system \u001b[0m\n",
       "\u001b[32moutputs.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Resources:\\n1. Internet access for searches and information gathering.\\n2. Long Term memory \u001b[0m\n",
       "\u001b[32mmanagement.\\n3. GPT-3.5 powered Agents for delegation of simple tasks.\\n4. File output.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[33mpage_content\u001b[0m=\u001b[32m'Almost 100 years later, Callison-Burch \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2009\u001b[0m\u001b[32m)\u001b[0m\u001b[32m did an early study on using Amazon Mechanical \u001b[0m\n",
       "\u001b[32mTurk \u001b[0m\u001b[32m(\u001b[0m\u001b[32mAMT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m to run non-expert human evaluation on Machine Translation \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m tasks and even to rely on non-experts to \u001b[0m\n",
       "\u001b[32mcreate new gold reference translations. The setup for human evaluation was simple: Each turker is shown a source \u001b[0m\n",
       "\u001b[32msentence, a reference translation, and 5 translations from 5 MT systems. They are asked to rank 5 translations from\u001b[0m\n",
       "\u001b[32mbest to worst. Each task is completed by 5 turkers.'\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 1:</span>\n",
       "Fig. 1. Two directions to approach high <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality.\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> Raters ↔ <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Data</span> Quality#\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Collecting</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> involve a set of operation <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">steps</span> and every step contributes to the <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 1:\u001b[0m\n",
       "Fig. 1. Two directions to approach high \u001b[4;31mdata\u001b[0m quality.\n",
       "\u001b[4;31mHuman\u001b[0m Raters ↔ \u001b[4;31mData\u001b[0m Quality#\n",
       "\u001b[4;31mCollecting\u001b[0m \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m involve a set of operation \u001b[4;31msteps\u001b[0m and every step contributes to the \u001b[4;31mdata\u001b[0m quality:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 2:</span>\n",
       "High-quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> is the fuel <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> modern <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> deep learning model training. Most of the task-specific labeled <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> \n",
       "comes from <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> LLM alignment training. Lots of ML techniques in the post can help with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> quality, \n",
       "but fundamentally <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>, but somehow we have this subtle\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 2:\u001b[0m\n",
       "High-quality \u001b[4;31mdata\u001b[0m is the fuel \u001b[4;31mfor\u001b[0m modern \u001b[4;31mdata\u001b[0m deep learning model training. Most of the task-specific labeled \u001b[4;31mdata\u001b[0m \n",
       "comes from \u001b[4;31mhuman\u001b[0m annotation, such as classification task or RLHF labeling (which can be constructed as \n",
       "classification format) \u001b[4;31mfor\u001b[0m LLM alignment training. Lots of ML techniques in the post can help with \u001b[4;31mdata\u001b[0m quality, \n",
       "but fundamentally \u001b[4;31mhuman\u001b[0m \u001b[4;31mdata\u001b[0m collection involves attention to details and careful execution. The community knows \n",
       "the value of high quality \u001b[4;31mdata\u001b[0m, but somehow we have this subtle\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 3:</span>\n",
       "Collect and aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span>. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> to identify the true labels.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 3:\u001b[0m\n",
       "Collect and aggregate \u001b[4;31mdata\u001b[0m. This is the stage where more ML techniques can be applied to clean, filter and smartly \n",
       "aggregate \u001b[4;31mdata\u001b[0m to identify the true labels.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 4:</span>\n",
       "Memory can be defined as the processes used to acquire, store, retain, and later retrieve information. There <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> \n",
       "several types of memory in <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> brains.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 4:\u001b[0m\n",
       "Memory can be defined as the processes used to acquire, store, retain, and later retrieve information. There \u001b[4;31mare\u001b[0m \n",
       "several types of memory in \u001b[4;31mhuman\u001b[0m brains.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 5:</span>\n",
       "Chain of Hindsight (CoH; Liu et al. 2023) encourages the model to improve on its own outputs by explicitly \n",
       "presenting it with a sequence of past outputs, each annotated with feedback. <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Human</span> feedback <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">data</span> is a collection of\n",
       "$D_h = \\{(x, y_i , r_i , z_i)\\}_{i=1}^n$, where $x$ is the prompt, each $y_i$ is a model completion, $r_i$ is the \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> rating of $y_i$, and $z_i$ is the corresponding <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span>-provided hindsight feedback. Assume the feedback tuples\n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> ranked by reward, $r_n \\geq r_{n-1} \\geq \\dots \\geq\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 5:\u001b[0m\n",
       "Chain of Hindsight (CoH; Liu et al. 2023) encourages the model to improve on its own outputs by explicitly \n",
       "presenting it with a sequence of past outputs, each annotated with feedback. \u001b[4;31mHuman\u001b[0m feedback \u001b[4;31mdata\u001b[0m is a collection of\n",
       "$D_h = \\{(x, y_i , r_i , z_i)\\}_{i=1}^n$, where $x$ is the prompt, each $y_i$ is a model completion, $r_i$ is the \n",
       "\u001b[4;31mhuman\u001b[0m rating of $y_i$, and $z_i$ is the corresponding \u001b[4;31mhuman\u001b[0m-provided hindsight feedback. Assume the feedback tuples\n",
       "\u001b[4;31mare\u001b[0m ranked by reward, $r_n \\geq r_{n-1} \\geq \\dots \\geq\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 6:</span>\n",
       "Aroyo &amp; Welty (2015) discussed a set of “myths” in the practice of <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> annotation collection and found all of \n",
       "them somewhat inaccurate, key findings including:\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 6:\u001b[0m\n",
       "Aroyo & Welty (2015) discussed a set of “myths” in the practice of \u001b[4;31mhuman\u001b[0m annotation collection and found all of \n",
       "them somewhat inaccurate, key findings including:\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 7:</span>\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">Steps</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> XYZ.\\n1.\", \"<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">What</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> the subgoals \n",
       "<span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> writing a novel, or\n",
       "(3) with <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> inputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 7:\u001b[0m\n",
       "Task decomposition can be done (1) by LLM with simple prompting like \"\u001b[4;31mSteps\u001b[0m \u001b[4;31mfor\u001b[0m XYZ.\\n1.\", \"\u001b[4;31mWhat\u001b[0m \u001b[4;31mare\u001b[0m the subgoals \n",
       "\u001b[4;31mfor\u001b[0m achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" \u001b[4;31mfor\u001b[0m writing a novel, or\n",
       "(3) with \u001b[4;31mhuman\u001b[0m inputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 8:</span>\n",
       "In a harder task, non-expert <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> annotators were asked to create new gold reference translations. Callison-Burch \n",
       "designed the task in two stages, where the first stage created new translations with reference to MT outputs and \n",
       "the second one filtered translations that may seem to be gerated by a MT system. The correlation between experts’ \n",
       "and crowdsourced translations is higher than that between expert and MT system outputs.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 8:\u001b[0m\n",
       "In a harder task, non-expert \u001b[4;31mhuman\u001b[0m annotators were asked to create new gold reference translations. Callison-Burch \n",
       "designed the task in two stages, where the first stage created new translations with reference to MT outputs and \n",
       "the second one filtered translations that may seem to be gerated by a MT system. The correlation between experts’ \n",
       "and crowdsourced translations is higher than that between expert and MT system outputs.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 9:</span>\n",
       "Resources:\n",
       "1. Internet access <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> searches and information gathering.\n",
       "2. Long Term memory management.\n",
       "3. GPT-3.5 powered Agents <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> delegation of simple tasks.\n",
       "4. File output.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 9:\u001b[0m\n",
       "Resources:\n",
       "1. Internet access \u001b[4;31mfor\u001b[0m searches and information gathering.\n",
       "2. Long Term memory management.\n",
       "3. GPT-3.5 powered Agents \u001b[4;31mfor\u001b[0m delegation of simple tasks.\n",
       "4. File output.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Chunk 10:</span>\n",
       "Almost 100 years later, Callison-Burch (2009) did an early study on using Amazon Mechanical Turk (AMT) to run \n",
       "non-expert <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> evaluation on Machine Translation (MT) tasks and even to rely on non-experts to create new gold \n",
       "reference translations. The setup <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">for</span> <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">human</span> evaluation was simple: Each turker is shown a source sentence, a \n",
       "reference translation, and 5 translations from 5 MT systems. They <span style=\"color: #800000; text-decoration-color: #800000; text-decoration: underline\">are</span> asked to rank 5 translations from best to \n",
       "worst. Each task is completed by 5 turkers.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;32mChunk 10:\u001b[0m\n",
       "Almost 100 years later, Callison-Burch (2009) did an early study on using Amazon Mechanical Turk (AMT) to run \n",
       "non-expert \u001b[4;31mhuman\u001b[0m evaluation on Machine Translation (MT) tasks and even to rely on non-experts to create new gold \n",
       "reference translations. The setup \u001b[4;31mfor\u001b[0m \u001b[4;31mhuman\u001b[0m evaluation was simple: Each turker is shown a source sentence, a \n",
       "reference translation, and 5 translations from 5 MT systems. They \u001b[4;31mare\u001b[0m asked to rank 5 translations from best to \n",
       "worst. Each task is completed by 5 turkers.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "retriever = RAG.as_langchain_retriever(k=search_kwargs[\"k\"])\n",
    "\n",
    "for query in queries:\n",
    "    print(query)\n",
    "    docs = retriever.invoke(query)\n",
    "    rprint(docs)\n",
    "\n",
    "    chunk_pattern = re.compile(r'^Chunk \\d+.*:$', flags=re.MULTILINE)\n",
    "    terms_pattern = re.compile(rf'\\b({\"|\".join(query.split())})\\b', flags=re.IGNORECASE)\n",
    "    \n",
    "    for chunk_id, doc in enumerate(docs, start=1):\n",
    "        text = Text(f\"Chunk {chunk_id}:\\n{doc.page_content}\")\n",
    "        text.highlight_regex(chunk_pattern, \"bold green\")\n",
    "        text.highlight_regex(terms_pattern, \"underline red\")\n",
    "        rprint(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7dace376-8dc7-473d-a81d-fc5066d2bfdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class State(TypedDict):\n",
    "    question: str\n",
    "    context: list[Document]\n",
    "    answer: str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "76b24d56-8816-4a47-ad31-1f5863bee37d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAG0AAAFNCAIAAACFQXaDAAAAAXNSR0IArs4c6QAAHw1JREFUeJztnXdYFGf+wN/tfYFdegcp0lUsRL0oUbHExAbRQz29JJfEckZjjZrTFONdLmp+XkxiNKfYY0NjOfUssSQxRkUR0AUEKStL2WV7L78/xuM42d2ZhXfZHZjPc8897s47M18+eafs274km80GCLoM2dMB9BAIj3AgPMKB8AgHwiMcCI9woEI5iqRGr1WatWqLxWQz6KxQjulWaEwylUJi8ylsHiUokkmmkLp4QFJX3h9Fd1RVD9TVJZroZI7NBthcil8Q3ajHgUc6iyxvNmqVFoPW8rRaH5HAjk3j9B3Mo1I7eYF20mPJL4qff5BGJ7Nj07gxqRwKtav/PT1LzUNN1QNNfYWu7yDeoBxBJ47gskfpU8P5PZKQGNbQV4UMFqUTp/Rmbp6V3r8qz5kdFJPKdWlH1zyK7qjuXGqd+GYIX0BzPUh8YDRYfzzS5BdId6liuuDxSZmm/I4qZ3ZwZyPEEzfPSmkMcuYoP4zlsXosutLaWGsYN6dXSET4+XSLTm0ZNSMIS2FMj6faR9pakbZXSQQADJ3oT6OT71+TYymM7lGtMN+/Lp/0ThiM2HDGi1MDpA1GcaUWtSS6x59OtiRm8iAFhj/ShvtcL2xBLYbisVlsaG00JgzovR4Dwhh+QfTyuyrnxVA8lvykGD7FH2pg+GPYq8KKoi54NBmtotuq8Dg27MBwBteXpm61NNXrnZRx5rG6RBOTynFDYM44fPjw+vXrO7HjypUrT5065YaIAAAgJo1T/UDjpIAzjw1Vuvj+rv086joPHz7s5h2xEJfBbRYbnBRw9h7+/ed12TMCAsOZ7oisqKho27ZtlZWVFoslISFhwYIFAwYMeOutt+7evYsU2L9/f2Ji4rlz5/bu3VtbW0un09PT05cuXRoeHo7UPhKJFB0dvW/fvo0bNy5ZsgTZi8vl/vjjj9CjNRutO9ZWz/usj6MCzuqjRmXm8OA0UD6HTqdbvHhxbGzsrl27CgoK4uPjFy1apFQqN2/e3Ldv35ycnIsXL8bFxZWWlq5du3bYsGF79+7dunWrTqdbvnw5cgQajVZZWfno0aOtW7empaWdPXsWALB8+fKTJ0+6I2AqnUyhkAw6i8MCTnbWqixsnltadCQSiUajmTBhQkxMDABg2bJlY8aModPpTCaTSqXS6XRfX18AQFRU1N69e+Pj46lUKgAgPz//vffek8lkAoEAAFBfX//dd9/5+PgAAAwGAwCAzWYjH90Bh0/RKC2OmrgcerRarSwOmUR2S8NiZGRkVFTU2rVrc3Nzs7KyEhMTMzMzOxbjcrlisfjLL7+sq6vT6/UmkwkAoFQqEY9RUVHus9YRJoditTi8Bzq8rslkss0GdGqHNbkrUCiUnTt3jh49urCwcNasWa+88sqZM2c6Frtw4cKqVatSU1O3bt164MCBNWvWtN/K5XbrM7C1ycjhO6x2zu6PbD5VqzS7Jyrg5+e3ePHikydPHj58ePDgwevWrev4wC0sLBw4cOC8efOio6P9/f31emdvcG7FarEZdFYW1+FdzpnH0Bim1j31USwWtz1VY2NjV69eTSaTHz9+jHzT9gphNBqRGyXCuXPn2m/tiPvGKqkV5uhkZ6/Szjz6hzEq76ndEBWQSCQrVqzYt2/fkydPampqdu7cSSaT09LSAAA8Hk8kEolEIrlcnpqaevPmzZKSkoaGho0bN/r7+wMAysrKOlZMBoPBYDDu3r0rEonMZvjXUNUDDV/g7JlMcfLjgeNDvXGipX821jZh7ISGhoaGhh47dmz37t0nT57UarWrVq1KT08HAPj4+Jw5c+b48eP9+/fPycmpqKj49ttvz549m5mZuWTJkuLi4u+//z46Orq2tlatVk+aNKntmFartbCw8Pz587m5uQwGA27Av5yWpg7zcdabYnPK+T0NTXU652V6PEa9ufDLOudlUNp7Egfyfjkjg/vfFnfcPCuLRus+RPm5EpXEuXtJLq7UhcWx7BZYuHBhSUmJ3U0Wi4VCsf+A+/DDD0eMGOH81J1m5MiRjuJBXrnsbr148SLytv8cGqW5okj9+kcxzk+K3s/VWKsvvqEYk2+/u0er1SLxdcRsNtuNDADAYrEcbeo6KpX9tkLk+ePovDye/bbqn0+3BIQy4tFasjH1Fz64oZBKDCNzA1FL9jCKr8tbm0wjpgWglsTUX5g23MdmBbfOSWHEhhsq76kr76uxSHRtHMCdS60Ws23w2M4Mf8Ed5XdVVSWacX/A2tXswvCqzFF+ZpP1/B5JZ2PDDb9dkFU9cEFiZ8ZJld9VXT3WNGScMP13vhiK44yKItXPp6Rpw/gDRrl22XVm3J7JYPn5tKzqgTp9uG9MGkcQRHf1CN6GqtVUXaJ5UqqhsyhDXxF2YhRY58eRquXm4hvy6gcaqxXEpHGoVBKHT+ULqBYcDCMFFApJJTdplRad2tJQpdNrrTGpnOQhvIDOdqJ0aTwugrzZKHmiV7WaNUozmUJSySA3E9y/fz8lJQXu+ybXl2o129h8CseXGhTJDAjr6u9xCB7dzejRo48ePdq+Ac0LIeYrwIHwCAcceExMTPR0COjgwKNIJPJ0COjgwGN3dq52Ghx4VCgUng4BHRx4DAkJ8XQI6ODAY0NDg6dDQAcHHlNSUjwdAjo48FhaWurpENDBgUdcgAOPyDAKLwcHHlta0KeveBwceCTqIxyI+tiLwIHHPn0czhLwHnDgsW18qTeDA4+4AAcek5KSPB0COjjw6NYJb7DAgUdcgAOPRHsPHIj2nl4EDjwS/a5wIPpdexE48Ej0X8OB6L+GA9HeAweivacXgQOPQUGYVmD0LDjw2NjY6OkQ0MGBx+TkZE+HgA4OPJaVlXk6BHRw4JGoj3Ag6iMckIXhvBzvnYc0YcIEZA5XS0uLQCAgk8k2m83f33/Xrl2eDs0O7lrcoOuQSKSnT58i/5ZIJMgycIsXL/Z0XPbx3uu6f//+z10rMTExo0aN8lxEzvBej7Nnzw4O/u9MchaLNXPmTI9G5Azv9ZiYmNivX7+2j3369MnJyfFoRM7wXo8AgFmzZiE/rtlsdn5+vqfDcYZXe0xKSsrIyLDZbDExMd5cGTvzvDYarC1ig17bTbP+x704p77cNDlnSlWJs2WnIUJnkIQhDCdLPdrFtffHf++XPC7WBEezyO5Z79UboLPIdSJNeBxrdH4QjYH1esXq0Wq1FX4l7pPO75PB71qc+KCxVvfr2eZpC8OYHEwVE6vHk1+L4zN9IxK7e3l7D6KWm87vFs9dF42lMKZ6W1OmYfKovUoiklYhfgC/+AakPD4AgJanRgazp6WGwwLHh9r4xFk6gDYwedRpLD4BuF/sqBP4+NONBkxvJpg8mo02i8lLm4XcitUC9NhWrPbq93AcQXiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeIQD4REO3u5x0pRRe/bu9HQU6HjeY3X14xn5Ex1tnf/Okqys4d0bUWfw/LiU8nJn06vHjnWo2KtwV32cPHX00WMHVr6/KGfcC2q1GgBw6fL5d+bNHv/y8Km5OV9u24Tk2NpdsP2vn61vbJRkjxp49NiBwhOHp0wb89NPV6dMG/P1N188d12XVzxasXLhpCmjXn7lxQ/+skwiaQAA7Pxu28RXRyCpDREOHipwflJ34C6PVCr11OnjsTFxWzZtZzKZN278+MmGNZmZQ3Z8e3DF8nXXrl/atGUDAGDG9DlTp84IDAw6cfziKxOn0Wg0vV53vPDQyhXrJ03Ka3/AxkbJe0vfJpHJWzZt3/T5N0qVYunyeUaj8aXssRqN5s7dW20lr127lDVkOJfLdXRSd+AujyQSiclgvv3WopSUdCqVeuDQ7oyMAX96c2F4WETWkGF/evPPFy/+q6mpkclkMugMEonk4+PLYDBIJJJer8+dlp81ZFhoSFj7A/5w6iiJRFq7ZkNsbFzfxOTVqz5uaBBfvXYpNjYuMjL6xo0rSLHGRskjUdmoUeMAAHZPKpW6ZVUlNz5nUlLSkX9Yrdby8ocDM7PaNvXLyAQAVFVV2N0xOTmt45cPH5b0TUzhcZ/lJQoKCg4JCausFAEAskfm/PTzVavVCgC4dv0Sh8PJGjLc0Umrn7hlVpMbnzMczrPcYHq93mKx7C7YvmfvjvYFpDL7VaNtx/ZoNOqKSlHOuBfavjGZTMgRXsrOKdjzbUnJ/fT0/levXRo+LJvBYCAJrzqetLXVLWnbuuN5jWQRnjplxssTJrf/3tfPhdwkHA43La3f0iX/k+KVxWIDACIjo2Nj467fuBIaGl5aWjznD285OalA4JZV57rDI5lMjo/v29jYEBn5rE/dZDI1NTfyeS4MzUhKSj1/4XRoaHhbwoq6uhqh8JmU7JE55y+cDg+P9PMTDOg/yMlJ3ZRdt5vew2dM/8O165cPHNxdV1dTUSn6dOMHi959Q6PRAAC4XJ5U2lJcXIS8xzjilYnTdDrt3z5bX1Epqq+v3bN35x/feO3Ro2dLgGRn59TX1546fWzkyDFtmfXsnlSr1brjD+wmjy/+7qXV73986fK519+cvnzFApPZtGXTdg6HAwAY9dK40NDwpcvn/eucs1TqwcEhmzdtl8mki9594535s2/99vMnH29ueyKFhYYnxPd9/Lhi9EvjnJ+UzWa74w/ENL7nxyPNXD964iAczMuHS1Ot/t7llmnvok888fzv654B4REOhEc4EB7hQHiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeIQDJo8sHoVM7bETCp1iwzjhBZNHvh+1qUbX5ZjwR1O9nsnBpAhTofBEllYJOas1LlA0GaOTMbX7YvLI86X1Hcy78j0O8ghC5NezzXwhNTwek0cX5l9X3lPfOi9LHOQjDGUy2T12uqHFZG0W6xuqtMIQ+uCxWHs0XZvHLm0w3L+mkDeblFIThuJwMBgMdDqdROqmB50ghMFkkxMGcKKTXehZ9N71pNog8tr3IgiPcMCBRyJvChyIvClwINZhhwOxDjsc+vbt6+kQ0MGBx0ePHnk6BHRw4JG4P8KBuD/2InDgMT4+3tMhoIMDjxUV9qeHeBU48IgLcOCRyWR6OgR0cODRfZMrIYIDj3w+DlZAxYFHpVLp6RDQwYFHXIADj2FhYRhKeRgceBSLxZ4OAR0ceMQFOPBItPfAgWjv6UXgwCPR7woHot+1F4EDj8TzGg7E8xoOXj5iDwEHHuVyTJlLPAsOPOICHHhMTEz0dAjo4MCjSCTydAjo4MBjUlKSp0NABwceHz50tvCrl4ADj8S4PTgQ4/bggIv7o/fOQ8rLy2MymWQyuby8PDw8HPk3k8ncvn27p0Ozg+fXD3fE48ePyeRnl0t1dTUAgEKhEHntXWbw4MHPfRMRETFjxgwPhYOC93qcO3du+xEpZDJ56tSp3TZb01W812NWVlZCQkLb7Ts8PHz69OmeDsoh3usRqZI+Pj7InTEvL69t4VsvxKs9ZmVlJSYm2my20NBQb66MWJ/XZpNVp+6mRPbPMSP3jzWPm/KmzNIorAB4IAYanYxlqQ+U98eHt5TF1xUyiZHF9d5ryq0w2BSjzpLyAn/gGGdrLDjzeOuCrOWpqd8IAU9Ac0+Q+EAtN1XdV6lajePmBDsq49Djr+dkSqk5a2KgOyPEE2U35bIG/fi59lXav/Jbm4wtYgMhsT3JWb50FuVJmcbuVvseW8QGm81L33g9CJ1JaayxP+jfvke1whIQgYPZFt2MMJSh19p/Z7D/3mMyWE04mGzR3VjNNkfrk3n1eziOIDzCgfAIB8IjHAiPcCA8woHwCAfCIxwIj3AgPMKB8AgHwiMcerjH9R+uPHf+VDecqId7LC/vprGT9vsVbp2XGfUgY6QL+YBbWpo3bdlQVPQbl8vLnZav0aivXb9csOsoAMBsNu/b/93lKxcaGxsCAoLycmdOejUXAFBTUz339bzNm745dvzggwf3yGRy9sgxC+YvRfqp5fLWr77Zcv/+HYVCHhsb/6c3F/bvNxAAUHji8J69O5a9t/bzzZ/kjHl53juLW1tlX2//4u7dWyqVMiAgaOrk6VOnzgAAZI8aiMTG5XJPnfwRSXN/5Mi+mtpqFov9UvbYN99Y4NKiNjVl6rpHqvF/DOm4Cdo4qc83f1JZKfr4o00CP+HOf26rrX1Cpz/L8PDN9v87c7Zw8aJVKakZd+78+uW2z6lU6ssTJlOoVADAtq82LXn3/U8+2nTn7q1ly+enpfXPHjnGarWuXPVntUa9csV6ocD/5A9HVr2/6Otte2Jj42g0ml6vO154aOWK9Ugu4c8+/6iu9skHaz4VCIQPSu5t2rwhMCh4+LCRhw+dfW3GhD8vXI6kZ0fS3Of/fu7atZ/W19du3rJBoZSvef9jKH8+nOtaJpPeuvXzrJlvDBqY1adP/NrVG5SKZ5Ne1Gr1yR+OTH9t9tixE8PDIia9mjs2Z+KBg7vb9h3x4mgkc3vmgMGhIWEiURkA4PadX8srHi1bunZA/0FRUTELFywLCgo5XngIAEAikfR6fe60/Kwhw0JDwgAAC+Yv/eyzbRkZAyIioiaMnxTXJ+H27ZsAAD7fBwDAZrN9+D6O0tw3NTVCMQCnPorFdTabLTUlA/nI4XAyM4fU1FYDAB4/Ljebze3zy2dkZJ45e6Itf3Kf2P8uA8fl8tRqFZLFnkajIZnokUFS6Wn9kSz2CG2ZhgEALCbrwKHd9+7dVijkVqtVpVKGhUU8FyGS5n7unLfbvkEOXlVVERgY1HUDcDwqFHIAAKtdSmSkLgAAtFoNAGDJ0rfbhoohd2RZqxT5SGcw2h8K2arVakwm09jxQ9u+t1gsAoGw7SOH82zRfrPZvGLVQovFsnDBssiIaAqFsvYvSztGqNfr7aa5l8paYAiA5BFxYWi3gJZK9WzxIuQPXrP6k9iYuPa7BAYENTU7vKY4HC6dTt+x/UD7L9uGlbbn4cOSqqrK/9uyIz29P/KNQt4aEhz6XDFHae59/Vx4ljoBjkfkOnokKo2NjQMAaDSaO3d+FfoHAABiY+NpNFprqyxyxLP88nJ5K4lEansK2aVv3xSj0WixWGJink0alkgafH39OpY0GA3tq39paXGD5GliYnJbAaSCO0pzz+fBWfQLznMGSYe+f/8/S0uLa2ufbPzbX/z+cw1yudyJE6fuLth++cqFpw3ionu3l62Y/9fP1js/YOaAwfFxiZ9u/ODevTsNkqcXL5176+38kz8c6Vgyrk8CnU4/XnhIKm357fbNrf/4bNDArLr6mtZWGYPBYDAY94vvVlSKzGaz3TT3Go39fn1Xgfbes3bNhr9v+njJ0rf9hQEzZ74uFPg/evRsPYT57yzhcXnf7tgqlbYIBMKhL7z4xusLnB+NQqH87a//+Hr7F+s+XKHX64KDQ2fPfjMvd2bHkr6+fiuWr9u588sL/z6TkJC0csX65pamjz95/71l7+z67vDvZ8w99H3BL79c37f3BJLm/uCh3bt2f8PhcFNTM7Zs2s7hcKD8+dDew/V6vcls4nF5yMf3lr7D5/usX/c3KFF6Cd3xHr56zWJZq3TpkjV+foJfbl4vund744YvYB3c+4F5XX/19eYP1i0zGPShoeGrVqzPyhoO6+DeDzSPAoFw7ZoNsI6GO3p4e0+3QXiEA+ERDoRHOBAe4UB4hAPhEQ6ERzgQHuFAeISD/d+FdCbJCoj5M89DppA4PvaN2a+PPD9ac69MZO+cFrHe0XxV+x4DIxjeuoCBJzHqLcEx9scNOKyPYXHMa8ckbg4MTxRdlpJIIMJBmntn84ZLf1FU3FNnjBD6BdEp1N77RJI26B/fV9JopBenBjgqgzKPvbpUc++qXFKtp1A9dp1brBYymeKp07M4FBqTnDqUlzrU2fKyWNeTMug8s64CAGDy5MkFBQXIgh/dD51JxvKowNoezmB57Lo2WbR0JsmDAWDBq4PDETjwSKzDDgdiHXY4EPk+4EDk+4ADUR/hQNRHOBB5SeFA5CXtReDAI/GcgQPxnOlF4MBjVFSUp0NABwcea2pqPB0COjjwiAtw4NFTLeEugQOPCoXC0yGggwOPdqcVehs4CNFq9VgXG3Zw4BEX4MAjkZcUDkRe0l4EDjwS/a5wIPpdexE48Ei048KBaMftReDAI4/H83QI6ODAo0ql8nQI6ODAI/GcgQPxnIFDWFiYp0NABwcexWKxp0NABwceQ0OfXzzPC8GBx6dPn3o6BHRw4DE5ORlDKQ+DA49lZWWeDgEdrPO5up/MzEybzUYmk61WK/L/FAplzpw5Cxcu9HRodvDe+hgXF4csqYv0u5LJ5PDw8Pz8fE/HZR/v9Th79uznFkkfN26cQABnOVvoeK/HiRMnxsTEtH2MiIjIy8vzaETO8F6PAICZM2ey/7OW9tixY722Mnq7x/HjxyNVMjo6+rXXXvN0OM7wao8AgOnTpzOZzPHjx3tzZYT23mM2WqtLNXUVBmmDQae2UOlkpdQIIzwAADCbTFQqFUBaeMQvkKHXmFlcqm8QLSSaEZfOdbQEikt01WNdubboirK+QsMLZPMDOGQqicagUhkUEtlL11shAZtRbzEbLBazVd2iVbdoffzp/Ub69B3YpVb3znuU1OivFUp1Gpt/tC9HwOpKEJ5FI9fL65UWo+l3U/xjku0vh4JKZzzabOD6D611Ip1PKJ8rxLHB9uhUBmm13C+QOn5OYCcGXHbG49ldEqWSHJwgxFAWZ8jqlEalZsaycFd3dNnjvw82K5UUYSQOxmx3DrVUp5Mp8xa51ujpWg0+v6dRperJEgEAXCGLJeAd/LzOpb1c8HjnUqtcThJE9GSJCFwhm+nDvbC/CfsuWD3KGg1lv6mD4nvgPdEufuF8WZO16gHWrnOsHq8XSn1Cen5NbI9fhM/1QhnGwpg8Sp7oW5st/EA4qVrwAoNDp3MZZTcxzd7B5LHoR7k33xaPn/r73//xe3cc2S/Cp/gnTJc2Jo/VJRqufw9533YJJpeuajUrZSbUkugea0VanpBBpnh7y5Cb4Pmzqx6g5/BCX2+vqUbPEbrxzlhUfOHqTwcam6sZDHb/tJzxo+fR6UwAwJ5Dq0kkkBj/wpVrexSq5kD/qCkTl0VFpAEAFMrmIyc2VFbfYTK5Lwya6r7YAAAcIatZjL5UMHota5GYSG5bxbKk7Or+Ix8kxA1eumDf9CkfFJdePvrDRmQThUKtrrlfW1e6eP6e9SvPsdk+3x//BNl08Nh6SVPVG7O3zPvjVxqN/EHZFTeFBwCg0CgtYgNqMXSPGoWZxoCWNuk5Ll/fExs9YMKY+f7CiKSEoS/nLLh7/5xc8Sw/pNGoe3X8YgadRaczB6SPa2p5YjTq5Yqmyqrb2b/7Q3zswKDAmCkTlzEZbrxcaAyKVmVGLYbukUIlURkQWjo7YrVa658+TIgb3PZNbPQAAECDpBL56C+MQK5xAACbxQcAaHXKpuYnAIDI8GeDLEgkUkS4GwdcUBkUJpeK2gqBXtEMOivN5JYZpyaT3mq1XLi8499Xvmv/vVL1LOcqlcrosJPNYNQ+t4lB72SjIRYsJqtGbiKhtcaje+T4UM0G9IrdCWg0JoVCHZ41fUjmq+2/53KcdcXQ6SwAgF6vbvtGp3fjwGezwcLioltCv655fhSTwQIpqv89N5kcFtK3Vd4QGBCN/E/gF0YmU9lsZwuaBQgjAQBPJRXIR4vF/Lj6rjvCQzAbLWw++m0N3WNwJNOkRX9gdY6Rw2c9KLty+VpBU3ON+KnowNF123a+pdc7e18T+IVERaRdvlYgqvxV/FR05MSnVCrNTeEBAHQKQ3BUx9vL86B7jEnlyCVaSFE9T3pK9u+nfVhUfGHTl/nfFiyyWEzzXv+KyUR5/s7M+yjAP/Kf+5bu2POur2/wgIzxNretGaCVaePSuajFMLWHf7+5nhvsx/Gzn1qgB2M2WKpvif/0aQxqSUy/9tKH81XNcPIb4wtFozplKKbVJzG9YCcN5v96rtWgMTI49pN/37x94vT5f9jdZDYZqDT795cZU9elJr2IJQAsVNfc+26fnYz2AACz2Uil0OyOJMh9dVW/tDGOjtnwSDbt7ThHW9uDtZ/r8QP1L/9ShqcF2d2q12u0OvvtdFqdis2y38XO5QjaXrO7jslkUKmlDsJT0+lsu+vXcDh+DLr9pqzGCllsEmXQGEzjYVzoL/xXgcRMZvP8e0Vrrl5jVNRKp7+HtQPWhdaw8XOCpVUygwa9Ma4HUPmT+LXFLsx/cq1VcfaaqMbyJrPRLa/l3kPtvYZZqyNdGqLkmkcKlZS/LLzqZr1a1jOzdxl1poeXn0x+O9A3wP4T1RGdHCd15It6MosljMTBCkXYkdUr5fWKWe9H0pkuN/53frzZbxdkt87LguMFwijv7QLDiLxB3fxYFtePm53nMHOUc7o0/tFitl073lIj0lIZNK6QwwtgUWhuaal0B1aLVS3VqZq1WrkuNJY1Yqo/17fzzdUQxuOajNaaMm15kVrVamkR6xgsKlfINOnd0tTWdZg8mrJJZ9RZeP50Lo+SmMmNTmFjaRlzDuT5XBazTaM0a1UWi8lLp4mRySQWj8zhU2kMmD2g3jsvDl/00l5p6BAe4UB4hAPhEQ6ERzgQHuHw/6dv5ct5mp8JAAAAAElFTkSuQmCC",
      "text/plain": [
       "<langgraph.graph.state.CompiledStateGraph object at 0x7f15efe87650>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def retrieve(state: State):\n",
    "    retrieved_docs = retriever.invoke(state[\"question\"])\n",
    "    return {\"context\": retrieved_docs}\n",
    "\n",
    "def generate(state: State):\n",
    "    docs_content = format_docs(state[\"context\"])\n",
    "    rag_prompt = rag_prompt_template.format(\n",
    "        question=state[\"question\"],\n",
    "        context=docs_content\n",
    "    )\n",
    "    response = llm.invoke([\n",
    "        HumanMessage(content=rag_prompt)\n",
    "    ])\n",
    "    return {\"answer\": response.content}\n",
    "\n",
    "\n",
    "graph_builder = StateGraph(State)\n",
    "\n",
    "graph_builder.add_node(\"retrieve\", retrieve)\n",
    "graph_builder.add_node(\"generate\", generate)\n",
    "\n",
    "graph_builder.add_edge(START, \"retrieve\")\n",
    "graph_builder.add_edge(\"retrieve\", \"generate\")\n",
    "graph_builder.add_edge(\"generate\", END)\n",
    "\n",
    "graph = graph_builder.compile()\n",
    "graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4c704939-fec1-4f2b-ba01-5e09004078f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is task decomposition for LLM agents?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'What is task decomposition for LLM agents?'</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'context'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Decomposition#'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition can be done (1) by LLM with simple prompting like \"Steps for </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">story outline.\" for writing a novel, or (3) with human inputs.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Challenges in long-term planning and task decomposition: Planning over a lengthy history </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">and effectively exploring the solution space remain challenging. LLMs struggle to adjust plans when faced with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">unexpected errors, making them less robust compared to humans who learn from trial and error.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">subgoals, enabling efficient handling of complex tasks.\\nReflection and refinement: The agent can do self-criticism</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">and self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">quality of final results.\\n\\n\\nMemory'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Case Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'(2) Model selection: LLM distributes the tasks to expert models, where the request is </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">context length, task type based filtration is needed.\\nInstruction:'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Generative Agents (Park, et al. 2023) is super fun experiment where 25 virtual </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">characters, each controlled by a LLM-powered agent, are living and interacting in a sandbox environment, inspired </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">by The Sims. Generative agents create believable simulacra of human behavior for interactive applications.\\nThe </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">design of generative agents combines LLM with memory, planning and reflection mechanisms to enable agents to behave</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">conditioned on past experience, as well as to interact with other agents.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'MRKL (Karpas et al. 2022), short for “Modular Reasoning, Knowledge and Language”, is a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of “expert” </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">API).'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'ReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">action space to be a combination of task-specific discrete actions and the language space. The former enables LLM </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'answer'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition for LLM (Large Language Model) agents refers to the process of breaking down a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">complicated task into smaller, manageable subgoals or steps. This allows the agent to efficiently tackle complex </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">tasks by providing a clearer structure for action. Task decomposition can be achieved in several ways:\\n\\n1. </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">**Simple Prompting**: LLMs can generate steps for a task by using prompts like \"Steps for XYZ.\\\\n1.\" or by asking </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">questions such as \"What are the subgoals for achieving XYZ?\".\\n\\n2. **Task-Specific Instructions**: Providing clear</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">and specific instructions tailored to the task at hand, such as \"Write a story outline.\" for writing a novel, helps</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">guide the LLM in creating a detailed task breakdown.\\n\\n3. **Human Inputs**: Human users can manually input their </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">understanding and breakdown of the task, aiding the LLM in the decomposition process.\\n\\nSuccessfully decomposing a</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">task enables long-term planning, making it easier for LLM agents to manage complex tasks. However, challenges exist</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">in adjusting plans when unexpected errors occur, as LLMs are less robust in learning from mistakes compared to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">humans.'</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'question'\u001b[0m: \u001b[32m'What is task decomposition for LLM agents?'\u001b[0m,\n",
       "    \u001b[32m'context'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA \u001b[0m\n",
       "\u001b[32mcomplicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask \u001b[0m\n",
       "\u001b[32mDecomposition#'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Task decomposition can be done \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by LLM with simple prompting like \"Steps for \u001b[0m\n",
       "\u001b[32mXYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by using task-specific instructions; e.g. \"Write a \u001b[0m\n",
       "\u001b[32mstory outline.\" for writing a novel, or \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with human inputs.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Challenges in long-term planning and task decomposition: Planning over a lengthy history \u001b[0m\n",
       "\u001b[32mand effectively exploring the solution space remain challenging. LLMs struggle to adjust plans when faced with \u001b[0m\n",
       "\u001b[32munexpected errors, making them less robust compared to humans who learn from trial and error.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Subgoal and decomposition: The agent breaks down large tasks into smaller, manageable \u001b[0m\n",
       "\u001b[32msubgoals, enabling efficient handling of complex tasks.\\nReflection and refinement: The agent can do self-criticism\u001b[0m\n",
       "\u001b[32mand self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the \u001b[0m\n",
       "\u001b[32mquality of final results.\\n\\n\\nMemory'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Case Studies#\\nScientific Discovery Agent#\\nChemCrow \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBran et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is a \u001b[0m\n",
       "\u001b[32mdomain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic \u001b[0m\n",
       "\u001b[32msynthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was \u001b[0m\n",
       "\u001b[32mpreviously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'\u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Model selection: LLM distributes the tasks to expert models, where the request is \u001b[0m\n",
       "\u001b[32mframed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited \u001b[0m\n",
       "\u001b[32mcontext length, task type based filtration is needed.\\nInstruction:'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Generative Agents \u001b[0m\u001b[32m(\u001b[0m\u001b[32mPark, et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is super fun experiment where 25 virtual \u001b[0m\n",
       "\u001b[32mcharacters, each controlled by a LLM-powered agent, are living and interacting in a sandbox environment, inspired \u001b[0m\n",
       "\u001b[32mby The Sims. Generative agents create believable simulacra of human behavior for interactive applications.\\nThe \u001b[0m\n",
       "\u001b[32mdesign of generative agents combines LLM with memory, planning and reflection mechanisms to enable agents to behave\u001b[0m\n",
       "\u001b[32mconditioned on past experience, as well as to interact with other agents.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'MRKL \u001b[0m\u001b[32m(\u001b[0m\u001b[32mKarpas et al. 2022\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, short for “Modular Reasoning, Knowledge and Language”, is a \u001b[0m\n",
       "\u001b[32mneuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of “expert” \u001b[0m\n",
       "\u001b[32mmodules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These \u001b[0m\n",
       "\u001b[32mmodules can be neural \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. deep learning models\u001b[0m\u001b[32m)\u001b[0m\u001b[32m or symbolic \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. math calculator, currency converter, weather \u001b[0m\n",
       "\u001b[32mAPI\u001b[0m\u001b[32m)\u001b[0m\u001b[32m.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'ReAct \u001b[0m\u001b[32m(\u001b[0m\u001b[32mYao et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m integrates reasoning and acting within LLM by extending the \u001b[0m\n",
       "\u001b[32maction space to be a combination of task-specific discrete actions and the language space. The former enables LLM \u001b[0m\n",
       "\u001b[32mto interact with the environment \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g. use Wikipedia search API\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, while the latter prompting LLM to generate \u001b[0m\n",
       "\u001b[32mreasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, \u001b[0m\n",
       "\u001b[32mroughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... \u001b[0m\u001b[32m(\u001b[0m\u001b[32mRepeated many times\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 11. Illustration of how HuggingGPT works. \u001b[0m\u001b[32m(\u001b[0m\u001b[32mImage source: Shen et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\nThe \u001b[0m\n",
       "\u001b[32msystem comprises of 4 stages:\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Task planning: LLM works as the brain and parses the user requests into multiple\u001b[0m\n",
       "\u001b[32mtasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use \u001b[0m\n",
       "\u001b[32mfew-shot examples to guide LLM to do task parsing and planning.\\nInstruction:'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'answer'\u001b[0m: \u001b[32m'Task decomposition for LLM \u001b[0m\u001b[32m(\u001b[0m\u001b[32mLarge Language Model\u001b[0m\u001b[32m)\u001b[0m\u001b[32m agents refers to the process of breaking down a \u001b[0m\n",
       "\u001b[32mcomplicated task into smaller, manageable subgoals or steps. This allows the agent to efficiently tackle complex \u001b[0m\n",
       "\u001b[32mtasks by providing a clearer structure for action. Task decomposition can be achieved in several ways:\\n\\n1. \u001b[0m\n",
       "\u001b[32m**Simple Prompting**: LLMs can generate steps for a task by using prompts like \"Steps for XYZ.\\\\n1.\" or by asking \u001b[0m\n",
       "\u001b[32mquestions such as \"What are the subgoals for achieving XYZ?\".\\n\\n2. **Task-Specific Instructions**: Providing clear\u001b[0m\n",
       "\u001b[32mand specific instructions tailored to the task at hand, such as \"Write a story outline.\" for writing a novel, helps\u001b[0m\n",
       "\u001b[32mguide the LLM in creating a detailed task breakdown.\\n\\n3. **Human Inputs**: Human users can manually input their \u001b[0m\n",
       "\u001b[32munderstanding and breakdown of the task, aiding the LLM in the decomposition process.\\n\\nSuccessfully decomposing a\u001b[0m\n",
       "\u001b[32mtask enables long-term planning, making it easier for LLM agents to manage complex tasks. However, challenges exist\u001b[0m\n",
       "\u001b[32min adjusting plans when unexpected errors occur, as LLMs are less robust in learning from mistakes compared to \u001b[0m\n",
       "\u001b[32mhumans.'\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Task decomposition for LLM (Large Language Model) agents refers to the process of breaking down a complicated task \n",
       "into smaller, manageable subgoals or steps. This allows the agent to efficiently tackle complex tasks by providing \n",
       "a clearer structure for action. Task decomposition can be achieved in several ways:                                \n",
       "\n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 1 </span><span style=\"font-weight: bold\">Simple Prompting</span>: LLMs can generate steps for a task by using prompts like \"Steps for XYZ.\\n1.\" or by asking    \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>questions such as \"What are the subgoals for achieving XYZ?\".                                                   \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 2 </span><span style=\"font-weight: bold\">Task-Specific Instructions</span>: Providing clear and specific instructions tailored to the task at hand, such as     \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>\"Write a story outline.\" for writing a novel, helps guide the LLM in creating a detailed task breakdown.        \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 3 </span><span style=\"font-weight: bold\">Human Inputs</span>: Human users can manually input their understanding and breakdown of the task, aiding the LLM in   \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>the decomposition process.                                                                                      \n",
       "\n",
       "Successfully decomposing a task enables long-term planning, making it easier for LLM agents to manage complex      \n",
       "tasks. However, challenges exist in adjusting plans when unexpected errors occur, as LLMs are less robust in       \n",
       "learning from mistakes compared to humans.                                                                         \n",
       "</pre>\n"
      ],
      "text/plain": [
       "Task decomposition for LLM (Large Language Model) agents refers to the process of breaking down a complicated task \n",
       "into smaller, manageable subgoals or steps. This allows the agent to efficiently tackle complex tasks by providing \n",
       "a clearer structure for action. Task decomposition can be achieved in several ways:                                \n",
       "\n",
       "\u001b[1;33m 1 \u001b[0m\u001b[1mSimple Prompting\u001b[0m: LLMs can generate steps for a task by using prompts like \"Steps for XYZ.\\n1.\" or by asking    \n",
       "\u001b[1;33m   \u001b[0mquestions such as \"What are the subgoals for achieving XYZ?\".                                                   \n",
       "\u001b[1;33m 2 \u001b[0m\u001b[1mTask-Specific Instructions\u001b[0m: Providing clear and specific instructions tailored to the task at hand, such as     \n",
       "\u001b[1;33m   \u001b[0m\"Write a story outline.\" for writing a novel, helps guide the LLM in creating a detailed task breakdown.        \n",
       "\u001b[1;33m 3 \u001b[0m\u001b[1mHuman Inputs\u001b[0m: Human users can manually input their understanding and breakdown of the task, aiding the LLM in   \n",
       "\u001b[1;33m   \u001b[0mthe decomposition process.                                                                                      \n",
       "\n",
       "Successfully decomposing a task enables long-term planning, making it easier for LLM agents to manage complex      \n",
       "tasks. However, challenges exist in adjusting plans when unexpected errors occur, as LLMs are less robust in       \n",
       "learning from mistakes compared to humans.                                                                         \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are main steps for collecting human data?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dlabazkin/projects/gpt/rag-explore/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:266: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'What are main steps for collecting human data?'</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'context'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Fig. 1. Two directions to approach high data quality.\\nHuman Raters ↔ Data </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Quality#\\nCollecting human data involve a set of operation steps and every step contributes to the data quality:'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'High-quality data is the fuel for modern data deep learning model training. Most of the </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">task-specific labeled data comes from human annotation, such as classification task or RLHF labeling (which can be </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">constructed as classification format) for LLM alignment training. Lots of ML techniques in the post can help with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">data quality, but fundamentally human data collection involves attention to details and careful execution. The </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">community knows the value of high quality data, but somehow we have this subtle'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Collect and aggregate data. This is the stage where more ML techniques can be applied to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">clean, filter and smartly aggregate data to identify the true labels.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Memory can be defined as the processes used to acquire, store, retain, and later retrieve</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">information. There are several types of memory in human brains.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Chain of Hindsight (CoH; Liu et al. 2023) encourages the model to improve on its own </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">outputs by explicitly presenting it with a sequence of past outputs, each annotated with feedback. Human feedback </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">data is a collection of $D_h = \\\\{(x, y_i , r_i , z_i)\\\\}_{i=1}^n$, where $x$ is the prompt, each $y_i$ is a model </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">completion, $r_i$ is the human rating of $y_i$, and $z_i$ is the corresponding human-provided hindsight feedback. </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Assume the feedback tuples are ranked by reward, $r_n \\\\geq r_{n-1} \\\\geq \\\\dots \\\\geq'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Aroyo &amp; Welty (2015) discussed a set of “myths” in the practice of human annotation </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">collection and found all of them somewhat inaccurate, key findings including:'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Task decomposition can be done (1) by LLM with simple prompting like \"Steps for </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">story outline.\" for writing a novel, or (3) with human inputs.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'In a harder task, non-expert human annotators were asked to create new gold reference </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">translations. Callison-Burch designed the task in two stages, where the first stage created new translations with </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">reference to MT outputs and the second one filtered translations that may seem to be gerated by a MT system. The </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">correlation between experts’ and crowdsourced translations is higher than that between expert and MT system </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">outputs.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2023-06-23-agent/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Resources:\\n1. Internet access for searches and information gathering.\\n2. Long Term </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">memory management.\\n3. GPT-3.5 powered Agents for delegation of simple tasks.\\n4. File output.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>,\n",
       "        <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'</span><span style=\"font-weight: bold\">}</span>,\n",
       "            <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Almost 100 years later, Callison-Burch (2009) did an early study on using Amazon </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">Mechanical Turk (AMT) to run non-expert human evaluation on Machine Translation (MT) tasks and even to rely on </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">non-experts to create new gold reference translations. The setup for human evaluation was simple: Each turker is </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">shown a source sentence, a reference translation, and 5 translations from 5 MT systems. They are asked to rank 5 </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">translations from best to worst. Each task is completed by 5 turkers.'</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'answer'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"The main steps for collecting human data, as described in the context, can be summarized as </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">follows:\\n\\n1. **Task Definition**: Clearly define the task that requires human annotation. This may involve </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">decomposing the task through prompting with specific instructions or using task-specific instructions tailored to </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">the desired output.\\n\\n2. **Data Collection**: Gather data by engaging non-expert human annotators or 'turkers' </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">through platforms like Amazon Mechanical Turk (AMT) to complete tasks. This may include having them rank or provide</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">feedback on various outputs.\\n\\n3. **Annotation and Feedback Process**: During the collection phase, collect not </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">only the primary annotations (e.g., ratings or rankings) but also any additional feedback that provides insights </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">into the annotations (hindsight feedback).\\n\\n4. **Aggregation and Filtering**: After data collection, apply </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">machine learning techniques to clean, filter, and aggregate the data to identify true labels and improve overall </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">data quality.\\n\\n5. **Quality Assessment**: Evaluate the reliability of the collected data, ensuring that </span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">non-expert annotations align well with expert evaluations where applicable.\\n\\nThese steps emphasize the importance</span>\n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">of careful preparation, execution, and evaluation to ensure high-quality human data collection.\"</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'question'\u001b[0m: \u001b[32m'What are main steps for collecting human data?'\u001b[0m,\n",
       "    \u001b[32m'context'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Fig. 1. Two directions to approach high data quality.\\nHuman Raters ↔ Data \u001b[0m\n",
       "\u001b[32mQuality#\\nCollecting human data involve a set of operation steps and every step contributes to the data quality:'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'High-quality data is the fuel for modern data deep learning model training. Most of the \u001b[0m\n",
       "\u001b[32mtask-specific labeled data comes from human annotation, such as classification task or RLHF labeling \u001b[0m\u001b[32m(\u001b[0m\u001b[32mwhich can be \u001b[0m\n",
       "\u001b[32mconstructed as classification format\u001b[0m\u001b[32m)\u001b[0m\u001b[32m for LLM alignment training. Lots of ML techniques in the post can help with \u001b[0m\n",
       "\u001b[32mdata quality, but fundamentally human data collection involves attention to details and careful execution. The \u001b[0m\n",
       "\u001b[32mcommunity knows the value of high quality data, but somehow we have this subtle'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Collect and aggregate data. This is the stage where more ML techniques can be applied to \u001b[0m\n",
       "\u001b[32mclean, filter and smartly aggregate data to identify the true labels.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Memory can be defined as the processes used to acquire, store, retain, and later retrieve\u001b[0m\n",
       "\u001b[32minformation. There are several types of memory in human brains.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Chain of Hindsight \u001b[0m\u001b[32m(\u001b[0m\u001b[32mCoH; Liu et al. 2023\u001b[0m\u001b[32m)\u001b[0m\u001b[32m encourages the model to improve on its own \u001b[0m\n",
       "\u001b[32moutputs by explicitly presenting it with a sequence of past outputs, each annotated with feedback. Human feedback \u001b[0m\n",
       "\u001b[32mdata is a collection of $D_h = \\\\\u001b[0m\u001b[32m{\u001b[0m\u001b[32m(\u001b[0m\u001b[32mx, y_i , r_i , z_i\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\u001b[0m\u001b[32m}\u001b[0m\u001b[32m_\u001b[0m\u001b[32m{\u001b[0m\u001b[32mi\u001b[0m\u001b[32m=\u001b[0m\u001b[32m1\u001b[0m\u001b[32m}\u001b[0m\u001b[32m^n$, where $x$ is the prompt, each $y_i$ is a model \u001b[0m\n",
       "\u001b[32mcompletion, $r_i$ is the human rating of $y_i$, and $z_i$ is the corresponding human-provided hindsight feedback. \u001b[0m\n",
       "\u001b[32mAssume the feedback tuples are ranked by reward, $r_n \\\\geq r_\u001b[0m\u001b[32m{\u001b[0m\u001b[32mn-1\u001b[0m\u001b[32m}\u001b[0m\u001b[32m \\\\geq \\\\dots \\\\geq'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Aroyo & Welty \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2015\u001b[0m\u001b[32m)\u001b[0m\u001b[32m discussed a set of “myths” in the practice of human annotation \u001b[0m\n",
       "\u001b[32mcollection and found all of them somewhat inaccurate, key findings including:'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Task decomposition can be done \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by LLM with simple prompting like \"Steps for \u001b[0m\n",
       "\u001b[32mXYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2\u001b[0m\u001b[32m)\u001b[0m\u001b[32m by using task-specific instructions; e.g. \"Write a \u001b[0m\n",
       "\u001b[32mstory outline.\" for writing a novel, or \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with human inputs.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'In a harder task, non-expert human annotators were asked to create new gold reference \u001b[0m\n",
       "\u001b[32mtranslations. Callison-Burch designed the task in two stages, where the first stage created new translations with \u001b[0m\n",
       "\u001b[32mreference to MT outputs and the second one filtered translations that may seem to be gerated by a MT system. The \u001b[0m\n",
       "\u001b[32mcorrelation between experts’ and crowdsourced translations is higher than that between expert and MT system \u001b[0m\n",
       "\u001b[32moutputs.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2023-06-23-agent/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Resources:\\n1. Internet access for searches and information gathering.\\n2. Long Term \u001b[0m\n",
       "\u001b[32mmemory management.\\n3. GPT-3.5 powered Agents for delegation of simple tasks.\\n4. File output.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m,\n",
       "        \u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
       "            \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'source'\u001b[0m: \u001b[32m'https://lilianweng.github.io/posts/2024-02-05-human-data-quality/'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[33mpage_content\u001b[0m=\u001b[32m'Almost 100 years later, Callison-Burch \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2009\u001b[0m\u001b[32m)\u001b[0m\u001b[32m did an early study on using Amazon \u001b[0m\n",
       "\u001b[32mMechanical Turk \u001b[0m\u001b[32m(\u001b[0m\u001b[32mAMT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m to run non-expert human evaluation on Machine Translation \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m tasks and even to rely on \u001b[0m\n",
       "\u001b[32mnon-experts to create new gold reference translations. The setup for human evaluation was simple: Each turker is \u001b[0m\n",
       "\u001b[32mshown a source sentence, a reference translation, and 5 translations from 5 MT systems. They are asked to rank 5 \u001b[0m\n",
       "\u001b[32mtranslations from best to worst. Each task is completed by 5 turkers.'\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "    \u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'answer'\u001b[0m: \u001b[32m\"The main steps for collecting human data, as described in the context, can be summarized as \u001b[0m\n",
       "\u001b[32mfollows:\\n\\n1. **Task Definition**: Clearly define the task that requires human annotation. This may involve \u001b[0m\n",
       "\u001b[32mdecomposing the task through prompting with specific instructions or using task-specific instructions tailored to \u001b[0m\n",
       "\u001b[32mthe desired output.\\n\\n2. **Data Collection**: Gather data by engaging non-expert human annotators or 'turkers' \u001b[0m\n",
       "\u001b[32mthrough platforms like Amazon Mechanical Turk \u001b[0m\u001b[32m(\u001b[0m\u001b[32mAMT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m to complete tasks. This may include having them rank or provide\u001b[0m\n",
       "\u001b[32mfeedback on various outputs.\\n\\n3. **Annotation and Feedback Process**: During the collection phase, collect not \u001b[0m\n",
       "\u001b[32monly the primary annotations \u001b[0m\u001b[32m(\u001b[0m\u001b[32me.g., ratings or rankings\u001b[0m\u001b[32m)\u001b[0m\u001b[32m but also any additional feedback that provides insights \u001b[0m\n",
       "\u001b[32minto the annotations \u001b[0m\u001b[32m(\u001b[0m\u001b[32mhindsight feedback\u001b[0m\u001b[32m)\u001b[0m\u001b[32m.\\n\\n4. **Aggregation and Filtering**: After data collection, apply \u001b[0m\n",
       "\u001b[32mmachine learning techniques to clean, filter, and aggregate the data to identify true labels and improve overall \u001b[0m\n",
       "\u001b[32mdata quality.\\n\\n5. **Quality Assessment**: Evaluate the reliability of the collected data, ensuring that \u001b[0m\n",
       "\u001b[32mnon-expert annotations align well with expert evaluations where applicable.\\n\\nThese steps emphasize the importance\u001b[0m\n",
       "\u001b[32mof careful preparation, execution, and evaluation to ensure high-quality human data collection.\"\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">The main steps for collecting human data, as described in the context, can be summarized as follows:               \n",
       "\n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 1 </span><span style=\"font-weight: bold\">Task Definition</span>: Clearly define the task that requires human annotation. This may involve decomposing the task  \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>through prompting with specific instructions or using task-specific instructions tailored to the desired output.\n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 2 </span><span style=\"font-weight: bold\">Data Collection</span>: Gather data by engaging non-expert human annotators or 'turkers' through platforms like Amazon \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>Mechanical Turk (AMT) to complete tasks. This may include having them rank or provide feedback on various       \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>outputs.                                                                                                        \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 3 </span><span style=\"font-weight: bold\">Annotation and Feedback Process</span>: During the collection phase, collect not only the primary annotations (e.g.,   \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>ratings or rankings) but also any additional feedback that provides insights into the annotations (hindsight    \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>feedback).                                                                                                      \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 4 </span><span style=\"font-weight: bold\">Aggregation and Filtering</span>: After data collection, apply machine learning techniques to clean, filter, and       \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>aggregate the data to identify true labels and improve overall data quality.                                    \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\"> 5 </span><span style=\"font-weight: bold\">Quality Assessment</span>: Evaluate the reliability of the collected data, ensuring that non-expert annotations align  \n",
       "<span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">   </span>well with expert evaluations where applicable.                                                                  \n",
       "\n",
       "These steps emphasize the importance of careful preparation, execution, and evaluation to ensure high-quality human\n",
       "data collection.                                                                                                   \n",
       "</pre>\n"
      ],
      "text/plain": [
       "The main steps for collecting human data, as described in the context, can be summarized as follows:               \n",
       "\n",
       "\u001b[1;33m 1 \u001b[0m\u001b[1mTask Definition\u001b[0m: Clearly define the task that requires human annotation. This may involve decomposing the task  \n",
       "\u001b[1;33m   \u001b[0mthrough prompting with specific instructions or using task-specific instructions tailored to the desired output.\n",
       "\u001b[1;33m 2 \u001b[0m\u001b[1mData Collection\u001b[0m: Gather data by engaging non-expert human annotators or 'turkers' through platforms like Amazon \n",
       "\u001b[1;33m   \u001b[0mMechanical Turk (AMT) to complete tasks. This may include having them rank or provide feedback on various       \n",
       "\u001b[1;33m   \u001b[0moutputs.                                                                                                        \n",
       "\u001b[1;33m 3 \u001b[0m\u001b[1mAnnotation and Feedback Process\u001b[0m: During the collection phase, collect not only the primary annotations (e.g.,   \n",
       "\u001b[1;33m   \u001b[0mratings or rankings) but also any additional feedback that provides insights into the annotations (hindsight    \n",
       "\u001b[1;33m   \u001b[0mfeedback).                                                                                                      \n",
       "\u001b[1;33m 4 \u001b[0m\u001b[1mAggregation and Filtering\u001b[0m: After data collection, apply machine learning techniques to clean, filter, and       \n",
       "\u001b[1;33m   \u001b[0maggregate the data to identify true labels and improve overall data quality.                                    \n",
       "\u001b[1;33m 5 \u001b[0m\u001b[1mQuality Assessment\u001b[0m: Evaluate the reliability of the collected data, ensuring that non-expert annotations align  \n",
       "\u001b[1;33m   \u001b[0mwell with expert evaluations where applicable.                                                                  \n",
       "\n",
       "These steps emphasize the importance of careful preparation, execution, and evaluation to ensure high-quality human\n",
       "data collection.                                                                                                   \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for query in queries:\n",
    "    print(query)\n",
    "    response = graph.invoke({\"question\": query})\n",
    "    rprint(Pretty(response, no_wrap=False))\n",
    "    rprint(Markdown(response[\"answer\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}