notebooks/part-4-rag.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0200e33a-8d0f-400a-a59a-4fb03b529442",
   "metadata": {},
   "source": [
    "# Install required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0591e29-92ea-4881-8b3f-1995987febdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -qU \\\n",
    "    datasets \\\n",
    "    apache_beam \\\n",
    "    mwparserfromhell \\\n",
    "    tiktoken \\\n",
    "    langchain openai"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f98e1bfa-6b09-4a20-b2d4-031970215507",
   "metadata": {},
   "source": [
    "# Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d29441a5-91e1-427f-a587-1c25f074f835",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "data = load_dataset(\"wikipedia\", \"20220301.simple\", split='train[:100]')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62d14d92-034d-47a8-9209-25f9bdd83b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "\n",
    "tiktoken.encoding_for_model('gpt-3.5-turbo')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93e5c509-10ff-490d-b5c9-43aa36dd4ff3",
   "metadata": {},
   "source": [
    "# Tokenize and split into chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d571a5c-5480-4ba5-9f54-b2f8ca9706d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "\n",
    "tokenizer = tiktoken.get_encoding('cl100k_base')\n",
    "\n",
    "# create the length function\n",
    "def tiktoken_len(text):\n",
    "    tokens = tokenizer.encode(\n",
    "        text,\n",
    "        disallowed_special=()\n",
    "    )\n",
    "    return len(tokens)\n",
    "\n",
    "tiktoken_len(\"hello I am a chunk of text and using the tiktoken_len function \"\n",
    "             \"we can find the length of this chunk of text in tokens\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "921e7e3e-a897-4712-973c-48a4c79dcce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=400,\n",
    "    chunk_overlap=20,\n",
    "    length_function=tiktoken_len,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00046f99-ef54-40cf-9b0c-84adb105ae22",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks = text_splitter.split_text(data[6]['text'])[:3]\n",
    "chunks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d83b00b2-c866-4df7-924b-020f5818b571",
   "metadata": {},
   "source": [
    "# Load the OpenAI embedding model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20d0dc90-7d8f-4b5d-8333-1be809ee2646",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "\n",
    "model_name = 'text-embedding-ada-002'\n",
    "os.environ['OPENAI_API_KEY'] = <OPENAI-API-KEY>\n",
    "\n",
    "embed = OpenAIEmbeddings(\n",
    "    model=model_name\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43de1086-eb23-44e3-9b78-add563a6b40a",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = [\n",
    "    'this is the first chunk of text',\n",
    "    'then another second chunk of text is here'\n",
    "]\n",
    "\n",
    "res = embed.embed_documents(texts)\n",
    "len(res), len(res[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72bfc7ff-c3e3-4218-bc07-66a2dc158265",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.docstore.document import Document\n",
    "\n",
    "doc =  Document(page_content=\"text\", metadata={\"source\": \"local\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9d317ba-2853-4cd2-b41e-10ffc62905a0",
   "metadata": {},
   "source": [
    "# Create documents to store in VectorDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab6c067-e181-441f-9c6c-0d4c0d6c6fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from uuid import uuid4\n",
    "\n",
    "batch_limit = 100\n",
    "\n",
    "docs = []\n",
    "\n",
    "for i, record in enumerate(tqdm(data)):\n",
    "    # first get metadata fields for this record\n",
    "    metadata = {\n",
    "        'wiki-id': str(record['id']),\n",
    "        'source': record['url'],\n",
    "        'title': record['title']\n",
    "    }\n",
    "    # now we create chunks from the record text\n",
    "    record_texts = text_splitter.split_text(record['text'])\n",
    "    # create individual metadata dicts for each chunk\n",
    "    record_docs = [ Document(page_content=text, metadata={\n",
    "        \"chunk\": j, \"text\": text, **metadata\n",
    "    }) for j, text in enumerate(record_texts)]\n",
    "    # append these to current batches\n",
    "    docs.extend(record_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9522fbb1-4d35-4cd7-be06-b79cbc079398",
   "metadata": {},
   "source": [
    "# Store it in vectorDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8d15b10-7d3f-441d-b256-a9a1d8b11f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "from qdrant_client import QdrantClient\n",
    "from langchain.vectorstores import Qdrant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce2fa424-9730-474a-84f7-e38feff98f97",
   "metadata": {},
   "outputs": [],
   "source": [
    "qdrant = Qdrant.from_documents(\n",
    "    docs, embed, \n",
    "    path=\"/tmp/local_qdrant\",\n",
    "    collection_name=\"my_documents\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2b96cda-f028-46d4-a26a-9695c0943b68",
   "metadata": {},
   "source": [
    "# Create QA using Langchain APIs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea6e5046-fcea-4cfe-b238-6dc1ec435706",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "\n",
    "# completion llm\n",
    "llm = ChatOpenAI(\n",
    "    model_name='gpt-3.5-turbo',\n",
    "    temperature=0.0\n",
    ")\n",
    "\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=qdrant.as_retriever()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39992c74-eccd-43e9-8c2d-2ca54ca60f2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa.run('who is ethel')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1586ba59-9e8d-4ff7-b765-ef4d7098a4fa",
   "metadata": {},
   "source": [
    "# Productionizing QA - starting with Qdrant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6943a6ac-4e8e-4214-a548-9cd58fdc9142",
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace this with your Qdrant URL\n",
    "# it will be of the form <name-of-deployment>.<workspace-name>.svc.cluster.local\n",
    "qdrant_url = 'qdrant.abhi-test.svc.cluster.local'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6144e586-8620-451d-bfa2-2f9ab6388369",
   "metadata": {},
   "outputs": [],
   "source": [
    "qdrant_remote = Qdrant.from_documents(\n",
    "    docs,\n",
    "    embed,\n",
    "    url=qdrant_url,\n",
    "    prefer_grpc=True,\n",
    "    collection_name=\"my_documents\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a55730dd-62d4-4350-ace3-10575846c75a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "\n",
    "# completion llm\n",
    "llm = ChatOpenAI(\n",
    "    model_name='gpt-3.5-turbo',\n",
    "    temperature=0.0\n",
    ")\n",
    "\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=qdrant_remote.as_retriever()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab8d359c-14f8-4c47-8f7a-f9040fa9b71e",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa.run('who is ethel')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccc27000-f85e-47d9-8616-1bfa9a53cc51",
   "metadata": {},
   "source": [
    "# Replace ChatGPT LLM with open-source LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7dad962-821a-4975-891f-1bac5b67b9d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install servicefoundry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62dc5863-3357-4ac5-ab0c-096493692ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!servicefoundry login --host https://avmc.truefoundry.cloud/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b611e0e-8dc0-4a48-b042-1d52058c0585",
   "metadata": {},
   "outputs": [],
   "source": [
    "from servicefoundry.langchain import TruefoundryPlaygroundLLM\n",
    "import os\n",
    "\n",
    "# Note: Login using servicefoundry login --host <https://example-domain.com>\n",
    "model = TruefoundryPlaygroundLLM(\n",
    "  model_name=\"llama-2-7b-chat\",\n",
    "  provider=\"truefoundry-self-hosted\",\n",
    "  parameters={\n",
    "    \"maximumLength\": 100,\n",
    "    \"temperature\": 0.7,\n",
    "    \"topP\": 0.9,\n",
    "    \"repetitionPenalty\": 1\n",
    "  }\n",
    ")\n",
    "response = model.predict(\"Enter the prompt here\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e71da6b5-fbe5-4bd1-97ae-265cbb633e14",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=model,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=qdrant_remote.as_retriever()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b9bac8b-f142-49ed-a366-2a0a19af94a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa.run('who is ethel')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cf72860-6869-49f3-be26-bf8cfd664f02",
   "metadata": {},
   "source": [
    "# Let's create a service\n",
    "1. Create main.py, requirements.txt\n",
    "2. Write TFY deployment spec as servicefoundry.yaml\n",
    "3. Deploy to your workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1115291-e90d-487f-89e4-6f53db2c98f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile main.py\n",
    "\n",
    "import os\n",
    "import servicefoundry\n",
    "from fastapi import FastAPI, HTTPException\n",
    "from pydantic import BaseModel\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.vectorstores import Qdrant\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from servicefoundry.langchain import TruefoundryPlaygroundLLM\n",
    "from qdrant_client import QdrantClient\n",
    "from servicefoundry.lib.auth.servicefoundry_session import ServiceFoundrySession\n",
    "\n",
    "app = FastAPI()\n",
    "\n",
    "EMBEDDING_API_KEY = os.environ['OPENAI_API_KEY']\n",
    "QDRANT_URL = os.environ['QDRANT_URL']\n",
    "EMBED_MODEL_NAME = os.environ['EMBED_MODEL_NAME']\n",
    "MODEL_NAME = os.environ['MODEL_NAME']\n",
    "COLLECTION_NAME = os.environ['COLLECTION_NAME']\n",
    "\n",
    "# Initialize components\n",
    "embed = OpenAIEmbeddings(model=EMBED_MODEL_NAME, api_key=EMBEDDING_API_KEY)\n",
    "client = QdrantClient(host=QDRANT_URL, port=6333)\n",
    "qdrant_remote = Qdrant(\n",
    "    client=client, collection_name=COLLECTION_NAME, embeddings=embed\n",
    ")\n",
    "\n",
    "model = TruefoundryPlaygroundLLM(\n",
    "  model_name=\"llama-2-7b-chat\",\n",
    "  provider=\"truefoundry-self-hosted\",\n",
    "  parameters={\n",
    "    \"maximumLength\": 100,\n",
    "    \"temperature\": 0.7,\n",
    "    \"topP\": 0.9,\n",
    "    \"repetitionPenalty\": 1\n",
    "  }\n",
    ")\n",
    "qa = RetrievalQA.from_chain_type(llm=model, chain_type=\"stuff\", retriever=qdrant_remote.as_retriever())\n",
    "\n",
    "@app.get(\"/\")\n",
    "def read_root():\n",
    "    return {\"Hello\": \"World\"}\n",
    "\n",
    "# Define the request model\n",
    "class QueryModel(BaseModel):\n",
    "    query: str\n",
    "\n",
    "@app.post(\"/query/\")\n",
    "def get_answer(body: QueryModel):\n",
    "    query = body.query\n",
    "    if not query:\n",
    "        raise HTTPException(status_code=400, detail=\"Query not provided\")\n",
    "\n",
    "    answer = qa.run(query)\n",
    "    return {\"answer\": answer}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb7ed733-54f9-4a30-bac1-ab7431786a6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile requirements.txt\n",
    "\n",
    "fastapi\n",
    "uvicorn\n",
    "langchain\n",
    "servicefoundry\n",
    "tiktoken\n",
    "datasets\n",
    "apache-beam\n",
    "mwparserfromhell\n",
    "openai\n",
    "qdrant-client\n",
    "python-dotenv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "720e7fce-27bc-40e4-b153-274109e3fcdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile servicefoundry.yaml\n",
    "\n",
    "type: service\n",
    "image:\n",
    "  type: build\n",
    "  build_source:\n",
    "    type: local\n",
    "  build_spec:\n",
    "    type: tfy-python-buildpack\n",
    "    build_context_path: ./\n",
    "    command: uvicorn main:app --host 0.0.0.0 --port 8000 --root-path /fastapi-mine/ # replace --root-path with your path\n",
    "    python_version: '3.9'\n",
    "    requirements_path: requirements.txt\n",
    "name: fastapi-app-mine\n",
    "ports:\n",
    "  - expose: true\n",
    "    port: 8000\n",
    "    protocol: TCP\n",
    "    host: ml.avmc.truefoundry.cloud\n",
    "    path: /fastapi-mine/ # replace this with a unique path for your service\n",
    "    app_protocol: http\n",
    "replicas: 1\n",
    "env:\n",
    "  OPENAI_API_KEY: <OPENAI-API-KEY>\n",
    "  QDRANT_URL: <QDRANT-URL> # your qdrant URL\n",
    "  MODEL_NAME: llama-2-7b-chat # change model name if you deployed something else\n",
    "  COLLECTION_NAME: my_documents\n",
    "  TFY_HOST: https://avmc.truefoundry.cloud\n",
    "  TFY_API_KEY: <API-KEY> # From TrueFoundry dashboard, go to Settings > API Key to fetch one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4ba8570-99d8-4bbb-afb9-a7a94b977696",
   "metadata": {},
   "outputs": [],
   "source": [
    "!servicefoundry deploy --workspace-fqn=<YOUR-WORKSPACE-FQN>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".conda-jupyter-base",
   "language": "python",
   "name": "conda-env-.conda-jupyter-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}