prompting_14_check_outputs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d128d34a-5d6a-4b33-9701-0422c9925f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import tiktoken\n",
    "from IPython.display import display, Markdown, Latex, HTML, JSON\n",
    "\n",
    "MODEL_NAME = 'gpt-3.5-turbo'\n",
    "\n",
    "with open('/home/loc/Documents/OPENAI_API_KEY.txt') as f:\n",
    "    OPENAI_API_KEY = f.read().strip()\n",
    "    os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY\n",
    "    openai.api_key = OPENAI_API_KEY\n",
    "\n",
    "def get_completion(prompt, model=MODEL_NAME, temperature=0):\n",
    "    # generate message\n",
    "    messages = [{'role':'user',\n",
    "                 'content':prompt}\n",
    "               ]\n",
    "    # receive the response\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model = model,\n",
    "        messages = messages,\n",
    "        temperature = temperature # this is the degree of randomness of model\n",
    "    )\n",
    "    return response.choices[0].message['content']\n",
    "\n",
    "\n",
    "def get_completion_from_messages(messages,model=MODEL_NAME,\n",
    "                                 temperature=0,\n",
    "                                 max_tokens=500):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model = model,\n",
    "        messages = messages,\n",
    "        temperature = temperature, # this is the degree of randomness of model\n",
    "        max_tokens = max_tokens # the maximum number of tokens the model can ouptut\n",
    "    )\n",
    "    return response.choices[0].message['content']\n",
    "\n",
    "def get_completion_and_token_count(messages, \n",
    "                                   model=\"gpt-3.5-turbo\", \n",
    "                                   temperature=0, \n",
    "                                   max_tokens=500):\n",
    "    \n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=model,\n",
    "        messages=messages,\n",
    "        temperature=temperature, \n",
    "        max_tokens=max_tokens,\n",
    "    )\n",
    "    \n",
    "    content = response.choices[0].message[\"content\"]\n",
    "    \n",
    "    token_dict = {\n",
    "'prompt_tokens':response['usage']['prompt_tokens'],\n",
    "'completion_tokens':response['usage']['completion_tokens'],\n",
    "'total_tokens':response['usage']['total_tokens'],\n",
    "    }\n",
    "\n",
    "    return content, token_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13ecae0d-6a6c-45a0-8cd1-b87665cddf7a",
   "metadata": {},
   "source": [
    "### Check output for potentially harmful content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "39a327b7-04d5-4659-8077-dfd845d8d2a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"flagged\": false,\n",
      "  \"categories\": {\n",
      "    \"sexual\": false,\n",
      "    \"hate\": false,\n",
      "    \"harassment\": false,\n",
      "    \"self-harm\": false,\n",
      "    \"sexual/minors\": false,\n",
      "    \"hate/threatening\": false,\n",
      "    \"violence/graphic\": false,\n",
      "    \"self-harm/intent\": false,\n",
      "    \"self-harm/instructions\": false,\n",
      "    \"harassment/threatening\": false,\n",
      "    \"violence\": false\n",
      "  },\n",
      "  \"category_scores\": {\n",
      "    \"sexual\": 0.0001537607895443216,\n",
      "    \"hate\": 7.3742098720686045e-06,\n",
      "    \"harassment\": 2.7755460905609652e-05,\n",
      "    \"self-harm\": 1.296598270528193e-06,\n",
      "    \"sexual/minors\": 1.1709506907209288e-05,\n",
      "    \"hate/threatening\": 2.071006292680977e-06,\n",
      "    \"violence/graphic\": 1.520041587355081e-05,\n",
      "    \"self-harm/intent\": 2.0532834241748787e-06,\n",
      "    \"self-harm/instructions\": 3.7833510191376263e-07,\n",
      "    \"harassment/threatening\": 1.0162367289012764e-05,\n",
      "    \"violence\": 0.0003052482206840068\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "final_response_to_customer = f\"\"\"\n",
    "The SmartX ProPhone has a 6.1-inch display, 128GB storage, \\\n",
    "12MP dual camera, and 5G. The FotoSnap DSLR Camera \\\n",
    "has a 24.2MP sensor, 1080p video, 3-inch LCD, and \\\n",
    "interchangeable lenses. We have a variety of TVs, including \\\n",
    "the CineView 4K TV with a 55-inch display, 4K resolution, \\\n",
    "HDR, and smart TV features. We also have the SoundMax \\\n",
    "Home Theater system with 5.1 channel, 1000W output, wireless \\\n",
    "subwoofer, and Bluetooth. Do you have any specific questions \\\n",
    "about these products or any other products we offer?\n",
    "\"\"\"\n",
    "\n",
    "response = openai.Moderation.create(\n",
    "    input = final_response_to_customer\n",
    ")\n",
    "\n",
    "moderation_output = response['results'][0]\n",
    "print(moderation_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a8d5a47-37ee-4dc1-89e2-e9b2cdc18bb9",
   "metadata": {},
   "source": [
    "### Check if output is factually based on the provided product information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1bd49d78-e356-442d-a096-bb3bc7f7fc81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Y\n"
     ]
    }
   ],
   "source": [
    "system_message = \"\"\"\n",
    "You are an assistant that evaluates whether \\\n",
    "customer service agent responses sufficiently \\\n",
    "answer customer questions, and also validates that \\\n",
    "all the facts the assistant cites from the product \\\n",
    "information are correct.\n",
    "The product information and user and customer \\\n",
    "service agent messages will be delimited by \\\n",
    "3 backticks, i.e. ```.\n",
    "Respond with a Y or N character, with no punctuation:\n",
    "Y - if the output sufficiently answers the question \\\n",
    "AND the response correctly uses product information\n",
    "N - otherwise\n",
    "\n",
    "Output a single letter only.\n",
    "\"\"\"\n",
    "\n",
    "customer_message = f\"\"\"\n",
    "tell me about the smartx pro phone and \\\n",
    "the fotosnap camera, the dslr one. \\\n",
    "Also tell me about your tvs\"\"\"\n",
    "\n",
    "product_information = \"\"\"{ \"name\": \"SmartX ProPhone\", \"category\": \"Smartphones and Accessories\", \"brand\": \"SmartX\", \"model_number\": \"SX-PP10\", \"warranty\": \"1 year\", \"rating\": 4.6, \"features\": [ \"6.1-inch display\", \"128GB storage\", \"12MP dual camera\", \"5G\" ], \"description\": \"A powerful smartphone with advanced camera features.\", \"price\": 899.99 } \n",
    "                        { \"name\": \"FotoSnap DSLR Camera\", \"category\": \"Cameras and Camcorders\", \"brand\": \"FotoSnap\", \"model_number\": \"FS-DSLR200\", \"warranty\": \"1 year\", \"rating\": 4.7, \"features\": [ \"24.2MP sensor\", \"1080p video\", \"3-inch LCD\", \"Interchangeable lenses\" ], \"description\": \"Capture stunning photos and videos with this versatile DSLR camera.\", \"price\": 599.99 } \n",
    "                        { \"name\": \"CineView 4K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-4K55\", \"warranty\": \"2 years\", \"rating\": 4.8, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"A stunning 4K TV with vibrant colors and smart features.\", \"price\": 599.99 } \n",
    "                        { \"name\": \"SoundMax Home Theater\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-HT100\", \"warranty\": \"1 year\", \"rating\": 4.4, \"features\": [ \"5.1 channel\", \"1000W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"A powerful home theater system for an immersive audio experience.\", \"price\": 399.99 } \n",
    "                        { \"name\": \"CineView 8K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-8K65\", \"warranty\": \"2 years\", \"rating\": 4.9, \"features\": [ \"65-inch display\", \"8K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience the future of television with this stunning 8K TV.\", \"price\": 2999.99 } \n",
    "                        { \"name\": \"SoundMax Soundbar\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-SB50\", \"warranty\": \"1 year\", \"rating\": 4.3, \"features\": [ \"2.1 channel\", \"300W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"Upgrade your TV's audio with this sleek and powerful soundbar.\", \"price\": 199.99 } \n",
    "                        { \"name\": \"CineView OLED TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-OLED55\", \"warranty\": \"2 years\", \"rating\": 4.7, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience true blacks and vibrant colors with this OLED TV.\", \"price\": 1499.99 }\"\"\"\n",
    "\n",
    "q_a_pair = f\"\"\"\n",
    "Customer message: ```{customer_message}```\n",
    "Product information: ```{product_information}```\n",
    "Agent response: ```{final_response_to_customer}```\n",
    "\n",
    "Does the response use the retrived information correctly?\n",
    "Does the response suffciently answer the question\n",
    "\n",
    "Output Y or N\n",
    "\"\"\"\n",
    "\n",
    "messages = [\n",
    "    {'role':'system','content':system_message},\n",
    "    {'role':'user','content':q_a_pair}\n",
    "]\n",
    "\n",
    "response = get_completion_from_messages(messages, max_tokens=1)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "73279a21-b3af-4349-96bc-8529e294fab2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N\n"
     ]
    }
   ],
   "source": [
    "another_response = \"life is like a box of chocolates\"\n",
    "\n",
    "q_a_pair = f\"\"\"\n",
    "Customer message: ```{customer_message}```\n",
    "Product information: ```{product_information}```\n",
    "Agent response: ```{another_response}```\n",
    "\n",
    "Does the response use the retrived information correctly?\n",
    "Does the response suffciently answer the question\n",
    "\n",
    "Output Y or N\n",
    "\"\"\"\n",
    "\n",
    "messages = [\n",
    "    {'role':'system','content':system_message},\n",
    "    {'role':'user','content':q_a_pair}\n",
    "]\n",
    "\n",
    "response = get_completion_from_messages(messages, max_tokens=1)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fcddcc4-5182-48b5-a8a6-1dcbcf1aeac0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}