check outputs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d0670f1d-a612-4c45-b8e8-9be339e552d1",
   "metadata": {},
   "source": [
    "# L6: Check outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8adb9f4-f9ed-4c9f-8495-049c16548003",
   "metadata": {},
   "source": [
    "## Setup\n",
    "#### Load the API key and relevant Python libaries.\n",
    "In this course, we've provided some code that loads the OpenAI API key for you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6af3568d",
   "metadata": {
    "height": 115
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "\n",
    "openai.api_key  = os.environ['OPENAI_API_KEY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a858d53",
   "metadata": {
    "height": 168
   },
   "outputs": [],
   "source": [
    "def get_completion_from_messages(messages, model=\"gpt-3.5-turbo\", temperature=0, max_tokens=500):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=model,\n",
    "        messages=messages,\n",
    "        temperature=temperature, \n",
    "        max_tokens=max_tokens, \n",
    "    )\n",
    "    return response.choices[0].message[\"content\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f7d3db3-2258-48c9-819e-cfd8780416e3",
   "metadata": {},
   "source": [
    "### Check output for potentially harmful content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6164c53b",
   "metadata": {
    "height": 285
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"flagged\": false,\n",
      "  \"categories\": {\n",
      "    \"sexual\": false,\n",
      "    \"hate\": false,\n",
      "    \"harassment\": false,\n",
      "    \"self-harm\": false,\n",
      "    \"sexual/minors\": false,\n",
      "    \"hate/threatening\": false,\n",
      "    \"violence/graphic\": false,\n",
      "    \"self-harm/intent\": false,\n",
      "    \"self-harm/instructions\": false,\n",
      "    \"harassment/threatening\": false,\n",
      "    \"violence\": false\n",
      "  },\n",
      "  \"category_scores\": {\n",
      "    \"sexual\": 0.00015211118443403393,\n",
      "    \"hate\": 7.229043148981873e-06,\n",
      "    \"harassment\": 2.696166302484926e-05,\n",
      "    \"self-harm\": 1.2812188288080506e-06,\n",
      "    \"sexual/minors\": 1.154503297584597e-05,\n",
      "    \"hate/threatening\": 2.0055701952514937e-06,\n",
      "    \"violence/graphic\": 1.5082588106452022e-05,\n",
      "    \"self-harm/intent\": 2.012526920225355e-06,\n",
      "    \"self-harm/instructions\": 3.672591049053153e-07,\n",
      "    \"harassment/threatening\": 9.87596831691917e-06,\n",
      "    \"violence\": 0.0002972284273710102\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "final_response_to_customer = f\"\"\"\n",
    "The SmartX ProPhone has a 6.1-inch display, 128GB storage, \\\n",
    "12MP dual camera, and 5G. The FotoSnap DSLR Camera \\\n",
    "has a 24.2MP sensor, 1080p video, 3-inch LCD, and \\\n",
    "interchangeable lenses. We have a variety of TVs, including \\\n",
    "the CineView 4K TV with a 55-inch display, 4K resolution, \\\n",
    "HDR, and smart TV features. We also have the SoundMax \\\n",
    "Home Theater system with 5.1 channel, 1000W output, wireless \\\n",
    "subwoofer, and Bluetooth. Do you have any specific questions \\\n",
    "about these products or any other products we offer?\n",
    "\"\"\"\n",
    "response = openai.Moderation.create(\n",
    "    input=final_response_to_customer\n",
    ")\n",
    "moderation_output = response[\"results\"][0]\n",
    "print(moderation_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dde612c-383e-444a-96d6-51c4f42dfb51",
   "metadata": {},
   "source": [
    "### Check if output is factually based on the provided product information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef6e394f",
   "metadata": {
    "height": 678
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Y\n"
     ]
    }
   ],
   "source": [
    "system_message = f\"\"\"\n",
    "You are an assistant that evaluates whether \\\n",
    "customer service agent responses sufficiently \\\n",
    "answer customer questions, and also validates that \\\n",
    "all the facts the assistant cites from the product \\\n",
    "information are correct.\n",
    "The product information and user and customer \\\n",
    "service agent messages will be delimited by \\\n",
    "3 backticks, i.e. ```.\n",
    "Respond with a Y or N character, with no punctuation:\n",
    "Y - if the output sufficiently answers the question \\\n",
    "AND the response correctly uses product information\n",
    "N - otherwise\n",
    "\n",
    "Output a single letter only.\n",
    "\"\"\"\n",
    "customer_message = f\"\"\"\n",
    "tell me about the smartx pro phone and \\\n",
    "the fotosnap camera, the dslr one. \\\n",
    "Also tell me about your tvs\"\"\"\n",
    "product_information = \"\"\"{ \"name\": \"SmartX ProPhone\", \"category\": \"Smartphones and Accessories\", \"brand\": \"SmartX\", \"model_number\": \"SX-PP10\", \"warranty\": \"1 year\", \"rating\": 4.6, \"features\": [ \"6.1-inch display\", \"128GB storage\", \"12MP dual camera\", \"5G\" ], \"description\": \"A powerful smartphone with advanced camera features.\", \"price\": 899.99 } { \"name\": \"FotoSnap DSLR Camera\", \"category\": \"Cameras and Camcorders\", \"brand\": \"FotoSnap\", \"model_number\": \"FS-DSLR200\", \"warranty\": \"1 year\", \"rating\": 4.7, \"features\": [ \"24.2MP sensor\", \"1080p video\", \"3-inch LCD\", \"Interchangeable lenses\" ], \"description\": \"Capture stunning photos and videos with this versatile DSLR camera.\", \"price\": 599.99 } { \"name\": \"CineView 4K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-4K55\", \"warranty\": \"2 years\", \"rating\": 4.8, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"A stunning 4K TV with vibrant colors and smart features.\", \"price\": 599.99 } { \"name\": \"SoundMax Home Theater\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-HT100\", \"warranty\": \"1 year\", \"rating\": 4.4, \"features\": [ \"5.1 channel\", \"1000W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"A powerful home theater system for an immersive audio experience.\", \"price\": 399.99 } { \"name\": \"CineView 8K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-8K65\", \"warranty\": \"2 years\", \"rating\": 4.9, \"features\": [ \"65-inch display\", \"8K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience the future of television with this stunning 8K TV.\", \"price\": 2999.99 } { \"name\": \"SoundMax Soundbar\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-SB50\", \"warranty\": \"1 year\", \"rating\": 4.3, \"features\": [ \"2.1 channel\", \"300W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"Upgrade your TV's audio with this sleek and powerful soundbar.\", \"price\": 199.99 } { \"name\": \"CineView OLED TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-OLED55\", \"warranty\": \"2 years\", \"rating\": 4.7, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience true blacks and vibrant colors with this OLED TV.\", \"price\": 1499.99 }\"\"\"\n",
    "q_a_pair = f\"\"\"\n",
    "Customer message: ```{customer_message}```\n",
    "Product information: ```{product_information}```\n",
    "Agent response: ```{final_response_to_customer}```\n",
    "\n",
    "Does the response use the retrieved information correctly?\n",
    "Does the response sufficiently answer the question\n",
    "\n",
    "Output Y or N\n",
    "\"\"\"\n",
    "messages = [\n",
    "    {'role': 'system', 'content': system_message},\n",
    "    {'role': 'user', 'content': q_a_pair}\n",
    "]\n",
    "\n",
    "response = get_completion_from_messages(messages, max_tokens=1)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c33911c",
   "metadata": {
    "height": 319
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N\n"
     ]
    }
   ],
   "source": [
    "another_response = \"life is like a box of chocolates\"\n",
    "q_a_pair = f\"\"\"\n",
    "Customer message: ```{customer_message}```\n",
    "Product information: ```{product_information}```\n",
    "Agent response: ```{another_response}```\n",
    "\n",
    "Does the response use the retrieved information correctly?\n",
    "Does the response sufficiently answer the question?\n",
    "\n",
    "Output Y or N\n",
    "\"\"\"\n",
    "messages = [\n",
    "    {'role': 'system', 'content': system_message},\n",
    "    {'role': 'user', 'content': q_a_pair}\n",
    "]\n",
    "\n",
    "response = get_completion_from_messages(messages)\n",
    "print(response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}