manually_annotate_polygon_wikipedia_article.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "466968d8-f275-4090-a105-903d1cad7c83",
   "metadata": {},
   "source": [
    "# Annotating geometric hyponyms/hypernyms in the \"Polygon\" Wikipedia article "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a380d85b-7d19-4f3b-a530-16365a49e2f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter\n",
    "import wikipedia\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from collections import defaultdict\n",
    "from io import StringIO\n",
    "import os\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a841fe1a-cca5-4643-854c-681376ac5b47",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Download the Polygon article (https://en.wikipedia.org/wiki/Polygon). Save it to a file.\n",
    "If the file already exists and its contents differs from the freshly loaded wiki article, raise an error.\n",
    "\"\"\"\n",
    "\n",
    "POLYGON_ARTICLE_FILE_PATH = \"polygon_wiki.txt\"\n",
    "\n",
    "# auto_suggest=False because https://stackoverflow.com/a/69886635\n",
    "page = wikipedia.page(title=\"Polygon\", auto_suggest=False, preload=True)\n",
    "# print(page.content[:200], \"\\n\\n\", page.content[-200:])\n",
    "\n",
    "if os.path.isfile(POLYGON_ARTICLE_FILE_PATH):\n",
    "    with open(POLYGON_ARTICLE_FILE_PATH) as file:\n",
    "        if file.read() != page.content:\n",
    "            raise RuntimeError(f\"The article has changed, check the file {POLYGON_ARTICLE_FILE_PATH}\")\n",
    "else:    \n",
    "    with open(POLYGON_ARTICLE_FILE_PATH, mode=\"w\") as file:\n",
    "        file.write(page.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7dd7a450-ce11-4227-85a7-0186b7f7112f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunk_wikipedia_article(\n",
    "    article_text: str,\n",
    "    chunk_size_limit: int = 2000,\n",
    "    chunk_overlap: int = 200,\n",
    ") -> list:  \n",
    "    headers_to_split_on = [\n",
    "        (\"==\", \"header\"),\n",
    "    ]\n",
    "    md_splitter = MarkdownHeaderTextSplitter(headers_to_split_on, strip_headers=False)\n",
    "    docs = md_splitter.split_text(article_text)\n",
    "\n",
    "    def is_aux_section(d: str) -> bool:\n",
    "        try:\n",
    "            title = d.metadata[\"header\"]\n",
    "        except KeyError:\n",
    "            return False\n",
    "            \n",
    "        return (\n",
    "            title.startswith(\"See also\")\n",
    "            or title.startswith(\"Notes\")\n",
    "            or title.startswith(\"References\")\n",
    "            or title.startswith(\"External links\")\n",
    "        )\n",
    "\n",
    "    docs = [d for d in docs if not is_aux_section(d)]\n",
    "\n",
    "    rec_splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size_limit,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "    )\n",
    "    docs = rec_splitter.split_documents(docs)\n",
    "    \n",
    "    return docs\n",
    "\n",
    "def get_chunk_hash(chunk: str) -> str:\n",
    "    return hashlib.md5(chunk.encode()).hexdigest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f272fc5b-2233-4a9d-99a9-57d5951c7d53",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Load the Polygon Wikipedia article from file\n",
    "and split it into chunks. \n",
    "\"\"\"\n",
    "\n",
    "with open(POLYGON_ARTICLE_FILE_PATH) as file:\n",
    "    article_text = file.read()\n",
    "\n",
    "docs = chunk_wikipedia_article(article_text)\n",
    "annotations = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b304814-1111-42b2-acff-793d2f00e2fd",
   "metadata": {},
   "source": [
    "## Go through the chunks (aka `docs`) and annotate each one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8bde79d7-4396-42a6-b0f7-915008f264ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import textwrap\n",
    "\n",
    "def pretty_print(document) -> None:\n",
    "    print(textwrap.fill(document.page_content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "079ebfa4-5e2f-488c-bb4e-4e95e19a0bbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In geometry, a polygon () is a plane figure made up of line segments\n",
      "connected to form a closed polygonal chain. The segments of a closed\n",
      "polygonal chain are called its edges or sides. The points where two\n",
      "edges meet are the polygon's vertices or corners. An n-gon is a\n",
      "polygon with n sides; for example, a triangle is a 3-gon. A simple\n",
      "polygon is one which does not intersect itself. More precisely, the\n",
      "only allowed intersections among the line segments that make up the\n",
      "polygon are the shared endpoints of consecutive segments in the\n",
      "polygonal chain. A simple polygon is the boundary of a region of the\n",
      "plane that is called a solid polygon. The interior of a solid polygon\n",
      "is its body, also known as a polygonal region or polygonal area. In\n",
      "contexts where one is concerned only with simple and solid polygons, a\n",
      "polygon may refer only to a simple polygon or to a solid polygon. A\n",
      "polygonal chain may cross over itself, creating star polygons and\n",
      "other self-intersecting polygons. Some sources also consider closed\n",
      "polygonal chains in Euclidean space to be a type of polygon (a skew\n",
      "polygon), even when the chain does not lie in a single plane. A\n",
      "polygon is a 2-dimensional example of the more general polytope in any\n",
      "number of dimensions. There are many more generalizations of polygons\n",
      "defined for different purposes.\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"polygon\", \"plane figure\"),\n",
    "    (\"closed polygonal chain\", \"polygon\"),\n",
    "    (\"vertex\", \"point\"),\n",
    "    (\"edge\", \"segment\"),\n",
    "    (\"side\", \"segment\"),\n",
    "    (\"n-gon\", \"polygon\"),\n",
    "    (\"triangle\", \"n-gon\"),   \n",
    "    (\"triangle\", \"polygon\"), # n-gon and polygon are same thing, dupe\n",
    "    (\"3-gon\", \"n-gon\"),\n",
    "    # (\"simple polygon\", \"boundary\"),\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"solid polygon\", \"region\"),\n",
    "    (\"solid polygon\", \"polygon\"),\n",
    "    (\"skew polygon\", \"polygon\"),\n",
    "    (\"self-intersecting polygon\", \"polygon\"),\n",
    "    (\"star polygon\", \"polygon\"),\n",
    "    (\"polygon\", \"polytope\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5fd778e8-44c1-44d6-9689-7a5a4322481e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Etymology == The word polygon derives from the Greek adjective\n",
      "πολύς (polús) 'much', 'many' and γωνία (gōnía) 'corner' or 'angle'. It\n",
      "has been suggested that γόνυ (gónu) 'knee' may be the origin of gon.\n"
     ]
    }
   ],
   "source": [
    "i = 1\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "]\n",
    "# No meaningful annotations. A short section on etymology"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7d6a8575-7548-413a-b1a4-0cf6b3c99e29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Classification ==   === Number of sides === Polygons are primarily\n",
      "classified by the number of sides.   === Convexity and intersection\n",
      "=== Polygons may be characterized by their convexity or type of non-\n",
      "convexity:   Convex: any line drawn through the polygon (and not\n",
      "tangent to an edge or corner) meets its boundary exactly twice. As a\n",
      "consequence, all its interior angles are less than 180°. Equivalently,\n",
      "any line segment with endpoints on the boundary passes through only\n",
      "interior points between its endpoints. This condition is true for\n",
      "polygons in any geometry, not just Euclidean. Non-convex: a line may\n",
      "be found which meets its boundary more than twice. Equivalently, there\n",
      "exists a line segment between two boundary points that passes outside\n",
      "the polygon. Simple: the boundary of the polygon does not cross\n",
      "itself. All convex polygons are simple. Concave: Non-convex and\n",
      "simple. There is at least one interior angle greater than 180°. Star-\n",
      "shaped: the whole interior is visible from at least one point, without\n",
      "crossing any edge. The polygon must be simple, and may be convex or\n",
      "concave. All convex polygons are star-shaped. Self-intersecting: the\n",
      "boundary of the polygon crosses itself. The term complex is sometimes\n",
      "used in contrast to simple, but this usage risks confusion with the\n",
      "idea of a complex polygon as one which exists in the complex Hilbert\n",
      "plane consisting of two complex dimensions. Star polygon: a polygon\n",
      "which self-intersects in a regular way. A polygon cannot be both a\n",
      "star and star-shaped.   === Equality and symmetry === Equiangular: all\n",
      "corner angles are equal. Equilateral: all edges are of the same\n",
      "length. Regular: both equilateral and equiangular. Cyclic: all corners\n",
      "lie on a single circle, called the circumcircle. Tangential: all sides\n",
      "are tangent to an inscribed circle. Isogonal or vertex-transitive: all\n",
      "corners lie within the same symmetry orbit. The polygon is also cyclic\n",
      "and equiangular.\n"
     ]
    }
   ],
   "source": [
    "i = 2\n",
    "pretty_print(docs[i])\n",
    "# Different kinds of polygons\n",
    "annotations[i] = [\n",
    "    (\"convex polygon\", \"polygon\"),\n",
    "    (\"non-convex polygon\", \"polygon\"),\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"concave polygon\", \"polygon\"),\n",
    "    # (\"Euclidian geometry\", \"geometry\"),\n",
    "    (\"star-shaped polygon\", \"polygon\"),\n",
    "    (\"self-intersecting polygon\", \"polygon\"),\n",
    "    (\"star polygon\", \"polygon\"),\n",
    "    (\"equiangular polygon\", \"polygon\"),\n",
    "    (\"equilateral polygon\", \"polygon\"),\n",
    "    (\"regular polygon\", \"polygon\"),\n",
    "    (\"cyclic polygon\", \"polygon\"),\n",
    "    (\"tangential polygon\", \"polygon\"),\n",
    "    (\"isogonal polygon\", \"polygon\"),\n",
    "    (\"vertex-transitive polygon\", \"polygon\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c84d48ea-5334-478c-ac04-93ac0af6f1ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tangential: all sides are tangent to an inscribed circle. Isogonal or\n",
      "vertex-transitive: all corners lie within the same symmetry orbit. The\n",
      "polygon is also cyclic and equiangular. Isotoxal or edge-transitive:\n",
      "all sides lie within the same symmetry orbit. The polygon is also\n",
      "equilateral and tangential.The property of regularity may be defined\n",
      "in other ways: a polygon is regular if and only if it is both isogonal\n",
      "and isotoxal, or equivalently it is both cyclic and equilateral. A\n",
      "non-convex regular polygon is called a regular star polygon.   ===\n",
      "Miscellaneous === Rectilinear: the polygon's sides meet at right\n",
      "angles, i.e. all its interior angles are 90 or 270 degrees. Monotone\n",
      "with respect to a given line L: every line orthogonal to L intersects\n",
      "the polygon not more than twice.\n"
     ]
    }
   ],
   "source": [
    "i = 3\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"tangential polygon\", \"polygon\"),\n",
    "    (\"isogonal polygon\", \"polygon\"),\n",
    "    (\"isotoxal polygon\", \"polygon\"),\n",
    "    (\"convex polygon\", \"polygon\"),\n",
    "    (\"rectilinear polygon\", \"polygon\"),\n",
    "    (\"non-convex regular polygon\", \"regular polygon\"),\n",
    "    (\"regular star polygon\", \"star polygon\"),\n",
    "    # ??? Monotone with respect to a given line L --- too long to be included\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cc4384b6-4768-4812-bd98-fcadd67de757",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Properties and formulas == Euclidean geometry is assumed\n",
      "throughout.   === Angles === Any polygon has as many corners as it has\n",
      "sides. Each corner has several angles. The two most important ones\n",
      "are:   Interior angle – The sum of the interior angles of a simple\n",
      "n-gon is (n − 2) × π radians or (n − 2) × 180 degrees. This is because\n",
      "any simple n-gon ( having n sides ) can be considered to be made up of\n",
      "(n − 2) triangles, each of which has an angle sum of π radians or 180\n",
      "degrees. The measure of any interior angle of a convex regular n-gon\n",
      "is   (   1 −   2 n   )   π   {\\displaystyle \\left(1-{\\tfrac\n",
      "{2}{n}}\\right)\\pi } radians or   180 −   360 n   {\\displaystyle\n",
      "180-{\\tfrac {360}{n}}} degrees. The interior angles of regular star\n",
      "polygons were first studied by Poinsot, in the same paper in which he\n",
      "describes the four regular star polyhedra: for a regular   p q\n",
      "{\\displaystyle {\\tfrac {p}{q}}} -gon (a p-gon with central density q),\n",
      "each interior angle is   π ( p − 2 q )   p   {\\displaystyle {\\tfrac\n",
      "{\\pi (p-2q)}{p}}} radians or   180 ( p − 2 q )   p   {\\displaystyle\n",
      "{\\tfrac {180(p-2q)}{p}}} degrees. Exterior angle – The exterior angle\n",
      "is the supplementary angle to the interior angle. Tracing around a\n",
      "convex n-gon, the angle \"turned\" at a corner is the exterior or\n",
      "external angle. Tracing all the way around the polygon makes one full\n",
      "turn, so the sum of the exterior angles must be 360°. This argument\n",
      "can be generalized to concave simple polygons, if external angles that\n",
      "turn in the opposite direction are subtracted from the total turned.\n",
      "Tracing around an n-gon in general, the sum of the exterior angles\n",
      "(the total amount one rotates at the vertices) can be any integer\n",
      "multiple d of 360°, e.g. 720° for a pentagram and 0° for an angular\n",
      "\"eight\" or antiparallelogram, where d is the density or turning number\n",
      "of the polygon.   === Area === In this section, the vertices of the\n",
      "polygon under consideration are taken to be   (   x   0   ,   y   0\n",
      ") , (   x   1   ,   y   1   ) ,\n"
     ]
    }
   ],
   "source": [
    "i = 4\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    # (\"Euclidian geometry\", \"geometry\"),\n",
    "    (\"interior angle\", \"angle\"),\n",
    "    (\"exterior angle\", \"angle\"),\n",
    "    (\"external polygon\", \"polygon\"),\n",
    "    (\"external polygon\", \"polygon\"),\n",
    "    (\"simple n-gon\", \"n-gon\"),\n",
    "    (\"convex n-gon\", \"n-gon\"),\n",
    "    (\"concave simple polygon\", \"polygon\"),\n",
    "    # pentagram, angular \"eight\" or antiparallelogram, turning number --- all without obvious hypernyms\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a66d931e-3b1b-4da6-8ce0-2297ffb1a15e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Area === In this section, the vertices of the polygon under\n",
      "consideration are taken to be   (   x   0   ,   y   0   ) , (   x   1\n",
      ",   y   1   ) , … , (   x   n − 1   ,   y   n − 1   )   {\\displaystyle\n",
      "(x_{0},y_{0}),(x_{1},y_{1}),\\ldots ,(x_{n-1},y_{n-1})} in order. For\n",
      "convenience in some formulas, the notation (xn, yn) = (x0, y0) will\n",
      "also be used.   ==== Simple polygons ====   If the polygon is non-\n",
      "self-intersecting (that is, simple), the signed area is   A =   1 2\n",
      "∑   i = 0   n − 1   (   x   i   y   i + 1   −   x   i + 1   y   i   )\n",
      "where   x   n   =   x   0   and   y   n   =   y   0   ,\n",
      "{\\displaystyle A={\\frac {1}{2}}\\sum\n",
      "_{i=0}^{n-1}(x_{i}y_{i+1}-x_{i+1}y_{i})\\quad {\\text{where\n",
      "}}x_{n}=x_{0}{\\text{ and }}y_{n}=y_{0},} or, using determinants   16\n",
      "A   2   =   ∑   i = 0   n − 1   ∑   j = 0   n − 1   |   Q   i , j   Q\n",
      "i , j + 1   Q   i + 1 , j   Q   i + 1 , j + 1   |   ,   {\\displaystyle\n",
      "16A^{2}=\\sum _{i=0}^{n-1}\\sum _{j=0}^{n-\n",
      "1}{\\begin{vmatrix}Q_{i,j}&Q_{i,j+1}\\\\Q_{i+1,j}&Q_{i+1,j+1}\\end{vmatrix\n",
      "}},} where   Q   i , j   {\\displaystyle Q_{i,j}} is the squared\n",
      "distance between   (   x   i   ,   y   i   )   {\\displaystyle\n",
      "(x_{i},y_{i})} and   (   x   j   ,   y   j   ) .   {\\displaystyle\n",
      "(x_{j},y_{j}).} The signed area depends on the ordering of the\n",
      "vertices and of the orientation of the plane. Commonly, the positive\n",
      "orientation is defined by the (counterclockwise) rotation that maps\n",
      "the positive x-axis to the positive y-axis. If the vertices are\n",
      "ordered counterclockwise (that is, according to positive orientation),\n",
      "the signed area is positive; otherwise, it is negative. In either\n",
      "case, the area formula is correct in absolute value. This is commonly\n",
      "called the shoelace formula or surveyor's formula.The area A of a\n",
      "simple polygon can also be computed if the lengths of the sides, a1,\n",
      "a2, ..., an and the exterior angles, θ1, θ2, ..., θn are known, from:\n",
      "A =   1 2   (   a   1   [   a   2   sin ⁡ (   θ   1   ) +   a   3\n",
      "sin ⁡ (   θ   1   +   θ   2   ) +\n"
     ]
    }
   ],
   "source": [
    "i = 5\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"non-self-intersecting polygon\", \"polygon\"),\n",
    "    # (\"signed area\", \"area\")\n",
    "    (\"exterior angle\", \"angle\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "103703f8-74b5-421a-92e4-d5524c9c8c36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A =   1 2   (   a   1   [   a   2   sin ⁡ (   θ   1   ) +   a   3\n",
      "sin ⁡ (   θ   1   +   θ   2   ) + ⋯ +   a   n − 1   sin ⁡ (   θ   1\n",
      "+   θ   2   + ⋯ +   θ   n − 2   ) ]   +   a   2   [   a   3   sin ⁡ (\n",
      "θ   2   ) +   a   4   sin ⁡ (   θ   2   +   θ   3   ) + ⋯ +   a   n −\n",
      "1   sin ⁡ (   θ   2   + ⋯ +   θ   n − 2   ) ]   + ⋯ +   a   n − 2   [\n",
      "a   n − 1   sin ⁡ (   θ   n − 2   ) ] ) .   {\\displaystyle\n",
      "{\\begin{aligned}A={\\frac {1}{2}}(a_{1}[a_{2}\\sin(\\theta\n",
      "_{1})+a_{3}\\sin(\\theta _{1}+\\theta _{2})+\\cdots +a_{n-1}\\sin(\\theta\n",
      "_{1}+\\theta _{2}+\\cdots +\\theta _{n-2})]\\\\{}+a_{2}[a_{3}\\sin(\\theta\n",
      "_{2})+a_{4}\\sin(\\theta _{2}+\\theta _{3})+\\cdots +a_{n-1}\\sin(\\theta\n",
      "_{2}+\\cdots +\\theta _{n-2})]\\\\{}+\\cdots +a_{n-2}[a_{n-1}\\sin(\\theta\n",
      "_{n-2})]).\\end{aligned}}} The formula was described by Lopshits in\n",
      "1963.If the polygon can be drawn on an equally spaced grid such that\n",
      "all its vertices are grid points, Pick's theorem gives a simple\n",
      "formula for the polygon's area based on the numbers of interior and\n",
      "boundary grid points: the former number plus one-half the latter\n",
      "number, minus 1. In every polygon with perimeter p and area A , the\n",
      "isoperimetric inequality   p   2   > 4 π A   {\\displaystyle p^{2}>4\\pi\n",
      "A} holds.For any two simple polygons of equal area, the Bolyai–Gerwien\n",
      "theorem asserts that the first can be cut into polygonal pieces which\n",
      "can be reassembled to form the second polygon. The lengths of the\n",
      "sides of a polygon do not in general determine its area. However, if\n",
      "the polygon is simple and cyclic then the sides do determine the area.\n",
      "Of all n-gons with given side lengths, the one with the largest area\n",
      "is cyclic. Of all n-gons with a given perimeter, the one with the\n",
      "largest area is regular (and therefore cyclic).   ==== Regular\n",
      "polygons ==== Many specialized formulas apply to the areas of regular\n",
      "polygons. The area of a regular polygon is given in terms of the\n",
      "radius r of its inscribed circle and its perimeter p by   A =   1 2\n",
      "⋅ p ⋅ r .\n"
     ]
    }
   ],
   "source": [
    "i = 6\n",
    "pretty_print(docs[i])\n",
    "# Problem: polygon is same as n-gon, they are used interchangeably\n",
    "annotations[i] = [\n",
    "    (\"interior grid points\", \"grid points\"),\n",
    "    (\"boundary grid points\", \"grid points\"),\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"cyclic n-gon\", \"n-gon\"),\n",
    "    (\"regular n-gon\", \"n-gon\"),\n",
    "    (\"regular polygon\", \"polygon\"),\n",
    "    (\"cyclic polygon\", \"polygon\"),\n",
    "    (\"inscribed circle\", \"circle\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "57fac607-778e-4eba-b583-d0b4b9699018",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The area of a regular polygon is given in terms of the radius r of its\n",
      "inscribed circle and its perimeter p by   A =   1 2   ⋅ p ⋅ r .\n",
      "{\\displaystyle A={\\tfrac {1}{2}}\\cdot p\\cdot r.} This radius is also\n",
      "termed its apothem and is often represented as a. The area of a\n",
      "regular n-gon in terms of the radius R of its circumscribed circle can\n",
      "be expressed trigonometrically as:   A =   R   2   ⋅   n 2   ⋅ sin ⁡\n",
      "2 π   n   =   R   2   ⋅ n ⋅ sin ⁡   π n   ⋅ cos ⁡   π n\n",
      "{\\displaystyle A=R^{2}\\cdot {\\frac {n}{2}}\\cdot \\sin {\\frac {2\\pi\n",
      "}{n}}=R^{2}\\cdot n\\cdot \\sin {\\frac {\\pi }{n}}\\cdot \\cos {\\frac {\\pi\n",
      "}{n}}} The area of a regular n-gon inscribed in a unit-radius circle,\n",
      "with side s and interior angle   α ,   {\\displaystyle \\alpha ,} can\n",
      "also be expressed trigonometrically as:   A =   n   s   2   4   cot ⁡\n",
      "π n   =   n   s   2   4   cot ⁡   α   n − 2   = n ⋅ sin ⁡   α   n − 2\n",
      "⋅ cos ⁡   α   n − 2   .   {\\displaystyle A={\\frac {ns^{2}}{4}}\\cot\n",
      "{\\frac {\\pi }{n}}={\\frac {ns^{2}}{4}}\\cot {\\frac {\\alpha\n",
      "}{n-2}}=n\\cdot \\sin {\\frac {\\alpha }{n-2}}\\cdot \\cos {\\frac {\\alpha\n",
      "}{n-2}}.}   ==== Self-intersecting ==== The area of a self-\n",
      "intersecting polygon can be defined in two different ways, giving\n",
      "different answers:   Using the formulas for simple polygons, we allow\n",
      "that particular regions within the polygon may have their area\n",
      "multiplied by a factor which we call the density of the region. For\n",
      "example, the central convex pentagon in the center of a pentagram has\n",
      "density 2. The two triangular regions of a cross-quadrilateral (like a\n",
      "figure 8) have opposite-signed densities, and adding their areas\n",
      "together can give a total area of zero for the whole figure.\n"
     ]
    }
   ],
   "source": [
    "i = 7\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"regular polygon\", \"polygon\"),\n",
    "    (\"inscribed circle\", \"circle\"),\n",
    "    (\"regular n-gon\", \"n-gon\"),\n",
    "    (\"unit-radius circle\", \"circle\"),\n",
    "    (\"self-intersecting polygon\", \"polygon\"),\n",
    "    # central convex pentagon\n",
    "    # center of a pentagram, density\n",
    "    # triangular regions,\n",
    "    # cross-quadrilateral, figure 8\n",
    "    # have opposite-signed densities\n",
    "    # whole figure\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a479689-9e91-45c8-8238-9b00ddb563e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Considering the enclosed regions as point sets, we can find the area\n",
      "of the enclosed point set. This corresponds to the area of the plane\n",
      "covered by the polygon or to the area of one or more simple polygons\n",
      "having the same outline as the self-intersecting one. In the case of\n",
      "the cross-quadrilateral, it is treated as two simple triangles.   ===\n",
      "Centroid === Using the same convention for vertex coordinates as in\n",
      "the previous section, the coordinates of the centroid of a solid\n",
      "simple polygon are   C   x   =   1   6 A   ∑   i = 0   n − 1   (   x\n",
      "i   +   x   i + 1   ) (   x   i   y   i + 1   −   x   i + 1   y   i\n",
      ") ,   {\\displaystyle C_{x}={\\frac {1}{6A}}\\sum\n",
      "_{i=0}^{n-1}(x_{i}+x_{i+1})(x_{i}y_{i+1}-x_{i+1}y_{i}),}   C   y   =\n",
      "1   6 A   ∑   i = 0   n − 1   (   y   i   +   y   i + 1   ) (   x   i\n",
      "y   i + 1   −   x   i + 1   y   i   ) .   {\\displaystyle C_{y}={\\frac\n",
      "{1}{6A}}\\sum _{i=0}^{n-1}(y_{i}+y_{i+1})(x_{i}y_{i+1}-x_{i+1}y_{i}).}\n",
      "In these formulas, the signed value of area   A   {\\displaystyle A}\n",
      "must be used. For triangles (n = 3), the centroids of the vertices and\n",
      "of the solid shape are the same, but, in general, this is not true for\n",
      "n > 3. The centroid of the vertex set of a polygon with n vertices has\n",
      "the coordinates   c   x   =   1 n   ∑   i = 0   n − 1   x   i   ,\n",
      "{\\displaystyle c_{x}={\\frac {1}{n}}\\sum _{i=0}^{n-1}x_{i},}   c   y\n",
      "=   1 n   ∑   i = 0   n − 1   y   i   .   {\\displaystyle c_{y}={\\frac\n",
      "{1}{n}}\\sum _{i=0}^{n-1}y_{i}.}\n"
     ]
    }
   ],
   "source": [
    "i = 8\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    # enclosed region, point set\n",
    "    # simple polygon\n",
    "    # self-intersecting polygon\n",
    "    # cross-quadrilateral\n",
    "    # simple triangles\n",
    "    (\"centroid\", \"point\"),\n",
    "    (\"cross-quadrilateral\", \"polygon\"),\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"solid simple polygon\", \"polygon\"),\n",
    "    (\"triangle\", \"polygon\"),\n",
    "    (\"vertex\", \"point\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e56058a0-a191-4c3e-a94b-7ebb86b03656",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Generalizations == The idea of a polygon has been generalized in\n",
      "various ways. Some of the more important include:   A spherical\n",
      "polygon is a circuit of arcs of great circles (sides) and vertices on\n",
      "the surface of a sphere. It allows the digon, a polygon having only\n",
      "two sides and two corners, which is impossible in a flat plane.\n",
      "Spherical polygons play an important role in cartography (map making)\n",
      "and in Wythoff's construction of the uniform polyhedra. A skew polygon\n",
      "does not lie in a flat plane, but zigzags in three (or more)\n",
      "dimensions. The Petrie polygons of the regular polytopes are well\n",
      "known examples. An apeirogon is an infinite sequence of sides and\n",
      "angles, which is not closed but has no ends because it extends\n",
      "indefinitely in both directions. A skew apeirogon is an infinite\n",
      "sequence of sides and angles that do not lie in a flat plane. A\n",
      "polygon with holes is an area-connected or multiply-connected planar\n",
      "polygon with one external boundary and one or more interior boundaries\n",
      "(holes). A complex polygon is a configuration analogous to an ordinary\n",
      "polygon, which exists in the complex plane of two real and two\n",
      "imaginary dimensions. An abstract polygon is an algebraic partially\n",
      "ordered set representing the various elements (sides, vertices, etc.)\n",
      "and their connectivity. A real geometric polygon is said to be a\n",
      "realization of the associated abstract polygon. Depending on the\n",
      "mapping, all the generalizations described here can be realized. A\n",
      "polyhedron is a three-dimensional solid bounded by flat polygonal\n",
      "faces, analogous to a polygon in two dimensions. The corresponding\n",
      "shapes in four or higher dimensions are called polytopes. (In other\n",
      "conventions, the words polyhedron and polytope are used in any\n",
      "dimension, with the distinction between the two that a polytope is\n",
      "necessarily bounded.)\n"
     ]
    }
   ],
   "source": [
    "i = 9\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"spherical polygon\", \"polygon\"),\n",
    "    (\"digon\", \"polygon\"),\n",
    "    # flat plane, uniform polyhedra, Wythoff's construction\n",
    "    (\"skew polygon\", \"polygon\"),\n",
    "    (\"Petrie polygon\", \"polygon\"),\n",
    "    (\"apeirogon\", \"polygon\"),\n",
    "    (\"skew apeirogon\", \"apeirogon\"),\n",
    "    (\"polygon with holes\", \"polygon\"),\n",
    "    (\"area-connected polygon\", \"polygon\"),\n",
    "    (\"multiply-connected polygon\", \"polygon\"),\n",
    "    (\"planar polygon\", \"polygon\"),\n",
    "    (\"complex polygon\", \"polygon\"),\n",
    "    (\"abstract polygon\", \"polygon\"),\n",
    "    (\"polyhedron\", \"three-dimensional solid\"),\n",
    "    (\"polytope\", \"polyhedron\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1bf043db-de55-4461-848d-350178c5038d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Naming == The word polygon comes from Late Latin polygōnum (a\n",
      "noun), from Greek πολύγωνον (polygōnon/polugōnon), noun use of neuter\n",
      "of πολύγωνος (polygōnos/polugōnos, the masculine adjective), meaning\n",
      "\"many-angled\". Individual polygons are named (and sometimes\n",
      "classified) according to the number of sides, combining a Greek-\n",
      "derived numerical prefix with the suffix -gon, e.g. pentagon,\n",
      "dodecagon. The triangle, quadrilateral and nonagon are exceptions.\n",
      "Beyond decagons (10-sided) and dodecagons (12-sided), mathematicians\n",
      "generally use numerical notation, for example 17-gon and\n",
      "257-gon.Exceptions exist for side counts that are easily expressed in\n",
      "verbal form (e.g. 20 and 30), or are used by non-mathematicians. Some\n",
      "special polygons also have their own names; for example the regular\n",
      "star pentagon is also known as the pentagram.   To construct the name\n",
      "of a polygon with more than 20 and fewer than 100 edges, combine the\n",
      "prefixes as follows. The \"kai\" term applies to 13-gons and higher and\n",
      "was used by Kepler, and advocated by John H. Conway for clarity of\n",
      "concatenated prefix numbers in the naming of quasiregular polyhedra,\n",
      "though not all sources use it.\n"
     ]
    }
   ],
   "source": [
    "i = 10\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"pentagon\", \"polygon\"),\n",
    "    (\"dodecagon\", \"polygon\"),\n",
    "    (\"triangle\", \"polygon\"),\n",
    "    (\"quadrilateral\", \"polygon\"),\n",
    "    (\"nonagon\", \"polygon\"),\n",
    "    (\"decagon\", \"polygon\"),\n",
    "    (\"pentagram\", \"polygon\"),\n",
    "    (\"regular star pentagon\", \"pentagon\"),\n",
    "    # regular star pentagon AKA pentagram\n",
    "    # quasiregular polyhedra\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1b97acef-4baa-4a80-a202-f2b47d811f34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== History == Polygons have been known since ancient times. The\n",
      "regular polygons were known to the ancient Greeks, with the pentagram,\n",
      "a non-convex regular polygon (star polygon), appearing as early as the\n",
      "7th century B.C. on a krater by Aristophanes, found at Caere and now\n",
      "in the Capitoline Museum.The first known systematic study of non-\n",
      "convex polygons in general was made by Thomas Bradwardine in the 14th\n",
      "century.In 1952, Geoffrey Colin Shephard generalized the idea of\n",
      "polygons to the complex plane, where each real dimension is\n",
      "accompanied by an imaginary one, to create complex polygons.\n"
     ]
    }
   ],
   "source": [
    "i = 11\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"regular polygon\", \"polygon\"),\n",
    "    (\"pentagram\", \"non-convex regular polygon\"),\n",
    "    (\"non-convex regular polygon\", \"polygon\"),\n",
    "    (\"star polygon\", \"polygon\"),\n",
    "    (\"non-convex polygon\", \"polygon\"),\n",
    "    (\"real dimension\", \"dimension\"),\n",
    "    (\"imaginary dimension\", \"dimension\"),\n",
    "    (\"complex polygon\", \"polygon\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7a3174d0-c864-4120-a6bb-fa02af58ba5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== In nature == Polygons appear in rock formations, most commonly as\n",
      "the flat facets of crystals, where the angles between the sides depend\n",
      "on the type of mineral from which the crystal is made. Regular\n",
      "hexagons can occur when the cooling of lava forms areas of tightly\n",
      "packed columns of basalt, which may be seen at the Giant's Causeway in\n",
      "Northern Ireland, or at the Devil's Postpile in California. In\n",
      "biology, the surface of the wax honeycomb made by bees is an array of\n",
      "hexagons, and the sides and base of each cell are also polygons.\n"
     ]
    }
   ],
   "source": [
    "i = 12\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"regular hexagon\", \"hexagon\"),\n",
    "    # array of hexagons\n",
    "    # surface\n",
    "    # sides and base of each cell are poolygons\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "65a17145-cf8a-4a9b-9280-3295fe96386c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Computer graphics ==   In computer graphics, a polygon is a\n",
      "primitive used in modelling and rendering. They are defined in a\n",
      "database, containing arrays of vertices (the coordinates of the\n",
      "geometrical vertices, as well as other attributes of the polygon, such\n",
      "as color, shading and texture), connectivity information, and\n",
      "materials.Any surface is modelled as a tessellation called polygon\n",
      "mesh. If a square mesh has n + 1 points (vertices) per side, there are\n",
      "n squared squares in the mesh, or 2n squared triangles since there are\n",
      "two triangles in a square. There are (n + 1)2 / 2(n2) vertices per\n",
      "triangle. Where n is large, this approaches one half. Or, each vertex\n",
      "inside the square mesh connects four edges (lines). The imaging system\n",
      "calls up the structure of polygons needed for the scene to be created\n",
      "from the database. This is transferred to active memory and finally,\n",
      "to the display system (screen, TV monitors etc.) so that the scene can\n",
      "be viewed. During this process, the imaging system renders polygons in\n",
      "correct perspective ready for transmission of the processed data to\n",
      "the display system. Although polygons are two-dimensional, through the\n",
      "system computer they are placed in a visual scene in the correct\n",
      "three-dimensional orientation. In computer graphics and computational\n",
      "geometry, it is often necessary to determine whether a given point   P\n",
      "= (   x   0   ,   y   0   )   {\\displaystyle P=(x_{0},y_{0})} lies\n",
      "inside a simple polygon given by a sequence of line segments. This is\n",
      "called the point in polygon test.\n"
     ]
    }
   ],
   "source": [
    "i = 13\n",
    "pretty_print(docs[i])\n",
    "annotations[i] = [\n",
    "    (\"polygon\", \"primitive\"),\n",
    "    (\"polygon mesh\", \"tesselation\"),\n",
    "    (\"square\", \"polygon\"),\n",
    "    # polygons are two-dimensional\n",
    "    (\"simple polygon\", \"polygon\"),\n",
    "    (\"vertex\", \"point\"),\n",
    "    # line segments\n",
    "    # arrays of vertices (the coordinates of the geometrical vertices,\n",
    "    # as well as other attributes of the polygon, such as color, shading and texture)\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd4486f4-9b9d-4045-9856-a67e4d1d4489",
   "metadata": {},
   "source": [
    "## Observations\n",
    "\n",
    "- there are chunks from which no meaningful pairs can be extracted (e.g. etymology of the term)\n",
    "- most annotated pairs are of the form (\\<adjective\\> X, X), e.g. `(\"simple polygon\", \"polygon\")`\n",
    "- if a phrase containing \"\\<adjective\\> X\" appears and X is a valid term without the adjective, then I require (\\<adjective\\> X, X) to be retrieved by the LLM, even though X might be used within the chunk with this adjective only\n",
    "- there are synonymous terms like n-gon and polygon that make annotations dubious for naive metric calculation. I don't count those as hyponym/hypernym pairs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d75e1399-da57-41f3-930d-7f6c9adba84a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('3-gon', 'n-gon'),\n",
       " ('Petrie polygon', 'polygon'),\n",
       " ('abstract polygon', 'polygon'),\n",
       " ('apeirogon', 'polygon'),\n",
       " ('area-connected polygon', 'polygon'),\n",
       " ('boundary grid points', 'grid points'),\n",
       " ('centroid', 'point'),\n",
       " ('closed polygonal chain', 'polygon'),\n",
       " ('complex polygon', 'polygon'),\n",
       " ('concave polygon', 'polygon'),\n",
       " ('concave simple polygon', 'polygon'),\n",
       " ('convex n-gon', 'n-gon'),\n",
       " ('convex polygon', 'polygon'),\n",
       " ('cross-quadrilateral', 'polygon'),\n",
       " ('cyclic n-gon', 'n-gon'),\n",
       " ('cyclic polygon', 'polygon'),\n",
       " ('decagon', 'polygon'),\n",
       " ('digon', 'polygon'),\n",
       " ('dodecagon', 'polygon'),\n",
       " ('edge', 'segment'),\n",
       " ('equiangular polygon', 'polygon'),\n",
       " ('equilateral polygon', 'polygon'),\n",
       " ('exterior angle', 'angle'),\n",
       " ('external polygon', 'polygon'),\n",
       " ('imaginary dimension', 'dimension'),\n",
       " ('inscribed circle', 'circle'),\n",
       " ('interior angle', 'angle'),\n",
       " ('interior grid points', 'grid points'),\n",
       " ('isogonal polygon', 'polygon'),\n",
       " ('isotoxal polygon', 'polygon'),\n",
       " ('multiply-connected polygon', 'polygon'),\n",
       " ('n-gon', 'polygon'),\n",
       " ('non-convex polygon', 'polygon'),\n",
       " ('non-convex regular polygon', 'polygon'),\n",
       " ('non-convex regular polygon', 'regular polygon'),\n",
       " ('non-self-intersecting polygon', 'polygon'),\n",
       " ('nonagon', 'polygon'),\n",
       " ('pentagon', 'polygon'),\n",
       " ('pentagram', 'non-convex regular polygon'),\n",
       " ('pentagram', 'polygon'),\n",
       " ('planar polygon', 'polygon'),\n",
       " ('polygon', 'plane figure'),\n",
       " ('polygon', 'polytope'),\n",
       " ('polygon', 'primitive'),\n",
       " ('polygon mesh', 'tesselation'),\n",
       " ('polygon with holes', 'polygon'),\n",
       " ('polyhedron', 'three-dimensional solid'),\n",
       " ('polytope', 'polyhedron'),\n",
       " ('quadrilateral', 'polygon'),\n",
       " ('real dimension', 'dimension'),\n",
       " ('rectilinear polygon', 'polygon'),\n",
       " ('regular hexagon', 'hexagon'),\n",
       " ('regular n-gon', 'n-gon'),\n",
       " ('regular polygon', 'polygon'),\n",
       " ('regular star pentagon', 'pentagon'),\n",
       " ('regular star polygon', 'star polygon'),\n",
       " ('self-intersecting polygon', 'polygon'),\n",
       " ('side', 'segment'),\n",
       " ('simple n-gon', 'n-gon'),\n",
       " ('simple polygon', 'polygon'),\n",
       " ('skew apeirogon', 'apeirogon'),\n",
       " ('skew polygon', 'polygon'),\n",
       " ('solid polygon', 'polygon'),\n",
       " ('solid polygon', 'region'),\n",
       " ('solid simple polygon', 'polygon'),\n",
       " ('spherical polygon', 'polygon'),\n",
       " ('square', 'polygon'),\n",
       " ('star polygon', 'polygon'),\n",
       " ('star-shaped polygon', 'polygon'),\n",
       " ('tangential polygon', 'polygon'),\n",
       " ('triangle', 'n-gon'),\n",
       " ('triangle', 'polygon'),\n",
       " ('unit-radius circle', 'circle'),\n",
       " ('vertex', 'point'),\n",
       " ('vertex-transitive polygon', 'polygon')}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_annots_in_one_list = set()\n",
    "for anns in annotations.values():\n",
    "    all_annots_in_one_list = all_annots_in_one_list.union(anns)\n",
    "\n",
    "all_annots_in_one_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "13c26ae4-c6aa-4da8-9b18-c83a434f4f58",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Save the annotations to disk for future reuse\n",
    "\"\"\"\n",
    "import json\n",
    "\n",
    "annotations_sorted = [annot for idx, annot in sorted(annotations.items())]\n",
    "chunks = [d.page_content for d in docs]\n",
    "chunks_with_pairs = [\n",
    "    {\n",
    "        \"chunk\" : c,\n",
    "        \"x_is_a_y\" : annot \n",
    "     } for c, annot in zip(chunks, annotations_sorted)\n",
    "]\n",
    "to_save = {\"annotated_chunks\" : chunks_with_pairs}\n",
    "\n",
    "with open(\"annotations.json\", \"w\") as file:\n",
    "    json.dump(to_save, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f6c9111b-a4fb-40f5-ad65-ee4700d1b701",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'annotated_chunks': [{'chunk': \"In geometry, a polygon () is a plane figure made up of line segments connected to form a closed polygonal chain.\\nThe segments of a closed polygonal chain are called its edges or sides. The points where two edges meet are the polygon's vertices or corners. An n-gon i\n"
     ]
    }
   ],
   "source": [
    "with open(\"annotations.json\") as file:\n",
    "    js = json.load(file)\n",
    "\n",
    "print(str(js)[:300])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a715bfa6-7049-4559-b3de-9670685534e0",
   "metadata": {},
   "source": [
    "## Example: how to calculate the metrics\n",
    "\n",
    "Suppose the LLM read a chunk of Wikipedia text and returned a set of pairs for this chunk that is similar to my annotations above.\n",
    "How do we compare the LLM's answer to the fuzzy ground truth?\n",
    "Denote the LLM answers $\\{(l_i, L_i)\\}_{i=1}^n$ ($l_i$ is a kind of $L_i$) and the ground truth by and $\\{(t_i, T_i)\\}_{i=1}^m$\n",
    "\n",
    "Here's an outline of the algorithm plus some ideas for the future:\n",
    "\n",
    "1. validate both sets of pairs for duplicates and other trivially detectable inconsistencies\n",
    "\n",
    "2. convert everything to lowercase and strip of spaces\n",
    "\n",
    "3. find exact matches, count them and exclude them from the sets.\n",
    "\n",
    "4. Calculate the \"recall\" metric, i.e. the fraction of the ground truth samples that are covered by the LLM answer.\n",
    "\n",
    "5. Calculate the \"trash rate\" metric, i.e. $\\frac{n-C}{n}$ where we have $n$ pairs from the LLM and $C$ of them match some pairs from the ground truth.\n",
    "\n",
    "6. How can we aggregate the results when we've calculated the metrics for each of the N chunks? We can simply concatenate the lists of pairs for them and calculate recall and trash rate for the whole article.\n",
    "We can also keep the metrics for each chunk and average them, perhaps weighted by the number of ground truth pairs in each chunk.\n",
    "It's useful to check which chunks were least successful, what the typical problems were.\n",
    "\n",
    "7. MAYBE LATER: find partial matches by pairing $(l_i, L_i)$ with $(t_j, T_j)$ if $d(l_i, t_j) < D$ and $d(l_i, t_j) < D$ for some distance $d$ and threshold $D$ (e.g. edit distance of 2). This might account for slight variations such as apostrophe or spelling mistakes.\n",
    "\n",
    "8. MAYBE LATER: go over unmatched LLM answers and check for skipped elements in the hypernym chain, i.e. (rhombus, polygon) is a partially correct answer if the ground truth contains both (rhombus, quadrilateral) and (quadrilateral, polygon) \n",
    "\n",
    "9. MAYBE LATER: For each wrong answer (not a match), find a closest unpaired ground truth answer. That might ease debugging.\n",
    "\n",
    "10. MAYBE LATER: use LLM as a judge.\n",
    "\n",
    "11. MAYBE LATER: support annotations with a list of equivalent acceptable pairs (e.g. acceptable spelling variations)\n",
    "\n",
    "12. MAYBE LATER: support annotations with possible but undesirable answers to catch wrong answers\n",
    "\n",
    "13. MAYBE LATER: add confidence scores for each ground truth pair, i.e. how important is it for the LLM to retrieve this pair. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ae273796-5712-475b-94f6-3011b464b8a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recall = 0.5, trash_rate = 0.5\n",
      "exact_matches = {('interior angle', 'angle'), ('cyclic polygon', 'polygon')}\n",
      "wrong_answers = {('bogus example', 'polygon'), ('regular_polygon', 'angle')}\n",
      "unpaired_ground_truth = {('regular polygon', 'polygon'), ('exterior angle', 'angle')}\n"
     ]
    }
   ],
   "source": [
    "# Some sample data to calculate the metrics on.\n",
    "ground_truth = [\n",
    "    (\"interior angle\", \"angle\"),\n",
    "    (\"exterior angle\", \"angle\"),\n",
    "    (\"regular polygon\", \"polygon\"),\n",
    "    (\"cyclic polygon\", \"polygon\"),\n",
    "]\n",
    "llm_answers = [\n",
    "    (\"interior angle\", \"angle\"),\n",
    "    (\"cyclic polygon\", \"polygon\"),\n",
    "    (\"bogus example\", \"polygon\"),\n",
    "    (\"regular_polygon\", \"angle\"),\n",
    "]\n",
    "\n",
    "# Convert to lowercase, strip\n",
    "ground_truth = [(t.lower().strip(), T.lower().strip()) for t, T in ground_truth]\n",
    "llm_answers = [(l.lower().strip(), L.lower().strip()) for l, L in llm_answers]\n",
    "\n",
    "# Validation step (skipped)\n",
    "# Duplicates removed when set is constructed.\n",
    "ground_truth = set(ground_truth)\n",
    "llm_answers = set(llm_answers)\n",
    "\n",
    "exact_matches = set()\n",
    "for gt_pair in ground_truth:\n",
    "    if gt_pair in llm_answers:\n",
    "        exact_matches.add(gt_pair)\n",
    "\n",
    "wrong_answers = llm_answers.difference(exact_matches)\n",
    "unpaired_ground_truth = ground_truth.difference(exact_matches)\n",
    "\n",
    "if len(ground_truth) == 0:\n",
    "    recall = None\n",
    "else:\n",
    "    recall = len(unpaired_ground_truth) / len(ground_truth)\n",
    "\n",
    "if len(llm_answers) == 0:\n",
    "    trash_rate = None\n",
    "else:\n",
    "    trash_rate = len(wrong_answers) / len(llm_answers)\n",
    "\n",
    "print(f\"{recall = }, {trash_rate = }\")\n",
    "print(f\"{exact_matches = }\")\n",
    "print(f\"{wrong_answers = }\")\n",
    "print(f\"{unpaired_ground_truth = }\")\n",
    "\n",
    "metrics = {\n",
    "    \"exact_matches\" : exact_matches,\n",
    "    \"wrong_answers\" : wrong_answers,\n",
    "    \"unpaired_ground_truth\" : unpaired_ground_truth,\n",
    "    \"recall\" : recall,\n",
    "    \"trash_rate\" : trash_rate,\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}