Image Matching Challenge 2023 - Inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa385b8d",
   "metadata": {
    "papermill": {
     "duration": 0.008415,
     "end_time": "2023-06-12T10:31:56.201590",
     "exception": false,
     "start_time": "2023-06-12T10:31:56.193175",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 1. Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6c203a43",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-06-12T10:31:56.219473Z",
     "iopub.status.busy": "2023-06-12T10:31:56.218854Z",
     "iopub.status.idle": "2023-06-12T10:32:10.040423Z",
     "shell.execute_reply": "2023-06-12T10:32:10.039074Z"
    },
    "papermill": {
     "duration": 13.834215,
     "end_time": "2023-06-12T10:32:10.044104",
     "exception": false,
     "start_time": "2023-06-12T10:31:56.209889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n",
      "  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n",
      "  warnings.warn(f\"file system plugins are not loaded: {e}\")\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "import h5py\n",
    "import gc\n",
    "from glob import glob\n",
    "from pathlib import Path\n",
    "from functools import lru_cache\n",
    "from collections import defaultdict\n",
    "from fastprogress import progress_bar\n",
    "from copy import deepcopy\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image, ExifTags\n",
    "import cv2\n",
    "import sqlite3\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.cluster import DBSCAN\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import timm\n",
    "from timm.data import resolve_data_config\n",
    "from timm.data.transforms_factory import create_transform\n",
    "import kornia\n",
    "import pycolmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ffc118fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.081286Z",
     "iopub.status.busy": "2023-06-12T10:32:10.080036Z",
     "iopub.status.idle": "2023-06-12T10:32:10.086467Z",
     "shell.execute_reply": "2023-06-12T10:32:10.085236Z"
    },
    "papermill": {
     "duration": 0.027997,
     "end_time": "2023-06-12T10:32:10.089678",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.061681",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "competition_dataset = Path('/kaggle/input/image-matching-challenge-2023')\n",
    "external_dataset = Path('/kaggle/input/image-matching-challenge-2023-dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4c9aefe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.115360Z",
     "iopub.status.busy": "2023-06-12T10:32:10.115103Z",
     "iopub.status.idle": "2023-06-12T10:32:10.181847Z",
     "shell.execute_reply": "2023-06-12T10:32:10.180996Z"
    },
    "papermill": {
     "duration": 0.077681,
     "end_time": "2023-06-12T10:32:10.183693",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.106012",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sys.path.append(str(external_dataset / 'packages' / 'SuperGluePretrainedNetwork'))\n",
    "from models.matching import Matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0dae8132",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.201262Z",
     "iopub.status.busy": "2023-06-12T10:32:10.200532Z",
     "iopub.status.idle": "2023-06-12T10:32:10.230062Z",
     "shell.execute_reply": "2023-06-12T10:32:10.228858Z"
    },
    "papermill": {
     "duration": 0.040156,
     "end_time": "2023-06-12T10:32:10.232010",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.191854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Shape: (15, 5)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(competition_dataset / 'sample_submission.csv')\n",
    "\n",
    "if df.shape[0] != 3:\n",
    "    # Enable submission mode and disable verbose\n",
    "    submission = True\n",
    "    verbose = False\n",
    "    train_or_test_directory = competition_dataset / 'test'\n",
    "else:\n",
    "    # Disable submission mode and enable verbose\n",
    "    submission = False\n",
    "    verbose = True\n",
    "    train_or_test_directory = competition_dataset / 'train'\n",
    "\n",
    "if submission is False:\n",
    "    df = pd.read_csv(competition_dataset / 'train' / 'train_labels.csv')\n",
    "    # Use only bike scene from the haiper dataset if it's not submission mode\n",
    "    df = df.loc[df['scene'] == 'bike', :]\n",
    "    \n",
    "print(f'Dataset Shape: {df.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2db0fe0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.249943Z",
     "iopub.status.busy": "2023-06-12T10:32:10.249236Z",
     "iopub.status.idle": "2023-06-12T10:32:10.257087Z",
     "shell.execute_reply": "2023-06-12T10:32:10.256256Z"
    },
    "papermill": {
     "duration": 0.018865,
     "end_time": "2023-06-12T10:32:10.259028",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.240163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_angle_detection_model():\n",
    "    \n",
    "    \"\"\"\n",
    "    Load angle detection model and processor\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    model: keras.engine.functional.Functional\n",
    "        Angle detection model\n",
    "        \n",
    "    processor: transformers.ViTFeatureExtractor\n",
    "        Image processor\n",
    "    \"\"\"\n",
    "    \n",
    "    vit = TFViTModel.from_pretrained(external_dataset / 'models' / 'vit-base-patch16-224')\n",
    "    \n",
    "    input_layer = Input(shape=(3, 224, 224))\n",
    "    x = vit(input_layer)\n",
    "    y = Dense(1, activation='linear')(x[-1])\n",
    "    \n",
    "    model = Model(input_layer, y)\n",
    "    model.load_weights(external_dataset / 'models' / 'deep-image-orientation-angle-detection' / 'model-vit-ang-loss.h5')\n",
    "    \n",
    "    processor = ViTFeatureExtractor.from_pretrained(external_dataset / 'models' / 'vit-base-patch16-224')\n",
    "    \n",
    "    return model, processor\n",
    "\n",
    "\n",
    "def detect_angle(image, model, processor):\n",
    "    \n",
    "    \"\"\"\n",
    "    Detect angle of a given image\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    image: numpy.ndarray of shape (height, width, channel)\n",
    "        Image array\n",
    "        \n",
    "    model: keras.engine.functional.Functional\n",
    "        Angle detection model\n",
    "        \n",
    "    processor: transformers.ViTFeatureExtractor\n",
    "        Image processor\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    outputs: float\n",
    "    \"\"\"\n",
    "    \n",
    "    inputs = processor(images=[image], return_tensors='np')['pixel_values']\n",
    "    outputs = model.predict(inputs, verbose=None)[0][0]\n",
    "        \n",
    "    return outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "352d04a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.276345Z",
     "iopub.status.busy": "2023-06-12T10:32:10.276098Z",
     "iopub.status.idle": "2023-06-12T10:32:10.843925Z",
     "shell.execute_reply": "2023-06-12T10:32:10.842993Z"
    },
    "papermill": {
     "duration": 0.579243,
     "end_time": "2023-06-12T10:32:10.846241",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.266998",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:00<00:00, 27.38it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\ndf['angle'] = df['angle'].astype(np.float32)\\nangles = df[['image_path', 'angle']].set_index('image_path').to_dict()['angle']\\ndel angle_detection_model, angle_detection_processor\\ngc.collect()\\ntorch.cuda.empty_cache()\\nK.clear_session()\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#angle_detection_model, angle_detection_processor = load_angle_detection_model()\n",
    "\n",
    "for idx, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "\n",
    "    image_path = train_or_test_directory / row['image_path']\n",
    "    image = np.array(Image.open(image_path))\n",
    "    \n",
    "    # Extract image dimensions and memory usage\n",
    "    df.loc[idx, 'image_height'] = image.shape[0]\n",
    "    df.loc[idx, 'image_width'] = image.shape[1]\n",
    "    df.loc[idx, 'memory_usage'] = image.nbytes\n",
    "    \n",
    "    # Extract image orientation\n",
    "    '''\n",
    "    try:\n",
    "        with tf.device('/cpu:0'):\n",
    "            angle = detect_angle(\n",
    "                image=image,\n",
    "                model=angle_detection_model,\n",
    "                processor=angle_detection_processor\n",
    "            )\n",
    "            df.loc[idx, 'angle'] = angle\n",
    "    except:\n",
    "        df.loc[idx, 'angle'] = 0\n",
    "    ''' \n",
    "\n",
    "\n",
    "df['image_id'] = df['image_path'].apply(lambda x: str(x).split('/')[-1])\n",
    "df['memory_usage'] /= (1024 ** 2)\n",
    "\n",
    "df['image_height'] = df['image_height'].astype(np.uint16)\n",
    "df['image_width'] = df['image_width'].astype(np.uint16)\n",
    "df['memory_usage'] = df['memory_usage'].astype(np.float32)\n",
    "'''\n",
    "df['angle'] = df['angle'].astype(np.float32)\n",
    "angles = df[['image_path', 'angle']].set_index('image_path').to_dict()['angle']\n",
    "del angle_detection_model, angle_detection_processor\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "K.clear_session()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5d094894",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.865682Z",
     "iopub.status.busy": "2023-06-12T10:32:10.865385Z",
     "iopub.status.idle": "2023-06-12T10:32:10.884065Z",
     "shell.execute_reply": "2023-06-12T10:32:10.882986Z"
    },
    "papermill": {
     "duration": 0.031177,
     "end_time": "2023-06-12T10:32:10.886776",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.855599",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>scene</th>\n",
       "      <th>image_path</th>\n",
       "      <th>rotation_matrix</th>\n",
       "      <th>translation_vector</th>\n",
       "      <th>image_height</th>\n",
       "      <th>image_width</th>\n",
       "      <th>memory_usage</th>\n",
       "      <th>image_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_004.jpeg</td>\n",
       "      <td>-0.0444086367008516;-0.9943622104926982;0.0962...</td>\n",
       "      <td>-0.1960672197513899;-1.7201514631645476;2.9038...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_004.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_029.jpeg</td>\n",
       "      <td>-0.30379429789458934;-0.780428110344117;0.5464...</td>\n",
       "      <td>-0.7743795049636117;-2.4063622468195978;4.2039...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_029.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_038.jpeg</td>\n",
       "      <td>-0.44075875074949056;-0.6141344795467536;0.654...</td>\n",
       "      <td>-0.8061619977689489;-1.3614914099692277;2.5759...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_038.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_049.jpeg</td>\n",
       "      <td>-0.4615070783507189;-0.037215312681071866;0.88...</td>\n",
       "      <td>-0.05029237007998646;-2.0953351075544067;3.128...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_049.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_062.jpeg</td>\n",
       "      <td>0.06454466684801141;0.9943063402602297;0.08478...</td>\n",
       "      <td>0.45813698932578173;-1.7352377000311676;3.6787...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_062.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_076.jpeg</td>\n",
       "      <td>0.566876909470875;-0.09510152245268588;-0.8182...</td>\n",
       "      <td>1.1496049380742543;-1.4930263438651057;2.57378...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_076.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_088.jpeg</td>\n",
       "      <td>0.253507028584536;-0.8760074072730466;-0.41029...</td>\n",
       "      <td>-0.44871563031823725;-1.750899302580256;2.6802...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_088.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_094.jpeg</td>\n",
       "      <td>0.15438718127182738;-0.9293944893965761;-0.335...</td>\n",
       "      <td>-0.37595896679394963;-2.714322519243274;4.1322...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_094.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_101.jpeg</td>\n",
       "      <td>0.4234658773801474;-0.6722258122106511;-0.6072...</td>\n",
       "      <td>-0.11276896793699917;-0.7117396520115244;1.914...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_101.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_115.jpeg</td>\n",
       "      <td>0.3904043657227909;0.16367189264027227;-0.9059...</td>\n",
       "      <td>0.8296975499797659;-2.3480417943432315;3.47250...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_115.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_119.jpeg</td>\n",
       "      <td>0.34139276573704824;0.81351456101145;-0.470791...</td>\n",
       "      <td>-0.10007794513469218;-2.544833204003964;4.0417...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_119.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_128.jpeg</td>\n",
       "      <td>-0.2177248850251865;0.694914687853886;0.685338...</td>\n",
       "      <td>-1.1094352034062702;-2.5456449855138703;4.5788...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_128.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_137.jpeg</td>\n",
       "      <td>0.22425244568562896;0.8687843075194276;-0.4415...</td>\n",
       "      <td>0.7847615822482943;-0.25685464008216424;3.3788...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_137.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_139.jpeg</td>\n",
       "      <td>0.45484758018631954;0.5331256298557034;-0.7133...</td>\n",
       "      <td>1.0869579831200566;-1.5594034733929556;3.04410...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_139.jpeg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>haiper</td>\n",
       "      <td>bike</td>\n",
       "      <td>haiper/bike/images/image_150.jpeg</td>\n",
       "      <td>0.5749716111003644;-0.2792314233637461;-0.7690...</td>\n",
       "      <td>0.42673787089296894;-1.8688015981412343;2.7976...</td>\n",
       "      <td>1920</td>\n",
       "      <td>1440</td>\n",
       "      <td>7.910156</td>\n",
       "      <td>image_150.jpeg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset scene                         image_path  \\\n",
       "230  haiper  bike  haiper/bike/images/image_004.jpeg   \n",
       "231  haiper  bike  haiper/bike/images/image_029.jpeg   \n",
       "232  haiper  bike  haiper/bike/images/image_038.jpeg   \n",
       "233  haiper  bike  haiper/bike/images/image_049.jpeg   \n",
       "234  haiper  bike  haiper/bike/images/image_062.jpeg   \n",
       "235  haiper  bike  haiper/bike/images/image_076.jpeg   \n",
       "236  haiper  bike  haiper/bike/images/image_088.jpeg   \n",
       "237  haiper  bike  haiper/bike/images/image_094.jpeg   \n",
       "238  haiper  bike  haiper/bike/images/image_101.jpeg   \n",
       "239  haiper  bike  haiper/bike/images/image_115.jpeg   \n",
       "240  haiper  bike  haiper/bike/images/image_119.jpeg   \n",
       "241  haiper  bike  haiper/bike/images/image_128.jpeg   \n",
       "242  haiper  bike  haiper/bike/images/image_137.jpeg   \n",
       "243  haiper  bike  haiper/bike/images/image_139.jpeg   \n",
       "244  haiper  bike  haiper/bike/images/image_150.jpeg   \n",
       "\n",
       "                                       rotation_matrix  \\\n",
       "230  -0.0444086367008516;-0.9943622104926982;0.0962...   \n",
       "231  -0.30379429789458934;-0.780428110344117;0.5464...   \n",
       "232  -0.44075875074949056;-0.6141344795467536;0.654...   \n",
       "233  -0.4615070783507189;-0.037215312681071866;0.88...   \n",
       "234  0.06454466684801141;0.9943063402602297;0.08478...   \n",
       "235  0.566876909470875;-0.09510152245268588;-0.8182...   \n",
       "236  0.253507028584536;-0.8760074072730466;-0.41029...   \n",
       "237  0.15438718127182738;-0.9293944893965761;-0.335...   \n",
       "238  0.4234658773801474;-0.6722258122106511;-0.6072...   \n",
       "239  0.3904043657227909;0.16367189264027227;-0.9059...   \n",
       "240  0.34139276573704824;0.81351456101145;-0.470791...   \n",
       "241  -0.2177248850251865;0.694914687853886;0.685338...   \n",
       "242  0.22425244568562896;0.8687843075194276;-0.4415...   \n",
       "243  0.45484758018631954;0.5331256298557034;-0.7133...   \n",
       "244  0.5749716111003644;-0.2792314233637461;-0.7690...   \n",
       "\n",
       "                                    translation_vector  image_height  \\\n",
       "230  -0.1960672197513899;-1.7201514631645476;2.9038...          1920   \n",
       "231  -0.7743795049636117;-2.4063622468195978;4.2039...          1920   \n",
       "232  -0.8061619977689489;-1.3614914099692277;2.5759...          1920   \n",
       "233  -0.05029237007998646;-2.0953351075544067;3.128...          1920   \n",
       "234  0.45813698932578173;-1.7352377000311676;3.6787...          1920   \n",
       "235  1.1496049380742543;-1.4930263438651057;2.57378...          1920   \n",
       "236  -0.44871563031823725;-1.750899302580256;2.6802...          1920   \n",
       "237  -0.37595896679394963;-2.714322519243274;4.1322...          1920   \n",
       "238  -0.11276896793699917;-0.7117396520115244;1.914...          1920   \n",
       "239  0.8296975499797659;-2.3480417943432315;3.47250...          1920   \n",
       "240  -0.10007794513469218;-2.544833204003964;4.0417...          1920   \n",
       "241  -1.1094352034062702;-2.5456449855138703;4.5788...          1920   \n",
       "242  0.7847615822482943;-0.25685464008216424;3.3788...          1920   \n",
       "243  1.0869579831200566;-1.5594034733929556;3.04410...          1920   \n",
       "244  0.42673787089296894;-1.8688015981412343;2.7976...          1920   \n",
       "\n",
       "     image_width  memory_usage        image_id  \n",
       "230         1440      7.910156  image_004.jpeg  \n",
       "231         1440      7.910156  image_029.jpeg  \n",
       "232         1440      7.910156  image_038.jpeg  \n",
       "233         1440      7.910156  image_049.jpeg  \n",
       "234         1440      7.910156  image_062.jpeg  \n",
       "235         1440      7.910156  image_076.jpeg  \n",
       "236         1440      7.910156  image_088.jpeg  \n",
       "237         1440      7.910156  image_094.jpeg  \n",
       "238         1440      7.910156  image_101.jpeg  \n",
       "239         1440      7.910156  image_115.jpeg  \n",
       "240         1440      7.910156  image_119.jpeg  \n",
       "241         1440      7.910156  image_128.jpeg  \n",
       "242         1440      7.910156  image_137.jpeg  \n",
       "243         1440      7.910156  image_139.jpeg  \n",
       "244         1440      7.910156  image_150.jpeg  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a5caaf6",
   "metadata": {
    "papermill": {
     "duration": 0.008932,
     "end_time": "2023-06-12T10:32:10.905536",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.896604",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 2. Image Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "55bf6acc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.925178Z",
     "iopub.status.busy": "2023-06-12T10:32:10.924921Z",
     "iopub.status.idle": "2023-06-12T10:32:10.958004Z",
     "shell.execute_reply": "2023-06-12T10:32:10.957186Z"
    },
    "papermill": {
     "duration": 0.045533,
     "end_time": "2023-06-12T10:32:10.960157",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.914624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_global_desc(image_paths, model, device):\n",
    "        \n",
    "    config = resolve_data_config({}, model=model)\n",
    "    transform = create_transform(**config)\n",
    "    \n",
    "    global_descs = []\n",
    "    for i, img_fname_full in tqdm(enumerate(image_paths),total=len(image_paths)):\n",
    "        key = os.path.splitext(os.path.basename(img_fname_full))[0]\n",
    "        img = Image.open(img_fname_full).convert('RGB')\n",
    "        img = transform(img).unsqueeze(0).to(device)\n",
    "        with torch.no_grad():\n",
    "            with torch.autocast(device_type=device.type, dtype=torch.float16):\n",
    "                desc = model.forward_features(img.to(device)).mean(dim=(-1, 2))\n",
    "            desc = desc.view(1, -1).detach().cpu().float()\n",
    "            desc = F.normalize(desc, dim=1, p=2)\n",
    "        global_descs.append(desc)\n",
    "    global_descs = torch.cat(global_descs, dim=0)\n",
    "    \n",
    "    return global_descs\n",
    "\n",
    "\n",
    "def create_image_pairs_exhaustive(image_paths):\n",
    "\n",
    "    \"\"\"\n",
    "    Create all possible image pairs from given list of image paths\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image_paths: list of shape (n_images)\n",
    "        List of image paths\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    image_pair_indices: list of shape (n_image_pairs)\n",
    "        List of tuples of image pair indices\n",
    "    \"\"\"\n",
    "\n",
    "    image_pair_indices = []\n",
    "\n",
    "    for i in range(len(image_paths)):\n",
    "        for j in range(i + 1, len(image_paths)):\n",
    "            image_pair_indices.append((i, j))\n",
    "\n",
    "    return image_pair_indices\n",
    "\n",
    "\n",
    "def create_image_pairs(image_paths, sim_th=0.6, min_pairs=20, exhaustive_if_less=20, device=torch.device('cuda')):\n",
    "    \n",
    "    n_images = len(image_paths)\n",
    "\n",
    "    if n_images <= exhaustive_if_less:\n",
    "        \n",
    "        image_pair_indices = []\n",
    "        # Create all possible image pairs from given list of image paths\n",
    "        for i in range(len(image_paths)):\n",
    "            for j in range(i + 1, len(image_paths)):\n",
    "                image_pair_indices.append((i, j))\n",
    "\n",
    "        return image_pair_indices\n",
    "    \n",
    "    else:\n",
    "\n",
    "        model = timm.create_model(\n",
    "            'tf_efficientnet_b7',\n",
    "            checkpoint_path='/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b7/1/tf_efficientnet_b7_ra-6c08e654.pth'\n",
    "        )\n",
    "        model = model.eval().to(image_matching_device)\n",
    "        descs = get_global_desc(image_paths, model, device=device)\n",
    "        dm = torch.cdist(descs, descs, p=2).numpy()\n",
    "        mask = dm <= sim_th\n",
    "        total = 0\n",
    "        image_pair_indices = []\n",
    "        ar = np.arange(n_images)\n",
    "        already_there_set = []\n",
    "        for st_idx in range(n_images - 1):\n",
    "            mask_idx = mask[st_idx]\n",
    "            to_match = ar[mask_idx]\n",
    "            if len(to_match) < min_pairs:\n",
    "                to_match = np.argsort(dm[st_idx])[:min_pairs]  \n",
    "            for idx in to_match:\n",
    "                if st_idx == idx:\n",
    "                    continue\n",
    "                if dm[st_idx, idx] < 1000:\n",
    "                    image_pair_indices.append(tuple(sorted((st_idx, idx.item()))))\n",
    "                    total += 1\n",
    "                    \n",
    "        image_pair_indices = sorted(list(set(image_pair_indices)))\n",
    "        \n",
    "        return image_pair_indices\n",
    "\n",
    "\n",
    "def resize_with_aspect_ratio(image, longest_edge):\n",
    "\n",
    "    \"\"\"\n",
    "    Resize image while preserving its aspect ratio\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image: numpy.ndarray of shape (height, width, 3)\n",
    "        Image array\n",
    "\n",
    "    longest_edge: int\n",
    "        Desired number of pixels on the longest edge\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    image: numpy.ndarray of shape (resized_height, resized_width, 3)\n",
    "        Resized image array\n",
    "    \"\"\"\n",
    "\n",
    "    height, width = image.shape[:2]\n",
    "    scale = longest_edge / max(height, width)\n",
    "    image = cv2.resize(image, dsize=(int(width * scale), int(height * scale)), interpolation=cv2.INTER_LANCZOS4)\n",
    "\n",
    "    return image\n",
    "\n",
    "\n",
    "def get_image_tensor(image_path_or_array, resize, resize_shape, resize_longest_edge, scale, grayscale):\n",
    "\n",
    "    \"\"\"\n",
    "    Load image and preprocess it\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image_path_or_array: str or numpy.ndarray of shape (height, width, 3)\n",
    "        Image path or image array\n",
    "\n",
    "    resize: bool\n",
    "        Whether to resize the image or not\n",
    "\n",
    "    resize_shape: tuple or int\n",
    "        Tuple of image height and width or number of pixels for both height and width\n",
    "\n",
    "    resize_longest_edge: bool\n",
    "        Whether to resize the longest edge or not\n",
    "\n",
    "    scale: bool\n",
    "        Whether to scale image pixel values by max 8-bit pixel value or not\n",
    "\n",
    "    grayscale: bool\n",
    "        Whether to convert RGB image to grayscale or not\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    image: torch.Tensor of shape (1, 1 or 3, height, width)\n",
    "        Image tensor\n",
    "    \"\"\"\n",
    "\n",
    "    if isinstance(image_path_or_array, Path) or isinstance(image_path_or_array, str):\n",
    "        # Read image from the given path if image_path_or_array is a path-like string\n",
    "        image = cv2.imread(str(image_path_or_array))\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    else:\n",
    "        image = image_path_or_array\n",
    "\n",
    "    if resize:\n",
    "        if resize_longest_edge:\n",
    "            image = resize_with_aspect_ratio(image=image, longest_edge=resize_shape)\n",
    "        else:\n",
    "            resize_shape = (resize_shape, resize_shape) if isinstance(resize_shape, int) else resize_shape\n",
    "            image = cv2.resize(image, resize_shape, interpolation=cv2.INTER_LANCZOS4)\n",
    "\n",
    "    if scale:\n",
    "        image = image / 255.\n",
    "\n",
    "    image = kornia.image_to_tensor(image, False).float()\n",
    "    if grayscale:\n",
    "        image = kornia.color.rgb_to_grayscale(image)\n",
    "\n",
    "    return image\n",
    "\n",
    "\n",
    "def crop(image, keypoints, perc_points=0.85, pad=5):\n",
    "\n",
    "    norm_keypoints = MinMaxScaler().fit_transform(keypoints)\n",
    "    total = len(keypoints)\n",
    "    best_dist = 1\n",
    "    best_clusters = None\n",
    "    best_asm = None\n",
    "    for eps in [0.01, 0.025, 0.05, 0.1, 0.2]:\n",
    "        clusters = DBSCAN(eps=eps).fit_predict(norm_keypoints)\n",
    "        counts = pd.Series(clusters).value_counts().sort_values(ascending=False)\n",
    "        counts = counts[counts.index > -1]\n",
    "        if len(counts) == 0:\n",
    "            continue\n",
    "\n",
    "        cumsums = np.cumsum(counts.values) / total\n",
    "        dists = np.abs(cumsums - perc_points)\n",
    "        best_ix = np.argmin(dists)\n",
    "\n",
    "        if dists[best_ix] < best_dist:\n",
    "            best_dist = dists[best_ix]\n",
    "            best_clusters = list(counts.head(best_ix + 1).index)\n",
    "            best_asm = clusters\n",
    "\n",
    "    mask = np.isin(best_asm, best_clusters)\n",
    "\n",
    "    miny = int(np.min(keypoints[mask][:, 1]))\n",
    "    miny = max(miny - pad, 0)\n",
    "\n",
    "    maxy = int(np.max(keypoints[mask][:, 1]))\n",
    "    maxy = min(maxy + pad, image.shape[0])\n",
    "\n",
    "    minx = int(np.min(keypoints[mask][:, 0]))\n",
    "    minx = max(minx - pad, 0)\n",
    "\n",
    "    maxx = int(np.max(keypoints[mask][:, 0]))\n",
    "    maxx = min(maxx + pad, image.shape[1])\n",
    "\n",
    "    image_cropped = image[miny:maxy + 1, minx:maxx + 1, :]\n",
    "    keypoints_cropped = np.copy(keypoints)\n",
    "    keypoints_cropped[:, 0] -= minx\n",
    "    keypoints_cropped[:, 1] -= miny\n",
    "\n",
    "    keypoints_cropped = keypoints_cropped[(keypoints_cropped[:, 0] > 0) & (keypoints_cropped[:, 1] > 0) & (keypoints_cropped[:, 0] < image_cropped.shape[1]) & (keypoints_cropped[:, 1] < image_cropped.shape[0])]\n",
    "\n",
    "    return image_cropped, keypoints_cropped, minx, miny\n",
    "\n",
    "\n",
    "def rotate_image(img, angle):\n",
    "    height, width = img.shape[:2]\n",
    "    matrix = cv2.getRotationMatrix2D((width / 2 - 0.5, height / 2 - 0.5), angle, 1.0)\n",
    "    matrix = np.vstack((matrix, [[0, 0, 1]]))\n",
    "    return cv2.warpPerspective(img, matrix, (width, height))\n",
    "\n",
    "\n",
    "def rotate_coordinates(img, angle, keypoints):\n",
    "    height, width = img.shape[:2]\n",
    "    inv_matrix = cv2.getRotationMatrix2D((width / 2 - 0.5, height / 2 - 0.5), angle, 1.0)\n",
    "    inv_matrix = np.vstack((inv_matrix, [[0, 0, 1]]))\n",
    "    return cv2.perspectiveTransform(keypoints[None, :, :], inv_matrix)[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e9e9d20",
   "metadata": {
    "papermill": {
     "duration": 0.008926,
     "end_time": "2023-06-12T10:32:10.978267",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.969341",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 3. Camera Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80b9afc3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:10.998045Z",
     "iopub.status.busy": "2023-06-12T10:32:10.997317Z",
     "iopub.status.idle": "2023-06-12T10:32:11.005274Z",
     "shell.execute_reply": "2023-06-12T10:32:11.004487Z"
    },
    "papermill": {
     "duration": 0.019793,
     "end_time": "2023-06-12T10:32:11.007182",
     "exception": false,
     "start_time": "2023-06-12T10:32:10.987389",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_focal_length(image_path):\n",
    "\n",
    "    \"\"\"\n",
    "    Get focal length from EXIF or calculate it using prior\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image_path: str\n",
    "        Image path\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    focal_length: float\n",
    "        Focal length extracted from EXIF or calculated using prior\n",
    "    \"\"\"\n",
    "\n",
    "    image = Image.open(image_path)\n",
    "    image_longest_edge = max(image.size)\n",
    "\n",
    "    focal_length = None\n",
    "    exif = image.getexif()\n",
    "    \n",
    "    try:\n",
    "        if image._getexif() is not None:\n",
    "            exif = {\n",
    "                ExifTags.TAGS[k]: v\n",
    "                for k, v in image._getexif().items()\n",
    "                if k in ExifTags.TAGS\n",
    "            }\n",
    "        else:\n",
    "            exif = None\n",
    "    except:\n",
    "        exif = None\n",
    "       \n",
    "    if exif is not None:\n",
    "\n",
    "        focal_length_35mm = None\n",
    "\n",
    "        for tag, value in exif.items():\n",
    "            if tag == 'FocalLengthIn35mmFilm':\n",
    "                focal_length_35mm = float(value)\n",
    "        \n",
    "        if focal_length_35mm is not None:\n",
    "            focal_length = focal_length_35mm / 35. * image_longest_edge\n",
    "\n",
    "    if focal_length is None:\n",
    "        prior_focal_length = 1.2\n",
    "        focal_length = prior_focal_length * image_longest_edge\n",
    "\n",
    "    return focal_length\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a22221f8",
   "metadata": {
    "papermill": {
     "duration": 0.009036,
     "end_time": "2023-06-12T10:32:11.025335",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.016299",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 4. Database Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e83dbe22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.045651Z",
     "iopub.status.busy": "2023-06-12T10:32:11.045015Z",
     "iopub.status.idle": "2023-06-12T10:32:11.068147Z",
     "shell.execute_reply": "2023-06-12T10:32:11.067371Z"
    },
    "papermill": {
     "duration": 0.035511,
     "end_time": "2023-06-12T10:32:11.070045",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.034534",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "MAX_IMAGE_ID = 2 ** 31 - 1\n",
    "\n",
    "CREATE_CAMERAS_TABLE = \"\"\"CREATE TABLE IF NOT EXISTS cameras (\n",
    "    camera_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,\n",
    "    model INTEGER NOT NULL,\n",
    "    width INTEGER NOT NULL,\n",
    "    height INTEGER NOT NULL,\n",
    "    params BLOB,\n",
    "    prior_focal_length INTEGER NOT NULL)\"\"\"\n",
    "\n",
    "CREATE_DESCRIPTORS_TABLE = \"\"\"CREATE TABLE IF NOT EXISTS descriptors (\n",
    "    image_id INTEGER PRIMARY KEY NOT NULL,\n",
    "    rows INTEGER NOT NULL,\n",
    "    cols INTEGER NOT NULL,\n",
    "    data BLOB,\n",
    "    FOREIGN KEY(image_id) REFERENCES images(image_id) ON DELETE CASCADE)\"\"\"\n",
    "\n",
    "CREATE_IMAGES_TABLE = \"\"\"CREATE TABLE IF NOT EXISTS images (\n",
    "    image_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,\n",
    "    name TEXT NOT NULL UNIQUE,\n",
    "    camera_id INTEGER NOT NULL,\n",
    "    prior_qw REAL,\n",
    "    prior_qx REAL,\n",
    "    prior_qy REAL,\n",
    "    prior_qz REAL,\n",
    "    prior_tx REAL,\n",
    "    prior_ty REAL,\n",
    "    prior_tz REAL,\n",
    "    CONSTRAINT image_id_check CHECK(image_id >= 0 and image_id < {}),\n",
    "    FOREIGN KEY(camera_id) REFERENCES cameras(camera_id))\n",
    "\"\"\".format(MAX_IMAGE_ID)\n",
    "\n",
    "CREATE_TWO_VIEW_GEOMETRIES_TABLE = \"\"\"\n",
    "CREATE TABLE IF NOT EXISTS two_view_geometries (\n",
    "    pair_id INTEGER PRIMARY KEY NOT NULL,\n",
    "    rows INTEGER NOT NULL,\n",
    "    cols INTEGER NOT NULL,\n",
    "    data BLOB,\n",
    "    config INTEGER NOT NULL,\n",
    "    F BLOB,\n",
    "    E BLOB,\n",
    "    H BLOB)\n",
    "\"\"\"\n",
    "\n",
    "CREATE_KEYPOINTS_TABLE = \"\"\"CREATE TABLE IF NOT EXISTS keypoints (\n",
    "    image_id INTEGER PRIMARY KEY NOT NULL,\n",
    "    rows INTEGER NOT NULL,\n",
    "    cols INTEGER NOT NULL,\n",
    "    data BLOB,\n",
    "    FOREIGN KEY(image_id) REFERENCES images(image_id) ON DELETE CASCADE)\n",
    "\"\"\"\n",
    "\n",
    "CREATE_MATCHES_TABLE = \"\"\"CREATE TABLE IF NOT EXISTS matches (\n",
    "    pair_id INTEGER PRIMARY KEY NOT NULL,\n",
    "    rows INTEGER NOT NULL,\n",
    "    cols INTEGER NOT NULL,\n",
    "    data BLOB)\"\"\"\n",
    "\n",
    "CREATE_NAME_INDEX = \\\n",
    "    \"CREATE UNIQUE INDEX IF NOT EXISTS index_name ON images(name)\"\n",
    "\n",
    "CREATE_ALL = \"; \".join([\n",
    "    CREATE_CAMERAS_TABLE,\n",
    "    CREATE_IMAGES_TABLE,\n",
    "    CREATE_KEYPOINTS_TABLE,\n",
    "    CREATE_DESCRIPTORS_TABLE,\n",
    "    CREATE_MATCHES_TABLE,\n",
    "    CREATE_TWO_VIEW_GEOMETRIES_TABLE,\n",
    "    CREATE_NAME_INDEX\n",
    "])\n",
    "\n",
    "\n",
    "def image_ids_to_pair_id(image_id1, image_id2):\n",
    "    if image_id1 > image_id2:\n",
    "        image_id1, image_id2 = image_id2, image_id1\n",
    "    return image_id1 * MAX_IMAGE_ID + image_id2\n",
    "\n",
    "\n",
    "def pair_id_to_image_ids(pair_id):\n",
    "    image_id2 = pair_id % MAX_IMAGE_ID\n",
    "    image_id1 = (pair_id - image_id2) / MAX_IMAGE_ID\n",
    "    return image_id1, image_id2\n",
    "\n",
    "\n",
    "def array_to_blob(array):\n",
    "    return array.tostring()\n",
    "\n",
    "\n",
    "def blob_to_array(blob, dtype, shape=(-1,)):\n",
    "    return np.fromstring(blob, dtype=dtype).reshape(*shape)\n",
    "\n",
    "\n",
    "class COLMAPDatabase(sqlite3.Connection):\n",
    "\n",
    "    @staticmethod\n",
    "    def connect(database_path):\n",
    "        return sqlite3.connect(database_path, factory=COLMAPDatabase)\n",
    "\n",
    "\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super(COLMAPDatabase, self).__init__(*args, **kwargs)\n",
    "\n",
    "        self.create_tables = lambda: self.executescript(CREATE_ALL)\n",
    "        self.create_cameras_table = \\\n",
    "            lambda: self.executescript(CREATE_CAMERAS_TABLE)\n",
    "        self.create_descriptors_table = \\\n",
    "            lambda: self.executescript(CREATE_DESCRIPTORS_TABLE)\n",
    "        self.create_images_table = \\\n",
    "            lambda: self.executescript(CREATE_IMAGES_TABLE)\n",
    "        self.create_two_view_geometries_table = \\\n",
    "            lambda: self.executescript(CREATE_TWO_VIEW_GEOMETRIES_TABLE)\n",
    "        self.create_keypoints_table = \\\n",
    "            lambda: self.executescript(CREATE_KEYPOINTS_TABLE)\n",
    "        self.create_matches_table = \\\n",
    "            lambda: self.executescript(CREATE_MATCHES_TABLE)\n",
    "        self.create_name_index = lambda: self.executescript(CREATE_NAME_INDEX)\n",
    "\n",
    "    def add_camera(self, model, width, height, params,\n",
    "                   prior_focal_length=False, camera_id=None):\n",
    "        params = np.asarray(params, np.float64)\n",
    "        cursor = self.execute(\n",
    "            \"INSERT INTO cameras VALUES (?, ?, ?, ?, ?, ?)\",\n",
    "            (camera_id, model, width, height, array_to_blob(params),\n",
    "             prior_focal_length))\n",
    "        return cursor.lastrowid\n",
    "\n",
    "    def add_image(self, name, camera_id,\n",
    "                  prior_q=np.zeros(4), prior_t=np.zeros(3), image_id=None):\n",
    "        cursor = self.execute(\n",
    "            \"INSERT INTO images VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\",\n",
    "            (image_id, name, camera_id, prior_q[0], prior_q[1], prior_q[2],\n",
    "             prior_q[3], prior_t[0], prior_t[1], prior_t[2]))\n",
    "        return cursor.lastrowid\n",
    "\n",
    "    def add_keypoints(self, image_id, keypoints):\n",
    "        assert(len(keypoints.shape) == 2)\n",
    "        assert(keypoints.shape[1] in [2, 4, 6])\n",
    "\n",
    "        keypoints = np.asarray(keypoints, np.float32)\n",
    "        self.execute(\n",
    "            \"INSERT INTO keypoints VALUES (?, ?, ?, ?)\",\n",
    "            (image_id,) + keypoints.shape + (array_to_blob(keypoints),))\n",
    "\n",
    "    def add_descriptors(self, image_id, descriptors):\n",
    "        descriptors = np.ascontiguousarray(descriptors, np.uint8)\n",
    "        self.execute(\n",
    "            \"INSERT INTO descriptors VALUES (?, ?, ?, ?)\",\n",
    "            (image_id,) + descriptors.shape + (array_to_blob(descriptors),))\n",
    "\n",
    "    def add_matches(self, image_id1, image_id2, matches):\n",
    "        assert(len(matches.shape) == 2)\n",
    "        assert(matches.shape[1] == 2)\n",
    "\n",
    "        if image_id1 > image_id2:\n",
    "            matches = matches[:,::-1]\n",
    "\n",
    "        pair_id = image_ids_to_pair_id(image_id1, image_id2)\n",
    "        matches = np.asarray(matches, np.uint32)\n",
    "        self.execute(\n",
    "            \"INSERT INTO matches VALUES (?, ?, ?, ?)\",\n",
    "            (pair_id,) + matches.shape + (array_to_blob(matches),))\n",
    "\n",
    "    def add_two_view_geometry(self, image_id1, image_id2, matches,\n",
    "                              F=np.eye(3), E=np.eye(3), H=np.eye(3), config=2):\n",
    "        assert(len(matches.shape) == 2)\n",
    "        assert(matches.shape[1] == 2)\n",
    "\n",
    "        if image_id1 > image_id2:\n",
    "            matches = matches[:,::-1]\n",
    "\n",
    "        pair_id = image_ids_to_pair_id(image_id1, image_id2)\n",
    "        matches = np.asarray(matches, np.uint32)\n",
    "        F = np.asarray(F, dtype=np.float64)\n",
    "        E = np.asarray(E, dtype=np.float64)\n",
    "        H = np.asarray(H, dtype=np.float64)\n",
    "        self.execute(\n",
    "            \"INSERT INTO two_view_geometries VALUES (?, ?, ?, ?, ?, ?, ?, ?)\",\n",
    "            (pair_id,) + matches.shape + (array_to_blob(matches), config,\n",
    "             array_to_blob(F), array_to_blob(E), array_to_blob(H)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b5981beb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.089364Z",
     "iopub.status.busy": "2023-06-12T10:32:11.089115Z",
     "iopub.status.idle": "2023-06-12T10:32:11.104424Z",
     "shell.execute_reply": "2023-06-12T10:32:11.103617Z"
    },
    "papermill": {
     "duration": 0.027328,
     "end_time": "2023-06-12T10:32:11.106329",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.079001",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_camera(db, image_path, camera_model):\n",
    "    \n",
    "    image = Image.open(image_path)\n",
    "    width, height = image.size\n",
    "\n",
    "    focal_length = get_focal_length(image_path)\n",
    "\n",
    "    if camera_model == 'simple-pinhole':\n",
    "        model = 0\n",
    "        param_arr = np.array([focal_length, width / 2, height / 2])\n",
    "    if camera_model == 'pinhole':\n",
    "        model = 1\n",
    "        param_arr = np.array([focal_length, focal, width / 2, height / 2])\n",
    "    elif camera_model == 'simple-radial':\n",
    "        model = 2\n",
    "        param_arr = np.array([focal_length, width / 2, height / 2, 0.1])\n",
    "    elif camera_model == 'opencv':\n",
    "        model = 4\n",
    "        param_arr = np.array([focal_length, focal_length, width / 2, height / 2, 0., 0., 0., 0.])\n",
    "         \n",
    "    return db.add_camera(model, width, height, param_arr)\n",
    "\n",
    "\n",
    "def add_keypoints(db, h5_path, image_path, camera_model, single_camera=True):\n",
    "    \n",
    "    keypoint_f = h5py.File(os.path.join(h5_path, 'keypoints.h5'), 'r')\n",
    "    camera_id = None\n",
    "    fname_to_id = {}\n",
    "    \n",
    "    for filename in tqdm(list(keypoint_f.keys())):\n",
    "        \n",
    "        keypoints = keypoint_f[filename][()]\n",
    "\n",
    "        path = os.path.join(image_path, filename)\n",
    "        if not os.path.isfile(path):\n",
    "            raise IOError(f'Invalid image path {path}')\n",
    "\n",
    "        if camera_id is None or not single_camera:\n",
    "            camera_id = create_camera(db, path, camera_model)\n",
    "        image_id = db.add_image(filename, camera_id)\n",
    "        fname_to_id[filename] = image_id\n",
    "\n",
    "        db.add_keypoints(image_id, keypoints)\n",
    "\n",
    "    return fname_to_id\n",
    "\n",
    "\n",
    "def add_matches(db, h5_path, fname_to_id):\n",
    "    \n",
    "    match_file = h5py.File(os.path.join(h5_path, 'matches.h5'), 'r')\n",
    "    \n",
    "    added = set()\n",
    "    n_keys = len(match_file.keys())\n",
    "    n_total = (n_keys * (n_keys - 1)) // 2\n",
    "\n",
    "    with tqdm(total=n_total) as pbar:\n",
    "        for key_1 in match_file.keys():\n",
    "            group = match_file[key_1]\n",
    "            for key_2 in group.keys():\n",
    "                id_1 = fname_to_id[key_1]\n",
    "                id_2 = fname_to_id[key_2]\n",
    "\n",
    "                pair_id = image_ids_to_pair_id(id_1, id_2)\n",
    "                if pair_id in added:\n",
    "                    continue\n",
    "            \n",
    "                matches = group[key_2][()]\n",
    "                db.add_matches(id_1, id_2, matches)\n",
    "                added.add(pair_id)\n",
    "                pbar.update(1)\n",
    "                \n",
    "                \n",
    "def get_unique_idxs(A, dim=0):\n",
    "    \n",
    "    unique, idx, counts = torch.unique(A, dim=dim, sorted=True, return_inverse=True, return_counts=True)\n",
    "    _, ind_sorted = torch.sort(idx, stable=True)\n",
    "    cum_sum = counts.cumsum(0)\n",
    "    cum_sum = torch.cat((torch.tensor([0],device=cum_sum.device), cum_sum[:-1]))\n",
    "    first_indices = ind_sorted[cum_sum]\n",
    "    \n",
    "    return first_indices\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe87366d",
   "metadata": {
    "papermill": {
     "duration": 0.008859,
     "end_time": "2023-06-12T10:32:11.124870",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.116011",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 5. Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "996e9b65",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.145365Z",
     "iopub.status.busy": "2023-06-12T10:32:11.144015Z",
     "iopub.status.idle": "2023-06-12T10:32:11.177170Z",
     "shell.execute_reply": "2023-06-12T10:32:11.176404Z"
    },
    "papermill": {
     "duration": 0.045314,
     "end_time": "2023-06-12T10:32:11.179229",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.133915",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def array_to_string(a):\n",
    "\n",
    "    \"\"\"\n",
    "    Flatten given array and convert it to a string with semicolon delimiters\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    a: np.ndarray\n",
    "        N-dimensional array\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    s: string\n",
    "        String form of the given array\n",
    "    \"\"\"\n",
    "\n",
    "    s = ';'.join([str(x) for x in a.reshape(-1)])\n",
    "\n",
    "    return s\n",
    "\n",
    "\n",
    "def string_to_array(s):\n",
    "\n",
    "    \"\"\"\n",
    "    Convert semicolon delimited string to an array\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    s: string\n",
    "        String form of the array\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    a: np.ndarray\n",
    "        N-dimensional array\n",
    "    \"\"\"\n",
    "\n",
    "    a = np.array(s.split(';')).astype(np.float64)\n",
    "\n",
    "    return a\n",
    "\n",
    "\n",
    "def rotation_matrix_to_quaternion(rotation_matrix):\n",
    "\n",
    "    \"\"\"\n",
    "    Convert rotation matrix to quaternion\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    rotation_matrix: numpy.ndarray of shape (3, 3)\n",
    "        Array of directions of the world-axes in camera coordinates\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    quaternion: numpy.ndarray of shape (4)\n",
    "        Array of quaternion\n",
    "    \"\"\"\n",
    "\n",
    "    r00 = rotation_matrix[0, 0]\n",
    "    r01 = rotation_matrix[0, 1]\n",
    "    r02 = rotation_matrix[0, 2]\n",
    "    r10 = rotation_matrix[1, 0]\n",
    "    r11 = rotation_matrix[1, 1]\n",
    "    r12 = rotation_matrix[1, 2]\n",
    "    r20 = rotation_matrix[2, 0]\n",
    "    r21 = rotation_matrix[2, 1]\n",
    "    r22 = rotation_matrix[2, 2]\n",
    "\n",
    "    k = np.array([\n",
    "        [r00 - r11 - r22, 0.0, 0.0, 0.0],\n",
    "        [r01 + r10, r11 - r00 - r22, 0.0, 0.0],\n",
    "        [r02 + r20, r12 + r21, r22 - r00 - r11, 0.0],\n",
    "        [r21 - r12, r02 - r20, r10 - r01, r00 + r11 + r22]\n",
    "    ])\n",
    "    k /= 3.0\n",
    "\n",
    "    # Quaternion is the eigenvector of k that corresponds to the largest eigenvalue\n",
    "    w, v = np.linalg.eigh(k)\n",
    "    quaternion = v[[3, 0, 1, 2], np.argmax(w)]\n",
    "\n",
    "    if quaternion[0] < 0:\n",
    "        np.negative(quaternion, quaternion)\n",
    "\n",
    "    return quaternion\n",
    "\n",
    "\n",
    "def pose_difference(r1, t1, r2, t2):\n",
    "\n",
    "    \"\"\"\n",
    "    Calculate relative pose difference from given rotation matrices and translation vectors\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    r1: numpy.ndarray of shape (3, 3)\n",
    "        First rotation matrix\n",
    "\n",
    "    t1: numpy.ndarray of shape (3)\n",
    "        First translation vector\n",
    "\n",
    "    r2: numpy.ndarray of shape (3, 3)\n",
    "        Second rotation matrix\n",
    "\n",
    "    t2: numpy.ndarray of shape (3)\n",
    "        Second translation vector\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    rotation_difference: float\n",
    "        Rotation difference in terms of degrees from the first image\n",
    "\n",
    "    translation_difference: float\n",
    "        Translation difference in terms of meters from the first image\n",
    "    \"\"\"\n",
    "\n",
    "    rotation_difference = np.dot(r2, r1.T)\n",
    "    translation_difference = t2 - np.dot(rotation_difference, t1)\n",
    "\n",
    "    return rotation_difference, translation_difference\n",
    "\n",
    "\n",
    "def rotation_and_translation_error(q_ground_truth, t_ground_truth, q_prediction, t_prediction, epsilon=1e-15):\n",
    "\n",
    "    \"\"\"\n",
    "    Calculate rotation and translation error\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    q_ground_truth: numpy.ndarray of shape (4)\n",
    "        Array of quaternion derived from ground truth rotation matrix\n",
    "\n",
    "    t_ground_truth: numpy.ndarray of shape (3)\n",
    "        Array of ground truth translation vector\n",
    "\n",
    "    q_prediction: numpy.ndarray of shape (4)\n",
    "        Array of quaternion derived from estimated rotation matrix\n",
    "\n",
    "    t_prediction: numpy.ndarray of shape (3)\n",
    "        Array of estimated translation vector\n",
    "\n",
    "    epsilon: float\n",
    "        A small number for preventing zero division\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    rotation_error: float\n",
    "        Rotation error in terms of degrees\n",
    "\n",
    "    translation_error: float\n",
    "        Translation error in terms of meters\n",
    "    \"\"\"\n",
    "\n",
    "    q_ground_truth_norm = q_ground_truth / (np.linalg.norm(q_ground_truth) + epsilon)\n",
    "    q_prediction_norm = q_prediction / (np.linalg.norm(q_prediction) + epsilon)\n",
    "    loss_q = np.maximum(epsilon, (1.0 - np.sum(q_prediction_norm * q_ground_truth_norm) ** 2))\n",
    "\n",
    "    rotation_error = np.degrees(np.arccos(1 - (2 * loss_q)))\n",
    "\n",
    "    scaling_factor = np.linalg.norm(t_ground_truth)\n",
    "    t_prediction = scaling_factor * (t_prediction / (np.linalg.norm(t_prediction) + epsilon))\n",
    "    translation_error = min(\n",
    "        np.linalg.norm(t_ground_truth - t_prediction),\n",
    "        np.linalg.norm(t_ground_truth + t_prediction)\n",
    "    )\n",
    "\n",
    "    return rotation_error, translation_error\n",
    "\n",
    "\n",
    "def mean_average_accuracy(rotation_errors, translation_errors, rotation_error_thresholds, translation_error_thresholds):\n",
    "\n",
    "    \"\"\"\n",
    "    Calculate mean average accuracies over a set of thresholds for rotation and translation\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    rotation_errors: list of shape (n_pairs)\n",
    "        List of rotation errors\n",
    "\n",
    "    translation_errors: list of shape (n_pairs)\n",
    "        List of translation errors\n",
    "\n",
    "    rotation_error_thresholds: numpy.ndarray of shape (10)\n",
    "        Array of rotation error thresholds\n",
    "\n",
    "    translation_error_thresholds: numpy.ndarray of shape (10)\n",
    "        Array of translation error thresholds\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    maa: float\n",
    "        Mean average accuracy calculated on both rotation and translation errors\n",
    "\n",
    "    rotation_maa: float\n",
    "        Mean average accuracy calculated on rotation errors\n",
    "\n",
    "    translation_maa: float\n",
    "        Mean average accuracy calculated on translation errors\n",
    "    \"\"\"\n",
    "\n",
    "    accuracies, rotation_accuracies, translation_accuracies = [], [], []\n",
    "\n",
    "    for rotation_error_threshold, translation_error_threshold in zip(rotation_error_thresholds, translation_error_thresholds):\n",
    "\n",
    "        # Calculate whether the errors are less than specified thresholds or not\n",
    "        rotation_accuracy = (rotation_errors <= rotation_error_threshold)\n",
    "        translation_accuracy = (translation_errors <= translation_error_threshold)\n",
    "        accuracy = rotation_accuracy & translation_accuracy\n",
    "\n",
    "        accuracies.append(accuracy.astype(np.float32).mean())\n",
    "        rotation_accuracies.append(rotation_accuracy.astype(np.float32).mean())\n",
    "        translation_accuracies.append(translation_accuracy.astype(np.float32).mean())\n",
    "\n",
    "    maa = np.array(accuracies).mean()\n",
    "    rotation_maa = np.array(rotation_accuracies).mean()\n",
    "    translation_maa = np.array(translation_accuracies).mean()\n",
    "\n",
    "    return maa, rotation_maa, translation_maa\n",
    "\n",
    "\n",
    "def evaluate(df, verbose=False):\n",
    "\n",
    "    \"\"\"\n",
    "    Calculate mean average accuracies over a set of thresholds for rotation and translation\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df: pandas.DataFrame\n",
    "        Dataframe with dataset, scene, rotation_matrix, translation_vector, rotation_matrix_prediction and translation_vector_prediction columns\n",
    "\n",
    "    verbose: bool\n",
    "        Whether to print scores or not\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    df_scores: pandas.DataFrame\n",
    "        Dataframe of scores\n",
    "    \"\"\"\n",
    "\n",
    "    rotation_error_thresholds = {\n",
    "        **{('haiper', scene): np.linspace(1, 10, 10) for scene in ['bike', 'chairs', 'fountain']},\n",
    "        **{('heritage', scene): np.linspace(1, 10, 10) for scene in ['cyprus', 'dioscuri']},\n",
    "        **{('heritage', 'wall'): np.linspace(0.2, 10, 10)},\n",
    "        **{('urban', 'kyiv-puppet-theater'): np.linspace(1, 10, 10)},\n",
    "    }\n",
    "    translation_error_thresholds = {\n",
    "        **{('haiper', scene): np.geomspace(0.05, 0.5, 10) for scene in ['bike', 'chairs', 'fountain']},\n",
    "        **{('heritage', scene): np.geomspace(0.1, 2, 10) for scene in ['cyprus', 'dioscuri']},\n",
    "        **{('heritage', 'wall'): np.geomspace(0.05, 1, 10)},\n",
    "        **{('urban', 'kyiv-puppet-theater'): np.geomspace(0.5, 5, 10)},\n",
    "    }\n",
    "    df_scores = pd.DataFrame(columns=['dataset', 'scene', 'image_pairs', 'maa', 'rotation_maa', 'translation_maa'])\n",
    "\n",
    "    for (dataset, scene), df_scene in tqdm(df.groupby(['dataset', 'scene'])):\n",
    "\n",
    "        scene_rotation_errors = []\n",
    "        scene_translation_errors = []\n",
    "\n",
    "        for i in range(df_scene.shape[0]):\n",
    "            for j in range(i + 1, df_scene.shape[0]):\n",
    "\n",
    "                rotation_matrix_difference_ground_truth, translation_vector_difference_ground_truth = pose_difference(\n",
    "                    r1=string_to_array((df_scene.iloc[i]['rotation_matrix'])).reshape(3, 3),\n",
    "                    t1=string_to_array((df_scene.iloc[i]['translation_vector'])),\n",
    "                    r2=string_to_array((df_scene.iloc[j]['rotation_matrix'])).reshape(3, 3),\n",
    "                    t2=string_to_array((df_scene.iloc[j]['translation_vector'])),\n",
    "                )\n",
    "                quaternion_ground_truth = rotation_matrix_to_quaternion(rotation_matrix=rotation_matrix_difference_ground_truth)\n",
    "\n",
    "                rotation_matrix_difference_prediction, translation_vector_difference_prediction = pose_difference(\n",
    "                    r1=string_to_array((df_scene.iloc[i]['rotation_matrix_prediction'])).reshape(3, 3),\n",
    "                    t1=string_to_array((df_scene.iloc[i]['translation_vector_prediction'])),\n",
    "                    r2=string_to_array((df_scene.iloc[j]['rotation_matrix_prediction'])).reshape(3, 3),\n",
    "                    t2=string_to_array((df_scene.iloc[j]['translation_vector_prediction'])),\n",
    "                )\n",
    "                quaternion_prediction = rotation_matrix_to_quaternion(rotation_matrix=rotation_matrix_difference_prediction)\n",
    "\n",
    "                rotation_error, translation_error = rotation_and_translation_error(\n",
    "                    q_ground_truth=quaternion_ground_truth,\n",
    "                    t_ground_truth=translation_vector_difference_ground_truth,\n",
    "                    q_prediction=quaternion_prediction,\n",
    "                    t_prediction=translation_vector_difference_prediction,\n",
    "                    epsilon=1e-15\n",
    "                )\n",
    "                scene_rotation_errors.append(rotation_error)\n",
    "                scene_translation_errors.append(translation_error)\n",
    "\n",
    "        scene_maa, scene_rotation_maa, scene_translation_maa = mean_average_accuracy(\n",
    "            rotation_errors=scene_rotation_errors,\n",
    "            translation_errors=scene_translation_errors,\n",
    "            rotation_error_thresholds=rotation_error_thresholds[(dataset, scene)],\n",
    "            translation_error_thresholds=translation_error_thresholds[(dataset, scene)]\n",
    "        )\n",
    "\n",
    "        if verbose:\n",
    "            settings.logger.info(\n",
    "                f'''\n",
    "                Dataset: {dataset} - Scene: {scene}\n",
    "                Number of image pairs: {len(scene_rotation_errors)}\n",
    "                mAA: {scene_maa:.6f} - rotation mAA: {scene_rotation_maa:.6f} - translation mAA: {scene_translation_maa:.6f}\n",
    "                '''\n",
    "            )\n",
    "\n",
    "        df_scores = pd.concat((\n",
    "            df_scores,\n",
    "            pd.DataFrame(\n",
    "                data=[[dataset, scene, len(scene_rotation_errors), scene_maa, scene_rotation_maa, scene_translation_maa]],\n",
    "                columns=['dataset', 'scene', 'image_pairs', 'maa', 'rotation_maa', 'translation_maa']\n",
    "            )\n",
    "        ), axis=0)\n",
    "\n",
    "    return df_scores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0ce4ca8",
   "metadata": {
    "papermill": {
     "duration": 0.008844,
     "end_time": "2023-06-12T10:32:11.197244",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.188400",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 6. Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "81f6e6b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.216444Z",
     "iopub.status.busy": "2023-06-12T10:32:11.216190Z",
     "iopub.status.idle": "2023-06-12T10:32:11.229587Z",
     "shell.execute_reply": "2023-06-12T10:32:11.228744Z"
    },
    "papermill": {
     "duration": 0.025208,
     "end_time": "2023-06-12T10:32:11.231478",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.206270",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def loftr_match_images(image1, image2, model, device, amp, transforms, confidence_threshold, top_k):\n",
    "\n",
    "    \"\"\"\n",
    "    Match given two images with each other using LoFTR model\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image1: numpy.ndarray of shape (3, height, width)\n",
    "        Array of first image\n",
    "\n",
    "    image2: numpy.ndarray of shape (3, height, width)\n",
    "        Array of second image\n",
    "\n",
    "    model: torch.nn.Module\n",
    "        LoFTR Model\n",
    "\n",
    "    device: torch.device\n",
    "        Location of the image1, image2 and the model\n",
    "\n",
    "    amp: bool\n",
    "        Whether to use auto mixed precision or not\n",
    "\n",
    "    transforms: dict\n",
    "        Dictionary of transform parameters\n",
    "\n",
    "    confidence_threshold: float or int\n",
    "        Confidence threshold to filter out low confidence matches\n",
    "\n",
    "    top_k: int\n",
    "        Number of matches to take\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    outputs: dict\n",
    "        Model outputs\n",
    "    \"\"\"\n",
    "\n",
    "    image1_raw_height, image1_raw_width = image1.shape[:2]\n",
    "    image1 = get_image_tensor(\n",
    "        image_path_or_array=image1,\n",
    "        resize=transforms['resize'],\n",
    "        resize_shape=transforms['resize_shape'],\n",
    "        resize_longest_edge=transforms['resize_longest_edge'],\n",
    "        scale=transforms['scale'],\n",
    "        grayscale=transforms['grayscale']\n",
    "    )\n",
    "    image1 = image1.to(device)\n",
    "    image1_transformed_height, image1_transformed_width = image1.shape[2:]\n",
    "\n",
    "    image2_raw_height, image2_raw_width = image2.shape[:2]\n",
    "    image2 = get_image_tensor(\n",
    "        image_path_or_array=image2,\n",
    "        resize=transforms['resize'],\n",
    "        resize_shape=transforms['resize_shape'],\n",
    "        resize_longest_edge=transforms['resize_longest_edge'],\n",
    "        scale=transforms['scale'],\n",
    "        grayscale=transforms['grayscale']\n",
    "    )\n",
    "    image2 = image2.to(device)\n",
    "    image2_transformed_height, image2_transformed_width = image2.shape[2:]\n",
    "\n",
    "    with torch.no_grad():\n",
    "        if amp:\n",
    "            with torch.autocast(device_type=device.type, dtype=torch.float16):\n",
    "                outputs = model({'image0': image1, 'image1': image2})\n",
    "        else:\n",
    "            outputs = model({'image0': image1, 'image1': image2})\n",
    "\n",
    "    for k in outputs.keys():\n",
    "        outputs[k] = outputs[k].detach().cpu().numpy()\n",
    "\n",
    "    if confidence_threshold is not None:\n",
    "        if isinstance(confidence_threshold, float):\n",
    "            # Select matched keypoints with above given confidence threshold\n",
    "            confidence_mask = outputs['confidence'] >= confidence_threshold\n",
    "        elif isinstance(confidence_threshold, int):\n",
    "            # Select keypoints dynamically based on confidence distribution\n",
    "            confidence_mean, confidence_std = outputs['confidence'].mean(), outputs['confidence'].std()\n",
    "            confidence_mask = outputs['confidence'] >= (confidence_mean + (confidence_std * confidence_threshold))\n",
    "        else:\n",
    "            raise ValueError(f'Invalid confidence_threshold {confidence_threshold}')\n",
    "\n",
    "        for k in outputs.keys():\n",
    "            outputs[k] = outputs[k][confidence_mask]\n",
    "\n",
    "    if top_k is not None:\n",
    "        # Select top-k keypoints based on their confidences\n",
    "        sorting_idx = outputs['matching_scores0'].argsort()[-top_k:]\n",
    "        for k in outputs.keys():\n",
    "            outputs[k] = outputs[k][sorting_idx]\n",
    "\n",
    "    outputs['keypoints0'][:, 0] *= image1_raw_width / image1_transformed_width\n",
    "    outputs['keypoints0'][:, 1] *= image1_raw_height / image1_transformed_height\n",
    "    outputs['keypoints1'][:, 0] *= image2_raw_width / image2_transformed_width\n",
    "    outputs['keypoints1'][:, 1] *= image2_raw_height / image2_transformed_height\n",
    "\n",
    "    return outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4178b8aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.251311Z",
     "iopub.status.busy": "2023-06-12T10:32:11.250871Z",
     "iopub.status.idle": "2023-06-12T10:32:11.267034Z",
     "shell.execute_reply": "2023-06-12T10:32:11.266198Z"
    },
    "papermill": {
     "duration": 0.028112,
     "end_time": "2023-06-12T10:32:11.268932",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.240820",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def superglue_match_images(image1, image2, model, device, amp, transforms, score_threshold, top_k):\n",
    "\n",
    "    \"\"\"\n",
    "    Match given two images with each other using SuperGlue model\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    image1: numpy.ndarray of shape (3, height, width)\n",
    "        Array of first image\n",
    "\n",
    "    image2: numpy.ndarray of shape (3, height, width)\n",
    "        Array of second image\n",
    "\n",
    "    model: torch.nn.Module\n",
    "        SuperGlue Model\n",
    "\n",
    "    device: torch.device\n",
    "        Location of the image1, image2 and the model\n",
    "\n",
    "    amp: bool\n",
    "        Whether to use auto mixed precision or not\n",
    "\n",
    "    transforms: dict\n",
    "        Dictionary of transform parameters\n",
    "\n",
    "    score_threshold: float, int or None\n",
    "        Confidence threshold\n",
    "\n",
    "    top_k: int or None\n",
    "        Number of keypoints to take\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    outputs: dict\n",
    "        Model outputs\n",
    "    \"\"\"\n",
    "\n",
    "    image1_raw_height, image1_raw_width = image1.shape[:2]\n",
    "    image1 = get_image_tensor(\n",
    "        image_path_or_array=image1,\n",
    "        resize=transforms[1]['resize'],\n",
    "        resize_shape=transforms[1]['resize_shape'],\n",
    "        resize_longest_edge=transforms[1]['resize_longest_edge'],\n",
    "        scale=transforms[1]['scale'],\n",
    "        grayscale=transforms[1]['grayscale']\n",
    "    )\n",
    "    image1 = image1.to(device)\n",
    "    image1_transformed_height, image1_transformed_width = image1.shape[2:]\n",
    "\n",
    "    image2_raw_height, image2_raw_width = image2.shape[:2]\n",
    "    image2 = get_image_tensor(\n",
    "        image_path_or_array=image2,\n",
    "        resize=transforms[2]['resize'],\n",
    "        resize_shape=transforms[2]['resize_shape'],\n",
    "        resize_longest_edge=transforms[2]['resize_longest_edge'],\n",
    "        scale=transforms[2]['scale'],\n",
    "        grayscale=transforms[2]['grayscale']\n",
    "    )\n",
    "    image2 = image2.to(device)\n",
    "    image2_transformed_height, image2_transformed_width = image2.shape[2:]\n",
    "\n",
    "    with torch.no_grad():\n",
    "        if amp:\n",
    "            with torch.autocast(device_type=device.type, dtype=torch.float16):\n",
    "                outputs = model({'image0': image1, 'image1': image2})\n",
    "        else:\n",
    "            outputs = model({'image0': image1, 'image1': image2})\n",
    "\n",
    "    for k in outputs.keys():\n",
    "        if k == 'descriptors0' or k == 'descriptors1':\n",
    "            outputs[k] = outputs[k][0].detach().cpu().numpy().T\n",
    "        else:\n",
    "            outputs[k] = outputs[k][0].detach().cpu().numpy()\n",
    "\n",
    "    matches_mask = outputs['matches0'] > -1\n",
    "\n",
    "    for k in ['keypoints1', 'scores1', 'descriptors1', 'matches1', 'matching_scores1']:\n",
    "        outputs[k] = outputs[k][outputs['matches0'][matches_mask]]\n",
    "\n",
    "    for k in ['keypoints0', 'scores0', 'descriptors0', 'matches0', 'matching_scores0']:\n",
    "        outputs[k] = outputs[k][matches_mask]\n",
    "\n",
    "    if score_threshold is not None:\n",
    "        if isinstance(score_threshold, float):\n",
    "            # Select matched keypoints with above given score threshold\n",
    "            score_mask = outputs['matching_scores0'] >= score_threshold\n",
    "        elif isinstance(score_threshold, int):\n",
    "            # Select keypoints dynamically based on score distribution\n",
    "            score_mean, score_std = outputs['matching_scores0'].mean(), outputs['matching_scores0'].std()\n",
    "            score_mask = outputs['matching_scores0'] >= (score_mean + (score_std * score_threshold))\n",
    "        else:\n",
    "            raise ValueError(f'Invalid score_threshold {score_threshold}')\n",
    "\n",
    "        for k in outputs.keys():\n",
    "            outputs[k] = outputs[k][score_mask]\n",
    "\n",
    "    if top_k is not None:\n",
    "        # Select top-k keypoints based on their scores\n",
    "        sorting_idx = outputs['matching_scores0'].argsort()[-top_k:]\n",
    "        for k in outputs.keys():\n",
    "            outputs[k] = outputs[k][sorting_idx]\n",
    "\n",
    "    outputs['keypoints0'][:, 0] *= image1_raw_width / image1_transformed_width\n",
    "    outputs['keypoints0'][:, 1] *= image1_raw_height / image1_transformed_height\n",
    "    outputs['keypoints1'][:, 0] *= image2_raw_width / image2_transformed_width\n",
    "    outputs['keypoints1'][:, 1] *= image2_raw_height / image2_transformed_height\n",
    "\n",
    "    return outputs\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5ba3084",
   "metadata": {
    "papermill": {
     "duration": 0.009121,
     "end_time": "2023-06-12T10:32:11.287013",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.277892",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 7. Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5219650b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:11.306508Z",
     "iopub.status.busy": "2023-06-12T10:32:11.306259Z",
     "iopub.status.idle": "2023-06-12T10:32:17.074426Z",
     "shell.execute_reply": "2023-06-12T10:32:17.073450Z"
    },
    "papermill": {
     "duration": 5.780665,
     "end_time": "2023-06-12T10:32:17.076944",
     "exception": false,
     "start_time": "2023-06-12T10:32:11.296279",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded SuperPoint model\n",
      "Loaded SuperGlue model (\"outdoor\" weights)\n"
     ]
    }
   ],
   "source": [
    "image_matching_device = torch.device('cuda')\n",
    "'''\n",
    "# Load LoFTR model with specified configurations\n",
    "loftr_model = LoFTR(\n",
    "    pretrained=None,\n",
    "    config={\n",
    "        'backbone_type': 'ResNetFPN',\n",
    "        'resolution': (8, 2),\n",
    "        'fine_window_size': 5,\n",
    "        'fine_concat_coarse_feat': True,\n",
    "        'resnetfpn': {\n",
    "            'initial_dim': 128,\n",
    "            'block_dims': [128, 196, 256]\n",
    "        },\n",
    "        'coarse': {\n",
    "            'd_model': 256,\n",
    "            'd_ffn': 256,\n",
    "            'nhead': 8,\n",
    "            'layer_names': ['self', 'cross', 'self', 'cross', 'self', 'cross', 'self', 'cross'],\n",
    "            'attention': 'linear',\n",
    "            'temp_bug_fix': False,\n",
    "        },\n",
    "        'match_coarse': {\n",
    "            'thr': 0.2,\n",
    "            'border_rm': 2,\n",
    "            'match_type': 'dual_softmax',\n",
    "            'dsmax_temperature': 0.1,\n",
    "            'skh_iters': 3,\n",
    "            'skh_init_bin_score': 1.0,\n",
    "            'skh_prefilter': True,\n",
    "            'train_coarse_percent': 0.4,\n",
    "            'train_pad_num_gt_min': 200,\n",
    "        },\n",
    "        'fine': {\n",
    "            'd_model': 128,\n",
    "            'd_ffn': 128,\n",
    "            'nhead': 8,\n",
    "            'layer_names': ['self', 'cross'],\n",
    "            'attention': 'linear'\n",
    "        }\n",
    "    }\n",
    ")\n",
    "loftr_model.load_state_dict(torch.load(external_dataset / 'models' / 'loftr' / 'loftr_outdoor.ckpt')['state_dict'], strict=False)\n",
    "loftr_model = loftr_model.eval().to(image_matching_device)\n",
    "'''\n",
    "# Load SuperPoint and SuperGlue model with specified configurations\n",
    "superglue_model = Matching(config={\n",
    "    'superpoint': {\n",
    "        'descriptor_dim': 256,\n",
    "        'nms_radius': 4,\n",
    "        'keypoint_threshold': 0.01,\n",
    "        'max_keypoints': -1,\n",
    "        'remove_borders': 4\n",
    "    },\n",
    "    'superglue': {\n",
    "        'descriptor_dim': 256,\n",
    "        'weights': 'outdoor',\n",
    "        'keypoint_encoder': [32, 64, 128, 256],\n",
    "        'sinkhorn_iterations': 100,\n",
    "        'match_threshold': 0.2\n",
    "    }\n",
    "})\n",
    "superglue_model = superglue_model.eval().to(image_matching_device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e2b820bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:17.097726Z",
     "iopub.status.busy": "2023-06-12T10:32:17.097042Z",
     "iopub.status.idle": "2023-06-12T10:32:17.137789Z",
     "shell.execute_reply": "2023-06-12T10:32:17.136964Z"
    },
    "papermill": {
     "duration": 0.053429,
     "end_time": "2023-06-12T10:32:17.139958",
     "exception": false,
     "start_time": "2023-06-12T10:32:17.086529",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def match(read_image_function, image_paths, image_pair_indices, feature_dir, superglue_model, stage2=False):\n",
    "    \n",
    "    with h5py.File(f'{feature_dir}/matches_loftr.h5', mode='w') as f_match:\n",
    "        for pair_idx in progress_bar(image_pair_indices):\n",
    "            \n",
    "            idx1, idx2 = pair_idx\n",
    "            fname1, fname2 = image_paths[idx1], image_paths[idx2]\n",
    "            key1, key2 = fname1.split('/')[-1], fname2.split('/')[-1]\n",
    "            \n",
    "            image1 = read_image_function(fname1)\n",
    "            image2 = read_image_function(fname2)\n",
    "            \n",
    "            keypoints1 = []\n",
    "            keypoints2 = []\n",
    "            \n",
    "            # Largest SuperGlue size that can be used on Kaggle GPU is 2560\n",
    "            superglue_longest_edge_limit = 2560\n",
    "            if (np.max(image1.shape[:2]) > superglue_longest_edge_limit) and (np.max(image2.shape[:2]) > superglue_longest_edge_limit):\n",
    "                # Both of the images have longest edges greater than 2560\n",
    "                first_stage_superglue_transforms = {\n",
    "                    1: {\n",
    "                        'resize': True,\n",
    "                        'resize_shape': superglue_longest_edge_limit,\n",
    "                        'resize_longest_edge': True,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    },\n",
    "                    2: {\n",
    "                        'resize': True,\n",
    "                        'resize_shape': superglue_longest_edge_limit,\n",
    "                        'resize_longest_edge': True,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    }\n",
    "                }\n",
    "            elif (np.max(image1.shape[:2]) > superglue_longest_edge_limit) and (np.max(image2.shape[:2]) <= superglue_longest_edge_limit):\n",
    "                # First image's longest edge is greater than 2560\n",
    "                first_stage_superglue_transforms = {\n",
    "                    1: {\n",
    "                        'resize': True,\n",
    "                        'resize_shape': superglue_longest_edge_limit,\n",
    "                        'resize_longest_edge': True,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    },\n",
    "                    2: {\n",
    "                        'resize': False,\n",
    "                        'resize_shape': None,\n",
    "                        'resize_longest_edge': None,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    }\n",
    "                }\n",
    "            elif (np.max(image1.shape[:2]) <= superglue_longest_edge_limit) and (np.max(image2.shape[:2]) > superglue_longest_edge_limit):\n",
    "                # Second image's longest edge is greater than 2560\n",
    "                first_stage_superglue_transforms = {\n",
    "                    1: {\n",
    "                        'resize': False,\n",
    "                        'resize_shape': None,\n",
    "                        'resize_longest_edge': None,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    },\n",
    "                    2: {\n",
    "                        'resize': True,\n",
    "                        'resize_shape': superglue_longest_edge_limit,\n",
    "                        'resize_longest_edge': True,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    }\n",
    "                }\n",
    "            else:\n",
    "                # Neither of the image's longest edge is greater than 2560\n",
    "                first_stage_superglue_transforms = {\n",
    "                    1: {\n",
    "                        'resize': False,\n",
    "                        'resize_shape': None,\n",
    "                        'resize_longest_edge': None,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    },\n",
    "                    2: {\n",
    "                        'resize': False,\n",
    "                        'resize_shape': None,\n",
    "                        'resize_longest_edge': None,\n",
    "                        'scale': True,\n",
    "                        'grayscale': True\n",
    "                    }\n",
    "                }\n",
    "                \n",
    "            first_stage_superglue_outputs = superglue_match_images(\n",
    "                image1=image1,\n",
    "                image2=image2,\n",
    "                model=superglue_model,\n",
    "                device=image_matching_device,\n",
    "                amp=True,\n",
    "                transforms=first_stage_superglue_transforms,\n",
    "                score_threshold=None,\n",
    "                top_k=None\n",
    "            )\n",
    "            \n",
    "            keypoints1.append(first_stage_superglue_outputs['keypoints0'])\n",
    "            keypoints2.append(first_stage_superglue_outputs['keypoints1'])\n",
    "            del first_stage_superglue_outputs\n",
    "            \n",
    "            keypoints1 = np.concatenate(keypoints1, axis=0)\n",
    "            keypoints2 = np.concatenate(keypoints2, axis=0)\n",
    "            \n",
    "            if stage2:\n",
    "                if (keypoints1.shape[0] > 10) & (keypoints1.shape[0] < 200):\n",
    "                    \n",
    "                    image1_cropped, keypoints1_cropped, x_offset1, y_offset1 = crop(image1, keypoints1)\n",
    "                    image2_cropped, keypoints2_cropped, x_offset2, y_offset2 = crop(image2, keypoints2)\n",
    "                    \n",
    "                    # Largest SuperGlue size that can be used on Kaggle GPU is 2560\n",
    "                    superglue_longest_edge_limit = 2560\n",
    "                    if (np.max(image1_cropped.shape[:2]) > superglue_longest_edge_limit) and (np.max(image2_cropped.shape[:2]) > superglue_longest_edge_limit):\n",
    "                        # Both of the images have longest edges greater than 2560\n",
    "                        second_stage_superglue_transforms = {\n",
    "                            1: {\n",
    "                                'resize': True,\n",
    "                                'resize_shape': superglue_longest_edge_limit,\n",
    "                                'resize_longest_edge': True,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            },\n",
    "                            2: {\n",
    "                                'resize': True,\n",
    "                                'resize_shape': superglue_longest_edge_limit,\n",
    "                                'resize_longest_edge': True,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            }\n",
    "                        }\n",
    "                    elif (np.max(image1_cropped.shape[:2]) > superglue_longest_edge_limit) and (np.max(image2_cropped.shape[:2]) <= superglue_longest_edge_limit):\n",
    "                        # First image's longest edge is greater than 2560\n",
    "                        second_stage_superglue_transforms = {\n",
    "                            1: {\n",
    "                                'resize': True,\n",
    "                                'resize_shape': superglue_longest_edge_limit,\n",
    "                                'resize_longest_edge': True,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            },\n",
    "                            2: {\n",
    "                                'resize': False,\n",
    "                                'resize_shape': None,\n",
    "                                'resize_longest_edge': None,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            }\n",
    "                        }\n",
    "                    elif (np.max(image1_cropped.shape[:2]) <= superglue_longest_edge_limit) and (np.max(image2_cropped.shape[:2]) > superglue_longest_edge_limit):\n",
    "                        # Second image's longest edge is greater than 2560\n",
    "                        second_stage_superglue_transforms = {\n",
    "                            1: {\n",
    "                                'resize': False,\n",
    "                                'resize_shape': None,\n",
    "                                'resize_longest_edge': None,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            },\n",
    "                            2: {\n",
    "                                'resize': True,\n",
    "                                'resize_shape': superglue_longest_edge_limit,\n",
    "                                'resize_longest_edge': True,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            }\n",
    "                        }\n",
    "                    else:\n",
    "                        # Neither of the image's longest edge is greater than 2560\n",
    "                        second_stage_superglue_transforms = {\n",
    "                            1: {\n",
    "                                'resize': False,\n",
    "                                'resize_shape': None,\n",
    "                                'resize_longest_edge': None,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            },\n",
    "                            2: {\n",
    "                                'resize': False,\n",
    "                                'resize_shape': None,\n",
    "                                'resize_longest_edge': None,\n",
    "                                'scale': True,\n",
    "                                'grayscale': True\n",
    "                            }\n",
    "                        }\n",
    "\n",
    "                    second_stage_superglue_outputs = superglue_match_images(\n",
    "                        image1=image1_cropped,\n",
    "                        image2=image2_cropped,\n",
    "                        model=superglue_model,\n",
    "                        device=image_matching_device,\n",
    "                        amp=True,\n",
    "                        transforms=second_stage_superglue_transforms,\n",
    "                        score_threshold=None,\n",
    "                        top_k=200\n",
    "                    )\n",
    "\n",
    "                    second_stage_superglue_outputs['keypoints0'][:, 0] += x_offset1\n",
    "                    second_stage_superglue_outputs['keypoints0'][:, 1] += y_offset1\n",
    "                    second_stage_superglue_outputs['keypoints1'][:, 0] += x_offset2\n",
    "                    second_stage_superglue_outputs['keypoints1'][:, 1] += y_offset2\n",
    "\n",
    "                    keypoints1 = np.concatenate([keypoints1, second_stage_superglue_outputs['keypoints0']], axis=0)\n",
    "                    keypoints2 = np.concatenate([keypoints2, second_stage_superglue_outputs['keypoints1']], axis=0)\n",
    "            \n",
    "            n_matches = len(keypoints1)\n",
    "            group  = f_match.require_group(key1)\n",
    "            if n_matches >= 15:\n",
    "                group.create_dataset(key2, data=np.concatenate([keypoints1, keypoints2], axis=1))\n",
    "\n",
    "    kpts = defaultdict(list)\n",
    "    match_indexes = defaultdict(dict)\n",
    "    total_kpts=defaultdict(int)\n",
    "    \n",
    "    with h5py.File(f'{feature_dir}/matches_loftr.h5', mode='r') as f_match:\n",
    "        for k1 in f_match.keys():\n",
    "            group  = f_match[k1]\n",
    "            for k2 in group.keys():\n",
    "                matches = group[k2][...]\n",
    "                total_kpts[k1]\n",
    "                kpts[k1].append(matches[:, :2])\n",
    "                kpts[k2].append(matches[:, 2:])\n",
    "                current_match = torch.arange(len(matches)).reshape(-1, 1).repeat(1, 2)\n",
    "                current_match[:, 0] += total_kpts[k1]\n",
    "                current_match[:, 1] += total_kpts[k2]\n",
    "                total_kpts[k1] += len(matches)\n",
    "                total_kpts[k2] += len(matches)\n",
    "                match_indexes[k1][k2]=current_match\n",
    "\n",
    "    for k in kpts.keys():\n",
    "        kpts[k] = np.round(np.concatenate(kpts[k], axis=0))\n",
    "        \n",
    "    unique_kpts = {}\n",
    "    unique_match_idxs = {}\n",
    "    out_match = defaultdict(dict)\n",
    "    \n",
    "    for k in kpts.keys():\n",
    "        uniq_kps, uniq_reverse_idxs = torch.unique(torch.from_numpy(kpts[k]),dim=0, return_inverse=True)\n",
    "        unique_match_idxs[k] = uniq_reverse_idxs\n",
    "        unique_kpts[k] = uniq_kps.numpy()\n",
    "        \n",
    "    for k1, group in match_indexes.items():\n",
    "        for k2, m in group.items():\n",
    "            m2 = deepcopy(m)\n",
    "            m2[:,0] = unique_match_idxs[k1][m2[:,0]]\n",
    "            m2[:,1] = unique_match_idxs[k2][m2[:,1]]\n",
    "            mkpts = np.concatenate([\n",
    "                unique_kpts[k1][m2[:,0]],\n",
    "                unique_kpts[k2][m2[:,1]]\n",
    "            ], axis=1)\n",
    "            unique_idxs_current = get_unique_idxs(torch.from_numpy(mkpts), dim=0)\n",
    "            m2_semiclean = m2[unique_idxs_current]\n",
    "            unique_idxs_current1 = get_unique_idxs(m2_semiclean[:, 0], dim=0)\n",
    "            m2_semiclean = m2_semiclean[unique_idxs_current1]\n",
    "            unique_idxs_current2 = get_unique_idxs(m2_semiclean[:, 1], dim=0)\n",
    "            m2_semiclean2 = m2_semiclean[unique_idxs_current2]\n",
    "            out_match[k1][k2] = m2_semiclean2.numpy()\n",
    "            \n",
    "    with h5py.File(f'{feature_dir}/keypoints.h5', mode='w') as f_kp:\n",
    "        for k, kpts1 in unique_kpts.items():\n",
    "            f_kp[k] = kpts1\n",
    "    \n",
    "    with h5py.File(f'{feature_dir}/matches.h5', mode='w') as f_match:\n",
    "        for k1, gr in out_match.items():\n",
    "            group  = f_match.require_group(k1)\n",
    "            for k2, match in gr.items():\n",
    "                group[k2] = match\n",
    "                \n",
    "    return\n",
    "\n",
    "\n",
    "def import_into_colmap(img_dir, feature_dir='.featureout', database_path='colmap.db'):\n",
    "    \n",
    "    db = COLMAPDatabase.connect(database_path)\n",
    "    db.create_tables()\n",
    "    single_camera = False\n",
    "    fname_to_id = add_keypoints(db, feature_dir, img_dir, 'simple-radial', single_camera)\n",
    "    add_matches(db, feature_dir, fname_to_id,)\n",
    "    db.commit()\n",
    "    \n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b06e9d77",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:32:17.159871Z",
     "iopub.status.busy": "2023-06-12T10:32:17.159605Z",
     "iopub.status.idle": "2023-06-12T10:34:59.405336Z",
     "shell.execute_reply": "2023-06-12T10:34:59.404253Z"
    },
    "papermill": {
     "duration": 162.258711,
     "end_time": "2023-06-12T10:34:59.407954",
     "exception": false,
     "start_time": "2023-06-12T10:32:17.149243",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='105' class='' max='105' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [105/105 02:13&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/15 [00:00<?, ?it/s]/tmp/ipykernel_23/2598211829.py:86: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  return array.tostring()\n",
      "100%|██████████| 15/15 [00:00<00:00, 452.59it/s]\n",
      " 88%|████████▊ | 80/91 [00:00<00:00, 4012.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "==============================================================================\n",
      "Exhaustive feature matching\n",
      "==============================================================================\n",
      "\n",
      "Matching block [1/1, 1/1] in 9.628s\n",
      "Elapsed time: 0.161 [minutes]\n",
      "\n",
      "==============================================================================\n",
      "Loading database\n",
      "==============================================================================\n",
      "\n",
      "Loading cameras... 15 in 0.000s\n",
      "Loading matches... 56 in 0.001s\n",
      "Loading images... 15 in 0.001s (connected 15)\n",
      "Building correspondence graph... in 0.004s (ignored 0)\n",
      "\n",
      "Elapsed time: 0.000 [minutes]\n",
      "\n",
      "\n",
      "==============================================================================\n",
      "Finding good initial image pair\n",
      "==============================================================================\n",
      "\n",
      "\n",
      "==============================================================================\n",
      "Initializing with image pair #7 and #1\n",
      "==============================================================================\n",
      "\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.115678e+04    0.00e+00    1.11e+06   0.00e+00   0.00e+00  1.00e+04        0    4.08e-03    1.75e-02\n",
      "   1  6.897078e+03    4.26e+03    1.07e+06   1.78e+02   4.72e-01  1.00e+04        1    1.05e-02    2.80e-02\n",
      "   2  1.959483e+03    4.94e+03    7.30e+04   3.96e+01   9.90e-01  3.00e+04        1    6.49e-03    3.46e-02\n",
      "   3  1.890919e+03    6.86e+01    3.65e+04   1.97e+01   8.61e-01  4.80e+04        1    6.35e-03    4.09e-02\n",
      "   4  1.876036e+03    1.49e+01    1.62e+04   1.32e+01   9.47e-01  1.44e+05        1    6.49e-03    4.74e-02\n",
      "   5  1.875935e+03    1.02e-01    2.66e+04   1.73e+01   3.94e-02  8.09e+04        1    6.27e-03    5.37e-02\n",
      "   6  1.872913e+03    3.02e+00    6.96e+03   8.42e+00   9.42e-01  2.43e+05        1    6.09e-03    5.99e-02\n",
      "   7  1.883642e+03   -1.07e+01    6.96e+03   2.34e+01  -4.84e+00  1.21e+05        1    3.40e-03    6.33e-02\n",
      "   8  1.872541e+03    3.72e-01    1.43e+04   1.18e+01   3.08e-01  1.15e+05        1    6.11e-03    6.94e-02\n",
      "   9  1.871401e+03    1.14e+00    1.26e+04   1.10e+01   6.31e-01  1.17e+05        1    6.29e-03    7.57e-02\n",
      "  10  1.870462e+03    9.39e-01    1.30e+04   1.11e+01   5.66e-01  1.17e+05        1    6.40e-03    8.22e-02\n",
      "  11  1.869477e+03    9.86e-01    1.31e+04   1.11e+01   5.74e-01  1.18e+05        1    6.32e-03    8.85e-02\n",
      "  12  1.868487e+03    9.89e-01    1.32e+04   1.11e+01   5.70e-01  1.18e+05        1    6.16e-03    9.47e-02\n",
      "  13  1.867491e+03    9.97e-01    1.33e+04   1.11e+01   5.67e-01  1.18e+05        1    6.06e-03    1.01e-01\n",
      "  14  1.866487e+03    1.00e+00    1.34e+04   1.11e+01   5.64e-01  1.18e+05        1    6.06e-03    1.07e-01\n",
      "  15  1.865478e+03    1.01e+00    1.35e+04   1.10e+01   5.62e-01  1.19e+05        1    6.18e-03    1.13e-01\n",
      "  16  1.864462e+03    1.02e+00    1.35e+04   1.10e+01   5.60e-01  1.19e+05        1    7.66e-03    1.21e-01\n",
      "  17  1.863441e+03    1.02e+00    1.36e+04   1.10e+01   5.58e-01  1.19e+05        1    6.41e-03    1.27e-01\n",
      "  18  1.862414e+03    1.03e+00    1.37e+04   1.10e+01   5.56e-01  1.19e+05        1    6.37e-03    1.34e-01\n",
      "  19  1.861382e+03    1.03e+00    1.37e+04   1.09e+01   5.54e-01  1.19e+05        1    6.39e-03    1.40e-01\n",
      "  20  1.860346e+03    1.04e+00    1.38e+04   1.09e+01   5.52e-01  1.19e+05        1    6.38e-03    1.46e-01\n",
      "  21  1.859305e+03    1.04e+00    1.38e+04   1.09e+01   5.51e-01  1.20e+05        1    6.47e-03    1.53e-01\n",
      "  22  1.858260e+03    1.05e+00    1.38e+04   1.09e+01   5.49e-01  1.20e+05        1    6.37e-03    1.59e-01\n",
      "  23  1.857210e+03    1.05e+00    1.39e+04   1.08e+01   5.48e-01  1.20e+05        1    6.57e-03    1.66e-01\n",
      "  24  1.856157e+03    1.05e+00    1.39e+04   1.08e+01   5.46e-01  1.20e+05        1    6.70e-03    1.73e-01\n",
      "  25  1.855100e+03    1.06e+00    1.39e+04   1.07e+01   5.45e-01  1.20e+05        1    6.79e-03    1.79e-01\n",
      "  26  1.854040e+03    1.06e+00    1.39e+04   1.07e+01   5.44e-01  1.20e+05        1    6.81e-03    1.86e-01\n",
      "  27  1.852976e+03    1.06e+00    1.39e+04   1.07e+01   5.43e-01  1.20e+05        1    6.51e-03    1.93e-01\n",
      "  28  1.851909e+03    1.07e+00    1.39e+04   1.06e+01   5.42e-01  1.20e+05        1    6.13e-03    1.99e-01\n",
      "  29  1.850840e+03    1.07e+00    1.39e+04   1.06e+01   5.41e-01  1.20e+05        1    6.34e-03    2.05e-01\n",
      "  30  1.849767e+03    1.07e+00    1.39e+04   1.05e+01   5.40e-01  1.20e+05        1    6.42e-03    2.12e-01\n",
      "  31  1.848692e+03    1.07e+00    1.39e+04   1.05e+01   5.39e-01  1.20e+05        1    6.42e-03    2.18e-01\n",
      "  32  1.847615e+03    1.08e+00    1.39e+04   1.04e+01   5.38e-01  1.20e+05        1    6.26e-03    2.24e-01\n",
      "  33  1.846536e+03    1.08e+00    1.39e+04   1.04e+01   5.38e-01  1.20e+05        1    6.25e-03    2.31e-01\n",
      "  34  1.845454e+03    1.08e+00    1.39e+04   1.04e+01   5.37e-01  1.21e+05        1    6.24e-03    2.37e-01\n",
      "  35  1.844371e+03    1.08e+00    1.39e+04   1.03e+01   5.36e-01  1.21e+05        1    6.33e-03    2.43e-01\n",
      "  36  1.843286e+03    1.08e+00    1.39e+04   1.03e+01   5.36e-01  1.21e+05        1    6.14e-03    2.50e-01\n",
      "  37  1.842200e+03    1.09e+00    1.38e+04   1.02e+01   5.35e-01  1.21e+05        1    6.22e-03    2.56e-01\n",
      "  38  1.841113e+03    1.09e+00    1.38e+04   1.02e+01   5.35e-01  1.21e+05        1    6.16e-03    2.62e-01\n",
      "  39  1.840024e+03    1.09e+00    1.38e+04   1.01e+01   5.35e-01  1.21e+05        1    6.22e-03    2.68e-01\n",
      "  40  1.838935e+03    1.09e+00    1.37e+04   1.01e+01   5.34e-01  1.21e+05        1    6.39e-03    2.75e-01\n",
      "  41  1.837844e+03    1.09e+00    1.37e+04   1.00e+01   5.34e-01  1.21e+05        1    6.19e-03    2.81e-01\n",
      "  42  1.836754e+03    1.09e+00    1.37e+04   9.95e+00   5.34e-01  1.21e+05        1    6.18e-03    2.87e-01\n",
      "  43  1.835663e+03    1.09e+00    1.36e+04   9.90e+00   5.34e-01  1.21e+05        1    6.30e-03    2.93e-01\n",
      "  44  1.834572e+03    1.09e+00    1.36e+04   9.85e+00   5.33e-01  1.21e+05        1    6.17e-03    3.00e-01\n",
      "  45  1.833480e+03    1.09e+00    1.35e+04   9.79e+00   5.33e-01  1.21e+05        1    6.39e-03    3.06e-01\n",
      "  46  1.832390e+03    1.09e+00    1.35e+04   9.74e+00   5.33e-01  1.21e+05        1    6.35e-03    3.12e-01\n",
      "  47  1.831299e+03    1.09e+00    1.34e+04   9.68e+00   5.34e-01  1.21e+05        1    6.32e-03    3.19e-01\n",
      "  48  1.830209e+03    1.09e+00    1.34e+04   9.63e+00   5.34e-01  1.21e+05        1    7.70e-03    3.26e-01\n",
      "  49  1.829120e+03    1.09e+00    1.33e+04   9.57e+00   5.34e-01  1.21e+05        1    6.32e-03    3.33e-01\n",
      "  50  1.828032e+03    1.09e+00    1.32e+04   9.51e+00   5.34e-01  1.21e+05        1    6.36e-03    3.39e-01\n",
      "  51  1.826945e+03    1.09e+00    1.32e+04   9.45e+00   5.34e-01  1.21e+05        1    6.19e-03    3.45e-01\n",
      "  52  1.825860e+03    1.09e+00    1.31e+04   9.40e+00   5.35e-01  1.21e+05        1    6.11e-03    3.51e-01\n",
      "  53  1.824776e+03    1.08e+00    1.30e+04   9.34e+00   5.35e-01  1.21e+05        1    6.10e-03    3.58e-01\n",
      "  54  1.823694e+03    1.08e+00    1.30e+04   9.28e+00   5.35e-01  1.21e+05        1    6.23e-03    3.64e-01\n",
      "  55  1.822614e+03    1.08e+00    1.29e+04   9.22e+00   5.36e-01  1.21e+05        1    6.22e-03    3.70e-01\n",
      "  56  1.821536e+03    1.08e+00    1.28e+04   9.16e+00   5.36e-01  1.21e+05        1    6.37e-03    3.76e-01\n",
      "  57  1.820461e+03    1.08e+00    1.27e+04   9.10e+00   5.37e-01  1.21e+05        1    6.19e-03    3.83e-01\n",
      "  58  1.819388e+03    1.07e+00    1.26e+04   9.04e+00   5.38e-01  1.21e+05        1    6.15e-03    3.89e-01\n",
      "  59  1.818318e+03    1.07e+00    1.26e+04   8.98e+00   5.38e-01  1.22e+05        1    6.19e-03    3.95e-01\n",
      "  60  1.817251e+03    1.07e+00    1.25e+04   8.92e+00   5.39e-01  1.22e+05        1    8.27e-03    4.03e-01\n",
      "  61  1.816187e+03    1.06e+00    1.24e+04   8.86e+00   5.40e-01  1.22e+05        1    1.14e-02    4.15e-01\n",
      "  62  1.815126e+03    1.06e+00    1.23e+04   8.79e+00   5.40e-01  1.22e+05        1    1.05e-02    4.25e-01\n",
      "  63  1.814069e+03    1.06e+00    1.22e+04   8.73e+00   5.41e-01  1.22e+05        1    8.79e-03    4.34e-01\n",
      "  64  1.813015e+03    1.05e+00    1.21e+04   8.67e+00   5.42e-01  1.22e+05        1    1.24e-02    4.47e-01\n",
      "  65  1.811966e+03    1.05e+00    1.20e+04   8.61e+00   5.43e-01  1.22e+05        1    1.22e-02    4.59e-01\n",
      "  66  1.810920e+03    1.05e+00    1.19e+04   8.55e+00   5.44e-01  1.22e+05        1    1.08e-02    4.70e-01\n",
      "  67  1.809879e+03    1.04e+00    1.18e+04   8.48e+00   5.45e-01  1.22e+05        1    8.66e-03    4.78e-01\n",
      "  68  1.808842e+03    1.04e+00    1.17e+04   8.42e+00   5.46e-01  1.22e+05        1    1.26e-02    4.91e-01\n",
      "  69  1.807810e+03    1.03e+00    1.16e+04   8.36e+00   5.47e-01  1.22e+05        1    1.01e-02    5.01e-01\n",
      "  70  1.806782e+03    1.03e+00    1.15e+04   8.30e+00   5.48e-01  1.22e+05        1    1.07e-02    5.12e-01\n",
      "  71  1.805759e+03    1.02e+00    1.14e+04   8.23e+00   5.49e-01  1.23e+05        1    9.08e-03    5.21e-01\n",
      "  72  1.804741e+03    1.02e+00    1.13e+04   8.17e+00   5.50e-01  1.23e+05        1    1.06e-02    5.32e-01\n",
      "  73  1.803728e+03    1.01e+00    1.12e+04   8.11e+00   5.51e-01  1.23e+05        1    1.05e-02    5.42e-01\n",
      "  74  1.802721e+03    1.01e+00    1.11e+04   8.05e+00   5.52e-01  1.23e+05        1    9.80e-03    5.52e-01\n",
      "  75  1.801719e+03    1.00e+00    1.10e+04   7.98e+00   5.53e-01  1.23e+05        1    1.27e-02    5.65e-01\n",
      "  76  1.800723e+03    9.96e-01    1.08e+04   7.92e+00   5.55e-01  1.23e+05        1    7.21e-03    5.72e-01\n",
      "  77  1.799732e+03    9.91e-01    1.07e+04   7.86e+00   5.56e-01  1.23e+05        1    9.01e-03    5.81e-01\n",
      "  78  1.798747e+03    9.85e-01    1.06e+04   7.80e+00   5.57e-01  1.24e+05        1    6.77e-03    5.88e-01\n",
      "  79  1.797768e+03    9.79e-01    1.05e+04   7.74e+00   5.58e-01  1.24e+05        1    6.37e-03    5.94e-01\n",
      "  80  1.796795e+03    9.73e-01    1.04e+04   7.68e+00   5.59e-01  1.24e+05        1    6.45e-03    6.01e-01\n",
      "  81  1.795828e+03    9.67e-01    1.03e+04   7.61e+00   5.60e-01  1.24e+05        1    6.40e-03    6.07e-01\n",
      "  82  1.794868e+03    9.61e-01    1.02e+04   7.55e+00   5.62e-01  1.24e+05        1    6.48e-03    6.14e-01\n",
      "  83  1.793913e+03    9.54e-01    1.01e+04   7.49e+00   5.63e-01  1.25e+05        1    6.42e-03    6.20e-01\n",
      "  84  1.792965e+03    9.48e-01    9.98e+03   7.43e+00   5.64e-01  1.25e+05        1    6.30e-03    6.27e-01\n",
      "  85  1.792024e+03    9.42e-01    9.87e+03   7.38e+00   5.65e-01  1.25e+05        1    6.09e-03    6.33e-01\n",
      "  86  1.791089e+03    9.35e-01    9.76e+03   7.32e+00   5.66e-01  1.26e+05        1    6.25e-03    6.39e-01\n",
      "  87  1.790160e+03    9.28e-01    9.65e+03   7.26e+00   5.67e-01  1.26e+05        1    6.37e-03    6.45e-01\n",
      "  88  1.789238e+03    9.22e-01    9.55e+03   7.20e+00   5.68e-01  1.26e+05        1    6.09e-03    6.51e-01\n",
      "  89  1.788323e+03    9.15e-01    9.44e+03   7.14e+00   5.69e-01  1.26e+05        1    6.37e-03    6.58e-01\n",
      "  90  1.787415e+03    9.08e-01    9.34e+03   7.09e+00   5.70e-01  1.27e+05        1    6.16e-03    6.64e-01\n",
      "  91  1.786514e+03    9.01e-01    9.24e+03   7.03e+00   5.71e-01  1.27e+05        1    6.34e-03    6.70e-01\n",
      "  92  1.785619e+03    8.94e-01    9.13e+03   6.97e+00   5.72e-01  1.28e+05        1    6.26e-03    6.77e-01\n",
      "  93  1.784732e+03    8.88e-01    9.03e+03   6.92e+00   5.73e-01  1.28e+05        1    6.28e-03    6.83e-01\n",
      "  94  1.783851e+03    8.81e-01    8.93e+03   6.86e+00   5.74e-01  1.28e+05        1    6.07e-03    6.89e-01\n",
      "  95  1.782978e+03    8.73e-01    8.83e+03   6.81e+00   5.75e-01  1.29e+05        1    6.23e-03    6.95e-01\n",
      "  96  1.782111e+03    8.66e-01    8.73e+03   6.75e+00   5.76e-01  1.29e+05        1    6.62e-03    7.02e-01\n",
      "  97  1.781252e+03    8.59e-01    8.63e+03   6.70e+00   5.77e-01  1.30e+05        1    6.07e-03    7.08e-01\n",
      "  98  1.780400e+03    8.52e-01    8.53e+03   6.64e+00   5.78e-01  1.30e+05        1    6.40e-03    7.14e-01\n",
      "  99  1.779555e+03    8.45e-01    8.43e+03   6.59e+00   5.79e-01  1.31e+05        1    6.26e-03    7.21e-01\n",
      " 100  1.778717e+03    8.38e-01    8.33e+03   6.54e+00   5.80e-01  1.31e+05        1    6.09e-03    7.27e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 6448\n",
      "   Parameters : 4845\n",
      "   Iterations : 101\n",
      "         Time : 0.727057 [s]\n",
      " Initial cost : 1.3154 [px]\n",
      "   Final cost : 0.52522 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Filtered observations: 0\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #9 (3)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 614 / 2170 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 762\n",
      "   Parameters : 8\n",
      "   Iterations : 16\n",
      "         Time : 0.00931096 [s]\n",
      " Initial cost : 0.82134 [px]\n",
      "   Final cost : 0.73741 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 380\n",
      "  => Added observations: 1069\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 9346\n",
      "   Parameters : 6224\n",
      "   Iterations : 26\n",
      "         Time : 0.26996 [s]\n",
      " Initial cost : 1.4557 [px]\n",
      "   Final cost : 0.564395 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 4\n",
      "  => Filtered observations: 155\n",
      "  => Changed observations: 0.034025\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 9044\n",
      "   Parameters : 6140\n",
      "   Iterations : 6\n",
      "         Time : 0.062813 [s]\n",
      " Initial cost : 0.573794 [px]\n",
      "   Final cost : 0.547504 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 52\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.011720\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 21\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  2.843460e+03    0.00e+00    1.19e+04   0.00e+00   0.00e+00  1.00e+04        0    4.72e-03    1.32e-02\n",
      "   1  2.774506e+03    6.90e+01    3.06e+02   3.01e+00   1.00e+00  3.00e+04        1    1.08e-02    2.40e-02\n",
      "   2  2.774085e+03    4.21e-01    4.96e+02   3.54e+00   9.99e-01  9.00e+04        1    9.71e-03    3.38e-02\n",
      "   3  2.773910e+03    1.75e-01    3.85e+02   3.11e+00   1.00e+00  2.70e+05        1    1.02e-02    4.39e-02\n",
      "   4  2.773889e+03    2.19e-02    5.90e+01   1.20e+00   1.02e+00  8.10e+05        1    1.02e-02    5.42e-02\n",
      "   5  2.773888e+03    5.31e-04    1.90e+00   1.99e-01   1.04e+00  2.43e+06        1    9.85e-03    6.41e-02\n",
      "   6  2.773888e+03    3.56e-06    1.75e-01   1.72e-02   1.05e+00  7.29e+06        1    9.62e-03    7.37e-02\n",
      "   7  2.773888e+03    1.59e-08    1.38e-02   1.08e-03   1.06e+00  2.19e+07        1    9.71e-03    8.35e-02\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 9188\n",
      "   Parameters : 6215\n",
      "   Iterations : 8\n",
      "         Time : 0.0838211 [s]\n",
      " Initial cost : 0.556305 [px]\n",
      "   Final cost : 0.549457 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 4\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000871\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  2.800598e+03    0.00e+00    4.17e+03   0.00e+00   0.00e+00  1.00e+04        0    4.46e-03    1.24e-02\n",
      "   1  2.780304e+03    2.03e+01    3.68e+01   8.10e-01   1.00e+00  3.00e+04        1    1.22e-02    2.46e-02\n",
      "   2  2.780270e+03    3.45e-02    4.02e+01   7.77e-01   1.01e+00  9.00e+04        1    9.96e-03    3.46e-02\n",
      "   3  2.780257e+03    1.29e-02    2.86e+01   6.44e-01   1.02e+00  2.70e+05        1    9.92e-03    4.45e-02\n",
      "   4  2.780255e+03    1.28e-03    4.16e+00   2.37e-01   1.03e+00  8.10e+05        1    9.76e-03    5.43e-02\n",
      "   5  2.780255e+03    2.82e-05    6.48e-01   3.75e-02   1.04e+00  2.43e+06        1    1.00e-02    6.44e-02\n",
      "   6  2.780255e+03    1.98e-07    3.67e-02   3.11e-03   1.05e+00  7.29e+06        1    9.83e-03    7.42e-02\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 9196\n",
      "   Parameters : 6215\n",
      "   Iterations : 7\n",
      "         Time : 0.0745091 [s]\n",
      " Initial cost : 0.551856 [px]\n",
      "   Final cost : 0.549848 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 4\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000870\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  2.809609e+03    0.00e+00    3.62e+03   0.00e+00   0.00e+00  1.00e+04        0    4.74e-03    1.30e-02\n",
      "   1  2.787929e+03    2.17e+01    7.24e+01   1.28e+00   1.00e+00  3.00e+04        1    1.03e-02    2.33e-02\n",
      "   2  2.787875e+03    5.46e-02    6.21e+01   9.73e-01   1.01e+00  9.00e+04        1    9.60e-03    3.30e-02\n",
      "   3  2.787856e+03    1.87e-02    3.95e+01   7.60e-01   1.02e+00  2.70e+05        1    1.04e-02    4.34e-02\n",
      "   4  2.787854e+03    1.70e-03    5.14e+00   2.65e-01   1.03e+00  8.10e+05        1    9.63e-03    5.31e-02\n",
      "   5  2.787854e+03    3.37e-05    7.54e-01   3.97e-02   1.04e+00  2.43e+06        1    9.80e-03    6.29e-02\n",
      "   6  2.787854e+03    2.11e-07    3.66e-02   3.11e-03   1.05e+00  7.29e+06        1    9.74e-03    7.26e-02\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 9204\n",
      "   Parameters : 6215\n",
      "   Iterations : 7\n",
      "         Time : 0.072927 [s]\n",
      " Initial cost : 0.552503 [px]\n",
      "   Final cost : 0.55036 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000217\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #8 (4)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 872 / 2133 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 1656\n",
      "   Parameters : 8\n",
      "   Iterations : 11\n",
      "         Time : 0.013628 [s]\n",
      " Initial cost : 0.857232 [px]\n",
      "   Final cost : 0.732747 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 789\n",
      "  => Added observations: 664\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 12112\n",
      "   Parameters : 7084\n",
      "   Iterations : 26\n",
      "         Time : 0.43734 [s]\n",
      " Initial cost : 0.703182 [px]\n",
      "   Final cost : 0.651257 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 10\n",
      "  => Completed observations: 7\n",
      "  => Filtered observations: 349\n",
      "  => Changed observations: 0.060436\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11428\n",
      "   Parameters : 6940\n",
      "   Iterations : 6\n",
      "         Time : 0.0987811 [s]\n",
      " Initial cost : 0.63588 [px]\n",
      "   Final cost : 0.591687 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 9\n",
      "  => Completed observations: 91\n",
      "  => Filtered observations: 2\n",
      "  => Changed observations: 0.017851\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 9\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  4.147601e+03    0.00e+00    1.84e+03   0.00e+00   0.00e+00  1.00e+04        0    6.08e-03    1.69e-02\n",
      "   1  4.109721e+03    3.79e+01    9.54e+00   3.84e-01   1.00e+00  3.00e+04        1    1.46e-02    3.15e-02\n",
      "   2  4.109717e+03    3.91e-03    2.38e+00   2.48e-01   1.00e+00  9.00e+04        1    1.38e-02    4.54e-02\n",
      "   3  4.109716e+03    8.64e-04    1.92e+00   1.67e-01   1.01e+00  2.70e+05        1    1.40e-02    5.94e-02\n",
      "   4  4.109716e+03    4.85e-05    4.79e-01   4.52e-02   1.01e+00  8.10e+05        1    1.39e-02    7.33e-02\n",
      "   5  4.109716e+03    4.99e-07    1.47e-02   4.82e-03   1.02e+00  2.43e+06        1    1.58e-02    8.91e-02\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11624\n",
      "   Parameters : 7060\n",
      "   Iterations : 6\n",
      "         Time : 0.089576 [s]\n",
      " Initial cost : 0.597339 [px]\n",
      "   Final cost : 0.594604 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.000172\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #15 (5)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 504 / 2519 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 664\n",
      "   Parameters : 8\n",
      "   Iterations : 15\n",
      "         Time : 0.00829911 [s]\n",
      " Initial cost : 0.992008 [px]\n",
      "   Final cost : 0.68912 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 328\n",
      "  => Added observations: 645\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 13568\n",
      "   Parameters : 7779\n",
      "   Iterations : 17\n",
      "         Time : 0.343583 [s]\n",
      " Initial cost : 0.610739 [px]\n",
      "   Final cost : 0.543416 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 4\n",
      "  => Completed observations: 2\n",
      "  => Filtered observations: 130\n",
      "  => Changed observations: 0.020047\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 13312\n",
      "   Parameters : 7713\n",
      "   Iterations : 5\n",
      "         Time : 0.0981209 [s]\n",
      " Initial cost : 0.618022 [px]\n",
      "   Final cost : 0.59653 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 6\n",
      "  => Completed observations: 29\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.005258\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 2\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  4.866174e+03    0.00e+00    1.43e+03   0.00e+00   0.00e+00  1.00e+04        0    7.37e-03    2.00e-02\n",
      "   1  4.844967e+03    2.12e+01    1.92e+01   7.10e-01   1.00e+00  3.00e+04        1    1.75e-02    3.76e-02\n",
      "   2  4.844955e+03    1.24e-02    4.50e+00   4.92e-01   1.00e+00  9.00e+04        1    1.65e-02    5.41e-02\n",
      "   3  4.844952e+03    2.28e-03    1.96e+00   2.82e-01   1.01e+00  2.70e+05        1    1.68e-02    7.10e-02\n",
      "   4  4.844952e+03    8.90e-05    5.63e-01   6.26e-02   1.01e+00  8.10e+05        1    1.81e-02    8.91e-02\n",
      "   5  4.844952e+03    5.74e-07    1.96e-02   5.21e-03   1.01e+00  2.43e+06        1    1.72e-02    1.06e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 13374\n",
      "   Parameters : 7746\n",
      "   Iterations : 6\n",
      "         Time : 0.106792 [s]\n",
      " Initial cost : 0.603203 [px]\n",
      "   Final cost : 0.601886 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.000150\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #6 (6)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 467 / 2515 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 736\n",
      "   Parameters : 8\n",
      "   Iterations : 11\n",
      "         Time : 0.00639415 [s]\n",
      " Initial cost : 0.921331 [px]\n",
      "   Final cost : 0.636043 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 366\n",
      "  => Added observations: 1047\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16198\n",
      "   Parameters : 9155\n",
      "   Iterations : 26\n",
      "         Time : 0.623193 [s]\n",
      " Initial cost : 0.56572 [px]\n",
      "   Final cost : 0.535654 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 5\n",
      "  => Filtered observations: 140\n",
      "  => Changed observations: 0.017903\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 15928\n",
      "   Parameters : 9086\n",
      "   Iterations : 5\n",
      "         Time : 0.120799 [s]\n",
      " Initial cost : 0.620544 [px]\n",
      "   Final cost : 0.599692 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 45\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.005776\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  5.811551e+03    0.00e+00    3.27e+03   0.00e+00   0.00e+00  1.00e+04        0    8.82e-03    2.64e-02\n",
      "   1  5.804244e+03    7.31e+00    6.38e+00   2.09e-01   1.00e+00  3.00e+04        1    2.38e-02    5.02e-02\n",
      "   2  5.804242e+03    2.77e-03    1.54e+00   1.68e-01   1.00e+00  9.00e+04        1    2.10e-02    7.13e-02\n",
      "   3  5.804241e+03    4.02e-04    1.86e+00   8.64e-02   9.99e-01  2.70e+05        1    2.12e-02    9.25e-02\n",
      "   4  5.804241e+03    1.24e-05    1.12e-01   1.68e-02   1.00e+00  8.10e+05        1    2.12e-02    1.14e-01\n",
      "   5  5.804241e+03    5.49e-08    6.95e-03   1.13e-03   1.01e+00  2.43e+06        1    2.19e-02    1.36e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16018\n",
      "   Parameters : 9152\n",
      "   Iterations : 6\n",
      "         Time : 0.136212 [s]\n",
      " Initial cost : 0.60234 [px]\n",
      "   Final cost : 0.601961 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000000\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #10 (7)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 523 / 1949 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 850\n",
      "   Parameters : 8\n",
      "   Iterations : 13\n",
      "         Time : 0.00849605 [s]\n",
      " Initial cost : 0.906824 [px]\n",
      "   Final cost : 0.67126 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 417\n",
      "  => Added observations: 430\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 15824\n",
      "   Parameters : 6851\n",
      "   Iterations : 26\n",
      "         Time : 0.553533 [s]\n",
      " Initial cost : 0.611362 [px]\n",
      "   Final cost : 0.578574 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 4\n",
      "  => Completed observations: 5\n",
      "  => Filtered observations: 175\n",
      "  => Changed observations: 0.023256\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 15484\n",
      "   Parameters : 6743\n",
      "   Iterations : 3\n",
      "         Time : 0.0609369 [s]\n",
      " Initial cost : 0.614646 [px]\n",
      "   Final cost : 0.596648 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 5\n",
      "  => Completed observations: 44\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.006329\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 41\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.173529e+05    0.00e+00    3.76e+05   0.00e+00   0.00e+00  1.00e+04        0    1.02e-02    2.85e-02\n",
      "   1  2.891535e+04    8.84e+04    2.32e+05   7.35e+01   9.69e-01  3.00e+04        1    2.69e-02    5.55e-02\n",
      "   2  2.553982e+04    3.38e+03    1.08e+05   6.25e+01   8.23e-01  4.11e+04        1    2.54e-02    8.09e-02\n",
      "   3  2.476479e+04    7.75e+02    1.26e+04   2.27e+01   9.89e-01  1.23e+05        1    2.54e-02    1.06e-01\n",
      "   4  2.475221e+04    1.26e+01    1.10e+03   6.54e+00   9.99e-01  3.70e+05        1    2.62e-02    1.32e-01\n",
      "   5  2.475209e+04    1.20e-01    1.74e+01   7.98e-01   1.01e+00  1.11e+06        1    2.48e-02    1.57e-01\n",
      "   6  2.475209e+04    2.23e-04    1.26e+00   5.08e-02   1.04e+00  3.33e+06        1    2.45e-02    1.82e-01\n",
      "   7  2.475209e+04    7.11e-07    1.02e-01   2.63e-03   1.06e+00  9.98e+06        1    2.49e-02    2.07e-01\n",
      "   8  2.475209e+04    2.51e-09    5.93e-03   1.40e-04   1.11e+00  2.99e+07        1    2.49e-02    2.32e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 17544\n",
      "   Parameters : 9670\n",
      "   Iterations : 9\n",
      "         Time : 0.232318 [s]\n",
      " Initial cost : 2.58632 [px]\n",
      "   Final cost : 1.1878 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 26\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 310\n",
      "  => Changed observations: 0.038304\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  9.654851e+03    0.00e+00    3.73e+04   0.00e+00   0.00e+00  1.00e+04        0    9.63e-03    2.65e-02\n",
      "   1  7.933596e+03    1.72e+03    2.89e+04   2.10e+01   9.72e-01  3.00e+04        1    2.51e-02    5.17e-02\n",
      "   2  7.823512e+03    1.10e+02    1.83e+04   2.69e+01   8.29e-01  4.19e+04        1    2.40e-02    7.57e-02\n",
      "   3  7.790657e+03    3.29e+01    3.40e+03   1.11e+01   9.78e-01  1.26e+05        1    2.32e-02    9.89e-02\n",
      "   4  7.788798e+03    1.86e+00    5.07e+02   4.30e+00   9.97e-01  3.77e+05        1    2.29e-02    1.22e-01\n",
      "   5  7.788759e+03    3.84e-02    1.09e+01   6.46e-01   1.01e+00  1.13e+06        1    2.32e-02    1.45e-01\n",
      "   6  7.788759e+03    9.75e-05    6.11e-01   4.15e-02   1.02e+00  3.39e+06        1    2.34e-02    1.68e-01\n",
      "   7  7.788759e+03    1.60e-07    1.64e-02   1.60e-03   1.03e+00  1.02e+07        1    2.37e-02    1.92e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16976\n",
      "   Parameters : 9454\n",
      "   Iterations : 8\n",
      "         Time : 0.192724 [s]\n",
      " Initial cost : 0.754145 [px]\n",
      "   Final cost : 0.677355 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 26\n",
      "  => Merged observations: 12\n",
      "  => Filtered observations: 25\n",
      "  => Changed observations: 0.007422\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  7.333612e+03    0.00e+00    2.24e+04   0.00e+00   0.00e+00  1.00e+04        0    1.03e-02    2.73e-02\n",
      "   1  6.372269e+03    9.61e+02    2.83e+04   2.41e+01   9.62e-01  3.00e+04        1    2.46e-02    5.20e-02\n",
      "   2  6.276435e+03    9.58e+01    2.23e+04   2.97e+01   7.35e-01  3.35e+04        1    2.98e-02    8.18e-02\n",
      "   3  6.228744e+03    4.77e+01    4.42e+03   1.26e+01   9.73e-01  1.00e+05        1    2.61e-02    1.08e-01\n",
      "   4  6.224947e+03    3.80e+00    1.25e+03   6.74e+00   9.76e-01  3.01e+05        1    2.36e-02    1.32e-01\n",
      "   5  6.224759e+03    1.88e-01    5.12e+01   1.37e+00   1.00e+00  9.04e+05        1    2.37e-02    1.55e-01\n",
      "   6  6.224759e+03    6.60e-04    1.85e+00   1.05e-01   1.01e+00  2.71e+06        1    2.30e-02    1.78e-01\n",
      "   7  6.224759e+03    6.48e-07    1.45e-02   3.50e-03   1.02e+00  8.14e+06        1    2.28e-02    2.01e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16978\n",
      "   Parameters : 9415\n",
      "   Iterations : 8\n",
      "         Time : 0.201729 [s]\n",
      " Initial cost : 0.657227 [px]\n",
      "   Final cost : 0.605505 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 53\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 2\n",
      "  => Changed observations: 0.006479\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  6.399655e+03    0.00e+00    7.09e+03   0.00e+00   0.00e+00  1.00e+04        0    1.02e-02    2.76e-02\n",
      "   1  6.312667e+03    8.70e+01    2.33e+02   2.27e+00   1.00e+00  3.00e+04        1    2.45e-02    5.21e-02\n",
      "   2  6.312168e+03    4.99e-01    1.05e+02   2.02e+00   1.00e+00  9.00e+04        1    2.49e-02    7.70e-02\n",
      "   3  6.312090e+03    7.80e-02    3.38e+01   1.12e+00   1.00e+00  2.70e+05        1    2.38e-02    1.01e-01\n",
      "   4  6.312087e+03    2.87e-03    1.93e+00   2.40e-01   1.01e+00  8.10e+05        1    2.38e-02    1.25e-01\n",
      "   5  6.312087e+03    1.69e-05    1.36e-01   1.91e-02   1.01e+00  2.43e+06        1    2.33e-02    1.48e-01\n",
      "   6  6.312087e+03    2.29e-08    2.56e-03   6.50e-04   1.02e+00  7.29e+06        1    2.31e-02    1.71e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 17080\n",
      "   Parameters : 9415\n",
      "   Iterations : 7\n",
      "         Time : 0.171634 [s]\n",
      " Initial cost : 0.612117 [px]\n",
      "   Final cost : 0.607915 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 5\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 2\n",
      "  => Changed observations: 0.000820\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  6.331338e+03    0.00e+00    2.96e+03   0.00e+00   0.00e+00  1.00e+04        0    9.56e-03    2.69e-02\n",
      "   1  6.316996e+03    1.43e+01    7.44e+00   4.60e-01   1.00e+00  3.00e+04        1    2.47e-02    5.16e-02\n",
      "   2  6.316985e+03    1.11e-02    1.99e+00   2.79e-01   1.00e+00  9.00e+04        1    2.36e-02    7.51e-02\n",
      "   3  6.316983e+03    1.62e-03    1.42e+00   1.62e-01   1.00e+00  2.70e+05        1    2.54e-02    1.01e-01\n",
      "   4  6.316983e+03    5.83e-05    3.21e-01   3.47e-02   1.01e+00  8.10e+05        1    2.38e-02    1.24e-01\n",
      "   5  6.316983e+03    3.48e-07    8.49e-03   2.75e-03   1.01e+00  2.43e+06        1    2.39e-02    1.48e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 17086\n",
      "   Parameters : 9415\n",
      "   Iterations : 6\n",
      "         Time : 0.148946 [s]\n",
      " Initial cost : 0.608734 [px]\n",
      "   Final cost : 0.608044 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000117\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #14 (8)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 415 / 2350 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 744\n",
      "   Parameters : 8\n",
      "   Iterations : 17\n",
      "         Time : 0.00915194 [s]\n",
      " Initial cost : 0.92203 [px]\n",
      "   Final cost : 0.729198 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 366\n",
      "  => Added observations: 664\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 13828\n",
      "   Parameters : 4259\n",
      "   Iterations : 26\n",
      "         Time : 0.407467 [s]\n",
      " Initial cost : 0.585052 [px]\n",
      "   Final cost : 0.536886 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 10\n",
      "  => Completed observations: 5\n",
      "  => Filtered observations: 120\n",
      "  => Changed observations: 0.019526\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 13598\n",
      "   Parameters : 4199\n",
      "   Iterations : 3\n",
      "         Time : 0.047503 [s]\n",
      " Initial cost : 0.606463 [px]\n",
      "   Final cost : 0.592284 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 6\n",
      "  => Completed observations: 12\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.002795\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 2\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  6.958779e+03    0.00e+00    1.75e+03   0.00e+00   0.00e+00  1.00e+04        0    1.02e-02    3.01e-02\n",
      "   1  6.905511e+03    5.33e+01    1.86e+03   8.53e+00   9.96e-01  3.00e+04        1    2.77e-02    5.78e-02\n",
      "   2  6.896900e+03    8.61e+00    1.79e+03   8.19e+00   9.75e-01  9.00e+04        1    2.68e-02    8.46e-02\n",
      "   3  6.895534e+03    1.37e+00    3.97e+02   3.84e+00   9.95e-01  2.70e+05        1    2.72e-02    1.12e-01\n",
      "   4  6.895494e+03    4.02e-02    1.19e+01   6.78e-01   1.00e+00  8.10e+05        1    2.84e-02    1.40e-01\n",
      "   5  6.895494e+03    1.28e-04    5.17e-01   4.39e-02   1.01e+00  2.43e+06        1    2.63e-02    1.67e-01\n",
      "   6  6.895494e+03    1.02e-07    6.94e-03   1.19e-03   1.02e+00  7.29e+06        1    2.71e-02    1.94e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 18944\n",
      "   Parameters : 10302\n",
      "   Iterations : 7\n",
      "         Time : 0.194438 [s]\n",
      " Initial cost : 0.606081 [px]\n",
      "   Final cost : 0.603319 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 5\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000528\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  6.933090e+03    0.00e+00    1.18e+03   0.00e+00   0.00e+00  1.00e+04        0    1.06e-02    3.01e-02\n",
      "   1  6.922016e+03    1.11e+01    9.24e+00   2.20e-01   1.00e+00  3.00e+04        1    2.74e-02    5.75e-02\n",
      "   2  6.922003e+03    1.27e-02    1.98e+00   2.81e-01   1.00e+00  9.00e+04        1    2.65e-02    8.40e-02\n",
      "   3  6.922001e+03    1.52e-03    1.88e+00   1.36e-01   1.00e+00  2.70e+05        1    2.68e-02    1.11e-01\n",
      "   4  6.922001e+03    3.74e-05    1.73e-01   2.38e-02   1.01e+00  8.10e+05        1    2.89e-02    1.40e-01\n",
      "   5  6.922001e+03    1.46e-07    6.10e-03   1.51e-03   1.01e+00  2.43e+06        1    2.68e-02    1.67e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 18954\n",
      "   Parameters : 10302\n",
      "   Iterations : 6\n",
      "         Time : 0.167273 [s]\n",
      " Initial cost : 0.604802 [px]\n",
      "   Final cost : 0.604318 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000106\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #11 (9)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 393 / 1922 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 610\n",
      "   Parameters : 8\n",
      "   Iterations : 14\n",
      "         Time : 0.00632215 [s]\n",
      " Initial cost : 0.961616 [px]\n",
      "   Final cost : 0.707332 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 305\n",
      "  => Added observations: 827\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11716\n",
      "   Parameters : 4433\n",
      "   Iterations : 26\n",
      "         Time : 0.383118 [s]\n",
      " Initial cost : 0.644935 [px]\n",
      "   Final cost : 0.593254 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 4\n",
      "  => Completed observations: 4\n",
      "  => Filtered observations: 161\n",
      "  => Changed observations: 0.028849\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11402\n",
      "   Parameters : 4265\n",
      "   Iterations : 3\n",
      "         Time : 0.043818 [s]\n",
      " Initial cost : 0.627628 [px]\n",
      "   Final cost : 0.612447 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 30\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.005262\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 53\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  4.309502e+04    0.00e+00    8.40e+04   0.00e+00   0.00e+00  1.00e+04        0    1.20e-02    3.45e-02\n",
      "   1  2.518671e+04    1.79e+04    7.40e+03   1.44e+01   9.96e-01  3.00e+04        1    3.23e-02    6.68e-02\n",
      "   2  2.506710e+04    1.20e+02    4.24e+02   4.61e+00   1.02e+00  9.00e+04        1    3.15e-02    9.83e-02\n",
      "   3  2.506659e+04    5.04e-01    4.96e+01   1.40e+00   1.03e+00  2.70e+05        1    3.13e-02    1.30e-01\n",
      "   4  2.506659e+04    5.56e-03    1.36e+00   2.15e-01   1.02e+00  8.10e+05        1    3.00e-02    1.60e-01\n",
      "   5  2.506659e+04    2.36e-05    6.64e-01   1.16e-02   1.03e+00  2.43e+06        1    3.01e-02    1.90e-01\n",
      "   6  2.506659e+04    7.89e-08    5.95e-02   5.68e-04   1.02e+00  7.29e+06        1    2.98e-02    2.20e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 21072\n",
      "   Parameters : 11420\n",
      "   Iterations : 7\n",
      "         Time : 0.220301 [s]\n",
      " Initial cost : 1.43008 [px]\n",
      "   Final cost : 1.09067 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 8\n",
      "  => Merged observations: 6\n",
      "  => Filtered observations: 85\n",
      "  => Changed observations: 0.009396\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  8.757340e+03    0.00e+00    6.42e+04   0.00e+00   0.00e+00  1.00e+04        0    1.20e-02    3.42e-02\n",
      "   1  7.988295e+03    7.69e+02    5.82e+03   1.34e+01   9.99e-01  3.00e+04        1    3.23e-02    6.65e-02\n",
      "   2  7.986659e+03    1.64e+00    7.54e+01   1.68e+00   1.00e+00  9.00e+04        1    3.13e-02    9.78e-02\n",
      "   3  7.986618e+03    4.02e-02    9.61e+00   5.54e-01   1.00e+00  2.70e+05        1    2.97e-02    1.28e-01\n",
      "   4  7.986618e+03    6.54e-04    1.48e+00   8.00e-02   1.01e+00  8.10e+05        1    2.97e-02    1.57e-01\n",
      "   5  7.986618e+03    1.75e-06    1.91e-02   4.27e-03   1.01e+00  2.43e+06        1    3.05e-02    1.88e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 20918\n",
      "   Parameters : 11300\n",
      "   Iterations : 6\n",
      "         Time : 0.188408 [s]\n",
      " Initial cost : 0.647032 [px]\n",
      "   Final cost : 0.617905 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 3\n",
      "  => Changed observations: 0.000287\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #3 (10)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 391 / 2118 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 514\n",
      "   Parameters : 8\n",
      "   Iterations : 15\n",
      "         Time : 0.00569606 [s]\n",
      " Initial cost : 0.903334 [px]\n",
      "   Final cost : 0.71785 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 257\n",
      "  => Added observations: 732\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16346\n",
      "   Parameters : 7526\n",
      "   Iterations : 20\n",
      "         Time : 0.535766 [s]\n",
      " Initial cost : 0.571366 [px]\n",
      "   Final cost : 0.531185 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 7\n",
      "  => Completed observations: 3\n",
      "  => Filtered observations: 89\n",
      "  => Changed observations: 0.012113\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16178\n",
      "   Parameters : 7499\n",
      "   Iterations : 3\n",
      "         Time : 0.0696681 [s]\n",
      " Initial cost : 0.639449 [px]\n",
      "   Final cost : 0.624444 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 10\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.001236\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 1\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  2.061020e+04    0.00e+00    4.47e+04   0.00e+00   0.00e+00  1.00e+04        0    1.37e-02    3.93e-02\n",
      "   1  1.696099e+04    3.65e+03    2.57e+03   2.40e+00   9.96e-01  3.00e+04        1    3.50e-02    7.43e-02\n",
      "   2  1.695151e+04    9.48e+00    9.85e+01   1.25e+00   9.79e-01  9.00e+04        1    3.49e-02    1.09e-01\n",
      "   3  1.695147e+04    3.84e-02    6.76e+00   2.79e-01   9.99e-01  2.70e+05        1    3.39e-02    1.43e-01\n",
      "   4  1.695147e+04    1.73e-04    1.67e+00   3.48e-02   1.01e+00  8.10e+05        1    3.28e-02    1.76e-01\n",
      "   5  1.695147e+04    4.62e-07    7.65e-02   1.60e-03   1.02e+00  2.43e+06        1    3.29e-02    2.09e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 22742\n",
      "   Parameters : 12205\n",
      "   Iterations : 6\n",
      "         Time : 0.209671 [s]\n",
      " Initial cost : 0.951978 [px]\n",
      "   Final cost : 0.863355 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 5\n",
      "  => Changed observations: 0.000440\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #2 (11)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 578 / 1337 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 1040\n",
      "   Parameters : 8\n",
      "   Iterations : 16\n",
      "         Time : 0.0135832 [s]\n",
      " Initial cost : 0.913311 [px]\n",
      "   Final cost : 0.708782 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 512\n",
      "  => Added observations: 577\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 16154\n",
      "   Parameters : 7268\n",
      "   Iterations : 26\n",
      "         Time : 0.564509 [s]\n",
      " Initial cost : 0.597252 [px]\n",
      "   Final cost : 0.561305 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 11\n",
      "  => Completed observations: 8\n",
      "  => Filtered observations: 171\n",
      "  => Changed observations: 0.023524\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 15830\n",
      "   Parameters : 7196\n",
      "   Iterations : 3\n",
      "         Time : 0.0673041 [s]\n",
      " Initial cost : 0.658149 [px]\n",
      "   Final cost : 0.6374 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 43\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.005433\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 4\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 1\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.115825e+04    0.00e+00    2.68e+04   0.00e+00   0.00e+00  1.00e+04        0    1.46e-02    4.23e-02\n",
      "   1  1.053759e+04    6.21e+02    1.23e+03   4.18e+00   9.94e-01  3.00e+04        1    3.80e-02    8.04e-02\n",
      "   2  1.053603e+04    1.57e+00    7.91e+01   9.15e-01   9.94e-01  9.00e+04        1    3.98e-02    1.20e-01\n",
      "   3  1.053599e+04    3.36e-02    3.27e+00   2.87e-01   1.00e+00  2.70e+05        1    4.10e-02    1.61e-01\n",
      "   4  1.053599e+04    3.12e-04    4.01e-01   4.16e-02   1.01e+00  8.10e+05        1    3.98e-02    2.01e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 24680\n",
      "   Parameters : 12957\n",
      "   Iterations : 5\n",
      "         Time : 0.201843 [s]\n",
      " Initial cost : 0.672397 [px]\n",
      "   Final cost : 0.653379 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 4\n",
      "  => Changed observations: 0.000324\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #5 (12)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 355 / 2094 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 560\n",
      "   Parameters : 8\n",
      "   Iterations : 20\n",
      "         Time : 0.0105951 [s]\n",
      " Initial cost : 0.885743 [px]\n",
      "   Final cost : 0.739577 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 278\n",
      "  => Added observations: 1003\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11644\n",
      "   Parameters : 4988\n",
      "   Iterations : 26\n",
      "         Time : 0.417449 [s]\n",
      " Initial cost : 0.67731 [px]\n",
      "   Final cost : 0.568326 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 4\n",
      "  => Filtered observations: 150\n",
      "  => Changed observations: 0.026451\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11352\n",
      "   Parameters : 4892\n",
      "   Iterations : 3\n",
      "         Time : 0.0454938 [s]\n",
      " Initial cost : 0.638074 [px]\n",
      "   Final cost : 0.619331 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 31\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.005462\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 0\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.074080e+04    0.00e+00    5.17e+03   0.00e+00   0.00e+00  1.00e+04        0    1.59e-02    4.65e-02\n",
      "   1  1.071124e+04    2.96e+01    9.49e+01   2.50e+00   1.00e+00  3.00e+04        1    4.32e-02    8.97e-02\n",
      "   2  1.071111e+04    1.26e-01    9.06e+00   6.27e-01   1.00e+00  9.00e+04        1    4.17e-02    1.31e-01\n",
      "   3  1.071111e+04    7.63e-03    1.76e+00   2.16e-01   1.00e+00  2.70e+05        1    4.05e-02    1.72e-01\n",
      "   4  1.071111e+04    1.08e-04    1.92e-01   2.88e-02   1.01e+00  8.10e+05        1    4.05e-02    2.12e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 27006\n",
      "   Parameters : 14303\n",
      "   Iterations : 5\n",
      "         Time : 0.213331 [s]\n",
      " Initial cost : 0.63065 [px]\n",
      "   Final cost : 0.629777 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000074\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #13 (13)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 600 / 2008 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 892\n",
      "   Parameters : 8\n",
      "   Iterations : 14\n",
      "         Time : 0.00940108 [s]\n",
      " Initial cost : 0.966893 [px]\n",
      "   Final cost : 0.692585 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 443\n",
      "  => Added observations: 805\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11704\n",
      "   Parameters : 4454\n",
      "   Iterations : 26\n",
      "         Time : 0.39163 [s]\n",
      " Initial cost : 0.704964 [px]\n",
      "   Final cost : 0.668708 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 17\n",
      "  => Completed observations: 18\n",
      "  => Filtered observations: 265\n",
      "  => Changed observations: 0.051265\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11210\n",
      "   Parameters : 4340\n",
      "   Iterations : 3\n",
      "         Time : 0.0446532 [s]\n",
      " Initial cost : 0.666744 [px]\n",
      "   Final cost : 0.637491 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 72\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.012846\n",
      "\n",
      "==============================================================================\n",
      "Registering image #12 (14)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 588 / 1688 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 972\n",
      "   Parameters : 8\n",
      "   Iterations : 21\n",
      "         Time : 0.0163739 [s]\n",
      " Initial cost : 0.886467 [px]\n",
      "   Final cost : 0.729393 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 471\n",
      "  => Added observations: 314\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 11322\n",
      "   Parameters : 3650\n",
      "   Iterations : 26\n",
      "         Time : 0.410856 [s]\n",
      " Initial cost : 0.720668 [px]\n",
      "   Final cost : 0.636634 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 13\n",
      "  => Completed observations: 13\n",
      "  => Filtered observations: 214\n",
      "  => Changed observations: 0.042395\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 10926\n",
      "   Parameters : 3530\n",
      "   Iterations : 3\n",
      "         Time : 0.044102 [s]\n",
      " Initial cost : 0.682403 [px]\n",
      "   Final cost : 0.660051 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 50\n",
      "  => Filtered observations: 1\n",
      "  => Changed observations: 0.009336\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 3\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 3\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.814664e+04    0.00e+00    1.28e+04   0.00e+00   0.00e+00  1.00e+04        0    1.94e-02    5.46e-02\n",
      "   1  1.622807e+04    1.92e+03    8.29e+02   7.18e+00   1.01e+00  3.00e+04        1    5.03e-02    1.05e-01\n",
      "   2  1.622592e+04    2.16e+00    5.84e+01   1.41e+00   1.01e+00  9.00e+04        1    4.88e-02    1.54e-01\n",
      "   3  1.622588e+04    3.97e-02    5.75e+00   4.29e-01   1.00e+00  2.70e+05        1    4.70e-02    2.01e-01\n",
      "   4  1.622588e+04    5.31e-04    5.12e-01   5.31e-02   1.01e+00  8.10e+05        1    4.65e-02    2.47e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 30432\n",
      "   Parameters : 15360\n",
      "   Iterations : 5\n",
      "         Time : 0.248237 [s]\n",
      " Initial cost : 0.772206 [px]\n",
      "   Final cost : 0.730195 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 2\n",
      "  => Merged observations: 4\n",
      "  => Filtered observations: 7\n",
      "  => Changed observations: 0.000854\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.275456e+04    0.00e+00    9.81e+03   0.00e+00   0.00e+00  1.00e+04        0    1.72e-02    5.00e-02\n",
      "   1  1.261848e+04    1.36e+02    2.49e+02   8.84e+00   1.00e+00  3.00e+04        1    5.00e-02    1.00e-01\n",
      "   2  1.261827e+04    2.08e-01    6.36e+00   8.58e-01   1.00e+00  9.00e+04        1    4.74e-02    1.47e-01\n",
      "   3  1.261827e+04    4.14e-03    1.81e+00   1.45e-01   1.00e+00  2.70e+05        1    4.68e-02    1.94e-01\n",
      "   4  1.261827e+04    5.66e-05    8.88e-02   1.73e-02   1.01e+00  8.10e+05        1    4.77e-02    2.42e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 30422\n",
      "   Parameters : 15351\n",
      "   Iterations : 5\n",
      "         Time : 0.242947 [s]\n",
      " Initial cost : 0.647499 [px]\n",
      "   Final cost : 0.64403 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 5\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.000329\n",
      "  => Filtered images: 0\n",
      "\n",
      "==============================================================================\n",
      "Registering image #4 (15)\n",
      "==============================================================================\n",
      "\n",
      "  => Image sees 403 / 1287 points\n",
      "\n",
      "Pose refinement report\n",
      "----------------------\n",
      "    Residuals : 696\n",
      "   Parameters : 8\n",
      "   Iterations : 16\n",
      "         Time : 0.00827312 [s]\n",
      " Initial cost : 0.886092 [px]\n",
      "   Final cost : 0.680687 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Continued observations: 347\n",
      "  => Added observations: 212\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 14164\n",
      "   Parameters : 4418\n",
      "   Iterations : 26\n",
      "         Time : 0.396812 [s]\n",
      " Initial cost : 0.628174 [px]\n",
      "   Final cost : 0.605156 [px]\n",
      "  Termination : No convergence\n",
      "\n",
      "  => Merged observations: 7\n",
      "  => Completed observations: 4\n",
      "  => Filtered observations: 139\n",
      "  => Changed observations: 0.021180\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 14976\n",
      "   Parameters : 5015\n",
      "   Iterations : 3\n",
      "         Time : 0.0553861 [s]\n",
      " Initial cost : 0.656098 [px]\n",
      "   Final cost : 0.64241 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Merged observations: 0\n",
      "  => Completed observations: 33\n",
      "  => Filtered observations: 0\n",
      "  => Changed observations: 0.004407\n",
      "\n",
      "==============================================================================\n",
      "Retriangulation\n",
      "==============================================================================\n",
      "\n",
      "  => Completed observations: 1\n",
      "  => Merged observations: 0\n",
      "  => Retriangulated observations: 32\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  8.080863e+04    0.00e+00    7.90e+04   0.00e+00   0.00e+00  1.00e+04        0    1.87e-02    5.47e-02\n",
      "   1  2.852925e+04    5.23e+04    8.58e+03   1.36e+01   9.87e-01  3.00e+04        1    5.40e-02    1.09e-01\n",
      "   2  2.809047e+04    4.39e+02    2.52e+02   2.34e+00   9.88e-01  9.00e+04        1    4.99e-02    1.59e-01\n",
      "   3  2.809029e+04    1.87e-01    2.20e+00   1.91e-01   1.00e+00  2.70e+05        1    4.97e-02    2.08e-01\n",
      "   4  2.809029e+04    1.82e-04    1.63e+00   6.91e-03   1.01e+00  8.10e+05        1    4.93e-02    2.58e-01\n",
      "   5  2.809029e+04    2.09e-07    1.64e-02   2.06e-04   1.01e+00  2.43e+06        1    4.90e-02    3.07e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 31402\n",
      "   Parameters : 15578\n",
      "   Iterations : 6\n",
      "         Time : 0.307759 [s]\n",
      " Initial cost : 1.60417 [px]\n",
      "   Final cost : 0.9458 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 2\n",
      "  => Merged observations: 6\n",
      "  => Filtered observations: 46\n",
      "  => Changed observations: 0.003439\n",
      "\n",
      "==============================================================================\n",
      "Global bundle adjustment\n",
      "==============================================================================\n",
      "\n",
      "iter      cost      cost_change  |gradient|   |step|    tr_ratio  tr_radius  ls_iter  iter_time  total_time\n",
      "   0  1.350816e+04    0.00e+00    1.83e+04   0.00e+00   0.00e+00  1.00e+04        0    1.88e-02    5.30e-02\n",
      "   1  1.323222e+04    2.76e+02    1.17e+03   9.44e+00   1.00e+00  3.00e+04        1    5.15e-02    1.04e-01\n",
      "   2  1.323156e+04    6.53e-01    5.12e+01   1.27e+00   1.00e+00  9.00e+04        1    4.83e-02    1.53e-01\n",
      "   3  1.323155e+04    1.55e-02    3.33e+00   2.95e-01   1.01e+00  2.70e+05        1    5.02e-02    2.03e-01\n",
      "   4  1.323155e+04    1.78e-04    1.37e-01   3.09e-02   1.01e+00  8.10e+05        1    4.87e-02    2.52e-01\n",
      "\n",
      "\n",
      "Bundle adjustment report\n",
      "------------------------\n",
      "    Residuals : 31314\n",
      "   Parameters : 15536\n",
      "   Iterations : 5\n",
      "         Time : 0.252663 [s]\n",
      " Initial cost : 0.656793 [px]\n",
      "   Final cost : 0.650034 [px]\n",
      "  Termination : Convergence\n",
      "\n",
      "  => Completed observations: 0\n",
      "  => Merged observations: 0\n",
      "  => Filtered observations: 5\n",
      "  => Changed observations: 0.000319\n",
      "  => Filtered images: 0\n",
      "\n",
      "Elapsed time: 0.308 [minutes]\n",
      "\n",
      "                Dataset: haiper - Scene: bike\n",
      "                Reconstruction count: 1\n",
      "                Best reconstruction registered image count: 15/15\n",
      "                \n"
     ]
    }
   ],
   "source": [
    "reconstruction_root_directory = Path('./inference')\n",
    "reconstruction_root_directory.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "datasets = [\n",
    "    directory for directory in os.listdir(train_or_test_directory)\n",
    "    if (train_or_test_directory / directory).is_dir() and (directory in df['dataset'].unique())\n",
    "]\n",
    "\n",
    "for dataset in datasets:\n",
    "        \n",
    "    dataset_directory = train_or_test_directory / dataset\n",
    "    scenes = [\n",
    "        directory for directory in os.listdir(dataset_directory)\n",
    "        if (dataset_directory / directory).is_dir() and (directory in df['scene'].unique())\n",
    "    ]\n",
    "    \n",
    "    for scene in scenes:\n",
    "        \n",
    "        df_scene = df.loc[df['scene'] == scene, :]\n",
    "                \n",
    "        scene_directory = dataset_directory / scene\n",
    "        scene_image_directory = scene_directory / 'images'\n",
    "        image_paths = sorted(glob(str(scene_image_directory / '*')))\n",
    "        scene_image_count = len(image_paths)\n",
    "        \n",
    "        scene_reconstruction_directory = reconstruction_root_directory / dataset / scene\n",
    "        scene_reconstruction_directory.mkdir(parents=True, exist_ok=True)                    \n",
    "        database_path = scene_reconstruction_directory / 'colmap.db'\n",
    "        if os.path.isfile(database_path):\n",
    "            os.remove(database_path)\n",
    "        \n",
    "        scene_mean_memory_usage = df_scene['memory_usage'].mean()\n",
    "        scene_mean_memory_usage_limit = 16\n",
    "        if scene_mean_memory_usage < scene_mean_memory_usage_limit:\n",
    "            \n",
    "            # Create image pair indices from given image paths\n",
    "            scene_image_pair_indices = create_image_pairs(\n",
    "                image_paths,\n",
    "                sim_th=0.85,\n",
    "                min_pairs=120,\n",
    "                exhaustive_if_less=200,\n",
    "                device=image_matching_device\n",
    "            )\n",
    "            \n",
    "            @lru_cache(maxsize=2)\n",
    "            def read_image(image_path):\n",
    "                return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n",
    "            \n",
    "            match(\n",
    "                read_image,\n",
    "                image_paths,\n",
    "                scene_image_pair_indices,\n",
    "                feature_dir=scene_reconstruction_directory,\n",
    "                superglue_model=superglue_model,\n",
    "                stage2=False\n",
    "            )\n",
    "            import_into_colmap(scene_image_directory, feature_dir=scene_reconstruction_directory, database_path=database_path)\n",
    "        else:\n",
    "            sift_extraction_options = pycolmap.SiftExtractionOptions()\n",
    "            sift_extraction_options.num_threads = -1\n",
    "            sift_extraction_options.max_image_size = 1400\n",
    "            sift_extraction_options.max_num_features = 8192\n",
    "            sift_extraction_options.estimate_affine_shape = False\n",
    "            sift_extraction_options.upright = False\n",
    "            sift_extraction_options.normalization = 'L2'\n",
    "            pycolmap.extract_features(\n",
    "                database_path=database_path,\n",
    "                image_path=scene_image_directory,\n",
    "                image_list=[image_path.split('/')[-1] for image_path in image_paths],\n",
    "                sift_options=sift_extraction_options,\n",
    "                device=pycolmap.Device('cpu'),\n",
    "                verbose=verbose\n",
    "            )\n",
    "    \n",
    "        pycolmap.match_exhaustive(database_path)\n",
    "        \n",
    "        output_path = scene_reconstruction_directory / 'colmap_rec'\n",
    "        output_path.mkdir(parents=True, exist_ok=True)                    \n",
    "\n",
    "        incremental_mapper_options = pycolmap.IncrementalMapperOptions()\n",
    "        incremental_mapper_options.min_model_size = 3\n",
    "        incremental_mapper_options.min_num_matches = 5\n",
    "        reconstructions = pycolmap.incremental_mapping(\n",
    "            database_path=database_path,\n",
    "            image_path=scene_image_directory,\n",
    "            output_path=output_path,\n",
    "            options=incremental_mapper_options\n",
    "        )\n",
    "        \n",
    "        if len(reconstructions) > 0:\n",
    "\n",
    "            best_registered_image_count = 0\n",
    "            best_reconstruction_idx = None\n",
    "\n",
    "            for reconstruction_idx in reconstructions.keys():\n",
    "                if reconstructions[reconstruction_idx].num_reg_images() > best_registered_image_count:\n",
    "                    best_reconstruction_idx = reconstruction_idx\n",
    "                    best_registered_image_count = reconstructions[reconstruction_idx].num_reg_images()\n",
    "\n",
    "            best_reconstruction = reconstructions[best_reconstruction_idx]\n",
    "        else:\n",
    "            best_registered_image_count = 0\n",
    "            best_reconstruction_idx = None\n",
    "            best_reconstruction = None\n",
    "\n",
    "        if verbose:\n",
    "            print(\n",
    "                f'''\n",
    "                Dataset: {dataset} - Scene: {scene}\n",
    "                Reconstruction count: {len(reconstructions)}\n",
    "                Best reconstruction registered image count: {best_registered_image_count}/{scene_image_count}\n",
    "                '''\n",
    "            )\n",
    "\n",
    "        if best_reconstruction is not None:\n",
    "            registered_images = {image.name: image for image in best_reconstruction.images.values()}\n",
    "        else:\n",
    "            registered_images = {}\n",
    "            \n",
    "        for idx, row in df.loc[(df['dataset'] == dataset) & (df['scene'] == scene)].iterrows():\n",
    "            if row['image_id'] in registered_images:\n",
    "                rotation_matrix_prediction = registered_images[row['image_id']].rotmat()\n",
    "                translation_vector_prediction = registered_images[row['image_id']].tvec\n",
    "                df.loc[idx, 'rotation_matrix'] = ';'.join([str(x) for x in rotation_matrix_prediction.reshape(-1)])\n",
    "                df.loc[idx, 'translation_vector'] = ';'.join([str(x) for x in translation_vector_prediction.reshape(-1)])\n",
    "            else:\n",
    "                df.loc[idx, 'rotation_matrix'] = np.nan\n",
    "                df.loc[idx, 'translation_vector'] = np.nan\n",
    "\n",
    "        # Fill unregistered images rotation matrices with the prediction mean or zeros\n",
    "        scene_rotation_matrix_predictions = df.loc[df['scene'] == scene, 'rotation_matrix'].dropna().apply(lambda x: np.array(str(x).split(';'), dtype=np.float64).reshape(1, 3, 3)).values\n",
    "        if scene_rotation_matrix_predictions.shape[0] == 0:\n",
    "            rotation_matrix_fill_value = np.zeros((3, 3))\n",
    "        else:\n",
    "            rotation_matrix_fill_value = np.mean(np.concatenate(scene_rotation_matrix_predictions, axis=0), axis=0)\n",
    "        df.loc[(df['scene'] == scene) & (df['rotation_matrix'].isnull()), 'rotation_matrix'] = ';'.join([str(x) for x in rotation_matrix_fill_value.reshape(-1)])\n",
    "\n",
    "        # Fill unregistered images translation vectors with the prediction mean or zeros\n",
    "        scene_translation_vector_predictions = df.loc[df['scene'] == scene, 'translation_vector'].dropna().apply(lambda x: np.array(str(x).split(';'), dtype=np.float64).reshape(1, 3)).values\n",
    "        if scene_translation_vector_predictions.shape[0] == 0:\n",
    "            translation_vector_fill_value = np.zeros((3, 1))\n",
    "        else:\n",
    "            translation_vector_fill_value = np.mean(np.concatenate(scene_translation_vector_predictions, axis=0), axis=0)\n",
    "        df.loc[(df['scene'] == scene) & (df['translation_vector'].isnull()), 'translation_vector'] = ';'.join([str(x) for x in translation_vector_fill_value.reshape(-1)])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e65e309b",
   "metadata": {
    "papermill": {
     "duration": 0.01501,
     "end_time": "2023-06-12T10:34:59.439008",
     "exception": false,
     "start_time": "2023-06-12T10:34:59.423998",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 8. Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b0ed9692",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-06-12T10:34:59.470836Z",
     "iopub.status.busy": "2023-06-12T10:34:59.470502Z",
     "iopub.status.idle": "2023-06-12T10:34:59.479243Z",
     "shell.execute_reply": "2023-06-12T10:34:59.478345Z"
    },
    "papermill": {
     "duration": 0.027296,
     "end_time": "2023-06-12T10:34:59.481318",
     "exception": false,
     "start_time": "2023-06-12T10:34:59.454022",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_submission = df.loc[:, ['image_path', 'dataset', 'scene', 'rotation_matrix', 'translation_vector']]\n",
    "df_submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d9c987b",
   "metadata": {
    "papermill": {
     "duration": 0.015702,
     "end_time": "2023-06-12T10:34:59.513950",
     "exception": false,
     "start_time": "2023-06-12T10:34:59.498248",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 196.764667,
   "end_time": "2023-06-12T10:35:02.655469",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-06-12T10:31:45.890802",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}