ptt_article_post_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "MgOkmEfBJ_Fz"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import bs4\n",
    "import pandas as pd\n",
    "import os\n",
    "import gc\n",
    "import re\n",
    "from datetime import datetime\n",
    "# 讀取 Google drive\n",
    "#from google.colab import drive\n",
    "#drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "45upQNZBw5xb",
    "outputId": "070d3742-54a9-402b-cee1-9bdaca9d3dc6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.25.3'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "w3lvt4xhC_B4"
   },
   "source": [
    "# add dict Simplified to traditional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Ry5OruEAXHok",
    "outputId": "2ba5ffbc-2fb9-4683-c221-3838f97d44f7"
   },
   "outputs": [],
   "source": [
    "path = r'G:\\py_workspace\\20191103_ptt_public_opinion\\data\\id_check\\\\'\n",
    "#os.listdir(path + 'source')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "source = pd.read_csv(path + 'source/gephi_trns_v2.csv')\n",
    "usr_lst = ['yuxds', 'omanorbo', 'kieranc', 'mark2165']\n",
    "usr_lst.extend(source.sort_values('pagerank', ascending=False).id.to_list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>community</th>\n",
       "      <th>id</th>\n",
       "      <th>pagerank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30771</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>Neptunium</td>\n",
       "      <td>254.015470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5012</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>kero2377</td>\n",
       "      <td>213.156853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21324</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Wojnarowski</td>\n",
       "      <td>210.151955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6121</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>nicholas0406</td>\n",
       "      <td>144.894421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41340</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>Kevin1103</td>\n",
       "      <td>141.652942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16957</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>xxxdog</td>\n",
       "      <td>0.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2611</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>captainmm</td>\n",
       "      <td>0.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2610</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>capsspac</td>\n",
       "      <td>0.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2609</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>capdog</td>\n",
       "      <td>0.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>AV8Dpachuchi</td>\n",
       "      <td>0.150000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42356 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  community            id    pagerank\n",
       "30771           0          4     Neptunium  254.015470\n",
       "5012            0          0      kero2377  213.156853\n",
       "21324           0          3   Wojnarowski  210.151955\n",
       "6121            0          0  nicholas0406  144.894421\n",
       "41340           0          4     Kevin1103  141.652942\n",
       "...           ...        ...           ...         ...\n",
       "16957           0          1        xxxdog    0.150000\n",
       "2611            0          0     captainmm    0.150000\n",
       "2610            0          0      capsspac    0.150000\n",
       "2609            0          0        capdog    0.150000\n",
       "0               0          0  AV8Dpachuchi    0.150000\n",
       "\n",
       "[42356 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source.sort_values('pagerank', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4], dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source.community.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0      int64\n",
       "community       int64\n",
       "id             object\n",
       "pagerank      float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['yuxds',\n",
       " 'omanorbo',\n",
       " 'kieranc',\n",
       " 'mark2165',\n",
       " 'Neptunium',\n",
       " 'kero2377',\n",
       " 'Wojnarowski',\n",
       " 'nicholas0406',\n",
       " 'Kevin1103',\n",
       " 'diefishfish',\n",
       " 'KingKingCold',\n",
       " 'hong888',\n",
       " 'papamama007',\n",
       " 'jobli',\n",
       " 'ray90910',\n",
       " 'ben108472',\n",
       " 'ice80712',\n",
       " 'AskWhat',\n",
       " 'jason748',\n",
       " 'oftisa']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usr_lst[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#usr_lst = ['yuxds', 'omanorbo', 'kieranc', 'mark2165']\n",
    "#usr_lst.extend(source.id.to_list())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# parser function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def content_parser(soup):\n",
    "    cont_dict = {}\n",
    "    url_cnt = 0\n",
    "    art_id = 'invalid'\n",
    "    ip = 'invalid'\n",
    "    ori_url_split = 'invalid'\n",
    "    \n",
    "    temp_title = soup.select('.thread-title')[0].text\n",
    "        \n",
    "    #content parsing\n",
    "    temp_content = soup.select('#main-content')\n",
    "\n",
    "    if temp_content:\n",
    "        t1 = temp_content[0]\n",
    "    else:\n",
    "        return 'NoContent'\n",
    "    \n",
    "    for x in soup.select('span.f2'):\n",
    "        if x.text.startswith('※ 文章網址: https://www.ptt.cc/bbs'):\n",
    "            ori_url_split = x.a['href'].split('/')\n",
    "            ip = x.previous_element.split(':')[-1].split()[0]\n",
    "            url_cnt = url_cnt + 1\n",
    "        elif x.text.startswith('※ 文章網址:'):\n",
    "            pre_ip = [x for x in x.previous_elements][1]\n",
    "            pre_url = x.next_sibling\n",
    "            \n",
    "            if type(pre_url) is bs4.element.NavigableString:\n",
    "                #old article\n",
    "                ori_url_split = x.a['href'].split('/')\n",
    "            else:\n",
    "                ori_url_split = x.next_sibling.text.split('/')\n",
    "                \n",
    "            if type(pre_ip) is bs4.element.NavigableString:\n",
    "                ip = pre_ip.split(':')[-1].split()[0]\n",
    "            else:\n",
    "                #old article\n",
    "                ip = pre_ip.text.split(':')[-1].split()[0]\n",
    "            url_cnt = url_cnt + 1            \n",
    "               \n",
    "        if url_cnt > 1:\n",
    "            ori_url_split = 'invalid'\n",
    "            ip = 'invalid'\n",
    "            break\n",
    "            #raise ValueError('More than one url....')\n",
    "    \n",
    "    if ori_url_split != 'invalid':\n",
    "        art_id = ori_url_split[-1].strip('.html')\n",
    "        #board = ori_url_split[-2]\n",
    "    for tag in remove_tags:\n",
    "        if len(tag) == 1:\n",
    "            iter_tags = t1.find_all(tag[0])\n",
    "        else:\n",
    "            iter_tags = t1.find_all(tag[0], tag[1])\n",
    "        \n",
    "        for x in iter_tags:\n",
    "            x.decompose()\n",
    "        \n",
    "    cont_dict.update({\n",
    "          'article_id' : art_id\n",
    "        , 'article_title' : temp_title \n",
    "        , 'board' : soup.select('.board')[0].text.strip()\n",
    "        , 'content' : t1.text.replace('\\n', ' ')\n",
    "        , 'date' : soup.select('.post-time')[0].text.strip()\n",
    "        , 'ip' : ip\n",
    "    })\n",
    "    return cont_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "HzD3z72daqXK",
    "outputId": "e5d8c2c7-cdca-47b2-ec13-54d6471a1cd8"
   },
   "outputs": [],
   "source": [
    "def message_cnt(soup):\n",
    "    for x in soup.select('.head-item .push-tag'):\n",
    "        tp = x.text[-1] \n",
    "        cont = int(x.text[:-1])\n",
    "        if '推' == tp:\n",
    "            push_cnt = cont\n",
    "        elif '噓' == tp:\n",
    "            boo_cnt = cont\n",
    "        elif '→' == tp:\n",
    "            neutral_cnt = cont\n",
    "\n",
    "    count_cnt = push_cnt - boo_cnt\n",
    "    all_cnt = push_cnt + boo_cnt + neutral_cnt\n",
    "\n",
    "    message_count = {\n",
    "          'all' : all_cnt\n",
    "        , 'boo' : boo_cnt\n",
    "        , 'count' : count_cnt\n",
    "        , 'neutral' : neutral_cnt\n",
    "        , 'push' : push_cnt        \n",
    "    }\n",
    "    return message_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get personal historical links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#usr_lst.remove('Neptunium')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# check what if only one page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "invalid_rec = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_lst = [\n",
    "      'yuxds'\n",
    ", 'omanorbo'\n",
    ", 'kieranc'\n",
    ", 'mark2165'\n",
    ", 'peter080808'\n",
    ", 'linhu8883324'\n",
    ", 'ebsd'\n",
    ", 'idcc'\n",
    ", 'OusakaMegumu'\n",
    ", 'kim'\n",
    ", 'kaminari22tw'\n",
    ", 'a0986188522'\n",
    ", 'monopoliest'\n",
    ", 'zalora'\n",
    ", 'ptt80357'\n",
    ", 'mureka'\n",
    ", 'dxoxb'\n",
    ", 'pled' \n",
    ", 'ebsd' \n",
    ", 'sofc' \n",
    ", 'saed' \n",
    ", 'exponential7' \n",
    ", 'sigmaaldrich'\n",
    ", 'Hartmann'\n",
    ", 'Wojnarowski'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_lst.extend([x for x in usr_lst if x not in test_lst])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fn_lst = []\n",
    "\n",
    "for x in os.listdir(path):\n",
    "    if x.endswith('id_check.json'):\n",
    "        fn_lst.append(x.split('_')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gogoog_lst = [x for x in test_lst if x not in fn_lst]\n",
    "#invalid_rec = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Neptunium',\n",
       " 'kero2377',\n",
       " 'nicholas0406',\n",
       " 'Kevin1103',\n",
       " 'diefishfish',\n",
       " 'KingKingCold',\n",
       " 'hong888',\n",
       " 'papamama007',\n",
       " 'jobli',\n",
       " 'ray90910',\n",
       " 'ben108472',\n",
       " 'ice80712',\n",
       " 'AskWhat',\n",
       " 'jason748',\n",
       " 'oftisa']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in usr_lst[:20] if x not in test_lst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42200"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([x for x in test_lst if x not in fn_lst])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "426"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#pd.DataFrame(invalid_rec, columns=['usr_id', 'url', 'text', 'err_ms']).to_excel('invalid_rec_20191213.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42259"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gogoog_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test_lst = ['KouChouChing']\n",
    "#BrownNoser\n",
    "#arsonlolita \n",
    "#tina81huang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GntqaWLgSrpJ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user BrownNoser starts!\n",
      "Total 54 pages\n",
      "1\n",
      "10\n",
      "19\n",
      "28\n",
      "37\n",
      "46\n",
      "1\n",
      "2019-12-15 12:44:47.901655\n",
      "301\n",
      "2019-12-15 12:50:03.368230\n",
      "901\n",
      "2019-12-15 13:00:06.421965\n"
     ]
    }
   ],
   "source": [
    "#get all post ranking https://pttweb.tw/user/\n",
    "#for user in usr_lst:\n",
    "for user in test_lst:\n",
    "    url = 'https://pttweb.tw/user/'\n",
    "    print('user {} starts!'.format(user))\n",
    "    url_lst = []\n",
    "    rs_lst = []\n",
    "    rs = requests.get(url + user, params={'page':1})\n",
    "    soup = BeautifulSoup(rs.text)\n",
    "    \n",
    "    try:\n",
    "        #check what if only one page\n",
    "        lst_p = int(soup.findAll('a', {'rel':'nofollow'}, string='尾頁')[0]['href'].split('?page=')[1])\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print('no data')\n",
    "        invalid_rec.append([user, 'no_url', 'no_text', 'no_err'])\n",
    "        continue\n",
    "        \n",
    "    print('Total {} pages'.format(lst_p))\n",
    "\n",
    "    for page in range(1, lst_p + 1):    \n",
    "        rs = requests.get(url + user, params={'page':page})\n",
    "        soup = BeautifulSoup(rs.text)      \n",
    "        #post urls\n",
    "        for x in soup.select('.thread-title a'):\n",
    "            url_lst.append('https://pttweb.tw' + x['href'])\n",
    "        if page % 9 == 1:\n",
    "            print(page)\n",
    "    #get personal fully publish history\n",
    "    for idx, url in enumerate(url_lst):\n",
    "        messages = []\n",
    "        remove_tags = [[True, {'class':['push', 'f2', 'f6']}], ['script']]\n",
    "        try:\n",
    "            rs = requests.get(url)\n",
    "        except Exception as e:\n",
    "            invalid_rec.append([user, url, x.text, e])\n",
    "            print(idx)\n",
    "            print(url)\n",
    "            print(x.text)\n",
    "            \n",
    "        soup = BeautifulSoup(rs.text)\n",
    "\n",
    "        #reply content\n",
    "        for x in soup.select('.push'):\n",
    "            try:\n",
    "                messages.append({\n",
    "                      'push_tag' : x.select('.push-tag')[0].text[0]\n",
    "                    , 'push_userid' : x.select('.push-userid')[0].text\n",
    "                    , 'push_content' : x.select('.push-content')[0].text[2:]\n",
    "                    , 'push_ipdatetime' : x.select('.push-ipdatetime')[0].text.strip()\n",
    "                })\n",
    "            except Exception as e:\n",
    "                invalid_rec.append([user, url, x.text, e])\n",
    "                print(idx)\n",
    "                print(url)\n",
    "                print(x.text)\n",
    "\n",
    "        try:\n",
    "            art = content_parser(soup)        \n",
    "            if art == 'NoContent':\n",
    "                continue    \n",
    "            message_aggr = message_cnt(soup)\n",
    "            art.update({'message_count':message_aggr, 'messages':messages, 'author':user})\n",
    "            temp_df = pd.DataFrame([art])\n",
    "            rs_lst.append(temp_df)\n",
    "        except Exception as e:\n",
    "            invalid_rec.append([user, url, x.text, e])\n",
    "            print(idx)\n",
    "            print(e)\n",
    "            print(url)  \n",
    "        if idx % 300 == 1:\n",
    "            print(idx)\n",
    "            print(datetime.now())\n",
    "    #output\n",
    "    if rs_lst == []:\n",
    "        continue\n",
    "    fn_df = pd.concat(rs_lst)\n",
    "    fn_df.reset_index(inplace=True)\n",
    "    fn_df.to_json(path + '{}_id_check.json'.format(user))\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://pttweb.tw/s/40kcA'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_lst[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# beerduck test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user beerduck starts!\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "user = 'beerduck'\n",
    "\n",
    "url = 'https://pttweb.tw/user/'\n",
    "print('user {} starts!'.format(user))\n",
    "url_lst = []\n",
    "rs_lst = []\n",
    "rs = requests.get(url + user, params={'page':1})\n",
    "soup = BeautifulSoup(rs.text)\n",
    "\n",
    "lst_p = int(soup.findAll('a', {'rel':'nofollow'}, string='尾頁')[0]['href'].split('?page=')[1])\n",
    "print(lst_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<!DOCTYPE html>\n",
       "<html lang=\"zh-hant\" prefix=\"og: http://ogp.me/ns# fb: http://ogp.me/ns/fb# article: http://ogp.me/ns/article#\">\n",
       "<head>\n",
       "<meta charset=\"utf-8\"/>\n",
       "<title>beerduck (beer鴨)發表的文章 | PTT Web</title>\n",
       "<meta content=\"IE=edge\" http-equiv=\"X-UA-Compatible\"/>\n",
       "<meta content=\"width=device-width, initial-scale=1\" name=\"viewport\"/>\n",
       "<meta content=\"beerduck總共發表了97篇文章，最近發表的文章:[爆卦]夾手藍蝦官方粉絲團成立啦！（近況更新）12/10,[爆卦]藍蝦自夾手痛苦倒地影片爆紅，外國妹震驚12/09,[爆卦]網軍指揮徐弘庭和指傷陳玉珍都關臉書了！12/08,[爆卦]佛山爆發森林大火影片曝光12/07,Re:[問卦]狗狗在度估要叫他起來嗎12/07,[新聞]強推執勤女警藍委陳宜民函送究辦12/07\" name=\"description\"/>\n",
       "<meta content=\"index,follow\" name=\"robots\"/>\n",
       "<link href=\"https://pttweb.tw/user/beerduck/\" rel=\"canonical\"/>\n",
       "<meta content=\"beerduck (beer鴨)發表的文章\" property=\"og:title\"/>\n",
       "<meta content=\"article\" property=\"og:type\"/>\n",
       "<meta content=\"2019-12-10T04:26:29+08:00\" property=\"og:updated_time\"/>\n",
       "<meta content=\"https://pttweb.tw/user/beerduck/\" property=\"og:url\"/>\n",
       "<meta content=\"beerduck總共發表了97篇文章，最近發表的文章:[爆卦]夾手藍蝦官方粉絲團成立啦！（近況更新）12/10,[爆卦]藍蝦自夾手痛苦倒地影片爆紅，外國妹震驚12/09,[爆卦]網軍指揮徐弘庭和指傷陳玉珍都關臉書了！12/08,[爆卦]佛山爆發森林大火影片曝光12/07,Re:[問卦]狗狗在度估要叫他起來嗎12/07,[新聞]強推執勤女警藍委陳宜民函送究辦12/07\" property=\"og:description\"/>\n",
       "<meta content=\"PTT Web\" property=\"og:site_name\"/>\n",
       "<meta content=\"2592000\" property=\"og:ttl\"/>\n",
       "<meta content=\"545118689021371\" property=\"fb:app_id\"/>\n",
       "<meta content=\"summary\" name=\"twitter:card\"/>\n",
       "<meta content=\"https://pttweb.tw/user/beerduck/\" name=\"twitter:url\"/>\n",
       "<meta content=\"beerduck (beer鴨)發表的文章\" name=\"twitter:title\"/>\n",
       "<meta content=\"beerduck總共發表了97篇文章，最近發表的文章:[爆卦]夾手藍蝦官方粉絲團成立啦！（近況更新）12/10,[爆卦]藍蝦自夾手痛苦倒地影片爆紅，外國妹震驚12/09,[爆卦]網軍指揮徐弘庭和指傷陳玉珍都關臉書了！12/08,[爆卦]佛山爆發森林大火影片曝光12/07,Re:[\" name=\"twitter:description\"/>\n",
       "<script async=\"\" src=\"//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js\"></script>\n",
       "<link href=\"/static/CACHE/css/051f24d2fcf8.css\" rel=\"stylesheet\" type=\"text/css\"/>\n",
       "<link href=\"/feeds/rss\" rel=\"alternate\" title=\"RSS Feed\" type=\"application/rss+xml\"/>\n",
       "<link href=\"/feeds/atom\" rel=\"alternate\" title=\"Atom\" type=\"application/atom+xml\"/>\n",
       "<style>\n",
       "        \n",
       "    \n",
       "    \n",
       "    \n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "@media(min-width:800px) {.ad-align-left {margin: 0} }\n",
       "\n",
       ".ad-rwd-h { width: 336px; height: 280px; margin:0 auto; }\n",
       "@media(min-width: 500px) { .ad-rwd-h { width: 336px; height: 280px; } }\n",
       "@media(min-width: 800px) { .ad-rwd-h { width: 728px; height: 90px; margin:0; } }\n",
       "\n",
       ".ad-rwd-v { width:336px; height:280px; margin:0 auto; }\n",
       "@media(min-width:500px) { .ad-rwd-v { width: 336px; height: 280px; } }\n",
       "@media(min-width:800px) { .ad-rwd-v { width: 300px; height: 600px; margin:0; } }\n",
       "\n",
       ".ad-rwd-r { width: 336px; height: 280px; margin:0 auto; }\n",
       "@media(min-width: 500px) { .ad-rwd-r { width: 336px; height: 280px; } }\n",
       "@media(min-width: 800px) { .ad-rwd-r { width: 336px; height: 280px; margin:0; } }\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    .sidebar-block {\n",
       "    border-bottom: 1px solid #666;\n",
       "    padding: 5px;\n",
       "    font-size: 18px;\n",
       "    }\n",
       "    .sidebar-block h3 {\n",
       "    font-weight: bold;\n",
       "    }\n",
       "    .sidebar-block h3 a {\n",
       "    text-decoration: underline;\n",
       "    }\n",
       "\n",
       "    .alink a {\n",
       "    text-decoration: underline;\n",
       "    }\n",
       "\n",
       "\n",
       "    h1.user-name {\n",
       "    display:inline;\n",
       "    font-size:16pt;\n",
       "    }\n",
       "\n",
       "    .post-boards {\n",
       "    display:inline-block;\n",
       "    }\n",
       "    .more-post-boards {\n",
       "    display:none;\n",
       "    }\n",
       "\n",
       "    </style>\n",
       "</head>\n",
       "<body>\n",
       "<nav class=\"navbar navbar-inverse navbar-fixed-top\">\n",
       "<div class=\"container\">\n",
       "<div class=\"navbar-header\">\n",
       "<button aria-controls=\"navbar\" aria-expanded=\"false\" class=\"navbar-toggle collapsed\" data-target=\"#navbar\" data-toggle=\"collapse\" type=\"button\">\n",
       "<span class=\"sr-only\">Toggle navigation</span>\n",
       "<span class=\"icon-bar\"></span>\n",
       "<span class=\"icon-bar\"></span>\n",
       "<span class=\"icon-bar\"></span>\n",
       "</button>\n",
       "<a class=\"navbar-brand\" href=\"/\">PTT Web</a>\n",
       "</div>\n",
       "<div class=\"collapse navbar-collapse\" id=\"navbar\">\n",
       "<ul class=\"nav navbar-nav\">\n",
       "<li><a href=\"/news\" title=\"新聞\">新聞</a></li>\n",
       "<li><a href=\"/hot-threads\" title=\"熱門文章\">熱門文章</a></li>\n",
       "<li><a href=\"/hot-boards\" title=\"熱門看板\">熱門看板</a></li>\n",
       "<li><a href=\"/board-list\" title=\"看板列表\">看板列表</a></li>\n",
       "<li><a href=\"/newest-threads\" title=\"最新文章\">最新文章</a></li>\n",
       "<li><a href=\"/contact\" title=\"回報&amp;意見\">回報&amp;意見</a></li>\n",
       "</ul>\n",
       "</div>\n",
       "</div>\n",
       "</nav>\n",
       "<div class=\"container\" id=\"main\">\n",
       "<div class=\"row\">\n",
       "<ol class=\"breadcrumb\">\n",
       "<li><a href=\"/\">PTT Web</a></li>\n",
       "<li><a href=\"/user/\" title=\"全部作者\">全部作者</a></li>\n",
       "<li><a href=\"/user/beerduck/\" title=\"beerduck\">beerduck</a></li>\n",
       "</ol>\n",
       "</div>\n",
       "<div class=\"row row-offcanvas row-offcanvas-right\">\n",
       "<div class=\"col-xs-12 col-sm-12 \">\n",
       "<div class=\"row\" style=\"padding:0;\">\n",
       "</div>\n",
       "<div class=\"row\"></div>\n",
       "<div class=\"row\" style=\"padding:5px;\">\n",
       "<div style=\"margin-top:20px;\">\n",
       "<h3 class=\"block-title\"><a href=\"/user/beerduck/\">beerduck</a>在各看板的發文</h3>\n",
       "<span class=\"post-boards\" style=\"margin-top:15px; margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/\" title=\"beerduck的全部發文\">全部</a>\n",
       "</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/gossiping/\" rel=\"nofollow\" title=\"beerduck在看板gossiping的發文\">gossiping</a> 70</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/boardgame/\" rel=\"nofollow\" title=\"beerduck在看板boardgame的發文\">boardgame</a> 5</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/actuary/\" rel=\"nofollow\" title=\"beerduck在看板actuary的發文\">actuary</a> 4</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/lcd/\" rel=\"nofollow\" title=\"beerduck在看板lcd的發文\">lcd</a> 3</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/palmar_drama/\" rel=\"nofollow\" title=\"beerduck在看板palmar_drama的發文\">palmar_drama</a> 2</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/ukulele/\" rel=\"nofollow\" title=\"beerduck在看板ukulele的發文\">ukulele</a> 2</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/marvel/\" rel=\"nofollow\" title=\"beerduck在看板marvel的發文\">marvel</a> 1</span>\n",
       "<span class=\"post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/distantlove/\" rel=\"nofollow\" title=\"beerduck在看板distantlove的發文\">distantlove</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/ph-sea/\" rel=\"nofollow\" title=\"beerduck在看板ph-sea的發文\">ph-sea</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/home-sale/\" rel=\"nofollow\" title=\"beerduck在看板home-sale的發文\">home-sale</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/lifeismoney/\" rel=\"nofollow\" title=\"beerduck在看板lifeismoney的發文\">lifeismoney</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/japan_travel/\" rel=\"nofollow\" title=\"beerduck在看板japan_travel的發文\">japan_travel</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/wine/\" rel=\"nofollow\" title=\"beerduck在看板wine的發文\">wine</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/dv/\" rel=\"nofollow\" title=\"beerduck在看板dv的發文\">dv</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/helpbuy/\" rel=\"nofollow\" title=\"beerduck在看板helpbuy的發文\">helpbuy</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/id_problem/\" rel=\"nofollow\" title=\"beerduck在看板id_problem的發文\">id_problem</a> 1</span>\n",
       "<span class=\"more-post-boards\" style=\"margin-bottom:7px; padding:5px; border:1px solid #666;\">\n",
       "<a href=\"/user/beerduck/hardware/\" rel=\"nofollow\" title=\"beerduck在看板hardware的發文\">hardware</a> 1</span>\n",
       "<span><a class=\"show-btn\" href=\"#\" onclick=\"toggle_post_boards();\" style=\"text-decoration:underline;\">顯示全部 &gt;</a> <span class=\"more-post-boards-info\" style=\"font-size:16px;\">(還有9個)</span></span>\n",
       "</div>\n",
       "<script>\n",
       "            function toggle_post_boards() {\n",
       "                var txt = $('.show-btn').text();\n",
       "\n",
       "                if (txt == \"顯示全部 >\") {\n",
       "                    $(\".more-post-boards\").css('display', 'inline-block');\n",
       "                    $(\".show-btn\").text(\"顯示部份 <\")\n",
       "                    $(\".more-post-boards-info\").hide();\n",
       "                } else {\n",
       "                    $(\".more-post-boards\").css('display', 'none');\n",
       "                    $(\".show-btn\").text(\"顯示全部 >\")\n",
       "                    $(\".more-post-boards-info\").show();\n",
       "                }\n",
       "            }\n",
       "        </script>\n",
       "<div style=\"margin:40px 0 20px 0;\">\n",
       "<ins class=\"adsbygoogle ad-rwd-r ad-align-left\" data-ad-client=\"ca-pub-7340274291447970\" data-ad-format=\"auto\" data-ad-slot=\"1184153650\" data-full-width-responsive=\"true\" style=\"display:block;\"></ins>\n",
       "<script>\n",
       "     (adsbygoogle = window.adsbygoogle || []).push({});\n",
       "</script>\n",
       "</div>\n",
       "<div class=\"posts\" style=\"margin-top:50px;\">\n",
       "<div class=\"u-title\"><h1 class=\"block-title\"><a href=\"/user/beerduck/\"><b class=\"user-name\">beerduck</b></a>\n",
       "發表的討論串，共\n",
       "97則</h1>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/40lFI\">[爆卦] 夾手藍蝦官方粉絲團成立啦！（近況更新）</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">161留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-10 11:43:43</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3ym81\">[爆卦] 藍蝦自夾手痛苦倒地影片爆紅，外國妹震驚</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">761留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-09 12:26:26</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3wzn6\">[爆卦] 網軍指揮徐弘庭和指傷陳玉珍都關臉書了！</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">139留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-08 16:26:22</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3vJN0\">[爆卦] 佛山爆發森林大火影片曝光</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">95留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-07 16:20:00</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3vEvU\">Re: [問卦] 狗狗在度估 要叫他起來嗎</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">3留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-07 15:33:21</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3uinE\">[新聞] 強推執勤女警 藍委陳宜民函送究辦</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">220留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-07 10:10:00</span>\n",
       "</div>\n",
       "</div>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<ins class=\"adsbygoogle ad-rwd-r\" data-ad-client=\"ca-pub-7340274291447970\" data-ad-format=\"auto\" data-ad-slot=\"1184153650\" data-full-width-responsive=\"true\" style=\"display:block;\"></ins>\n",
       "<script>\n",
       "     (adsbygoogle = window.adsbygoogle || []).push({});\n",
       "</script>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3sGMn\">Re: [新聞] 韓國瑜愛將潘恒旭曾邀楊蕙如當評審　臉弱</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">723留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-06 14:52:54</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3raOP\">[新聞] 愛妻大義滅親！謝和弦吸毒自貼遭警逮上車</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">66留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-06 09:54:30</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3qEw7\">[新聞] 歌手謝和弦吸大麻 警接獲檢舉到場人贓俱</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">60留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-05 22:19:53</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3oSw3\">[新聞] 黃國昌爆2001年空拍照：請問韓國瑜，大洞</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">263留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-04 20:15:11</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3o3Lc\">Re: [新聞] 邱昱凱是網軍頭？柯：網路寫文章都有 但</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">278留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-04 12:39:46</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3lfAq\">[問卦] 有人不知道劍橋分析是一個公司嗎？</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">29留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-02 14:56:57</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3lSia\">[新聞] 為砂石案被告！黃國昌酸韓辦：別按錯鈴「</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">114留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-02 10:52:40</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3koIa\">[爆卦] 聯合報用最不可信澳洲報帶王力強案風向</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">380留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-01 22:32:39</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3khic\">[新聞] 「王立強並不是胡扯！」前國代爆：向心是</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">227留言</span>\n",
       "<span class=\"thread-posttime\">2019-12-01 20:43:55</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3hxFK\">Re: [新聞] 台灣國王獻極衝上台抗議 柯文哲：欠扁</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">321留言</span>\n",
       "<span class=\"thread-posttime\">2019-11-28 13:14:20</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3gkU2\">[爆卦] 香港地鐵站加裝鐵幕！</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">397留言</span>\n",
       "<span class=\"thread-posttime\">2019-11-27 16:50:00</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3gZga\">[新聞] 韓國瑜提「和平協議」需陸放棄武統 國台S</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">697留言</span>\n",
       "<span class=\"thread-posttime\">2019-11-27 14:10:39</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3ejvm\">[爆卦] 港警今天又開始拘捕上班上課的年輕人</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">103留言</span>\n",
       "<span class=\"thread-posttime\">2019-11-26 10:55:32</span>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"thread-item\">\n",
       "<span class=\"thread-title\"><a href=\"/s/3dwE6\">[爆卦] 妖西和蔡壁如同台幫李婉鈺站台</a></span>\n",
       "<div style=\"margin-top:10px;\">\n",
       "<span class=\"thread-list-board\"><a href=\"/gossiping/\" title=\"看板:gossiping\">Gossiping</a></span>\n",
       "<span class=\"thread-list-author\" style=\"display:inline-block; padding-left:5px; padding-right:5px;\">\n",
       "<a href=\"/user/beerduck/\" rel=\"nofollow\" title=\"作者:beerduck\">beerduck</a>\n",
       "</span>\n",
       "<span class=\"thread-popularity\" style=\"padding-left:5px; padding-right:5px;\">503留言</span>\n",
       "<span class=\"thread-posttime\">2019-11-25 21:22:57</span>\n",
       "</div>\n",
       "</div>\n",
       "</div>\n",
       "<div style=\"margin-top:20px;\">以上為批踢踢<a href=\"/user/beerduck/\"><i>beerduck</i></a>\n",
       "發表的討論串，共 97則\n",
       "</div>\n",
       "<div style=\"width:90%; margin:30px auto 0; text-align:center;\">\n",
       "<ul class=\"pagination\">\n",
       "<li class=\"disabled\"><span>«</span></li>\n",
       "<li class=\"active\"><span>1 <span class=\"sr-only\">(current)</span></span></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=2\" rel=\"nofollow\">2</a></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=3\" rel=\"nofollow\">3</a></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=4\" rel=\"nofollow\">4</a></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=5\" rel=\"nofollow\">5</a></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=2\" rel=\"nofollow\" title=\"下一頁\">»</a></li>\n",
       "<li><a href=\"http://pttweb.tw/user/beerduck/?page=5\" rel=\"nofollow\">尾頁</a></li>\n",
       "</ul>\n",
       "<div style=\"clear:both;\"></div>\n",
       "</div>\n",
       "<div style=\"margin:30px auto 0;\">\n",
       "<ins class=\"adsbygoogle ad-rwd-v\" data-ad-client=\"ca-pub-7340274291447970\" data-ad-format=\"auto\" data-ad-slot=\"1184153650\" data-full-width-responsive=\"true\" style=\"display:block;margin:0 auto;\"></ins>\n",
       "<script>\n",
       "     (adsbygoogle = window.adsbygoogle || []).push({});\n",
       "</script>\n",
       "</div>\n",
       "<div style=\"margin-top:20px;\">\n",
       "<a href=\"https://www.google.com.tw/search?q=site:ptt.cc+%22%3Ebeerduck%3C%22\" target=\"_blank\">Google\n",
       "查看更多批踢踢beerduck的留言</a>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"row\"></div>\n",
       "</div>\n",
       "</div>\n",
       "<hr/>\n",
       "<footer>\n",
       "<p>© <a href=\"/\">pttweb.tw</a> 2019\n",
       "</p>\n",
       "<a href=\"/contact\">問題回報&amp;意見反應</a>\n",
       "</footer>\n",
       "</div>\n",
       "<script src=\"https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js\"></script>\n",
       "<script async=\"\" src=\"/static/CACHE/js/ff011ee97f06.js\" type=\"text/javascript\"></script>\n",
       "<script async=\"\" src=\"https://www.googletagmanager.com/gtag/js?id=UA-272347-102\"></script>\n",
       "<script>\n",
       "  window.dataLayer = window.dataLayer || [];\n",
       "  function gtag(){dataLayer.push(arguments);}\n",
       "  gtag('js', new Date());\n",
       "\n",
       "  gtag('config', 'UA-272347-102');\n",
       "</script>\n",
       "<script>var ttt = '0.17093729972839355';</script>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://pttweb.tw/s/31WJ7',\n",
       " 'https://pttweb.twhttps://www.ptt.cc/bbs/tabletennis/M.1373104111.A.70F.html',\n",
       " 'https://pttweb.tw/s/31V6d',\n",
       " 'https://pttweb.tw/s/31V5k',\n",
       " 'https://pttweb.tw/s/31V2r',\n",
       " 'https://pttweb.tw/s/31UVa',\n",
       " 'https://pttweb.tw/s/31TbB']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#'https://pttweb.twhttps://www.ptt.cc/bbs/tabletennis/M.1373104111.A.70F.html'\n",
    "#url_lst[1713:1720]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1714"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#one no reply record\n",
    "#fn_df.to_json(path + '{}_id_check.json'.format(user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "343"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(invalid_rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File b'invalid_20191204.csv' does not exist: b'invalid_20191204.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-00808e6438c7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'invalid_20191204.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    683\u001b[0m         )\n\u001b[0;32m    684\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 685\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    686\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    687\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    455\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    456\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 457\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    458\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    893\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    894\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 895\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    896\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    897\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"c\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"c\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1135\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1136\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1137\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"python\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1915\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"usecols\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1916\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1917\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1918\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1919\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] File b'invalid_20191204.csv' does not exist: b'invalid_20191204.csv'"
     ]
    }
   ],
   "source": [
    "pd.read_csv('invalid_20191204.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yuxds</td>\n",
       "      <td>https://pttweb.tw/s/2qb7l</td>\n",
       "      <td>990F檔案過大！部分文章無法顯示</td>\n",
       "      <td>list index out of range</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>omanorbo</td>\n",
       "      <td>no_url</td>\n",
       "      <td>no_text</td>\n",
       "      <td>no_err</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kieranc</td>\n",
       "      <td>no_url</td>\n",
       "      <td>no_text</td>\n",
       "      <td>no_err</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mark2165</td>\n",
       "      <td>https://pttweb.tw/s/3X1Br</td>\n",
       "      <td>56F→ nnkj: 不是警告而已嗎 1.171.165.24 10/15 19:36\\n</td>\n",
       "      <td>More than one url....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mark2165</td>\n",
       "      <td>https://pttweb.tw/s/3T7eq</td>\n",
       "      <td>1F推 qooqoo2021: 不認識 IP看起來也沒有重複 09/21 16:23\\n</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8566</th>\n",
       "      <td>wizardfizban</td>\n",
       "      <td>https://pttweb.tw/s/1NSTX</td>\n",
       "      <td>57F推 yasakihayate:還以為直接捅了!!? 03/25 14:44\\n</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8567</th>\n",
       "      <td>wizardfizban</td>\n",
       "      <td>https://pttweb.tw/s/1LWFd</td>\n",
       "      <td>17F推 gfdsa:更正，回合制或者是wow like? 03/25 19:59\\n</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8568</th>\n",
       "      <td>wizardfizban</td>\n",
       "      <td>https://pttweb.tw/s/1L9UA</td>\n",
       "      <td>28F推 pulsar1:小時候誰沒偷看過A片 跟長大變性犯罪者還是兩回事 03/18 13...</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8569</th>\n",
       "      <td>wizardfizban</td>\n",
       "      <td>https://pttweb.tw/s/1Kvuq</td>\n",
       "      <td>4F推 supermars:了解~~ 03/16 14:07\\n</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8570</th>\n",
       "      <td>wizardfizban</td>\n",
       "      <td>https://pttweb.tw/s/1KrrG</td>\n",
       "      <td>3F→ supermars:七個技能@[email protected] 03/16 11:...</td>\n",
       "      <td>local variable 'ori_url_split' referenced befo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8571 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0                          1  \\\n",
       "0            yuxds  https://pttweb.tw/s/2qb7l   \n",
       "1         omanorbo                     no_url   \n",
       "2          kieranc                     no_url   \n",
       "3         mark2165  https://pttweb.tw/s/3X1Br   \n",
       "4         mark2165  https://pttweb.tw/s/3T7eq   \n",
       "...            ...                        ...   \n",
       "8566  wizardfizban  https://pttweb.tw/s/1NSTX   \n",
       "8567  wizardfizban  https://pttweb.tw/s/1LWFd   \n",
       "8568  wizardfizban  https://pttweb.tw/s/1L9UA   \n",
       "8569  wizardfizban  https://pttweb.tw/s/1Kvuq   \n",
       "8570  wizardfizban  https://pttweb.tw/s/1KrrG   \n",
       "\n",
       "                                                      2  \\\n",
       "0                                     990F檔案過大！部分文章無法顯示   \n",
       "1                                               no_text   \n",
       "2                                               no_text   \n",
       "3         56F→ nnkj: 不是警告而已嗎 1.171.165.24 10/15 19:36\\n   \n",
       "4          1F推 qooqoo2021: 不認識 IP看起來也沒有重複 09/21 16:23\\n   \n",
       "...                                                 ...   \n",
       "8566         57F推 yasakihayate:還以為直接捅了!!? 03/25 14:44\\n   \n",
       "8567        17F推 gfdsa:更正，回合制或者是wow like? 03/25 19:59\\n   \n",
       "8568  28F推 pulsar1:小時候誰沒偷看過A片 跟長大變性犯罪者還是兩回事 03/18 13...   \n",
       "8569                   4F推 supermars:了解~~ 03/16 14:07\\n   \n",
       "8570  3F→ supermars:七個技能@[email protected] 03/16 11:...   \n",
       "\n",
       "                                                      3  \n",
       "0                               list index out of range  \n",
       "1                                                no_err  \n",
       "2                                                no_err  \n",
       "3                                 More than one url....  \n",
       "4     local variable 'ori_url_split' referenced befo...  \n",
       "...                                                 ...  \n",
       "8566  local variable 'ori_url_split' referenced befo...  \n",
       "8567  local variable 'ori_url_split' referenced befo...  \n",
       "8568  local variable 'ori_url_split' referenced befo...  \n",
       "8569  local variable 'ori_url_split' referenced befo...  \n",
       "8570  local variable 'ori_url_split' referenced befo...  \n",
       "\n",
       "[8571 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#pd.DataFrame(invlid_rec).replace(',', '，').to_csv('invalid_20191204.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# some unknown rec should save by just date and no year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://pttweb.tw/s/10QWn'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hatepolitics\n",
      "<span class=\"f2\">※ 發信站: 批踢踢實業坊(ptt.cc), 來自: 36.236.144.21\n",
      "</span>\n",
      "※ 發信站: 批踢踢實業坊(ptt.cc), 來自: 36.236.144.21\n",
      "\n",
      "**********\n",
      "<span class=\"f2\">※ 文章網址: <a href=\"https://www.ptt.cc/bbs/HatePolitics/M.1555123536.A.947.html\" rel=\"nofollow\" target=\"_blank\">https://www.ptt.cc/bbs/HatePolitics/M.1555123536.A.947.html</a>\n",
      "</span>\n",
      "※ 文章網址: https://www.ptt.cc/bbs/HatePolitics/M.1555123536.A.947.html\n",
      "\n",
      "**********\n",
      "<span class=\"f2\">※ 編輯: geordie (36.236.144.21), 04/13/2019 10:53:07\n",
      "</span>\n",
      "※ 編輯: geordie (36.236.144.21), 04/13/2019 10:53:07\n",
      "\n",
      "**********\n",
      "<span class=\"f2\">※ 編輯: geordie (36.236.144.21), 04/13/2019 10:56:14\n",
      "</span>\n",
      "※ 編輯: geordie (36.236.144.21), 04/13/2019 10:56:14\n",
      "\n",
      "**********\n",
      "['https:', '', 'www.ptt.cc', 'bbs', 'HatePolitics', 'M.1555123536.A.947.html']\n"
     ]
    }
   ],
   "source": [
    "url_cnt = 0\n",
    "\n",
    "#ori\n",
    "#url = 'https://pttweb.tw/s/3n4eR'\n",
    "#<span class=\"f2\">※ 文章網址: <a href=\"https://www.ptt.cc/bbs/C_Chat/M.1575366675.A.20B.html\" rel=\"nofollow\" target=\"_blank\">https://www.ptt.cc/bbs/C_Chat/M.1575366675.A.20B.html</a>\n",
    "#</span>\n",
    "\n",
    "#2017\n",
    "#url = 'https://pttweb.tw/s/2LP5Y'\n",
    "\n",
    "#hist\n",
    "#url = 'https://pttweb.tw/s/10QWn'\n",
    "#url = 'https://pttweb.tw/s/1yXfV'\n",
    "#url = 'https://pttweb.tw/s/1tvdY'\n",
    "#url = 'https://pttweb.tw/s/1caNp'\n",
    "#url = 'https://pttweb.tw/s/2qb7l'\n",
    "#url = 'https://pttweb.tw/s/31n9Q'\n",
    "\n",
    "url = 'https://pttweb.tw/s/3JMji'\n",
    "\n",
    "rs = requests.get(url)\n",
    "soup = BeautifulSoup(rs.text)\n",
    "\n",
    "ori_url_split = []\n",
    "print(soup.select('.board')[0].text.strip())\n",
    "\n",
    "ori_url_split = ''\n",
    "ip = ''\n",
    "\n",
    "for x in soup.select('span.f2'):\n",
    "    print(x)\n",
    "    print(x.text)\n",
    "    print('*'*10)\n",
    "    \n",
    "    if x.text.startswith('※ 文章網址: https://www.ptt.cc/bbs'):\n",
    "        ori_url_split = x.a['href'].split('/')\n",
    "        ip = x.previous_element.split(':')[-1].split()[0]\n",
    "        url_cnt = url_cnt + 1    \n",
    "    elif x.text.startswith('※ 文章網址:'):\n",
    "        pre_ip = [x for x in x.previous_elements][1]\n",
    "        pre_url = x.next_sibling\n",
    "        \n",
    "        if type(pre_url) is bs4.element.NavigableString:\n",
    "            #old article\n",
    "            ori_url_split = x.a['href'].split('/')\n",
    "        else:\n",
    "            ori_url_split = x.next_sibling.text.split('/')\n",
    "        \n",
    "        if type(pre_ip) is bs4.element.NavigableString:\n",
    "            ip = pre_ip.split(':')[-1].split()[0]\n",
    "        else:\n",
    "            #old article\n",
    "            ip = pre_ip.text.split(':')[-1].split()[0]\n",
    "            \n",
    "            \n",
    "        url_cnt = url_cnt + 1\n",
    "    if url_cnt > 1:\n",
    "        raise ValueError('More than one url....')\n",
    "        \n",
    "print(ori_url_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x.next_sibling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.element.Tag"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type([x for x in x.previous_elements][1]) \n",
    "#is bs4.element.NavigableString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('.thread-posttime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://pttweb.tw/s/3n4eR'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#soup.findAll('a', {'rel':'nofollow'}, string='尾頁')#[0]['href']#.split('?page=')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# url parsing full test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#url_lst = ['https://pttweb.tw/s/3XsPy']\n",
    "\n",
    "#\n",
    "#url_lst = ['https://pttweb.tw/s/3JMji']\n",
    "\n",
    "invalid_rec = []\n",
    "user = 'testing'\n",
    "rs_lst = ['beer_duck']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "art parsing error\n",
      "0\n",
      "list index out of range\n",
      "https://pttweb.tw/s/3JMji\n"
     ]
    }
   ],
   "source": [
    "for idx, url in enumerate(url_lst):\n",
    "    messages = []\n",
    "    remove_tags = [[True, {'class':['push', 'f2', 'f6']}], ['script']]\n",
    "    rs = requests.get(url)\n",
    "    soup = BeautifulSoup(rs.text)\n",
    "\n",
    "    #reply content\n",
    "    for x in soup.select('.push'):\n",
    "        try:\n",
    "            messages.append({\n",
    "                  'push_tag' : x.select('.push-tag')[0].text[0]\n",
    "                , 'push_userid' : x.select('.push-userid')[0].text\n",
    "                , 'push_content' : x.select('.push-content')[0].text[2:]\n",
    "                , 'push_ipdatetime' : x.select('.push-ipdatetime')[0].text.strip()\n",
    "            })\n",
    "        except Exception as e:\n",
    "            invalid_rec.append([user, url, x.text, e])\n",
    "            print('message parsing error')\n",
    "            print(idx)\n",
    "            print(url)\n",
    "            print(x.text)\n",
    "\n",
    "    try:\n",
    "        art = content_parser(soup)        \n",
    "        if art == 'NoContent':\n",
    "            continue    \n",
    "        message_aggr = message_cnt(soup)\n",
    "        art.update({'message_count':message_aggr, 'messages':messages, 'author':user})\n",
    "        temp_df = pd.DataFrame([art])\n",
    "        rs_lst.append(temp_df)\n",
    "    except Exception as e:\n",
    "        print('art parsing error')\n",
    "        invalid_rec.append([user, url, x.text, e])\n",
    "        print(idx)\n",
    "        print(e)\n",
    "        print(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('.post-time')\n",
    "#.text.strip()\n",
    "#art = content_parser(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1209"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.concat(rs_lst).messages.values[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fFPTO-NYhYNo"
   },
   "source": [
    "# get personal fully publish history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "wNV_tpZyUz2c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "36\n",
      "More than one url....\n",
      "https://pttweb.tw/s/3X1Br\n",
      "51\n",
      "local variable 'ori_url_split' referenced before assignment\n",
      "https://pttweb.tw/s/3T7eq\n",
      "52\n",
      "More than one url....\n",
      "https://pttweb.tw/s/3T7ep\n",
      "62\n",
      "local variable 'ori_url_split' referenced before assignment\n",
      "https://pttweb.tw/s/3SNRN\n",
      "91\n",
      "local variable 'ori_url_split' referenced before assignment\n",
      "https://pttweb.tw/s/3Op3R\n",
      "101\n",
      "121\n",
      "local variable 'ori_url_split' referenced before assignment\n",
      "https://pttweb.tw/s/3M5jI\n",
      "201\n",
      "301\n",
      "401\n",
      "501\n",
      "601\n",
      "701\n",
      "801\n",
      "901\n",
      "1001\n"
     ]
    }
   ],
   "source": [
    "# for idx, url in enumerate(url_lst):\n",
    "#     messages = []\n",
    "#     remove_tags = [[True, {'class':['push', 'f2', 'f6']}], ['script']]\n",
    "#     rs = requests.get(url)\n",
    "#     soup = BeautifulSoup(rs.text)\n",
    "\n",
    "#     #reply content\n",
    "#     for x in soup.select('.push'):\n",
    "#         try:\n",
    "#             messages.append({\n",
    "#                   'push_tag' : x.select('.push-tag')[0].text[0]\n",
    "#                 , 'push_userid' : x.select('.push-userid')[0].text\n",
    "#                 , 'push_content' : x.select('.push-content')[0].text[2:]\n",
    "#                 , 'push_ipdatetime' : x.select('.push-ipdatetime')[0].text.strip()\n",
    "#             })\n",
    "#         except:\n",
    "#             print(idx)\n",
    "#             print(url)\n",
    "#             print(x.text)\n",
    "    \n",
    "#     try:\n",
    "#         art = content_parser(soup)        \n",
    "#         if art == 'NoContent':\n",
    "#             continue    \n",
    "#         message_aggr = message_cnt(soup)\n",
    "#         art.update({'message_count':message_aggr, 'messages':messages, 'author':user})\n",
    "#         temp_df = pd.DataFrame([art])\n",
    "#         rs_lst.append(temp_df)\n",
    "#     except Exception as e:\n",
    "#         print(idx)\n",
    "#         print(e)\n",
    "#         print(url)  \n",
    "#     if idx % 300 == 1:\n",
    "#         print(idx)\n",
    "#         print(datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test re\n",
    "#v url = 'https://pttweb.tw/s/3a6AF' #re with ori\n",
    "#v url = 'https://pttweb.tw/s/3XuNB' #re without ori\n",
    "#v url = 'https://pttweb.tw/s/3aYJ9' #colorful content\n",
    "#v url = 'https://pttweb.tw/s/3Yk0l' #simple news\n",
    "#url = 'https://pttweb.tw/s/3Yjaj' #forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 131
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ankzQ_1_VEcr",
    "outputId": "a08289f2-6186-4bbe-fcfd-9b3cb6f5f530"
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-35-ad7bdc39d36a>\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m    \"article_id\": 文章 ID,\u001b[0m\n\u001b[0m                      ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "#M.1119222611.A.7A9\n",
    "# {\n",
    "#     \"article_id\": 文章 ID,\n",
    "#     \"article_title\": 文章標題 ,\n",
    "#     \"author\": 作者,\n",
    "#     \"board\": 板名,\n",
    "#     \"content\": 文章內容,\n",
    "#     \"date\": 發文時間,\n",
    "#     \"ip\": 發文位址,\n",
    "#     \"message_count\": { # 推文\n",
    "#         \"all\": 總數,\n",
    "#         \"boo\": 噓文數,\n",
    "#         \"count\": 推文數-噓文數,\n",
    "#         \"neutral\": → 數,\n",
    "#         \"push\": 推文數\n",
    "#     },\n",
    "#     \"messages\": [ # 推文內容\n",
    "#       {\n",
    "#         \"push_content\": 推文內容,\n",
    "#         \"push_ipdatetime\": 推文時間及位址,\n",
    "#         \"push_tag\": 推/噓/→ ,\n",
    "#         \"push_userid\": 推文者 ID\n",
    "#       },\n",
    "#       ...\n",
    "#       ]\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "n71vo6p6v8tY"
   },
   "outputs": [],
   "source": [
    "#轉流量\n",
    "#https://www.ptt.cc/bbs/Gossiping/M.1574431993.A.6DE.html\n",
    "#certain media's forward\n",
    "#https://pttweb.tw/s/3a6AF\n",
    "#ws([t1.text])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "ptt_id_check.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}