step6_article_sentiment_check.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "step5_article_sentiment_check.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "-407XNVD3qa_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import gc\n",
        "import os\n",
        "import time\n",
        "from datetime import datetime"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFNVYce-joEL",
        "colab_type": "code",
        "outputId": "761a2bb2-d2a9-4764-c683-76095d1ddb88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "try:\n",
        "    from ckiptagger import data_utils, construct_dictionary, WS, POS, NER\n",
        "except:\n",
        "    !pip install -U ckiptagger[tfgpu,gdown]\n",
        "    from ckiptagger import data_utils, construct_dictionary, WS, POS, NER"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "628wPcN7stQw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#set GPU accelerate\n",
        "#Runtime > Change runtime type > Hardward accelerator > GPU"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBMNGfwbnlu2",
        "colab_type": "code",
        "outputId": "7fff8786-7b4b-45ac-c652-fa1a585fea60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "drive.mount('/content/drive/', force_remount=True)\n",
        "exec_dt = datetime.strftime(datetime.now(), '%Y%m%d_%H%M%S')\n",
        "path = '/content/drive/My Drive/side_project/20191109_ptt_analysis/'\n",
        "neg_df = pd.read_table(path + 'ntusd-negative.txt', names=['word'])\n",
        "pos_df = pd.read_table(path + 'ntusd-positive.txt', names=['word'])"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O04PB3O13sk8",
        "colab_type": "code",
        "outputId": "b6b17d14-fec4-4fc1-9c98-8a381b230de8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#todo: change hatepolitics as example\n",
        "for x in os.listdir(path):\n",
        "    if 'pol_check' in x:\n",
        "        print(x)\n",
        "        pol_df = pd.read_csv(path + x)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Gossiping-37700-39294_1207_article_20191229_192300_pol_check_20191231_003120.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d_ppWLX13ttT",
        "colab_type": "code",
        "outputId": "a21d46aa-d750-4617-f412-00b4f6aa8464",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "data_utils.download_data_gdown(\"./\")\n",
        "\n",
        "#CUDA_VISIBLE_DEVICES environment variable, e.g.\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
        "#disable_cuda=False, e.g. ws = WS(\"./data\", disable_cuda=False)\n",
        "\n",
        "ws = WS(\"./data\", disable_cuda=False)\n",
        "#pos = POS(\"./data\", disable_cuda=False)\n",
        "#ner = NER(\"./data\", disable_cuda=False)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1efHsY16pxK0lBD2gYCgCTnv1Swstq771\n",
            "To: /content/data.zip\n",
            "1.88GB [00:14, 133MB/s] \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dp7sMJ4intNH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#ws(['我很不開心', '不快樂'])\n",
        "#[['我', '很', '不', '開心'], ['不', '快樂']]\n",
        "\n",
        "#todo: \n",
        "#1. check segement & non-segment percision by identified abnormal user\n",
        "#2. strip counted string for calculating sentiment words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EOFqUt5e7fqm",
        "colab_type": "text"
      },
      "source": [
        "# get neo4j group"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hBEFzDBp-H3Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#currently community is in hatepolitics\n",
        "gp_df = pd.read_csv(path + 'neo4j_trns_20191231_135302.csv')\n",
        "gp_df.pagerank = gp_df.pagerank.astype(float)\n",
        "gp_df['rnk'] = gp_df.groupby(['community'])['pagerank'].rank(method = 'dense', ascending=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqrmDJWSgxFE",
        "colab_type": "text"
      },
      "source": [
        "# select part of DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xf99vVoPHVW1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_df['corpus'] = pol_df.article_title + ' ' + pol_df.content"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hvn0Mdj9HMcn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#slc_df = art_df[art_df.author.isin(set(cand_usr))].copy()\n",
        "start_idx = 1\n",
        "end_idx = 20000\n",
        "slc_id = set(gp_df[gp_df.rnk.between(start_idx, end_idx)].id)\n",
        "\n",
        "cond_1 = pol_df.author.isin(slc_id)\n",
        "cond_2 = pol_df.party != 'undef'\n",
        "\n",
        "slc_df = pol_df[cond_1 & cond_2].copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T7ZG_lOyiXZe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "370abb95-e5eb-492e-e745-55abd470892c"
      },
      "source": [
        "slc_df.shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(145, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6qtmKL1hXkq4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#selected \n",
        "#slc_idx = 200\n",
        "#slc_df = pol_df[pol_df.party != 'undef'][:slc_idx]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGC1oIsYG_gP",
        "colab_type": "code",
        "outputId": "45932fff-5179-41e2-fcfb-335f39544427",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "#if memory insufficient, try to delete and garbage collect => run 10% data\n",
        "print(time.ctime())\n",
        "print(pol_df.shape)\n",
        "cont_ws = ws(slc_df.corpus)\n",
        "print(time.ctime())"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Jan  1 01:48:32 2020\n",
            "(31206, 18)\n",
            "Wed Jan  1 01:49:45 2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ecfscwYuofbX",
        "colab_type": "code",
        "outputId": "942e0576-6937-4dec-953e-03db38aff48a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sent_dict = { \n",
        "    'neg' : neg_df.word.to_list()\n",
        "    , 'pos' : pos_df.word.to_list()\n",
        "}\n",
        "\n",
        "rs_lst = []\n",
        "\n",
        "for idx, ws_item in enumerate(cont_ws):\n",
        "    row = []\n",
        "    row.append(ws_item)\n",
        "    for sent in sent_dict:\n",
        "        cnt = 0\n",
        "        for word in sent_dict[sent]:\n",
        "            cnt = cnt + ws_item.count(word)\n",
        "        row.append(cnt)\n",
        "    rs_lst.append(row)\n",
        "    if idx % 500 == 1:\n",
        "        print(idx)\n",
        "        \n",
        "sent_df = pd.DataFrame(rs_lst, columns=['ws_lst', 'neg', 'pos'])\n",
        "sent_df['senti_sum'] = sent_df.pos - sent_df.neg"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NdWMq-iungvU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "slc_df.reset_index(inplace=True, drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xk4akqdenkbb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_sent_df = pd.concat([slc_df, sent_df], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FQEPjZkm3BlV",
        "colab_type": "text"
      },
      "source": [
        "# personal political spectrum (this is step6)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LydonR4Bu-6W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_sent_df.loc[pol_sent_df.senti_sum > 0, 'senti_flag'] = 'pos' \n",
        "pol_sent_df.loc[pol_sent_df.senti_sum < 0, 'senti_flag'] = 'neg' "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzBTBHF8pTuM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_dict = {\n",
        " 'deep_dpp_kw' : 'dpp',\n",
        " 'light_dpp_kw' : 'dpp',\n",
        " 'tsai_kw' : 'dpp',\n",
        " 'han_kw' : 'kmt',\n",
        " 'light_kmt_kw' : 'kmt',\n",
        " 'deep_kmt_kw' : 'kmt',\n",
        " 'third_party_kw' : '3rd',\n",
        " 'unrelated_kw' : 'undef',\n",
        " 'hk_protest' : 'event_hk',\n",
        " 'china_spy' : 'event_chn'\n",
        " }\n",
        "\n",
        "pol_sent_df['party_aggr'] = pol_sent_df.party.replace(pol_dict)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJSMgtjBrGjb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cond_kmt = pol_sent_df.party_aggr == 'kmt'\n",
        "cond_dpp = pol_sent_df.party_aggr == 'dpp'\n",
        "cond_3rd = pol_sent_df.party_aggr == '3rd'\n",
        "\n",
        "cond_pos = pol_sent_df.senti_flag == 'pos'\n",
        "cond_neg = pol_sent_df.senti_flag == 'neg'\n",
        "\n",
        "pol_sent_df.loc[cond_kmt & cond_pos, 'party_aggr_sent'] = 'kmt'\n",
        "pol_sent_df.loc[cond_dpp & cond_neg, 'party_aggr_sent'] = 'kmt'\n",
        "\n",
        "pol_sent_df.loc[cond_dpp & cond_pos, 'party_aggr_sent'] = 'dpp'\n",
        "pol_sent_df.loc[cond_kmt & cond_neg, 'party_aggr_sent'] = 'dpp'\n",
        "\n",
        "pol_sent_df.loc[cond_3rd & cond_pos, 'party_aggr_sent'] = '3rd_pos'\n",
        "pol_sent_df.loc[cond_3rd & cond_neg, 'party_aggr_sent'] = '3rd_neg'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rg-o3K4dfe5O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_sent_aggr_df = pol_sent_df.groupby(['author', 'party_aggr_sent'], as_index=False)\\\n",
        "                   .agg({'article_id':'count'}).rename(columns={'article_id':'cnt'})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWvDj6Sq-H6E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pol_sent_aggr_df['rnk'] = pol_sent_aggr_df.groupby(['author'])['cnt'].rank(method = 'dense', ascending=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZX7AsfdVVXQY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cond_1 = pol_sent_aggr_df.rnk == 1\n",
        "cond_2 = pol_sent_aggr_df.cnt != 1\n",
        "pol_sent_aggr_df[cond_1 & cond_2].to_csv(path + 'pol_sent_aggr_{}.csv'.format(exec_dt))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}