stimmenfryslan/notebooks/Gabmap Pronunciation Tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geographical pronunciation tables\n",
    "\n",
    "Creates gabmap files with region centroids, percentages and pronunciations for wijken in Friesland."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "\n",
    "import pandas\n",
    "import MySQLdb\n",
    "import json\n",
    "import copy\n",
    "\n",
    "db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n",
    "\n",
    "from shapely.geometry import shape, Point\n",
    "\n",
    "from gabmap import create_gabmap_dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/Friesland_wijken.geojson') as f:\n",
    "    regions = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Answers to how participants state a word should be pronounced\n",
    "\n",
    "answers = pandas.read_sql('''\n",
    "SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n",
    "FROM       core_surveyresult as survey\n",
    "INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n",
    "INNER JOIN core_predictionquizresultquestionanswer as answer\n",
    "    ON result.id = answer.prediction_quiz_id\n",
    "''', db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "zero_latlng_questions = {\n",
    "    q\n",
    "    for q, row in answers.groupby('question_text').agg('std').iterrows()\n",
    "    if row['user_lat'] == 0 and row['user_lng'] == 0\n",
    "}\n",
    "answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['gegaan', 'avond', 'heel', 'dag', 'bij (insect)', 'sprak (toe)',\n",
       "       'oog', 'armen (lichaamsdeel)', 'kaas', 'deurtje', 'koken',\n",
       "       'borst (lichaamsdeel)', 'vis', 'zaterdag', 'trein', 'geel', 'tand',\n",
       "       'gezet', 'blad (aan een boom)'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers_filtered['question_text'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_filtered['question_text'] = answers_filtered['question_text'].map(\n",
    "    lambda x: x.replace('\"', '').replace('*', ''))\n",
    "\n",
    "answers_filtered['answer_text'] = answers_filtered['answer_text'].map(\n",
    "    lambda x: x[x.find('('):x.find(')')][1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "centroids, pronunciations, counts = create_gabmap_dataframes(\n",
    "    regions, answers_filtered,\n",
    "    latitude_column='user_lat', longitude_column='user_lng',\n",
    "    word_column='question_text', pronunciation_column='answer_text',\n",
    "    region_name_property='gemeente_en_wijk_naam'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pronunciations.to_csv('../data/Friesland_wijken_pronunciations.gabmap.tsv', sep='\\t')\n",
    "counts.to_csv('../data/Friesland_wijken_pronunciation_percentages.gabmap.tsv', sep='\\t')\n",
    "centroids.to_csv('../data/Friesland_wijken_centroids.gabmap.tsv', sep='\\t', columns=['longitude', 'latitude'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}