{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Geographical pronunciation statistics" ] }, { "cell_type": "code", "execution_count": 128, "metadata": {}, "outputs": [], "source": [ "import pandas\n", "import MySQLdb\n", "import numpy\n", "import json\n", "\n", "db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n", "\n", "%matplotlib notebook\n", "from matplotlib import pyplot\n", "import folium\n", "from IPython.display import display\n", "from shapely.geometry import Polygon, MultiPolygon, shape, Point\n", "from jsbutton import JsButton\n", "from shapely.geometry import LineString, MultiLineString\n", "from jupyter_progressbar import ProgressBar\n", "from collections import defaultdict, Counter\n", "from ipy_table import make_table\n", "from html import escape\n", "\n", "import numpy as np\n", "from random import shuffle\n", "import pickle\n", "from jupyter_progressbar import ProgressBar" ] }, { "cell_type": "code", "execution_count": 129, "metadata": {}, "outputs": [], "source": [ "with open('friesland_wijken.p3', 'rb') as f:\n", " wijken, wijk_shapes = pickle.load(f)\n", "\n", "wijk_names = [wijk['properties']['GM_NAAM'] + ', ' + wijk['properties'].get('WK_NAAM', '') for wijk in wijken['features']]\n", "\n", "def get_wijk(point):\n", " for i, shape in enumerate(wijk_shapes):\n", " if shape.contains(point):\n", " return i\n", " return -1" ] }, { "cell_type": "code", "execution_count": 130, "metadata": {}, "outputs": [], "source": [ "def listify(rd_multipolygon):\n", " if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):\n", " return list(rd_multipolygon)\n", " return [\n", " listify(element)\n", " for element in rd_multipolygon\n", " ]" ] }, { "cell_type": "code", "execution_count": 131, "metadata": {}, "outputs": [], "source": [ "# Answers to how participants state a word should be pronounces.\n", "\n", "answers = pandas.read_sql('''\n", "SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n", "FROM core_surveyresult as survey\n", "INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n", "INNER JOIN core_predictionquizresultquestionanswer as answer\n", " ON result.id = answer.prediction_quiz_id\n", "''', db)" ] }, { "cell_type": "code", "execution_count": 132, "metadata": {}, "outputs": [], "source": [ "zero_latlng_questions = {\n", " q\n", " for q, row in answers.groupby('question_text').agg('std').iterrows()\n", " if row['user_lat'] == 0 and row['user_lng'] == 0\n", "}\n", "answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]" ] }, { "cell_type": "code", "execution_count": 133, "metadata": {}, "outputs": [], "source": [ "def reverse(rd_multipolygon):\n", " if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):\n", " return rd_multipolygon[::-1]\n", " return [\n", " reverse(element)\n", " for element in rd_multipolygon\n", " ]" ] }, { "cell_type": "code", "execution_count": 134, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " # Remove the CWD from sys.path while we load stuff.\n" ] } ], "source": [ "# Takes approximately 2 minutes\n", "points = set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))\n", "\n", "wijk_map = dict()\n", "for lng, lat in points:\n", " wijk_map[(lng, lat)] = get_wijk(Point(lng, lat))\n", "\n", "answers_filtered['wijk'] = [\n", " wijk_map[(lng, lat)]\n", " for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])\n", "]" ] }, { "cell_type": "code", "execution_count": 135, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " \n" ] } ], "source": [ "answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n", " lambda x: x.replace('\"', '').replace('*', ''))" ] }, { "cell_type": "code", "execution_count": 137, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ea89078b81da4daba82bcd4b1ddbe8c2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='0s passed', placeholder='0…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "cmap = pyplot.get_cmap('YlOrRd')\n", "\n", "for question, rows in ProgressBar(\n", " answers_filtered.groupby('question_text_url'),\n", " size=len(answers_filtered['question_text_url'].unique())\n", "):\n", " m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)\n", " order = [a for _, a in sorted((\n", " (r['user_lat'], answer)\n", " for answer, r in rows.groupby('answer_text').count().iterrows()\n", " ), reverse=True)]\n", " wijk_normalizer = {\n", " wijk: r['user_lat']\n", " for wijk, r in rows.groupby('wijk').count().iterrows()\n", " }\n", " for answer_text in (order):\n", " rows_ = rows[rows['answer_text'] == answer_text]\n", " if (rows_['wijk'] >= 0).sum() <= 0:\n", " continue\n", "\n", " spread = {\n", " wijk: r['user_lat']\n", " for wijk, r in rows_.groupby('wijk').count().iterrows()\n", " if wijk >= 0\n", " }\n", " n_answers = sum(spread.values())\n", " \n", " name = '{} ({})'.format(answer_text, n_answers)\n", " group = folium.FeatureGroup(name=name, overlay=False)\n", " folium.TileLayer(tiles='stamentoner').add_to(group)\n", " \n", " max_value = max(value / wijk_normalizer[wijk] for wijk, value in spread.items())\n", " \n", " for wijk, wijk_name in enumerate(wijk_names):\n", " coordinates = reverse(wijken['features'][wijk]['geometry']['coordinates'])\n", " if wijk in spread:\n", " value = spread[wijk]\n", " percentage = value / wijk_normalizer[wijk]\n", " color_value = percentage / max_value\n", " color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])\n", " \n", " polygon = folium.Polygon(\n", " coordinates, fill_color=color, fill_opacity=0.8,\n", " color='#555555', popup='{} ({}, {: 3d}%)'.format(wijk_name, value, int(100*percentage))\n", " \n", " )\n", " centroid = wijk_shapes[wijk].centroid\n", " centroid = (centroid.y, centroid.x)\n", " folium.map.Marker(\n", " [wijk_shapes[wijk].centroid.y, wijk_shapes[wijk].centroid.x],\n", " icon=folium.DivIcon(\n", " icon_size=(30, 16),\n", " icon_anchor=(15, 8),\n", " html='
{:d}%
'.format(int(100 * percentage)),\n", " )\n", " ).add_to(group)\n", " else:\n", " polygon = folium.Polygon(coordinates, fill_color=None, fill_opacity=0, color='#555555')\n", " polygon.add_to(group)\n", " group.add_to(m)\n", " JsButton(\n", " title='',\n", " function=\"\"\"\n", " function(btn, map){\n", " $('.percentage-label').toggle();\n", " }\n", " \"\"\"\n", " ).add_to(m)\n", " folium.map.LayerControl('topright', collapsed=False).add_to(m)\n", "# display(m)\n", " m.save('maps/heatmaps-wijk/{}.html'.format(question))\n", "# break" ] }, { "cell_type": "code", "execution_count": 138, "metadata": {}, "outputs": [], "source": [ "import glob\n", "with open('maps/heatmaps-wijk/index.html', 'w') as f:\n", " f.write('' + \n", " '
\\n'.join(\n", " '\\t{}'.format(fn, fn[:-5].replace('_', ' '))\n", " for fn in sorted(\n", " glob.glob('maps/heatmaps-wijk/*.html')\n", " )\n", " for fn in [fn[len('maps/heatmaps-wijk/'):]]\n", " ) + \"\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }