{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Geographical pronunciation statistics" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "import pandas\n", "import MySQLdb\n", "import numpy\n", "import json\n", "\n", "db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n", "\n", "%matplotlib notebook\n", "from matplotlib import pyplot\n", "import folium\n", "from IPython.display import display\n", "from shapely.geometry import Polygon, MultiPolygon, shape, Point\n", "from jsbutton import JsButton\n", "from jupyter_progressbar import ProgressBar\n", "from collections import defaultdict\n", "from ipy_table import make_table\n", "from html import escape\n", "\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from matplotlib.colors import LogNorm\n", "from sklearn import mixture\n", "from skimage.measure import find_contours\n", "from collections import Counter\n", "from random import shuffle" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# Borders of Frysian municipalities\n", "\n", "with open('Friesland_AL8.GeoJson') as f:\n", " gemeentes = json.load(f)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "coords = [feature['geometry'] for feature in gemeentes['features']]\n", "coords_folium = [[[[c__[::-1] for c__ in c_] for c_ in c] for c in coords_['coordinates']] for coords_ in coords]\n", "shapes = [shape(coords_) for coords_ in coords]\n", "gemeente_names = [feature['properties']['name'] for feature in gemeentes['features']]\n", "\n", "def get_gemeente(point):\n", " for i, shape in enumerate(shapes):\n", " if shape.contains(point):\n", " return i\n", " return -1" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# Answers to how participants state a word should be pronounces.\n", "\n", "answers = pandas.read_sql('''\n", "SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n", "FROM core_surveyresult as survey\n", "INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n", "INNER JOIN core_predictionquizresultquestionanswer as answer\n", " ON result.id = answer.prediction_quiz_id\n", "''', db)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "zero_latlng_questions = {\n", " q\n", " for q, row in answers.groupby('question_text').agg('std').iterrows()\n", " if row['user_lat'] == 0 and row['user_lng'] == 0\n", "}\n", "answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " # Remove the CWD from sys.path while we load stuff.\n" ] } ], "source": [ "# Takes approximately 2 minutes\n", "\n", "gemeente_map = {\n", " (lng, lat): get_gemeente(Point(lng, lat))\n", " for lng, lat in set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))\n", "}\n", "\n", "answers_filtered['gemeente'] = [\n", " gemeente_map[(lng, lat)]\n", " for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])\n", "]" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n", " \n" ] } ], "source": [ "answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n", " lambda x: x.replace('\"', '').replace('*', ''))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e48eb24f5c43434bad4241d4bea53074", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='0s passed', placeholder='0…" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "cmap = pyplot.get_cmap('YlOrRd')\n", "\n", "for question, rows in ProgressBar(\n", " answers_filtered.groupby('question_text_url'),\n", " size=len(answers_filtered['question_text_url'].unique())\n", "):\n", " m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)\n", " pecentage_labels = folium.FeatureGroup(name='pecentages', overlay=True)\n", " order = [a for _, a in sorted((\n", " (r['user_lat'], answer)\n", " for answer, r in rows.groupby('answer_text').count().iterrows()\n", " ), reverse=True)]\n", " gemeente_normalizer = {\n", " gemeente: r['user_lat']\n", " for gemeente, r in rows.groupby('gemeente').count().iterrows()\n", " }\n", " for answer_text in order:\n", " rows_ = rows[rows['answer_text'] == answer_text]\n", " if (rows_['gemeente'] >= 0).sum() <= 0:\n", " continue\n", "\n", " spread = {\n", " gemeente: r['user_lat']\n", " for gemeente, r in rows_.groupby('gemeente').count().iterrows()\n", " if gemeente >= 0\n", " }\n", " n_answers = sum(spread.values())\n", " \n", " name = '{} ({})'.format(answer_text, n_answers)\n", " group = folium.FeatureGroup(name=name, overlay=False)\n", " folium.TileLayer(tiles='stamentoner').add_to(group)\n", " \n", " max_value = max(value / gemeente_normalizer[gemeente] for gemeente, value in spread.items())\n", " for gemeente, gemeente_name in enumerate(gemeente_names):\n", " if gemeente in spread:\n", " value = spread[gemeente]\n", " percentage = value / gemeente_normalizer[gemeente]\n", " color_value = percentage / max_value\n", " color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])\n", " \n", " polygon = folium.Polygon(coords_folium[gemeente], fill_color=color, fill_opacity=0.8,\n", " color='#555555', popup='{} ({}, {}%)'.format(gemeente_name, value, round(100*percentage)))\n", " centroid = shapes[gemeente].centroid\n", " centroid = (centroid.y, centroid.x)\n", "# folium.Circle(centroid, color='green', radius=200).add_to(group)\n", " folium.map.Marker(\n", " [shapes[gemeente].centroid.y, shapes[gemeente].centroid.x],\n", " icon=folium.DivIcon(\n", " icon_size=(50, 24),\n", " icon_anchor=(25, 12),\n", " html='
{:d}%
'.format(int(100 * percentage)),\n", " )\n", " ).add_to(group)\n", " else:\n", " polygon = folium.Polygon(coords_folium[gemeente], fill_color=None, fill_opacity=0, color='#555555')\n", " polygon.add_to(group)\n", " group.add_to(m)\n", " pecentage_labels.add_to(m)\n", " folium.map.LayerControl('topright', collapsed=False).add_to(m)\n", " JsButton(\n", " title='',\n", " function=\"\"\"\n", " function(btn, map){\n", " $('.percentage-label').toggle();\n", " }\n", " \"\"\"\n", " ).add_to(m)\n", "# display(m)\n", " m.save('maps/heatmaps/{}.html'.format(question))\n", "# break" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import glob\n", "with open('maps/heatmaps/index.html', 'w') as f:\n", " f.write('' + \n", " '
\\n'.join(\n", " '\\t{}'.format(fn[5:], fn[14:-5].replace('_', ' '))\n", " for fn in sorted(\n", " glob.glob('maps/heatmaps/*.html')\n", " )\n", " ) + \"\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }