{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Geographical pronunciation statistics"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"import pandas\n",
"import MySQLdb\n",
"import numpy\n",
"import json\n",
"\n",
"db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n",
"\n",
"%matplotlib notebook\n",
"from matplotlib import pyplot\n",
"import folium\n",
"from IPython.display import display\n",
"from shapely.geometry import Polygon, MultiPolygon, shape, Point, box, mapping\n",
"from jupyter_progressbar import ProgressBar\n",
"from collections import defaultdict\n",
"from ipy_table import make_table\n",
"from html import escape\n",
"\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib.colors import LogNorm\n",
"from sklearn import mixture\n",
"from skimage.measure import find_contours\n",
"from collections import Counter\n",
"from random import shuffle"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# Borders of Frysian municipalities\n",
"\n",
"with open('Friesland_AL8.GeoJson') as f:\n",
" gemeentes = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"coords = [feature['geometry'] for feature in gemeentes['features']]\n",
"coords_folium = [[[[c__[::-1] for c__ in c_] for c_ in c] for c in coords_['coordinates']] for coords_ in coords]\n",
"shapes = [shape(coords_).simplify(tolerance=0.001) for coords_ in coords]\n",
"gemeente_names = [feature['properties']['name'] for feature in gemeentes['features']]\n",
"\n",
"def get_gemeente(point):\n",
" for i, shape in enumerate(shapes):\n",
" if shape.contains(point):\n",
" return i\n",
" return -1"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"# Answers to how participants state a word should be pronounces.\n",
"\n",
"answers = pandas.read_sql('''\n",
"SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n",
"FROM core_surveyresult as survey\n",
"INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n",
"INNER JOIN core_predictionquizresultquestionanswer as answer\n",
" ON result.id = answer.prediction_quiz_id\n",
"''', db)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"zero_latlng_questions = {\n",
" q\n",
" for q, row in answers.groupby('question_text').agg('std').iterrows()\n",
" if row['user_lat'] == 0 and row['user_lng'] == 0\n",
"}\n",
"answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" # Remove the CWD from sys.path while we load stuff.\n"
]
}
],
"source": [
"# Takes approximately 2 minutes\n",
"\n",
"gemeente_map = {\n",
" (lng, lat): get_gemeente(Point(lng, lat))\n",
" for lng, lat in set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))\n",
"}\n",
"\n",
"answers_filtered['gemeente'] = [\n",
" gemeente_map[(lng, lat)]\n",
" for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" \n"
]
}
],
"source": [
"answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n",
" lambda x: x.replace('\"', '').replace('*', ''))"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [],
"source": [
"def reverse(rd_multipolygon):\n",
" if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):\n",
" return rd_multipolygon[::-1]\n",
" return [\n",
" reverse(element)\n",
" for element in rd_multipolygon\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
"def get_palette(n, no_black=True, no_white=True):\n",
" with open('glasbey/{}_colors.txt'.format(n + no_black + no_white)) as f:\n",
" return [\n",
" '#%02x%02x%02x' % tuple(int(c) for c in line.replace('\\n', '').split(','))\n",
" for line in f\n",
" if not no_black or line != '0,0,0\\n'\n",
" if not no_white or line != '255,255,255\\n'\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "56dab238a8864488a66a23b26dfa4ce3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='0s passed', placeholder='0…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"cdfs = [\n",
" np.array([\n",
" box(xmin, ymin, xmax_, ymax).intersection(shape).area / shape.area\n",
" for xmax_ in numpy.linspace(xmin, xmax, 10001)\n",
" ][1:])\n",
" for shape in ProgressBar(shapes)\n",
" for xmin, ymin, xmax, ymax in [shape.bounds]\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6a7c2ca1bb1042ff8cfd4b6efbb14c2a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='0s passed', placeholder='0…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"cutoff_percentage = 0.05\n",
"\n",
"for question, rows in ProgressBar(\n",
" answers_filtered.groupby('question_text_url'),\n",
" size=len(answers_filtered['question_text_url'].unique())\n",
"):\n",
" m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles='stamentoner', zoom_start=10)\n",
" answer_texts = {\n",
" answer_text\n",
" for gemeente, _ in enumerate(gemeente_names)\n",
" for rows_ in [rows[rows['gemeente'] == gemeente]]\n",
" for answer_text, rows__ in rows_.groupby('answer_text')\n",
" if len(rows__) / len(rows_) >= cutoff_percentage\n",
" }\n",
" palette = dict(zip(answer_texts, get_palette(len(answer_texts))))\n",
" n_other = len(rows) - sum(sum(rows['answer_text'] == answer_text) for answer_text in answer_texts)\n",
" \n",
" groups = {\n",
" answer_text: folium.FeatureGroup(name=name, overlay=True)\n",
" for answer_text, color in palette.items()\n",
" for name in [\n",
" '{} ({})'.format(\n",
" color,\n",
" escape(answer_text),\n",
" sum(rows['answer_text'] == answer_text)\n",
" )\n",
" ]\n",
" }\n",
" groups['other'] = folium.FeatureGroup(\n",
" name='{} ({})'.format(\n",
" escape('other'),n_other),\n",
" overlay=True\n",
" )\n",
" \n",
" for gemeente, gemeente_name in enumerate(gemeente_names):\n",
" rows_ = rows[rows['gemeente'] == gemeente]\n",
" order = [a for _, a in sorted((\n",
" (r['user_lat'], answer)\n",
" for answer, r in rows_.groupby('answer_text').count().iterrows()\n",
" ), reverse=True)]\n",
" gemeente_shape = shapes[gemeente]\n",
" xmin, ymin, xmax, ymax = gemeente_shape.bounds\n",
" xmin_cum = xmin\n",
"# print(sum(sum(rows_['answer_text'] == answer_text) for answer_text in order))\n",
"# print(len(rows_))\n",
" cum_percentage = 0\n",
" for i, answer_text in enumerate(order):\n",
" total = sum(rows_['answer_text'] == answer_text)\n",
" percentage = total / len(rows_)\n",
" cum_percentage += percentage\n",
" if i == 0:\n",
" max_percentage = percentage\n",
" \n",
" name = '{} ({}, {}%)'.format(answer_text, total, int(round(100*percentage)))\n",
" if percentage < cutoff_percentage:\n",
" xmax_ = xmax\n",
" color = '#ffffff'\n",
" answer_text = 'other'\n",
" total = sum(sum(rows_['answer_text'] == answer_text) for answer_text in order[i:])\n",
" percentage = total / len(rows_)\n",
" done = True\n",
" else:\n",
" percentage_corrected = numpy.abs(cdfs[gemeente] - cum_percentage).argmin() / 10000\n",
" xmax_ = xmin + percentage_corrected * (xmax - xmin)\n",
" color = palette[answer_text]\n",
" done = False\n",
" \n",
" answer_shape = gemeente_shape.intersection(box(xmin_cum, ymin, xmax_, ymax))\n",
" xmin_cum = xmax_\n",
"# color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(percentage / max_percentage)[:3])\n",
" \n",
" polygon = folium.Polygon(\n",
" reverse(mapping(answer_shape)['coordinates']),\n",
" fill_color=color,\n",
" fill_opacity=0.8,\n",
" color=None,\n",
" popup='{} ({}, {: 3d}%)'.format(answer_text, total, round(100*percentage))\n",
" )\n",
" polygon.add_to(groups[answer_text])\n",
" if done:\n",
" break\n",
" \n",
" polygon = folium.Polygon(\n",
" reverse(mapping(gemeente_shape)['coordinates']),\n",
" fill_color=None, color='#000000'\n",
" )\n",
" polygon.add_to(m)\n",
" \n",
" for _, group in sorted(groups.items(), key=lambda x: sum(rows['answer_text'] == x[0]), reverse=True):\n",
" group.add_to(m)\n",
" \n",
" folium.map.LayerControl('topright', collapsed=False).add_to(m)\n",
" display(m)\n",
" m.save('maps/heatmaps-combined/{}.html'.format(question))\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [],
"source": [
"import glob\n",
"with open('maps/heatmaps-combined/index.html', 'w') as f:\n",
" f.write('' + \n",
" '
\\n'.join(\n",
" '\\t{}'.format(fn, fn[:-5].replace('_', ' '))\n",
" for fn in sorted(\n",
" glob.glob('maps/heatmaps-combined/*.html')\n",
" )\n",
" for fn in [fn[len('maps/heatmaps-combined/'):]]\n",
" ) + \"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}