294 lines
10 KiB
Plaintext
294 lines
10 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Geographical pronunciation statistics"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 12,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import pandas\n",
|
||
|
|
"import MySQLdb\n",
|
||
|
|
"import numpy\n",
|
||
|
|
"import json\n",
|
||
|
|
"\n",
|
||
|
|
"db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n",
|
||
|
|
"\n",
|
||
|
|
"%matplotlib notebook\n",
|
||
|
|
"from matplotlib import pyplot\n",
|
||
|
|
"import folium\n",
|
||
|
|
"from IPython.display import display\n",
|
||
|
|
"from shapely.geometry import Polygon, MultiPolygon, shape, Point\n",
|
||
|
|
"from jsbutton import JsButton\n",
|
||
|
|
"from jupyter_progressbar import ProgressBar\n",
|
||
|
|
"from collections import defaultdict\n",
|
||
|
|
"from ipy_table import make_table\n",
|
||
|
|
"from html import escape\n",
|
||
|
|
"\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import matplotlib.pyplot as plt\n",
|
||
|
|
"from matplotlib.colors import LogNorm\n",
|
||
|
|
"from sklearn import mixture\n",
|
||
|
|
"from skimage.measure import find_contours\n",
|
||
|
|
"from collections import Counter\n",
|
||
|
|
"from random import shuffle"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 13,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Borders of Frysian municipalities\n",
|
||
|
|
"\n",
|
||
|
|
"with open('Friesland_AL8.GeoJson') as f:\n",
|
||
|
|
" gemeentes = json.load(f)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"coords = [feature['geometry'] for feature in gemeentes['features']]\n",
|
||
|
|
"coords_folium = [[[[c__[::-1] for c__ in c_] for c_ in c] for c in coords_['coordinates']] for coords_ in coords]\n",
|
||
|
|
"shapes = [shape(coords_) for coords_ in coords]\n",
|
||
|
|
"gemeente_names = [feature['properties']['name'] for feature in gemeentes['features']]\n",
|
||
|
|
"\n",
|
||
|
|
"def get_gemeente(point):\n",
|
||
|
|
" for i, shape in enumerate(shapes):\n",
|
||
|
|
" if shape.contains(point):\n",
|
||
|
|
" return i\n",
|
||
|
|
" return -1"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Answers to how participants state a word should be pronounces.\n",
|
||
|
|
"\n",
|
||
|
|
"answers = pandas.read_sql('''\n",
|
||
|
|
"SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n",
|
||
|
|
"FROM core_surveyresult as survey\n",
|
||
|
|
"INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n",
|
||
|
|
"INNER JOIN core_predictionquizresultquestionanswer as answer\n",
|
||
|
|
" ON result.id = answer.prediction_quiz_id\n",
|
||
|
|
"''', db)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"zero_latlng_questions = {\n",
|
||
|
|
" q\n",
|
||
|
|
" for q, row in answers.groupby('question_text').agg('std').iterrows()\n",
|
||
|
|
" if row['user_lat'] == 0 and row['user_lng'] == 0\n",
|
||
|
|
"}\n",
|
||
|
|
"answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 6,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" # Remove the CWD from sys.path while we load stuff.\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"# Takes approximately 2 minutes\n",
|
||
|
|
"\n",
|
||
|
|
"gemeente_map = {\n",
|
||
|
|
" (lng, lat): get_gemeente(Point(lng, lat))\n",
|
||
|
|
" for lng, lat in set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))\n",
|
||
|
|
"}\n",
|
||
|
|
"\n",
|
||
|
|
"answers_filtered['gemeente'] = [\n",
|
||
|
|
" gemeente_map[(lng, lat)]\n",
|
||
|
|
" for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])\n",
|
||
|
|
"]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 8,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" \n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n",
|
||
|
|
" lambda x: x.replace('\"', '').replace('*', ''))"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 9,
|
||
|
|
"metadata": {
|
||
|
|
"scrolled": false
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"application/vnd.jupyter.widget-view+json": {
|
||
|
|
"model_id": "e48eb24f5c43434bad4241d4bea53074",
|
||
|
|
"version_major": 2,
|
||
|
|
"version_minor": 0
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='<b>0</b>s passed', placeholder='0…"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"cmap = pyplot.get_cmap('YlOrRd')\n",
|
||
|
|
"\n",
|
||
|
|
"for question, rows in ProgressBar(\n",
|
||
|
|
" answers_filtered.groupby('question_text_url'),\n",
|
||
|
|
" size=len(answers_filtered['question_text_url'].unique())\n",
|
||
|
|
"):\n",
|
||
|
|
" m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)\n",
|
||
|
|
" pecentage_labels = folium.FeatureGroup(name='pecentages', overlay=True)\n",
|
||
|
|
" order = [a for _, a in sorted((\n",
|
||
|
|
" (r['user_lat'], answer)\n",
|
||
|
|
" for answer, r in rows.groupby('answer_text').count().iterrows()\n",
|
||
|
|
" ), reverse=True)]\n",
|
||
|
|
" gemeente_normalizer = {\n",
|
||
|
|
" gemeente: r['user_lat']\n",
|
||
|
|
" for gemeente, r in rows.groupby('gemeente').count().iterrows()\n",
|
||
|
|
" }\n",
|
||
|
|
" for answer_text in order:\n",
|
||
|
|
" rows_ = rows[rows['answer_text'] == answer_text]\n",
|
||
|
|
" if (rows_['gemeente'] >= 0).sum() <= 0:\n",
|
||
|
|
" continue\n",
|
||
|
|
"\n",
|
||
|
|
" spread = {\n",
|
||
|
|
" gemeente: r['user_lat']\n",
|
||
|
|
" for gemeente, r in rows_.groupby('gemeente').count().iterrows()\n",
|
||
|
|
" if gemeente >= 0\n",
|
||
|
|
" }\n",
|
||
|
|
" n_answers = sum(spread.values())\n",
|
||
|
|
" \n",
|
||
|
|
" name = '{} ({})'.format(answer_text, n_answers)\n",
|
||
|
|
" group = folium.FeatureGroup(name=name, overlay=False)\n",
|
||
|
|
" folium.TileLayer(tiles='stamentoner').add_to(group)\n",
|
||
|
|
" \n",
|
||
|
|
" max_value = max(value / gemeente_normalizer[gemeente] for gemeente, value in spread.items())\n",
|
||
|
|
" for gemeente, gemeente_name in enumerate(gemeente_names):\n",
|
||
|
|
" if gemeente in spread:\n",
|
||
|
|
" value = spread[gemeente]\n",
|
||
|
|
" percentage = value / gemeente_normalizer[gemeente]\n",
|
||
|
|
" color_value = percentage / max_value\n",
|
||
|
|
" color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])\n",
|
||
|
|
" \n",
|
||
|
|
" polygon = folium.Polygon(coords_folium[gemeente], fill_color=color, fill_opacity=0.8,\n",
|
||
|
|
" color='#555555', popup='{} ({}, {}%)'.format(gemeente_name, value, round(100*percentage)))\n",
|
||
|
|
" centroid = shapes[gemeente].centroid\n",
|
||
|
|
" centroid = (centroid.y, centroid.x)\n",
|
||
|
|
"# folium.Circle(centroid, color='green', radius=200).add_to(group)\n",
|
||
|
|
" folium.map.Marker(\n",
|
||
|
|
" [shapes[gemeente].centroid.y, shapes[gemeente].centroid.x],\n",
|
||
|
|
" icon=folium.DivIcon(\n",
|
||
|
|
" icon_size=(50, 24),\n",
|
||
|
|
" icon_anchor=(25, 12),\n",
|
||
|
|
" html='<div class=\"percentage-label\" style=\"font-size: 12pt; background-color: rgba(255,255,255,0.8); border-radius: 12px; text-align: center;\">{:d}%</div>'.format(int(100 * percentage)),\n",
|
||
|
|
" )\n",
|
||
|
|
" ).add_to(group)\n",
|
||
|
|
" else:\n",
|
||
|
|
" polygon = folium.Polygon(coords_folium[gemeente], fill_color=None, fill_opacity=0, color='#555555')\n",
|
||
|
|
" polygon.add_to(group)\n",
|
||
|
|
" group.add_to(m)\n",
|
||
|
|
" pecentage_labels.add_to(m)\n",
|
||
|
|
" folium.map.LayerControl('topright', collapsed=False).add_to(m)\n",
|
||
|
|
" JsButton(\n",
|
||
|
|
" title='<i class=\"fas fa-tags\"></i>',\n",
|
||
|
|
" function=\"\"\"\n",
|
||
|
|
" function(btn, map){\n",
|
||
|
|
" $('.percentage-label').toggle();\n",
|
||
|
|
" }\n",
|
||
|
|
" \"\"\"\n",
|
||
|
|
" ).add_to(m)\n",
|
||
|
|
"# display(m)\n",
|
||
|
|
" m.save('maps/heatmaps/{}.html'.format(question))\n",
|
||
|
|
"# break"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 11,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import glob\n",
|
||
|
|
"with open('maps/heatmaps/index.html', 'w') as f:\n",
|
||
|
|
" f.write('<html><head></head><body>' + \n",
|
||
|
|
" '<br/>\\n'.join(\n",
|
||
|
|
" '\\t<a href=\"http://herbertkruitbosch.com/pronunciation_maps/{}\">{}<a>'.format(fn[5:], fn[14:-5].replace('_', ' '))\n",
|
||
|
|
" for fn in sorted(\n",
|
||
|
|
" glob.glob('maps/heatmaps/*.html')\n",
|
||
|
|
" )\n",
|
||
|
|
" ) + \"</body></html>\")"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.6.5"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 2
|
||
|
|
}
|