328 lines
11 KiB
Plaintext
328 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Geographical pronunciation statistics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas\n",
|
|
"import MySQLdb\n",
|
|
"import numpy\n",
|
|
"import json\n",
|
|
"\n",
|
|
"db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n",
|
|
"\n",
|
|
"%matplotlib notebook\n",
|
|
"from matplotlib import pyplot\n",
|
|
"import folium\n",
|
|
"from IPython.display import display\n",
|
|
"from shapely.geometry import Polygon, MultiPolygon, shape, Point\n",
|
|
"from jupyter_progressbar import ProgressBar\n",
|
|
"from collections import defaultdict\n",
|
|
"from ipy_table import make_table\n",
|
|
"from html import escape\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"from matplotlib.colors import LogNorm\n",
|
|
"from sklearn import mixture\n",
|
|
"from skimage.measure import find_contours\n",
|
|
"from collections import Counter\n",
|
|
"from random import shuffle"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Borders of Frysian municipalities\n",
|
|
"\n",
|
|
"with open('Friesland_AL8.GeoJson') as f:\n",
|
|
" gemeentes = json.load(f)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"shapes = [shape(feature['geometry']) for feature in gemeentes['features']]\n",
|
|
"gemeente_names = [feature['properties']['name'] for feature in gemeentes['features']]\n",
|
|
"\n",
|
|
"def get_gemeente(point):\n",
|
|
" for i, shape in enumerate(shapes):\n",
|
|
" if shape.contains(point):\n",
|
|
" return i\n",
|
|
" return -1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Answers to how participants state a word should be pronounces.\n",
|
|
"\n",
|
|
"answers = pandas.read_sql('''\n",
|
|
"SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n",
|
|
"FROM core_surveyresult as survey\n",
|
|
"INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n",
|
|
"INNER JOIN core_predictionquizresultquestionanswer as answer\n",
|
|
" ON result.id = answer.prediction_quiz_id\n",
|
|
"''', db)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Takes approximately 2 minutes\n",
|
|
"\n",
|
|
"gemeente_map = {\n",
|
|
" (lng, lat): get_gemeente(Point(lng, lat))\n",
|
|
" for lng, lat in set(zip(answers['user_lng'], answers['user_lat']))\n",
|
|
"}\n",
|
|
"\n",
|
|
"answers['gemeente'] = [\n",
|
|
" gemeente_map[(lng, lat)]\n",
|
|
" for lat, lng in zip(answers['user_lat'], answers['user_lng'])\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Mapping pronunciations\n",
|
|
"\n",
|
|
"The idea is to plot each pronunciation as a point of a different color, now only seems to show participation density.\n",
|
|
"\n",
|
|
"Slow, so started with the first question."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# cmap = pyplot.get_cmap('gist_rainbow')\n",
|
|
"\n",
|
|
"# std = (1.89, 1.35)\n",
|
|
"\n",
|
|
"# for _, (question, rows) in zip(range(3), answers.groupby('question_text')):\n",
|
|
"# plt.figure()\n",
|
|
"# n_answers = len(rows.groupby('answer_text').count())\n",
|
|
"# colors = cmap(range(256))[::256 // n_answers]\n",
|
|
"# for (answer, rows_), color in zip(rows.groupby('answer_text'), colors):\n",
|
|
"# if len(rows_) < 100:\n",
|
|
"# continue\n",
|
|
"# color = '#%02x%02x%02x' % tuple(int(c*255) for c in color[:3])\n",
|
|
"# X = rows_[['user_lat', 'user_lng']].as_matrix()\n",
|
|
"\n",
|
|
"# clf = mixture.GaussianMixture(n_components=5, covariance_type='full')\n",
|
|
"# clf.fit(X)\n",
|
|
"# xlim = numpy.percentile(X[:, 0], [1, 99.5])\n",
|
|
"# ylim = numpy.percentile(X[:, 1], [1, 99.5])\n",
|
|
"# xlim = [2*xlim[0] - xlim[1], 2*xlim[1] - xlim[0]]\n",
|
|
"# ylim = [2*ylim[0] - ylim[1], 2*ylim[1] - ylim[0]]\n",
|
|
" \n",
|
|
"# x = np.linspace(*xlim, 1000)\n",
|
|
"# y = np.linspace(*ylim, 1000)\n",
|
|
"# xx, yy = np.meshgrid(x, y)\n",
|
|
"# xxyy = np.array([xx.ravel(), yy.ravel()]).T\n",
|
|
"# z = np.exp(clf.score_samples(xxyy))\n",
|
|
"# z = z.reshape(xx.shape)\n",
|
|
" \n",
|
|
"# z_sorted = sorted(z.ravel(), reverse=True)\n",
|
|
"# z_sorted_cumsum = np.cumsum(z_sorted)\n",
|
|
"# split = np.where(z_sorted_cumsum > (z_sorted_cumsum[-1] * 0.5))[0][0]\n",
|
|
"# threshold = z_sorted[split]\n",
|
|
"# threshold\n",
|
|
"\n",
|
|
"# # p = list(range(0, 100, 5))\n",
|
|
"\n",
|
|
"# p = [80]\n",
|
|
"# plt.contour(xx, yy, z, levels=[threshold], colors=[color])\n",
|
|
"# plt.plot(X[:, 0], X[:, 1], '.', c=color)\n",
|
|
"# plt.xlim(*xlim)\n",
|
|
"# plt.ylim(*ylim)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"zero_latlng_questions = {\n",
|
|
" q\n",
|
|
" for q, row in answers.groupby('question_text').agg('std').iterrows()\n",
|
|
" if row['user_lat'] == 0 and row['user_lng'] == 0\n",
|
|
"}\n",
|
|
"answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n",
|
|
" lambda x: x.replace('\"', '').replace('*', ''))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_palette(n, no_black=True, no_white=True):\n",
|
|
" with open('glasbey/{}_colors.txt'.format(n + no_black + no_white)) as f:\n",
|
|
" return [\n",
|
|
" '#%02x%02x%02x' % tuple(int(c) for c in line.replace('\\n', '').split(','))\n",
|
|
" for line in f\n",
|
|
" if not no_black or line != '0,0,0\\n'\n",
|
|
" if not no_white or line != '255,255,255\\n'\n",
|
|
" ]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"options = [x[1] for x in sorted([\n",
|
|
" (row['user_lng'], answer_text)\n",
|
|
" for answer_text, row in rows.groupby('answer_text').agg({'user_lng': 'count'}).iterrows()\n",
|
|
"], reverse=True)]\n",
|
|
"\n",
|
|
"groups = [options[:len(options) // 2], options[len(options) // 2:]]\n",
|
|
"groups"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"80000 / 350"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import glob\n",
|
|
"with open('index.html', 'w') as f:\n",
|
|
" f.write('<html><head></head><body>' + \n",
|
|
" '<br/>\\n'.join(\n",
|
|
" '\\t<a href=\"http://herbertkruitbosch.com/pronunciation_maps/{}\">{}<a>'.format(fn, fn[:-4].replace('_', ' '))\n",
|
|
" for fn in sorted(\n",
|
|
" glob.glob('*_all.html') +\n",
|
|
" glob.glob('*_larger.html') +\n",
|
|
" glob.glob('*_smaller.html')\n",
|
|
" )\n",
|
|
" ) + \"</body></html>\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# cmap = pyplot.get_cmap('gist_rainbow')\n",
|
|
"# colors = pyplot.get_cmap('tab20')\n",
|
|
"# colors = ['#e6194b', '#3cb44b', '#ffe119', '#0082c8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#d2f53c', '#fabebe', '#008080', '#e6beff', '#aa6e28', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000080', '#808080']\n",
|
|
"\n",
|
|
"std = (1.89, 1.35)\n",
|
|
"\n",
|
|
"for question, rows in answers_filtered.groupby('question_text_url'):\n",
|
|
"# question = rows['question_text_url'][0]\n",
|
|
" n_answers = len(rows.groupby('answer_text').count())\n",
|
|
" \n",
|
|
" \n",
|
|
" options = [x[1] for x in sorted([\n",
|
|
" (row['user_lng'], answer_text)\n",
|
|
" for answer_text, row in rows.groupby('answer_text').agg({'user_lng': 'count'}).iterrows()\n",
|
|
" ], reverse=True)]\n",
|
|
" groups = [options]\n",
|
|
" if n_answers > 6:\n",
|
|
" groups.extend([options[:6], options[6:]])\n",
|
|
" \n",
|
|
" for group, group_name in zip(groups, ['all', 'larger', 'smaller']):\n",
|
|
" m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles='stamentoner', zoom_start=9)\n",
|
|
" # colors = cmap(range(256))[::256 // n_answers]\n",
|
|
" colors = get_palette(len(group))\n",
|
|
" for answer, color in zip(group, colors):\n",
|
|
" rows_ = rows[rows['answer_text'] == answer]\n",
|
|
" # color = '#%02x%02x%02x' % tuple(int(c*255) for c in color[:3])\n",
|
|
" name = '<span style=\\\\\"color:{}; \\\\\">{} ({})'.format(color, escape(answer), len(rows_))\n",
|
|
"\n",
|
|
" group = folium.FeatureGroup(name=name)\n",
|
|
" colormap[name] = color\n",
|
|
"\n",
|
|
" for point in zip(rows_['user_lat'], rows_['user_lng']):\n",
|
|
" point = tuple(p + 0.01 * s * numpy.random.randn() for p, s in zip(point, std))\n",
|
|
" folium.Circle(\n",
|
|
" point, color=None, fill_color=color,\n",
|
|
" radius=400*min(1, 100 / len(rows_)), fill_opacity=1 #1 - 0.5 * len(rows_) / len(rows)\n",
|
|
" ).add_to(group)\n",
|
|
" group.add_to(m)\n",
|
|
" folium.map.LayerControl('topright', collapsed=False).add_to(m)\n",
|
|
" \n",
|
|
" print(group_name, question)\n",
|
|
" if group_name == 'larger':\n",
|
|
" display(m)\n",
|
|
" m.save('{}_{}.html'.format(question, group_name))"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|