398 lines
12 KiB
Plaintext
398 lines
12 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Geographical pronunciation statistics"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import pandas\n",
|
||
|
|
"import MySQLdb\n",
|
||
|
|
"import numpy\n",
|
||
|
|
"import json\n",
|
||
|
|
"\n",
|
||
|
|
"db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')\n",
|
||
|
|
"\n",
|
||
|
|
"%matplotlib notebook\n",
|
||
|
|
"from matplotlib import pyplot\n",
|
||
|
|
"import folium\n",
|
||
|
|
"from IPython.display import display\n",
|
||
|
|
"from shapely.geometry import Polygon, MultiPolygon, shape, Point\n",
|
||
|
|
"from jsbutton import JsButton\n",
|
||
|
|
"from shapely.geometry import LineString, MultiLineString\n",
|
||
|
|
"from jupyter_progressbar import ProgressBar\n",
|
||
|
|
"from collections import defaultdict, Counter\n",
|
||
|
|
"from ipy_table import make_table\n",
|
||
|
|
"from html import escape\n",
|
||
|
|
"\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"from random import shuffle\n",
|
||
|
|
"import pickle\n",
|
||
|
|
"from jupyter_progressbar import ProgressBar"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"with open('friesland_wijken_land_only.p3', 'rb') as f:\n",
|
||
|
|
" wijken, wijk_shapes = pickle.load(f)\n",
|
||
|
|
"\n",
|
||
|
|
"for x in wijken['features']:\n",
|
||
|
|
" x['type'] = 'Feature'\n",
|
||
|
|
"\n",
|
||
|
|
"with open('friesland_wijken_geojson.json', 'w') as f:\n",
|
||
|
|
" wijken['features'] = wijken['features']\n",
|
||
|
|
" json.dump(wijken, f, indent=1)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"from osgeo import gdal, ogr\n",
|
||
|
|
"\n",
|
||
|
|
"srcDS = gdal.OpenEx('friesland_wijken_geojson.json')\n",
|
||
|
|
"ds = gdal.VectorTranslate('friesland_wijken_geojson.kml', srcDS, format='kml')"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"'k4luâ7mWBAgDSKhCVaysNdr TjeoE85JzëGúcM.,IRtp2-bLû69Un0wZF3Hv1iOfô'"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"''.join({\n",
|
||
|
|
" c\n",
|
||
|
|
" for wijk in wijken['features']\n",
|
||
|
|
" for c in wijk['properties']['gemeente_en_wijk_naam']\n",
|
||
|
|
"})"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"with open('friesland_wijken_land_only.p3', 'rb') as f:\n",
|
||
|
|
" wijken, wijk_shapes = pickle.load(f)\n",
|
||
|
|
"\n",
|
||
|
|
"wijk_names = [wijk['properties']['gemeente_en_wijk_naam'] for wijk in wijken['features']]\n",
|
||
|
|
"\n",
|
||
|
|
"def get_wijk(point):\n",
|
||
|
|
" for i, shape in enumerate(wijk_shapes):\n",
|
||
|
|
" if shape.contains(point):\n",
|
||
|
|
" return i\n",
|
||
|
|
" return -1"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 6,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"def listify(rd_multipolygon):\n",
|
||
|
|
" if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):\n",
|
||
|
|
" return list(rd_multipolygon)\n",
|
||
|
|
" return [\n",
|
||
|
|
" listify(element)\n",
|
||
|
|
" for element in rd_multipolygon\n",
|
||
|
|
" ]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 7,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Answers to how participants state a word should be pronounces.\n",
|
||
|
|
"\n",
|
||
|
|
"answers = pandas.read_sql('''\n",
|
||
|
|
"SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text\n",
|
||
|
|
"FROM core_surveyresult as survey\n",
|
||
|
|
"INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id\n",
|
||
|
|
"INNER JOIN core_predictionquizresultquestionanswer as answer\n",
|
||
|
|
" ON result.id = answer.prediction_quiz_id\n",
|
||
|
|
"''', db)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 8,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"zero_latlng_questions = {\n",
|
||
|
|
" q\n",
|
||
|
|
" for q, row in answers.groupby('question_text').agg('std').iterrows()\n",
|
||
|
|
" if row['user_lat'] == 0 and row['user_lng'] == 0\n",
|
||
|
|
"}\n",
|
||
|
|
"answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 9,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"def reverse(rd_multipolygon):\n",
|
||
|
|
" if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):\n",
|
||
|
|
" return rd_multipolygon[::-1]\n",
|
||
|
|
" return [\n",
|
||
|
|
" reverse(element)\n",
|
||
|
|
" for element in rd_multipolygon\n",
|
||
|
|
" ]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 10,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" # Remove the CWD from sys.path while we load stuff.\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"# Takes approximately 2 minutes\n",
|
||
|
|
"points = set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))\n",
|
||
|
|
"\n",
|
||
|
|
"wijk_map = dict()\n",
|
||
|
|
"for lng, lat in points:\n",
|
||
|
|
" wijk_map[(lng, lat)] = get_wijk(Point(lng, lat))\n",
|
||
|
|
"\n",
|
||
|
|
"answers_filtered['wijk'] = [\n",
|
||
|
|
" wijk_map[(lng, lat)]\n",
|
||
|
|
" for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])\n",
|
||
|
|
"]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 11,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" \n",
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" \"\"\"\n",
|
||
|
|
"/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
|
||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
|
|
"\n",
|
||
|
|
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
|
||
|
|
" \n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"answers_filtered['question_text_url'] = answers_filtered['question_text'].map(\n",
|
||
|
|
" lambda x: x.replace('\"', '').replace('*', ''))\n",
|
||
|
|
"\n",
|
||
|
|
"answers_filtered['wijk_name'] = answers_filtered['wijk'].map(\n",
|
||
|
|
" lambda x: wijk_names[x])\n",
|
||
|
|
"\n",
|
||
|
|
"answers_filtered['answer_text_url'] = answers_filtered['answer_text'].map(\n",
|
||
|
|
" lambda x: x[x.find('('):x.find(')')][1:])"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 12,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"wijken = pandas.DataFrame([\n",
|
||
|
|
" {'#name': name, 'longitude': shape.centroid.xy[0][0], 'latitude': shape.centroid.xy[1][0]}\n",
|
||
|
|
" for name, shape in zip(wijk_names, wijk_shapes)\n",
|
||
|
|
"])\n",
|
||
|
|
"\n",
|
||
|
|
"wijken.set_index('#name', inplace=True)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 23,
|
||
|
|
"metadata": {
|
||
|
|
"scrolled": true
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"def merge_dicts(*args):\n",
|
||
|
|
" for arg in args[1:]:\n",
|
||
|
|
" args[0].update(arg)\n",
|
||
|
|
" return args[0]\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"pronunciations = pandas.DataFrame([\n",
|
||
|
|
" merge_dicts(\n",
|
||
|
|
" {\n",
|
||
|
|
" question: answers['answer_text_url']\n",
|
||
|
|
" for question, answers in rows.groupby(\n",
|
||
|
|
" 'question_text_url'\n",
|
||
|
|
" ).agg(\n",
|
||
|
|
" {\n",
|
||
|
|
" 'answer_text_url': lambda x: [\n",
|
||
|
|
" {\n",
|
||
|
|
" 'pronunciation': answer_text,\n",
|
||
|
|
" 'count': answer_texts.count(answer_text)\n",
|
||
|
|
" }\n",
|
||
|
|
" for answer_texts in [list(x)]\n",
|
||
|
|
" for answer_text in sorted(set(x))\n",
|
||
|
|
" \n",
|
||
|
|
" ] \n",
|
||
|
|
" }\n",
|
||
|
|
" ).iterrows()\n",
|
||
|
|
" }, {\n",
|
||
|
|
" 'wijk': wijk_names[wijk]\n",
|
||
|
|
" })\n",
|
||
|
|
" for wijk, rows in answers_filtered.groupby('wijk')\n",
|
||
|
|
" if wijk >= 0\n",
|
||
|
|
"])\n",
|
||
|
|
"\n",
|
||
|
|
"pronunciations.set_index('wijk', inplace=True)\n",
|
||
|
|
"pronunciations\n",
|
||
|
|
"\n",
|
||
|
|
"columns = list(pronunciations.columns)\n",
|
||
|
|
"\n",
|
||
|
|
"counts = pandas.DataFrame([\n",
|
||
|
|
" merge_dicts({\n",
|
||
|
|
" column + \": \" + x['pronunciation']: 100 * x['count'] / total\n",
|
||
|
|
" for column in columns\n",
|
||
|
|
" for total in [sum(x['count'] for x in row[column])]\n",
|
||
|
|
" for x in row[column]\n",
|
||
|
|
" }, {'': wijk})\n",
|
||
|
|
" for wijk, row in pronunciations.iterrows()\n",
|
||
|
|
"])\n",
|
||
|
|
"\n",
|
||
|
|
"pronunciations = pandas.DataFrame([\n",
|
||
|
|
" merge_dicts({\n",
|
||
|
|
" column: ' / '.join(str(x['pronunciation']) for x in row[column])\n",
|
||
|
|
" for column in columns\n",
|
||
|
|
" }, {'': wijk})\n",
|
||
|
|
" for wijk, row in pronunciations.iterrows()\n",
|
||
|
|
"])\n",
|
||
|
|
"\n",
|
||
|
|
"pronunciations.set_index('', inplace=True)\n",
|
||
|
|
"counts.set_index('', inplace=True)\n",
|
||
|
|
"counts[counts != counts] = 0"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 29,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"<function shapely.geometry.geo.shape(context)>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 29,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"shape"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 26,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"pronunciations.to_csv('pronunciations_by_wijk.tsv', sep='\\t')\n",
|
||
|
|
"counts.to_csv('pronunciation_percentages_by_wijk.tsv', sep='\\t')\n",
|
||
|
|
"wijken.to_csv('wijk_centroid.tsv', sep='\\t', columns=['longitude', 'latitude'])"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 27,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"with open('pronunciations_by_wijk.tsv') as f:\n",
|
||
|
|
" p = list(f)\n",
|
||
|
|
" \n",
|
||
|
|
"with open('pronunciation_count_by_wijk.tsv') as f:\n",
|
||
|
|
" c = list(f)\n",
|
||
|
|
"\n",
|
||
|
|
"with open('wijk_centroid.tsv') as f:\n",
|
||
|
|
" w = list(f)"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.6.5"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 2
|
||
|
|
}
|