10 KiB
10 KiB
In [128]:
import pandas
import MySQLdb
import numpy
import json
db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')
%matplotlib notebook
from matplotlib import pyplot
import folium
from IPython.display import display
from shapely.geometry import Polygon, MultiPolygon, shape, Point
from jsbutton import JsButton
from shapely.geometry import LineString, MultiLineString
from jupyter_progressbar import ProgressBar
from collections import defaultdict, Counter
from ipy_table import make_table
from html import escape
import numpy as np
from random import shuffle
import pickle
from jupyter_progressbar import ProgressBarIn [129]:
with open('friesland_wijken.p3', 'rb') as f:
wijken, wijk_shapes = pickle.load(f)
wijk_names = [wijk['properties']['GM_NAAM'] + ', ' + wijk['properties'].get('WK_NAAM', '') for wijk in wijken['features']]
def get_wijk(point):
for i, shape in enumerate(wijk_shapes):
if shape.contains(point):
return i
return -1In [130]:
def listify(rd_multipolygon):
if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):
return list(rd_multipolygon)
return [
listify(element)
for element in rd_multipolygon
]In [131]:
# Answers to how participants state a word should be pronounces.
answers = pandas.read_sql('''
SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text
FROM core_surveyresult as survey
INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id
INNER JOIN core_predictionquizresultquestionanswer as answer
ON result.id = answer.prediction_quiz_id
''', db)In [132]:
zero_latlng_questions = {
q
for q, row in answers.groupby('question_text').agg('std').iterrows()
if row['user_lat'] == 0 and row['user_lng'] == 0
}
answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]In [133]:
def reverse(rd_multipolygon):
if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):
return rd_multipolygon[::-1]
return [
reverse(element)
for element in rd_multipolygon
]In [134]:
# Takes approximately 2 minutes
points = set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))
wijk_map = dict()
for lng, lat in points:
wijk_map[(lng, lat)] = get_wijk(Point(lng, lat))
answers_filtered['wijk'] = [
wijk_map[(lng, lat)]
for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])
]/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy # Remove the CWD from sys.path while we load stuff.
In [135]:
answers_filtered['question_text_url'] = answers_filtered['question_text'].map(
lambda x: x.replace('"', '').replace('*', ''))/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
In [137]:
cmap = pyplot.get_cmap('YlOrRd')
for question, rows in ProgressBar(
answers_filtered.groupby('question_text_url'),
size=len(answers_filtered['question_text_url'].unique())
):
m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)
order = [a for _, a in sorted((
(r['user_lat'], answer)
for answer, r in rows.groupby('answer_text').count().iterrows()
), reverse=True)]
wijk_normalizer = {
wijk: r['user_lat']
for wijk, r in rows.groupby('wijk').count().iterrows()
}
for answer_text in (order):
rows_ = rows[rows['answer_text'] == answer_text]
if (rows_['wijk'] >= 0).sum() <= 0:
continue
spread = {
wijk: r['user_lat']
for wijk, r in rows_.groupby('wijk').count().iterrows()
if wijk >= 0
}
n_answers = sum(spread.values())
name = '{} ({})'.format(answer_text, n_answers)
group = folium.FeatureGroup(name=name, overlay=False)
folium.TileLayer(tiles='stamentoner').add_to(group)
max_value = max(value / wijk_normalizer[wijk] for wijk, value in spread.items())
for wijk, wijk_name in enumerate(wijk_names):
coordinates = reverse(wijken['features'][wijk]['geometry']['coordinates'])
if wijk in spread:
value = spread[wijk]
percentage = value / wijk_normalizer[wijk]
color_value = percentage / max_value
color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])
polygon = folium.Polygon(
coordinates, fill_color=color, fill_opacity=0.8,
color='#555555', popup='{} ({}, {: 3d}%)'.format(wijk_name, value, int(100*percentage))
)
centroid = wijk_shapes[wijk].centroid
centroid = (centroid.y, centroid.x)
folium.map.Marker(
[wijk_shapes[wijk].centroid.y, wijk_shapes[wijk].centroid.x],
icon=folium.DivIcon(
icon_size=(30, 16),
icon_anchor=(15, 8),
html='<div class="percentage-label" style="font-size: 8pt; background-color: rgba(255,255,255,0.8); border-radius: 4px; text-align: center;">{:d}%</div>'.format(int(100 * percentage)),
)
).add_to(group)
else:
polygon = folium.Polygon(coordinates, fill_color=None, fill_opacity=0, color='#555555')
polygon.add_to(group)
group.add_to(m)
JsButton(
title='<i class="fas fa-tags"></i>',
function="""
function(btn, map){
$('.percentage-label').toggle();
}
"""
).add_to(m)
folium.map.LayerControl('topright', collapsed=False).add_to(m)
# display(m)
m.save('maps/heatmaps-wijk/{}.html'.format(question))
# breakVBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='<b>0</b>s passed', placeholder='0…
In [138]:
import glob
with open('maps/heatmaps-wijk/index.html', 'w') as f:
f.write('<html><head></head><body>' +
'<br/>\n'.join(
'\t<a href="{}">{}<a>'.format(fn, fn[:-5].replace('_', ' '))
for fn in sorted(
glob.glob('maps/heatmaps-wijk/*.html')
)
for fn in [fn[len('maps/heatmaps-wijk/'):]]
) + "</body></html>")