p253591
/
stimmenfryslan
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
stimmenfryslan/notebooks/Pronunciation heat maps Wij...

10 KiB
Raw Blame History

Geographical pronunciation statistics

In [128]:
import pandas
import MySQLdb
import numpy
import json

db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')

%matplotlib notebook
from matplotlib import pyplot
import folium
from IPython.display import display
from shapely.geometry import Polygon, MultiPolygon, shape, Point
from jsbutton import JsButton
from shapely.geometry import LineString, MultiLineString
from jupyter_progressbar import ProgressBar
from collections import defaultdict, Counter
from ipy_table import make_table
from html import escape

import numpy as np
from random import shuffle
import pickle
from jupyter_progressbar import ProgressBar
In [129]:
with open('friesland_wijken.p3', 'rb') as f:
    wijken, wijk_shapes = pickle.load(f)

wijk_names = [wijk['properties']['GM_NAAM'] + ', ' + wijk['properties'].get('WK_NAAM', '') for wijk in wijken['features']]

def get_wijk(point):
    for i, shape in enumerate(wijk_shapes):
        if shape.contains(point):
            return i
    return -1
In [130]:
def listify(rd_multipolygon):
    if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):
        return list(rd_multipolygon)
    return [
        listify(element)
        for element in rd_multipolygon
    ]
In [131]:
# Answers to how participants state a word should be pronounces.

answers = pandas.read_sql('''
SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text
FROM       core_surveyresult as survey
INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id
INNER JOIN core_predictionquizresultquestionanswer as answer
    ON result.id = answer.prediction_quiz_id
''', db)
In [132]:
zero_latlng_questions = {
    q
    for q, row in answers.groupby('question_text').agg('std').iterrows()
    if row['user_lat'] == 0 and row['user_lng'] == 0
}
answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]
In [133]:
def reverse(rd_multipolygon):
    if len(rd_multipolygon) == 2 and tuple(map(type, rd_multipolygon)) == (float, float):
        return rd_multipolygon[::-1]
    return [
        reverse(element)
        for element in rd_multipolygon
    ]
In [134]:
# Takes approximately 2 minutes
points = set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))

wijk_map = dict()
for lng, lat in points:
    wijk_map[(lng, lat)] = get_wijk(Point(lng, lat))

answers_filtered['wijk'] = [
    wijk_map[(lng, lat)]
    for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])
]

/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  # Remove the CWD from sys.path while we load stuff.
In [135]:
answers_filtered['question_text_url'] = answers_filtered['question_text'].map(
    lambda x: x.replace('"', '').replace('*', ''))

/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
In [137]:
cmap = pyplot.get_cmap('YlOrRd')

for question, rows in ProgressBar(
    answers_filtered.groupby('question_text_url'),
    size=len(answers_filtered['question_text_url'].unique())
):
    m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)
    order = [a for _, a in sorted((
        (r['user_lat'], answer)
        for answer, r in rows.groupby('answer_text').count().iterrows()
    ), reverse=True)]
    wijk_normalizer = {
        wijk: r['user_lat']
        for wijk, r in rows.groupby('wijk').count().iterrows()
    }
    for answer_text in (order):
        rows_ = rows[rows['answer_text'] == answer_text]
        if (rows_['wijk'] >= 0).sum() <= 0:
            continue

        spread = {
            wijk: r['user_lat']
            for wijk, r in rows_.groupby('wijk').count().iterrows()
            if wijk >= 0
        }
        n_answers = sum(spread.values())
        
        name = '{} ({})'.format(answer_text, n_answers)
        group = folium.FeatureGroup(name=name, overlay=False)
        folium.TileLayer(tiles='stamentoner').add_to(group)
        
        max_value = max(value / wijk_normalizer[wijk] for wijk, value in spread.items())
        
        for wijk, wijk_name in enumerate(wijk_names):
            coordinates = reverse(wijken['features'][wijk]['geometry']['coordinates'])
            if wijk in spread:
                value = spread[wijk]
                percentage = value / wijk_normalizer[wijk]
                color_value = percentage / max_value
                color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])
                
                polygon = folium.Polygon(
                    coordinates, fill_color=color, fill_opacity=0.8,
                    color='#555555', popup='{} ({}, {: 3d}%)'.format(wijk_name, value, int(100*percentage))
                    
                )
                centroid = wijk_shapes[wijk].centroid
                centroid = (centroid.y, centroid.x)
                folium.map.Marker(
                    [wijk_shapes[wijk].centroid.y, wijk_shapes[wijk].centroid.x],
                    icon=folium.DivIcon(
                        icon_size=(30, 16),
                        icon_anchor=(15, 8),
                        html='<div class="percentage-label" style="font-size: 8pt; background-color: rgba(255,255,255,0.8); border-radius: 4px; text-align: center;">{:d}%</div>'.format(int(100 * percentage)),
                    )
                ).add_to(group)
            else:
                polygon = folium.Polygon(coordinates, fill_color=None, fill_opacity=0, color='#555555')
            polygon.add_to(group)
        group.add_to(m)
    JsButton(
        title='<i class="fas fa-tags"></i>',
        function="""
            function(btn, map){
                $('.percentage-label').toggle();
            }
        """
    ).add_to(m)
    folium.map.LayerControl('topright', collapsed=False).add_to(m)
#     display(m)
    m.save('maps/heatmaps-wijk/{}.html'.format(question))
#     break

VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='<b>0</b>s passed', placeholder='0…
In [138]:
import glob
with open('maps/heatmaps-wijk/index.html', 'w') as f:
    f.write('<html><head></head><body>' + 
        '<br/>\n'.join(
            '\t<a href="{}">{}<a>'.format(fn, fn[:-5].replace('_', ' '))
            for fn in sorted(
                glob.glob('maps/heatmaps-wijk/*.html')
            )
            for fn in [fn[len('maps/heatmaps-wijk/'):]]
    ) + "</body></html>")