Geographical pronunciation statistics

In [12]:

import pandas
import MySQLdb
import numpy
import json

db = MySQLdb.connect(user='root', passwd='Nmmxhjgt1@', db='stimmen', charset='utf8')

%matplotlib notebook
from matplotlib import pyplot
import folium
from IPython.display import display
from shapely.geometry import Polygon, MultiPolygon, shape, Point
from jsbutton import JsButton
from jupyter_progressbar import ProgressBar
from collections import defaultdict
from ipy_table import make_table
from html import escape

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
from sklearn import mixture
from skimage.measure import find_contours
from collections import Counter
from random import shuffle

In [13]:

# Borders of Frysian municipalities

with open('Friesland_AL8.GeoJson') as f:
    gemeentes = json.load(f)

In [3]:

coords = [feature['geometry'] for feature in gemeentes['features']]
coords_folium = [[[[c__[::-1] for c__ in c_] for c_ in c] for c in coords_['coordinates']] for coords_ in coords]
shapes = [shape(coords_) for coords_ in coords]
gemeente_names = [feature['properties']['name'] for feature in gemeentes['features']]

def get_gemeente(point):
    for i, shape in enumerate(shapes):
        if shape.contains(point):
            return i
    return -1

In [4]:

# Answers to how participants state a word should be pronounces.

answers = pandas.read_sql('''
SELECT prediction_quiz_id, user_lat, user_lng, question_text, answer_text
FROM       core_surveyresult as survey
INNER JOIN core_predictionquizresult as result ON survey.id = result.survey_result_id
INNER JOIN core_predictionquizresultquestionanswer as answer
    ON result.id = answer.prediction_quiz_id
''', db)

In [5]:

zero_latlng_questions = {
    q
    for q, row in answers.groupby('question_text').agg('std').iterrows()
    if row['user_lat'] == 0 and row['user_lng'] == 0
}
answers_filtered = answers[answers['question_text'].map(lambda x: x not in zero_latlng_questions)]

In [6]:

# Takes approximately 2 minutes

gemeente_map = {
    (lng, lat): get_gemeente(Point(lng, lat))
    for lng, lat in set(zip(answers_filtered['user_lng'], answers_filtered['user_lat']))
}

answers_filtered['gemeente'] = [
    gemeente_map[(lng, lat)]
    for lat, lng in zip(answers_filtered['user_lat'], answers_filtered['user_lng'])
]

/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  # Remove the CWD from sys.path while we load stuff.

In [8]:

answers_filtered['question_text_url'] = answers_filtered['question_text'].map(
    lambda x: x.replace('"', '').replace('*', ''))

/home/herbert/.virtualenvs/stimmenfryslan/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy

In [9]:

cmap = pyplot.get_cmap('YlOrRd')

for question, rows in ProgressBar(
    answers_filtered.groupby('question_text_url'),
    size=len(answers_filtered['question_text_url'].unique())
):
    m = folium.Map((rows['user_lat'].median(), rows['user_lng'].median()), tiles=None, zoom_start=9)
    pecentage_labels = folium.FeatureGroup(name='pecentages', overlay=True)
    order = [a for _, a in sorted((
        (r['user_lat'], answer)
        for answer, r in rows.groupby('answer_text').count().iterrows()
    ), reverse=True)]
    gemeente_normalizer = {
        gemeente: r['user_lat']
        for gemeente, r in rows.groupby('gemeente').count().iterrows()
    }
    for answer_text in order:
        rows_ = rows[rows['answer_text'] == answer_text]
        if (rows_['gemeente'] >= 0).sum() <= 0:
            continue

        spread = {
            gemeente: r['user_lat']
            for gemeente, r in rows_.groupby('gemeente').count().iterrows()
            if gemeente >= 0
        }
        n_answers = sum(spread.values())
        
        name = '{} ({})'.format(answer_text, n_answers)
        group = folium.FeatureGroup(name=name, overlay=False)
        folium.TileLayer(tiles='stamentoner').add_to(group)
        
        max_value = max(value / gemeente_normalizer[gemeente] for gemeente, value in spread.items())
        for gemeente, gemeente_name in enumerate(gemeente_names):
            if gemeente in spread:
                value = spread[gemeente]
                percentage = value / gemeente_normalizer[gemeente]
                color_value = percentage / max_value
                color = '#%02x%02x%02x' % tuple(int(255 * c) for c in cmap(color_value)[:3])
                
                polygon = folium.Polygon(coords_folium[gemeente], fill_color=color, fill_opacity=0.8,
                              color='#555555', popup='{} ({}, {}%)'.format(gemeente_name, value, round(100*percentage)))
                centroid = shapes[gemeente].centroid
                centroid = (centroid.y, centroid.x)
#                 folium.Circle(centroid, color='green', radius=200).add_to(group)
                folium.map.Marker(
                    [shapes[gemeente].centroid.y, shapes[gemeente].centroid.x],
                    icon=folium.DivIcon(
                        icon_size=(50, 24),
                        icon_anchor=(25, 12),
                        html='<div class="percentage-label" style="font-size: 12pt; background-color: rgba(255,255,255,0.8); border-radius: 12px; text-align: center;">{:d}%</div>'.format(int(100 * percentage)),
                    )
                ).add_to(group)
            else:
                polygon = folium.Polygon(coords_folium[gemeente], fill_color=None, fill_opacity=0, color='#555555')
            polygon.add_to(group)
        group.add_to(m)
    pecentage_labels.add_to(m)
    folium.map.LayerControl('topright', collapsed=False).add_to(m)
    JsButton(
        title='<i class="fas fa-tags"></i>',
        function="""
            function(btn, map){
                $('.percentage-label').toggle();
            }
        """
    ).add_to(m)
#     display(m)
    m.save('maps/heatmaps/{}.html'.format(question))
#     break

VBox(children=(HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='<b>0</b>s passed', placeholder='0…

In [11]:

import glob
with open('maps/heatmaps/index.html', 'w') as f:
    f.write('<html><head></head><body>' + 
        '<br/>\n'.join(
            '\t<a href="http://herbertkruitbosch.com/pronunciation_maps/{}">{}<a>'.format(fn[5:], fn[14:-5].replace('_', ' '))
            for fn in sorted(
                glob.glob('maps/heatmaps/*.html')
            )
    ) + "</body></html>")

10 KiB Raw Blame History

Geographical pronunciation statistics

10 KiB

Raw Blame History