resturcured for reproducability

stimmen/cbs.py

@@ -24,7 +24,7 @@ def clear_cache_poll():
     global __cache, __last_access
     while True:
         time.sleep(60*60)
-        if (time.time() - __last_access) > 60*60:
+        if (__last_access is not None and time.time() - __last_access) > 60*60:
             __cache = {}
 
 
@@ -60,9 +60,15 @@ def province_geojson(province, with_water=False):
     return __cache[(province, with_water)]
 
 
+def expand_box(x0, y0, x1, y1, f=0.1):
+    return (x0 - (x1 - x0)*f, y0 - (y1 - y0)*f,
+            x1 + (x1 - x0)*f, y1 + (y1 - y0)*f)
+
+
 def gwb_in_province(
         province='Friesland', region_level='wijk', region_year='2018',
-        polygon_simplification=0.001, province_dilation=0.0005
+        polygon_simplification=0.001, province_dilation=0.0005,
+        bounding_box_dilation=0.01
 ):
     assert region_level in {'gem', 'wijk', 'buurt'}, (
         "region_level {} not supported, must be gem, wijk or buurt".format(region_level))
@@ -70,7 +76,7 @@ def gwb_in_province(
         "region_year {} not supported, must 2017 or 2018".format(region_year))
 
     province_with_water = shape(province_geojson(province, with_water=True)['geometry'])
-    province_bounding_box = box(*province_with_water.bounds)
+    province_bounding_box = box(*expand_box(*province_with_water.bounds, f=bounding_box_dilation))
 
     province_land_only = shape(province_geojson(province, with_water=False)['geometry'])
     province_land_only_dilated = province_land_only.buffer(-province_dilation)
@@ -80,7 +86,10 @@ def gwb_in_province(
     shapes = [shape(geojson_['geometry']) for geojson_ in geojson['features']]
 
     shapes, geojson = map(list, zip(*(
-        (intersection.simplify(tolerance=polygon_simplification), geojson_)
+        (
+            (intersection.simplify(tolerance=polygon_simplification), geojson_)
+            if polygon_simplification is not None else (intersection, geojson_)
+        )
         for shape_, geojson_ in zip(shapes, geojson['features'])
         if province_bounding_box.contains(shape_)
         for intersection in [shape_.intersection(province_land_only_dilated)] # alias

stimmen/folium.py

@@ -1,5 +1,6 @@
 import folium
 from jupyter_progressbar import ProgressBar
+from matplotlib import pyplot
 from pygeoif.geometry import mapping
 from shapely.geometry.geo import shape, box
 
@@ -9,6 +10,12 @@ import numpy as np
 
 from stimmen.latitude_longitude import reverse_latitude_longitude
 
+import tempfile
+import time
+from selenium import webdriver
+from .folium_injections import *
+from .folium_colorbar import *
+
 
 def get_palette(n, no_black=True, no_white=True):
     with open(data_file('data', 'glasbey', '{}_colors.txt'.format(n + no_black + no_white))) as f:
@@ -21,7 +28,7 @@ def get_palette(n, no_black=True, no_white=True):
 
 
 def colored_name(name, color):
-    return '<span style=\\"color:{}; \\">{}</span>'.format(color, name)
+    return '<span class=\\"with-block\\" style=\\"color:{}; \\"><span class=\\"blackable; \\">{}</span></span>'.format(color, name)
 
 
 def region_area_cdf(region_shape, resolution=10000):
@@ -75,19 +82,22 @@ def width_adjust_boundaries(region_shape, boundaries):
 
 
 def pronunciation_bars(
-        regions, dataframe,
-        region_name_property, region_name_column,
-        group_column='answer_text',
-        cutoff_percentage=0.05,
-        normalize_area=True,
-        progress_bar=False,
+    regions, dataframe,
+    region_name_property, region_name_column,
+    group_column='answer_text',
+    count_column=None,
+    cutoff_percentage=0.05,
+    normalize_area=True,
+    progress_bar=False,
+    area_adjust_resolution=10000,
+    simplify_shapes=None,
 ):
     # all values of group_column that appear at least cutoff_percentage in one of the regions
     relevant_groups = {
         group
         for region_name, region_rows in dataframe.groupby(region_name_column)
         for group, aggregation in region_rows.groupby(
-        group_column).agg({group_column: len}).iterrows()
+            group_column).agg({group_column: len}).iterrows()
         if aggregation[group_column] >= cutoff_percentage * len(region_rows)
     }
 
@@ -103,7 +113,7 @@ def pronunciation_bars(
     feature_groups = {
         group_value: folium.FeatureGroup(
             name=colored_name(
-                '{value} ({amount})'.format(value=escape(group_value), amount=amount),
+                '{value} <span class=\\"amount\\">({amount})</span>'.format(value=escape(group_value), amount=amount),
                 color
             ),
             overlay=True
@@ -114,6 +124,7 @@ def pronunciation_bars(
         if group_value != 'other' else
         n_other
     ]  # alias
+        if amount > 0
     }
 
     progress_bar = ProgressBar if progress_bar else lambda x: x
@@ -123,6 +134,8 @@ def pronunciation_bars(
         region_name = feature['properties'][region_name_property]
         region_rows = dataframe[dataframe[region_name_column] == region_name]
         region_shape = shape(feature['geometry'])
+        if simplify_shapes:
+            region_shape = region_shape.simplify(simplify_shapes)
         _, ymin, _, ymax = region_shape.bounds
 
         group_values_occurrence = {
@@ -136,14 +149,16 @@ def pronunciation_bars(
             key=lambda x: (x[0] == 'other', -x[1])
         ))
 
-        group_percentages = np.array(group_occurrences) / len(region_rows)
-        group_boundaries = np.cumsum((0,) + group_occurrences) / len(region_rows)
+        group_percentages = np.array(group_occurrences) / max(1, len(region_rows))
+        group_boundaries = np.cumsum((0,) + group_occurrences) / max(1, len(region_rows))
         if normalize_area:
             if '__region_shape_cdf_cache' not in feature['properties']:
-                feature['properties']['__region_shape_cdf_cache'] = region_area_cdf(region_shape).tolist()
+                feature['properties']['__region_shape_cdf_cache'] = region_area_cdf(
+                    region_shape, resolution=area_adjust_resolution).tolist()
             group_boundaries = area_adjust_boundaries(
                 region_shape, group_boundaries,
-                region_cdf_cache=feature['properties']['__region_shape_cdf_cache']
+                region_cdf_cache=feature['properties']['__region_shape_cdf_cache'],
+                resolution=area_adjust_resolution
             )
         else:
             group_boundaries = width_adjust_boundaries(region_shape, group_boundaries)
@@ -158,7 +173,7 @@ def pronunciation_bars(
                 continue
 
             bar_shape = region_shape.intersection(box(left_boundary, ymin, right_boundary, ymax))
-            if bar_shape.area == 0:
+            if bar_shape.area == 0 or group_occurrences == 0:
                 continue
             polygon = folium.Polygon(
                 reverse_latitude_longitude(mapping(bar_shape)['coordinates']),
@@ -167,6 +182,213 @@ def pronunciation_bars(
                 color=None,
                 popup='{} ({}, {: 3d}%)'.format(group_value, count, int(round(100 * percentage)))
             )
+            polygon._bar_shape = bar_shape
             polygon.add_to(feature_groups[group_value])
 
     return feature_groups
+
+
+def shape_label(region_shape, label, font_size=12):
+    return folium.map.Marker(
+        [region_shape.centroid.y, region_shape.centroid.x],
+        icon=folium.DivIcon(
+            icon_size=(50 / 12 * font_size, 24 / 12 * font_size),
+            icon_anchor=(25 / 12 * font_size, font_size),
+            html=(
+                '<div class="percentage-label" style="font-size: {}pt; '
+                'background-color: rgba(255,255,255,0.8); border-radius: {}px; text-align: center;">'
+                '{}</div>').format(font_size, font_size, label),
+        )
+    )
+
+
+def pronunciation_heatmaps(
+    regions, dataframe,
+    region_name_property, region_name_column,
+    group_column='answer_text',
+    cmap=pyplot.get_cmap('YlOrRd'),
+    label_font_size=12,
+    min_percentage=None, max_percentage=None,
+    show_labels=False
+):
+    def hex_color(percentage):
+        return '#{:02x}{:02x}{:02x}'.format(*(
+            int(255 * c)
+            for c in cmap(percentage)[:3]
+        ))
+
+    group_value_order, group_value_occurrence = zip(*sorted(
+        ((group_value, len(rows)) for group_value, rows in dataframe.groupby(group_column)),
+        key=lambda x: -x[1]
+    ))
+
+    occurrence_in_region = {
+        region_name: len(region_rows)
+        for region_name, region_rows in dataframe.groupby(region_name_column)
+    }
+
+    max_group_value_occurrence_in_region = [
+        max(
+            (region_rows[group_column] == group_value).sum() / occurrence_in_region[region_name]
+            for region_name, region_rows in dataframe.groupby(region_name_column)
+        )
+        for group_value in group_value_order
+        # for _ in [print(group_value)] # hack
+    ]
+
+    feature_groups = [
+        folium.FeatureGroup(
+            name='{} ({})'.format(group_value, occurrence),
+            overlay=False
+        )
+        for group_value, occurrence in zip(group_value_order, group_value_occurrence)
+    ]
+    for group in feature_groups:
+        folium.TileLayer(tiles='stamentoner').add_to(group)
+
+    for feature in regions['features']:
+        region_name = feature['properties'][region_name_property]
+        region_rows = dataframe[dataframe[region_name_column] == region_name]
+        region_shape = shape(feature['geometry'])
+        region_occurrence = occurrence_in_region.get(region_name, 1);
+
+        group_value_occurrence_in_region = [
+            (region_rows[group_column] == group_value).sum()
+            for group_value in group_value_order
+        ]
+
+        for group_value, value_occurrence_in_region, value_occurrence, max_group_value_occurrence, feature_group in zip(
+                group_value_order,
+                group_value_occurrence_in_region,
+                group_value_occurrence,
+                max_group_value_occurrence_in_region,
+                feature_groups
+        ):
+            percentage = value_occurrence_in_region / region_occurrence
+            if max_percentage is not None:
+                max_group_value_occurrence = max_percentage / 100
+            min_value = min_percentage / 100 if min_percentage is not None else 0
+            scale_value = percentage - min_value / (max_group_value_occurrence - min_value)
+            polygon = folium.Polygon(
+                reverse_latitude_longitude(feature['geometry']['coordinates']),
+                fill_color=hex_color(scale_value) if value_occurrence_in_region > 0 else '#888',
+                color='#000000',
+                fill_opacity=0.8,
+                popup='{} ({}, {: 3d}%)'.format( # ‰
+                    region_name[:50], value_occurrence_in_region,
+                    int(round(100 * percentage))
+                )
+            )
+            polygon.add_to(feature_group)
+            if show_labels and value_occurrence_in_region > 0:
+                shape_label(
+                    region_shape,
+                    '{:d}%'.format(int(round(100 * percentage))),  # ‰
+                    font_size=label_font_size
+                ).add_to(feature_group)
+
+    return dict(zip(group_value_order, feature_groups))
+
+
+def scatter_pronunciation_map(
+        dataframe,
+        latitude_column, longitude_column,
+        group_column,
+        split_at_groups=6
+):
+    std = (0.0189, 0.0135)
+
+    group_values, group_value_occurrences = zip(*sorted(
+        ((group_value, len(group_rows)) for group_value, group_rows in dataframe.groupby(group_column)),
+        key=lambda x: -x[1]
+    ))
+
+    maps = (
+        [group_values, group_values[:split_at_groups], group_values[split_at_groups:]]
+        if len(group_values) > split_at_groups else [group_values]
+    )
+    result_names = ['all', 'most_occurring', 'least_occurring']
+
+    results = {name: [] for name in result_names}
+
+    for map, map_name in zip(maps, result_names):
+        colors = get_palette(len(map))
+        for group_value, group_color in zip(map, colors):
+            group_rows = dataframe[dataframe[group_column] == group_value]
+
+            group_name = '<span style=\\"color: {}; \\">{}  ({})</span>'.format(
+                group_color, escape(group_value), len(group_rows))
+
+            results[map_name].append(folium.FeatureGroup(name=group_name))
+
+            for point in zip(group_rows[latitude_column], group_rows[longitude_column]):
+                point = tuple(p + s * np.random.randn() for p, s in zip(point, std))
+                folium.Circle(
+                    point,
+                    color=None,
+                    fill_color=group_color,
+                    radius=400 * min(1., 100 / len(group_rows)),
+                    fill_opacity=1
+                ).add_to(results[map_name][-1])
+
+    return results
+
+
+def bar_map_css(legend_fontsize='30pt', attribution_fontsize='14pt'):
+    return FoliumCSS("""
+.leaflet-control-container .leaflet-control-layers-base {{
+    display: none;
+}}
+
+.leaflet-control-container .leaflet-control-layers-separator {{
+    display: none;
+}}
+
+.leaflet-control-container .leaflet-control-layers-overlays {{
+    display: flex
+}}
+
+.leaflet-control-container .leaflet-control-layers-overlays label:not(:last-child) {{
+    margin-right: 15px;
+}}
+
+.leaflet-control-container .leaflet-control-layers-overlays label span.with-block::before {{
+    content: '■ '; color: inherit;
+}}
+
+.leaflet-control-container .leaflet-control-layers-overlays label {{
+    margin-bottom: 0px; font-size: {legend_fontsize};
+}}
+
+.leaflet-control-container .leaflet-control-layers-overlays label input {{
+    display: none;
+}}
+
+.leaflet-control-attribution a {{
+    display: none;
+}}
+
+.leaflet-control-attribution.leaflet-control-attribution.leaflet-control-attribution.leaflet-control-attribution {{
+    background-color: white;
+    font-size: {attribution_fontsize};
+}}
+""".format(legend_fontsize=legend_fontsize, attribution_fontsize=attribution_fontsize))
+
+
+def save_map(m, filename, resolution=(1600, 1400), headless=True):
+    f = tempfile.NamedTemporaryFile(delete=False, suffix='.html')
+    f.close()
+    m.save(f.name)
+
+    options = webdriver.ChromeOptions()
+    options.add_argument('--window-size={1},{0}'.format(*resolution))
+    if headless:
+        options.add_argument('--headless')
+
+    browser = webdriver.Chrome(options=options)
+    browser.get("file://" + f.name)
+    time.sleep(1)
+
+    browser.save_screenshot(filename)
+    browser.quit()
+    f.delete

stimmen/folium_colorbar.py

@@ -0,0 +1,127 @@
+import json
+from .folium_injections import FoliumCSSAndJavaScript
+
+
+def color_bar(colorbar_ticks, fontsize='16px', scale=1):
+    return FoliumCSSAndJavaScript(code="""
+jQuery.prototype.colorbar_legend = function(colormap, smooth) {{
+    $(this).html('')
+    var colormap_ = {{}}
+    for (var idx in colormap) {{
+        colormap_[parseFloat(idx)] = colormap[idx];
+    }}
+    colormap = colormap_
+
+    var colors_div =  $('<div class="colors"></div>');
+    var tick_locations = Object.keys(colormap).map(parseFloat);
+    tick_locations.sort();
+    tick_locations.reverse();
+    var previous = undefined;
+    var color_style = (
+        'linear-gradient(' +
+        tick_locations.map(function(tick) {{
+            var prepend = '';
+            if (!smooth)
+                prepend = previous ? previous + ' ' + (100 * (1-tick) + 0.001) + '%, ' : '';
+        previous = colormap[tick].color;
+        return (
+            prepend + colormap[tick].color + ' ' + (100 * (1-tick)) + '%'
+        );
+        }}).join(', ') + ')'
+    );
+    colors_div.css('background-image', color_style);
+    $(this).append(colors_div);
+
+    for(var i in tick_locations) {{
+        var tick = tick_locations[i];
+        var top = tick == 0 ? 'calc(100% - 1px)' : (100 * (1-tick)) + '%';
+        if ('value' in colormap[tick]) {{
+            var label = $('<div class="label">' + colormap[tick].value + '</div>');
+            label.css('top', top);
+            label.css('left', '130%');
+
+            var tick_ = $('<div class="tick"></div>');
+            tick_.css('top', top);
+            $(this).append(tick_);
+    
+            // label.css('width', '100%');
+            $(this).append(label);
+        }}
+        
+    }}
+}};
+
+var color_to_rgba = function(color) {{
+    var rgba = $('div').css('color', color).css('color');
+    rgba = rgba.replace('rgb(', '').replace('rgba(', '').replace(')', '').split(',').map(parseFloat);
+    if (rgba.length < 4) // jquery's color sometimes does not return an alpha if it is 1.
+        rgba.push(1);
+    return rgba;
+}};
+
+var colormap = function(start, end) {{
+    return (function(start, end, value) {{
+        var value = Math.max(0, Math.min(value, 1));
+        return (
+            'rgba(' + [0, 1, 2, 3].map(
+                i => start[i] + value * (end[i] - start[i])
+            ).join(',') + ')');
+    }}).bind(null, color_to_rgba(start), color_to_rgba(end))
+}};
+
+$(function() {{
+    var legend = $('<div class="colorbar"></div>');
+    $('body').append(legend);
+
+    legend.colorbar_legend({colorbar_ticks}, true);
+}});
+""".format(colorbar_ticks=json.dumps(colorbar_ticks)), style="""
+table.leaflet-filter-table td {{ padding-left: 10px; padding-right: 10px; border-bottom: 5px;}}
+
+table.leaflet-filter-table label {{ display: inline-block; min-width: 50px; }}
+
+.colorbar {{
+  width: calc({scale} * 25px);
+  position: relative;
+  background-color: rgba(255, 255, 255, 1);
+  position: fixed;
+  top: calc(50px * {scale});
+  width: calc(20px * {scale});
+  left: calc(5px * {scale});
+  height: calc(100% - {scale} * 100px);
+  z-index: 1000;
+}}
+
+.colorbar .colors {{
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  top: 0; left:0;
+  background-color: rgba(255, 255, 255, 1);
+}}
+
+.colorbar .tick {{
+  left: 100%;
+  position: absolute;
+  width: 20%;
+  border-style: solid;
+  border-color: black;
+  border-width: {scale}px 0 0 0;
+  height: 10px;
+}}
+
+.colorbar .label {{
+  border-radius: calc({fontsize} / 4);
+  background-color: #fff;
+  color: #000;
+  left:130%;
+  position: absolute;
+  border: 0;
+  height: {fontsize};
+  margin-top: calc({fontsize} / -2);
+  line-height: {fontsize};
+  vertical-align: middle;
+  font-size: {fontsize};
+  padding: 0px 3px;
+  font-family: Garamond;
+}}""".format(fontsize=fontsize, scale=scale))

stimmen/folium_injections.py

@@ -0,0 +1,55 @@
+from branca.element import CssLink, Figure, JavascriptLink, MacroElement
+
+from jinja2 import Template
+
+
+class FoliumCSS(MacroElement):
+    _template = Template("""
+        {% macro header(this, kwargs) %}
+        <style>
+        {{ this.style }}
+        </style>
+        {% endmacro %}
+        """)
+
+    def __init__(self, style=""""""):
+        super(FoliumCSS, self).__init__()
+        self.style = style
+
+
+class FoliumJavascript(MacroElement):
+    _template = Template("""
+        {% macro script(this, kwargs) %}
+        {{ this.code }}
+        {% endmacro %}
+        """)
+
+    def __init__(self, code="""alert('no code here');"""):
+        super(FoliumJavascript, self).__init__()
+        self.code = code
+
+    def add_to(self, m):
+        self.map_name = m.get_name()
+        super(FoliumJavascript, self).add_to(m)
+
+
+class FoliumCSSAndJavaScript(MacroElement):
+    _template = Template("""
+        {% macro header(this, kwargs) %}
+        <style>
+        {{ this.style }}
+        </style>
+        <script langauge="JavaScript">
+        {{ this.code }}
+        </script>
+        {% endmacro %}
+        """)
+
+    def __init__(self, style="""""", code=""""""):
+        super(FoliumCSSAndJavaScript, self).__init__()
+        self.style=style
+        self.code=code
+
+    def add_to(self, m):
+        self.map_name = m.get_name()
+        super(FoliumCSSAndJavaScript, self).add_to(m)