{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "eec04efd-3f8f-4032-8038-83d61bff1792", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from scipy.stats import chi2_contingency\n", "from statsmodels.stats.multitest import multipletests\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "id": "e3fcbc08-25aa-47fd-bee3-b468982bf04b", "metadata": {}, "outputs": [], "source": [ "def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):\n", " \"\"\"\n", " word : the word being tested\n", " counts_recent : raw count in week 5\n", " counts_reference : raw count in weeks 1-4\n", " total_recent : total tokens in week 5\n", " total_reference : total tokens in weeks 1-4\n", " \"\"\"\n", " a = counts_recent # word in recent\n", " b = counts_reference # word in reference\n", " c = total_recent - a # non-word in recent\n", " d = total_reference - b # non-word in reference\n", "\n", " contingency = np.array([[a, b],\n", " [c, d]])\n", "\n", " # --- Chi-Squared ---\n", " chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)\n", "\n", " # --- Log-Likelihood (G²) ---\n", " # G² = 2 * sum(observed * log(observed / expected))\n", " # scipy's chi2_contingency with lambda_=\"log-likelihood\" computes this\n", " g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_=\"log-likelihood\")\n", "\n", " # --- Effect sizes ---\n", " freq_recent = a / total_recent\n", " freq_reference = b / total_reference\n", "\n", " pct_diff = (freq_recent - freq_reference) / freq_reference * 100\n", "\n", " # Avoid log(0) with a small epsilon\n", " eps = 1e-9\n", " log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))\n", "\n", " return {\n", " \"word\": word,\n", " \"freq_recent\": freq_recent,\n", " \"freq_reference\": freq_reference,\n", " \"pct_diff\": pct_diff,\n", " \"log_ratio\": log_ratio,\n", " \"chi2\": chi2_stat,\n", " \"p_chi2\": p_chi2,\n", " \"g2\": g2_stat,\n", " \"p_g2\": p_g2,\n", " }" ] }, { "cell_type": "code", "execution_count": 3, "id": "2888f740-1539-4f62-a13e-73512a2be290", "metadata": {}, "outputs": [], "source": [ "counts_recent = {}\n", "counts_reference = {}\n", "with open(\"data.txt\", \"rt\", encoding=\"utf-8\") as fp:\n", " for line in fp:\n", " aa = line.split(\"\\t\")\n", " counts_reference[aa[0]] = max(int(aa[1]), 0.5)\n", " counts_recent[aa[0]] = max(int(aa[2]), 0.5)\n", "total_recent = sum(counts_recent.values())\n", "total_reference = sum(counts_reference.values())" ] }, { "cell_type": "code", "execution_count": 4, "id": "5ef99c2b-50d9-450b-854c-bfa9d688c63b", "metadata": {}, "outputs": [], "source": [ "results = [\n", " corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),\n", " total_recent, total_reference)\n", " for word in counts_recent]" ] }, { "cell_type": "code", "execution_count": 5, "id": "285064c8-bc52-4096-9aa5-bc57115fb813", "metadata": {}, "outputs": [], "source": [ "# FDR correction across all words\n", "p_values = [r[\"p_g2\"] for r in results]\n", "_, p_adjusted, _, _ = multipletests(p_values, method=\"fdr_bh\")\n", "\n", "for r, p_adj in zip(results, p_adjusted):\n", " r[\"p_g2_adjusted\"] = p_adj" ] }, { "cell_type": "code", "execution_count": 6, "id": "c0146062-608a-40dd-80b7-742df67ca9ee", "metadata": {}, "outputs": [], "source": [ "results = pd.DataFrame(results)" ] }, { "cell_type": "code", "execution_count": 7, "id": "8805479d-461f-4e56-8b85-b5aa65fc7f33", "metadata": {}, "outputs": [], "source": [ "#pd.set_option('display.max_rows', None)\n", "#pd.set_option('display.max_columns', None)" ] }, { "cell_type": "code", "execution_count": 8, "id": "f992bc03-6529-4521-9c98-3f7fe36eb7e8", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
wordfreq_recentfreq_referencepct_difflog_ratiochi2p_chi2g2p_g2p_g2_adjusted
4666Ye0.0091910.001412551.0306392.70272588.8730134.209875e-2173.6110949.512852e-182.265010e-14
2233Kanye West0.0061750.001150436.8228492.42444551.8609975.957208e-1342.5127957.022134e-111.114647e-07
2521Lieke Marsman0.0022980.0000268688.9136216.45755841.6750921.077733e-1034.0460505.382305e-096.407634e-06
55Ahmed Marcouch0.0030160.0002611053.5449133.52799738.8974014.466750e-1030.7741982.898665e-082.760689e-05
1343Flavio Cobolli0.0020100.0000267590.2994186.26491336.1864561.793125e-0929.0239267.148986e-085.673912e-05
4362Van Dissel0.0025850.0002091135.9409783.62753234.1971574.980167e-0926.7579032.306067e-071.546231e-04
2797Marsman0.0018670.0000267040.9923176.15799833.4436417.335924e-0926.5278832.597616e-071.546231e-04
2503Lewis Hamilton0.0040210.000784412.6866282.35807632.5002001.191803e-0826.1044233.234422e-071.711368e-04
673Charles Leclerc0.0028720.000418586.6338772.77953828.6706388.579474e-0822.4429922.164739e-069.371352e-04
1858Jaap van Dissel0.0028720.000418586.6338772.77953828.6706388.579474e-0822.4429922.164739e-069.371352e-04
4516Vollering0.0018670.0001051685.2480794.15804027.5905891.499064e-0721.0705614.426770e-061.716105e-03
2923Mette-Marit0.0045950.001307251.5565451.81375625.3186814.859791e-0720.7135105.333836e-061.716105e-03
1801Ingrid Alexandra0.0020100.0001571181.7165703.67999726.9350702.104062e-0720.5643495.765997e-061.716105e-03
3466Pols0.0020100.0001571181.7165703.67999726.9350702.104062e-0720.5643495.765997e-061.716105e-03
2717Marcouch0.0024410.000314678.1850602.96010926.2342663.024086e-0720.2564686.772409e-061.897071e-03
1053Donald Pols0.0021540.000261723.9606523.04257123.8719441.029615e-0618.2092271.979176e-055.236021e-03
3971Sjoerdsma0.0018670.000209792.6240403.15804721.5428623.460080e-0616.1783005.765077e-051.372665e-02
2734Marianne Thieme0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
2583Lotte van Kruistum0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
3003Mirra Andreeva0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
2351Kluytmans0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
249Antonia Niedermaier0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
1791Ilse Kuijt0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
742Cobolli0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
4724Zoë Slagter0.0011490.0000264294.4568115.45755919.7557668.799584e-0614.2812401.574261e-042.404300e-02
278Arjan Veurink0.0015800.000157907.0630193.33207419.2680291.135926e-0514.1744841.666147e-042.404300e-02
3098Naomi Mestrum0.0012920.0001051135.9409783.62752517.0913663.562389e-0512.2215434.724085e-046.103910e-02
2665Maja Chwalinska0.0015800.000209655.2972642.91703916.6866824.408944e-0512.2142284.742643e-046.103910e-02
4420Van der Breggen0.0014360.000157815.5118363.19457016.7722354.214548e-0512.1315684.957543e-046.212584e-02
1731Hitler0.0010050.0000263745.1497095.26491417.0277303.683788e-0511.9056415.596421e-046.345276e-02
2519Lieke0.0010050.0000263745.1497095.26491417.0277303.683788e-0511.9056415.596421e-046.345276e-02
4176Teunissen0.0010050.0000263745.1497095.26491417.0277303.683788e-0511.9056415.596421e-046.345276e-02
4464Veurink0.0010050.0000263745.1497095.26491417.0277303.683788e-0511.9056415.596421e-046.345276e-02
968Denzel Dumfries0.0020100.000418380.6437142.26496515.3594548.887510e-0511.6000926.594857e-047.303421e-02
2329Kimi Antonelli0.0034470.001203186.5950091.51901314.2940651.563571e-0411.5398966.811844e-047.372272e-02
2465Leclerc0.0012920.000157723.9606523.04256714.3187721.543183e-0410.1457811.446319e-031.306897e-01
1250Esther Ouwehand0.0012920.000157723.9606523.04256714.3187721.543183e-0410.1457811.446319e-031.306897e-01
713Christine Teunissen0.0011490.000105998.6142033.45760014.5290341.380160e-0410.1228011.464463e-031.306897e-01
1917Jan Kluytmans0.0011490.000105998.6142033.45760014.5290341.380160e-0410.1228011.464463e-031.306897e-01
2124Joost Luiten0.0011490.000105998.6142033.45760014.5290341.380160e-0410.1228011.464463e-031.306897e-01
\n", "
" ], "text/plain": [ " word freq_recent freq_reference pct_diff \\\n", "4666 Ye 0.009191 0.001412 551.030639 \n", "2233 Kanye West 0.006175 0.001150 436.822849 \n", "2521 Lieke Marsman 0.002298 0.000026 8688.913621 \n", "55 Ahmed Marcouch 0.003016 0.000261 1053.544913 \n", "1343 Flavio Cobolli 0.002010 0.000026 7590.299418 \n", "4362 Van Dissel 0.002585 0.000209 1135.940978 \n", "2797 Marsman 0.001867 0.000026 7040.992317 \n", "2503 Lewis Hamilton 0.004021 0.000784 412.686628 \n", "673 Charles Leclerc 0.002872 0.000418 586.633877 \n", "1858 Jaap van Dissel 0.002872 0.000418 586.633877 \n", "4516 Vollering 0.001867 0.000105 1685.248079 \n", "2923 Mette-Marit 0.004595 0.001307 251.556545 \n", "1801 Ingrid Alexandra 0.002010 0.000157 1181.716570 \n", "3466 Pols 0.002010 0.000157 1181.716570 \n", "2717 Marcouch 0.002441 0.000314 678.185060 \n", "1053 Donald Pols 0.002154 0.000261 723.960652 \n", "3971 Sjoerdsma 0.001867 0.000209 792.624040 \n", "2734 Marianne Thieme 0.001149 0.000026 4294.456811 \n", "2583 Lotte van Kruistum 0.001149 0.000026 4294.456811 \n", "3003 Mirra Andreeva 0.001149 0.000026 4294.456811 \n", "2351 Kluytmans 0.001149 0.000026 4294.456811 \n", "249 Antonia Niedermaier 0.001149 0.000026 4294.456811 \n", "1791 Ilse Kuijt 0.001149 0.000026 4294.456811 \n", "742 Cobolli 0.001149 0.000026 4294.456811 \n", "4724 Zoë Slagter 0.001149 0.000026 4294.456811 \n", "278 Arjan Veurink 0.001580 0.000157 907.063019 \n", "3098 Naomi Mestrum 0.001292 0.000105 1135.940978 \n", "2665 Maja Chwalinska 0.001580 0.000209 655.297264 \n", "4420 Van der Breggen 0.001436 0.000157 815.511836 \n", "1731 Hitler 0.001005 0.000026 3745.149709 \n", "2519 Lieke 0.001005 0.000026 3745.149709 \n", "4176 Teunissen 0.001005 0.000026 3745.149709 \n", "4464 Veurink 0.001005 0.000026 3745.149709 \n", "968 Denzel Dumfries 0.002010 0.000418 380.643714 \n", "2329 Kimi Antonelli 0.003447 0.001203 186.595009 \n", "2465 Leclerc 0.001292 0.000157 723.960652 \n", "1250 Esther Ouwehand 0.001292 0.000157 723.960652 \n", "713 Christine Teunissen 0.001149 0.000105 998.614203 \n", "1917 Jan Kluytmans 0.001149 0.000105 998.614203 \n", "2124 Joost Luiten 0.001149 0.000105 998.614203 \n", "\n", " log_ratio chi2 p_chi2 g2 p_g2 \\\n", "4666 2.702725 88.873013 4.209875e-21 73.611094 9.512852e-18 \n", "2233 2.424445 51.860997 5.957208e-13 42.512795 7.022134e-11 \n", "2521 6.457558 41.675092 1.077733e-10 34.046050 5.382305e-09 \n", "55 3.527997 38.897401 4.466750e-10 30.774198 2.898665e-08 \n", "1343 6.264913 36.186456 1.793125e-09 29.023926 7.148986e-08 \n", "4362 3.627532 34.197157 4.980167e-09 26.757903 2.306067e-07 \n", "2797 6.157998 33.443641 7.335924e-09 26.527883 2.597616e-07 \n", "2503 2.358076 32.500200 1.191803e-08 26.104423 3.234422e-07 \n", "673 2.779538 28.670638 8.579474e-08 22.442992 2.164739e-06 \n", "1858 2.779538 28.670638 8.579474e-08 22.442992 2.164739e-06 \n", "4516 4.158040 27.590589 1.499064e-07 21.070561 4.426770e-06 \n", "2923 1.813756 25.318681 4.859791e-07 20.713510 5.333836e-06 \n", "1801 3.679997 26.935070 2.104062e-07 20.564349 5.765997e-06 \n", "3466 3.679997 26.935070 2.104062e-07 20.564349 5.765997e-06 \n", "2717 2.960109 26.234266 3.024086e-07 20.256468 6.772409e-06 \n", "1053 3.042571 23.871944 1.029615e-06 18.209227 1.979176e-05 \n", "3971 3.158047 21.542862 3.460080e-06 16.178300 5.765077e-05 \n", "2734 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "2583 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "3003 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "2351 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "249 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "1791 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "742 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "4724 5.457559 19.755766 8.799584e-06 14.281240 1.574261e-04 \n", "278 3.332074 19.268029 1.135926e-05 14.174484 1.666147e-04 \n", "3098 3.627525 17.091366 3.562389e-05 12.221543 4.724085e-04 \n", "2665 2.917039 16.686682 4.408944e-05 12.214228 4.742643e-04 \n", "4420 3.194570 16.772235 4.214548e-05 12.131568 4.957543e-04 \n", "1731 5.264914 17.027730 3.683788e-05 11.905641 5.596421e-04 \n", "2519 5.264914 17.027730 3.683788e-05 11.905641 5.596421e-04 \n", "4176 5.264914 17.027730 3.683788e-05 11.905641 5.596421e-04 \n", "4464 5.264914 17.027730 3.683788e-05 11.905641 5.596421e-04 \n", "968 2.264965 15.359454 8.887510e-05 11.600092 6.594857e-04 \n", "2329 1.519013 14.294065 1.563571e-04 11.539896 6.811844e-04 \n", "2465 3.042567 14.318772 1.543183e-04 10.145781 1.446319e-03 \n", "1250 3.042567 14.318772 1.543183e-04 10.145781 1.446319e-03 \n", "713 3.457600 14.529034 1.380160e-04 10.122801 1.464463e-03 \n", "1917 3.457600 14.529034 1.380160e-04 10.122801 1.464463e-03 \n", "2124 3.457600 14.529034 1.380160e-04 10.122801 1.464463e-03 \n", "\n", " p_g2_adjusted \n", "4666 2.265010e-14 \n", "2233 1.114647e-07 \n", "2521 6.407634e-06 \n", "55 2.760689e-05 \n", "1343 5.673912e-05 \n", "4362 1.546231e-04 \n", "2797 1.546231e-04 \n", "2503 1.711368e-04 \n", "673 9.371352e-04 \n", "1858 9.371352e-04 \n", "4516 1.716105e-03 \n", "2923 1.716105e-03 \n", "1801 1.716105e-03 \n", "3466 1.716105e-03 \n", "2717 1.897071e-03 \n", "1053 5.236021e-03 \n", "3971 1.372665e-02 \n", "2734 2.404300e-02 \n", "2583 2.404300e-02 \n", "3003 2.404300e-02 \n", "2351 2.404300e-02 \n", "249 2.404300e-02 \n", "1791 2.404300e-02 \n", "742 2.404300e-02 \n", "4724 2.404300e-02 \n", "278 2.404300e-02 \n", "3098 6.103910e-02 \n", "2665 6.103910e-02 \n", "4420 6.212584e-02 \n", "1731 6.345276e-02 \n", "2519 6.345276e-02 \n", "4176 6.345276e-02 \n", "4464 6.345276e-02 \n", "968 7.303421e-02 \n", "2329 7.372272e-02 \n", "2465 1.306897e-01 \n", "1250 1.306897e-01 \n", "713 1.306897e-01 \n", "1917 1.306897e-01 \n", "2124 1.306897e-01 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# stijgers\n", "results[results.pct_diff > 0].sort_values('g2', ascending=False)[:40]" ] }, { "cell_type": "code", "execution_count": 9, "id": "bccce269-9540-445e-a5f9-775d55b469b9", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
wordfreq_recentfreq_referencepct_difflog_ratiochi2p_chi2g2p_g2p_g2_adjusted
1743Hondius0.0004310.010457-95.880197-4.60127866.4705173.551664e-1698.2757923.639886e-231.733314e-19
4215Thymen Arensman0.0000720.002353-96.948294-5.03422115.2569199.383239e-0520.6864645.409706e-061.716105e-03
1508Gidi Markuszower0.0000720.002091-96.566831-4.86429613.4341572.470840e-0417.8116402.438866e-056.112569e-03
4495Vingegaard0.0000720.001882-96.185367-4.71229311.9771365.385731e-0415.5345578.101072e-051.837014e-02
2782Markuszower0.0000720.001830-96.076378-4.67165111.6130726.548989e-0414.9690181.092909e-042.365652e-02
2109Jonas Vingegaard0.0004310.002667-83.843909-2.62984712.3539054.400650e-0414.7071791.255674e-042.404300e-02
3750Rubio0.0002870.002301-87.515748-3.00181411.7571256.061077e-0414.2123001.632996e-042.404300e-02
115Ali B0.0000720.001725-95.838583-4.58676210.8852069.693541e-0413.8430231.987332e-042.783434e-02
512Botic van de Zandschulp0.0000720.001621-95.570104-4.49656410.1577241.436979e-0312.7245023.608960e-044.910248e-02
921De Jong0.0005740.002562-77.579302-2.1570959.9473171.610840e-0311.2208378.088400e-048.559325e-02
4034Starmer0.0005740.002510-77.112204-2.1273489.6118421.933266e-0310.7780881.027087e-031.060152e-01
1054Donald Trump0.0101960.015529-34.342080-0.60695910.4400521.233118e-0310.7437001.046349e-031.060152e-01
4169Tedros Adhanom Ghebreyesus0.0000720.001412-94.913823-4.2972558.7041543.174857e-0310.5140211.184722e-031.175343e-01
269Arensman0.0000720.001359-94.718201-4.2428088.3411103.875791e-039.9678781.592950e-031.306897e-01
3185Noam Bettan0.0000720.001307-94.506929-4.1862247.9782314.734318e-039.4247252.140792e-031.396500e-01
3082Máxima0.0020100.004497-55.288957-1.1612968.2633544.045333e-038.6399083.288780e-032.088156e-01
1099Dylan Groenewegen0.0000720.001203-94.029271-4.0659307.2530407.078107e-038.3483883.860297e-032.418781e-01
4258Tom Berendsen0.0005740.002196-73.842519-1.9347037.6266705.751139e-038.2043694.178964e-032.456818e-01
4598Willem-Alexander0.0014360.003555-59.609772-1.3079217.6926035.544761e-038.0442264.564892e-032.587859e-01
3222Oceanwide Expeditions0.0000720.001150-93.757874-4.0018006.8907718.664187e-037.8157865.179183e-032.603795e-01
1411Freek0.0000720.001150-93.757874-4.0018006.8907718.664187e-037.8157865.179183e-032.603795e-01
2226Kaja Kallas0.0000720.001098-93.460630-3.9346866.5287551.061442e-027.2873506.944183e-032.603795e-01
4454Verhoeven0.0000720.001098-93.460630-3.9346866.5287551.061442e-027.2873506.944183e-032.603795e-01
1401Fred Rutten0.0000720.001098-93.460630-3.9346866.5287551.061442e-027.2873506.944183e-032.603795e-01
1981Jerdy Schouten0.0000720.001046-93.133661-3.8642966.1670241.301534e-026.7634859.304187e-033.370342e-01
2977Mikel Arteta0.0000720.001046-93.133661-3.8642966.1670241.301534e-026.7634859.304187e-033.370342e-01
3013Modi0.0000720.001046-93.133661-3.8642966.1670241.301534e-026.7634859.304187e-033.370342e-01
3237Oleksandr Usyk0.0000720.001046-93.133661-3.8642966.1670241.301534e-026.7634859.304187e-033.370342e-01
417Berendsen0.0007180.002248-68.063541-1.6467226.5093751.073072e-026.7561719.342400e-033.370342e-01
2016Jetten0.0024410.004601-46.941928-0.9143565.9410621.479210e-025.9690031.455951e-024.952314e-01
1310Felix Gall0.0000720.000941-92.370735-3.7122935.4445871.962906e-025.7314101.666410e-025.501480e-01
1167Ellen ten Damme0.0000720.000941-92.370735-3.7122935.4445871.962906e-025.7314101.666410e-025.501480e-01
4101Suzan & Freek0.0000720.000941-92.370735-3.7122935.4445871.962906e-025.7314101.666410e-025.501480e-01
4141Tallon Griekspoor0.0004310.001621-73.420624-1.9116185.5547151.843098e-025.6500351.745502e-025.501480e-01
4655Xi Jinping0.0011490.002719-57.745608-1.2428265.4841341.918982e-025.4885241.914170e-025.501480e-01
133Amalia0.0004310.001569-72.534645-1.8643135.2309922.218788e-025.2491972.195690e-025.501480e-01
3767Rutten0.0000720.000889-91.921954-3.6298315.0839882.414777e-025.2243772.227242e-025.501480e-01
2970Mika Godts0.0000720.000889-91.921954-3.6298315.0839882.414777e-025.2243772.227242e-025.501480e-01
537Brekelmans0.0000720.000889-91.921954-3.6298315.0839882.414777e-025.2243772.227242e-025.501480e-01
2712Marco Rubio0.0008620.002248-61.676249-1.3836885.2361002.212281e-025.2139442.240645e-025.501480e-01
\n", "
" ], "text/plain": [ " word freq_recent freq_reference pct_diff \\\n", "1743 Hondius 0.000431 0.010457 -95.880197 \n", "4215 Thymen Arensman 0.000072 0.002353 -96.948294 \n", "1508 Gidi Markuszower 0.000072 0.002091 -96.566831 \n", "4495 Vingegaard 0.000072 0.001882 -96.185367 \n", "2782 Markuszower 0.000072 0.001830 -96.076378 \n", "2109 Jonas Vingegaard 0.000431 0.002667 -83.843909 \n", "3750 Rubio 0.000287 0.002301 -87.515748 \n", "115 Ali B 0.000072 0.001725 -95.838583 \n", "512 Botic van de Zandschulp 0.000072 0.001621 -95.570104 \n", "921 De Jong 0.000574 0.002562 -77.579302 \n", "4034 Starmer 0.000574 0.002510 -77.112204 \n", "1054 Donald Trump 0.010196 0.015529 -34.342080 \n", "4169 Tedros Adhanom Ghebreyesus 0.000072 0.001412 -94.913823 \n", "269 Arensman 0.000072 0.001359 -94.718201 \n", "3185 Noam Bettan 0.000072 0.001307 -94.506929 \n", "3082 Máxima 0.002010 0.004497 -55.288957 \n", "1099 Dylan Groenewegen 0.000072 0.001203 -94.029271 \n", "4258 Tom Berendsen 0.000574 0.002196 -73.842519 \n", "4598 Willem-Alexander 0.001436 0.003555 -59.609772 \n", "3222 Oceanwide Expeditions 0.000072 0.001150 -93.757874 \n", "1411 Freek 0.000072 0.001150 -93.757874 \n", "2226 Kaja Kallas 0.000072 0.001098 -93.460630 \n", "4454 Verhoeven 0.000072 0.001098 -93.460630 \n", "1401 Fred Rutten 0.000072 0.001098 -93.460630 \n", "1981 Jerdy Schouten 0.000072 0.001046 -93.133661 \n", "2977 Mikel Arteta 0.000072 0.001046 -93.133661 \n", "3013 Modi 0.000072 0.001046 -93.133661 \n", "3237 Oleksandr Usyk 0.000072 0.001046 -93.133661 \n", "417 Berendsen 0.000718 0.002248 -68.063541 \n", "2016 Jetten 0.002441 0.004601 -46.941928 \n", "1310 Felix Gall 0.000072 0.000941 -92.370735 \n", "1167 Ellen ten Damme 0.000072 0.000941 -92.370735 \n", "4101 Suzan & Freek 0.000072 0.000941 -92.370735 \n", "4141 Tallon Griekspoor 0.000431 0.001621 -73.420624 \n", "4655 Xi Jinping 0.001149 0.002719 -57.745608 \n", "133 Amalia 0.000431 0.001569 -72.534645 \n", "3767 Rutten 0.000072 0.000889 -91.921954 \n", "2970 Mika Godts 0.000072 0.000889 -91.921954 \n", "537 Brekelmans 0.000072 0.000889 -91.921954 \n", "2712 Marco Rubio 0.000862 0.002248 -61.676249 \n", "\n", " log_ratio chi2 p_chi2 g2 p_g2 \\\n", "1743 -4.601278 66.470517 3.551664e-16 98.275792 3.639886e-23 \n", "4215 -5.034221 15.256919 9.383239e-05 20.686464 5.409706e-06 \n", "1508 -4.864296 13.434157 2.470840e-04 17.811640 2.438866e-05 \n", "4495 -4.712293 11.977136 5.385731e-04 15.534557 8.101072e-05 \n", "2782 -4.671651 11.613072 6.548989e-04 14.969018 1.092909e-04 \n", "2109 -2.629847 12.353905 4.400650e-04 14.707179 1.255674e-04 \n", "3750 -3.001814 11.757125 6.061077e-04 14.212300 1.632996e-04 \n", "115 -4.586762 10.885206 9.693541e-04 13.843023 1.987332e-04 \n", "512 -4.496564 10.157724 1.436979e-03 12.724502 3.608960e-04 \n", "921 -2.157095 9.947317 1.610840e-03 11.220837 8.088400e-04 \n", "4034 -2.127348 9.611842 1.933266e-03 10.778088 1.027087e-03 \n", "1054 -0.606959 10.440052 1.233118e-03 10.743700 1.046349e-03 \n", "4169 -4.297255 8.704154 3.174857e-03 10.514021 1.184722e-03 \n", "269 -4.242808 8.341110 3.875791e-03 9.967878 1.592950e-03 \n", "3185 -4.186224 7.978231 4.734318e-03 9.424725 2.140792e-03 \n", "3082 -1.161296 8.263354 4.045333e-03 8.639908 3.288780e-03 \n", "1099 -4.065930 7.253040 7.078107e-03 8.348388 3.860297e-03 \n", "4258 -1.934703 7.626670 5.751139e-03 8.204369 4.178964e-03 \n", "4598 -1.307921 7.692603 5.544761e-03 8.044226 4.564892e-03 \n", "3222 -4.001800 6.890771 8.664187e-03 7.815786 5.179183e-03 \n", "1411 -4.001800 6.890771 8.664187e-03 7.815786 5.179183e-03 \n", "2226 -3.934686 6.528755 1.061442e-02 7.287350 6.944183e-03 \n", "4454 -3.934686 6.528755 1.061442e-02 7.287350 6.944183e-03 \n", "1401 -3.934686 6.528755 1.061442e-02 7.287350 6.944183e-03 \n", "1981 -3.864296 6.167024 1.301534e-02 6.763485 9.304187e-03 \n", "2977 -3.864296 6.167024 1.301534e-02 6.763485 9.304187e-03 \n", "3013 -3.864296 6.167024 1.301534e-02 6.763485 9.304187e-03 \n", "3237 -3.864296 6.167024 1.301534e-02 6.763485 9.304187e-03 \n", "417 -1.646722 6.509375 1.073072e-02 6.756171 9.342400e-03 \n", "2016 -0.914356 5.941062 1.479210e-02 5.969003 1.455951e-02 \n", "1310 -3.712293 5.444587 1.962906e-02 5.731410 1.666410e-02 \n", "1167 -3.712293 5.444587 1.962906e-02 5.731410 1.666410e-02 \n", "4101 -3.712293 5.444587 1.962906e-02 5.731410 1.666410e-02 \n", "4141 -1.911618 5.554715 1.843098e-02 5.650035 1.745502e-02 \n", "4655 -1.242826 5.484134 1.918982e-02 5.488524 1.914170e-02 \n", "133 -1.864313 5.230992 2.218788e-02 5.249197 2.195690e-02 \n", "3767 -3.629831 5.083988 2.414777e-02 5.224377 2.227242e-02 \n", "2970 -3.629831 5.083988 2.414777e-02 5.224377 2.227242e-02 \n", "537 -3.629831 5.083988 2.414777e-02 5.224377 2.227242e-02 \n", "2712 -1.383688 5.236100 2.212281e-02 5.213944 2.240645e-02 \n", "\n", " p_g2_adjusted \n", "1743 1.733314e-19 \n", "4215 1.716105e-03 \n", "1508 6.112569e-03 \n", "4495 1.837014e-02 \n", "2782 2.365652e-02 \n", "2109 2.404300e-02 \n", "3750 2.404300e-02 \n", "115 2.783434e-02 \n", "512 4.910248e-02 \n", "921 8.559325e-02 \n", "4034 1.060152e-01 \n", "1054 1.060152e-01 \n", "4169 1.175343e-01 \n", "269 1.306897e-01 \n", "3185 1.396500e-01 \n", "3082 2.088156e-01 \n", "1099 2.418781e-01 \n", "4258 2.456818e-01 \n", "4598 2.587859e-01 \n", "3222 2.603795e-01 \n", "1411 2.603795e-01 \n", "2226 2.603795e-01 \n", "4454 2.603795e-01 \n", "1401 2.603795e-01 \n", "1981 3.370342e-01 \n", "2977 3.370342e-01 \n", "3013 3.370342e-01 \n", "3237 3.370342e-01 \n", "417 3.370342e-01 \n", "2016 4.952314e-01 \n", "1310 5.501480e-01 \n", "1167 5.501480e-01 \n", "4101 5.501480e-01 \n", "4141 5.501480e-01 \n", "4655 5.501480e-01 \n", "133 5.501480e-01 \n", "3767 5.501480e-01 \n", "2970 5.501480e-01 \n", "537 5.501480e-01 \n", "2712 5.501480e-01 " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# dalers\n", "results[results.pct_diff < 0].sort_values('g2', ascending=False)[:40]" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 5 }