update

README.admin.md

@@ -12,7 +12,7 @@ query.sh
 
 ## 1. Verzamelen van berichten
 
-Berichten van NieuwsNL in `NieuwsNL/yyyy/mm/dd/`
+Berichten van NieuwsNL in `NieuwsNL/yyyy/mm/dd/` TODO
 
 Overigen in `[A-Z]*/yyyy/ww/` (weeknummer)
 

README.user.md

@@ -0,0 +1 @@
+TODO

collect.sh

@@ -143,6 +143,14 @@ do
             | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
             | items2count > $part-overige-namen-$ds-$i
 
+        say $part-allewoorden-$ds-$i
+        alto \
+            'fp://node[(@pt and not(@pt="let" or @rel="mwp" or @neclass)) or (@cat="mwu" and not(.//node[@neclass]))]' \
+            'tt:%l\ti%d\t%I' $files \
+            | sed -e 's/pubdate: "[-0-9]*"//' \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
+            | items2count > $part-allewoorden-$ds-$i
+
         # tellingen met tags en postags
 
         say $part-nieuwe-woorden-extra-$ds-$i
@@ -166,18 +174,6 @@ do
             | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
             > $part-nieuwe-adjww-extra-$ds-$i
 
-        # kale tellingen
-
-        say $part-allewoorden-$ds-$i
-        alto \
-            'fp://node[(@pt and not(@pt="let" or @rel="mwp" or @neclass)) or (@cat="mwu" and not(.//node[@neclass]))]' \
-            'tt:%l\t%I' $files \
-            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
-            | sed -e 's/\t.*//' | uniq -c \
-            | grep -v '^ *1 ' \
-            | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
-            > $part-allewoorden-$ds-$i
-
     done
 
     # score

python/stijgers2json.py

@@ -56,17 +56,24 @@ oudfile = sys.argv[1]
 nieuwfile = sys.argv[2]
 jsonfile = sys.argv[3]
 
+extras = {}
 counts_recent = {}
 counts_reference = {}
 
 with open(oudfile, "rt", encoding="utf-8") as fp:
     for line in fp:
         aa = line.split("\t")
-        counts_reference[aa[1].strip()] = int(aa[0])
+        w = aa[1].strip()
+        counts_reference[w] = int(aa[0])
+        if len(aa) > 2:
+            extras[w] = '\t'.join(aa[2:]).strip()
 with open(nieuwfile, "rt", encoding="utf-8") as fp:
     for line in fp:
         aa = line.split("\t")
-        counts_recent[aa[1].strip()] = int(aa[0])
+        w = aa[1].strip()
+        counts_recent[w] = int(aa[0])
+        if len(aa) > 2:
+            extras[w] = '\t'.join(aa[2:]).strip()
 
 for key in counts_recent:
     if not key in counts_reference:
@@ -90,6 +97,11 @@ _, p_adjusted, _, _ = multipletests(p_values, method="fdr_bh")
 for r, p_adj in zip(results, p_adjusted):
     r["p_g2_adjusted"] = p_adj
 
+for i in range(len(results)):
+    results[i]["n"] = int(counts_recent[results[i]["word"]])
+    if len(extras) > 0:
+        results[i]["extras"] = extras.get(results[i]["word"], '')
+
 o = {}
 #o['up'] = sorted([x for x in results if x['p_g2'] < .05 and x['pct_diff'] > 0], key=lambda x: x['g2'], reverse=True)[:40]
 #o['dn'] = sorted([x for x in results if x['p_g2'] < .05 and x['pct_diff'] < 0], key=lambda x: x['g2'], reverse=True)[:40]

r/test-count.R

@@ -1,5 +1,5 @@
-nw <- read.table('data/2026/algemeen-count-per-2026.23-1', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
+nw <- read.table('data/2026/algemeen-allewoorden-2026.23-1', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word", "tags"))
-od <- read.table('data/2026/algemeen-count-per-2026.22-4', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
+od <- read.table('data/2026/algemeen-allewoorden-2026.22-4', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word", "tags"))
 words <- unique(c(od$word, nw$word))
 o <- order(words)
 words <- words[o]
@@ -26,3 +26,8 @@ nieuw[nieuw == 0] <- 0.5
 plot(log(oud), log(nieuw))
 lines(log(range(oud)), log(range(nieuw)))
 identify(log(oud), log(nieuw), labels=words)
+
+
+#plot(oud, nieuw)
+#lines(range(oud), range(nieuw))
+#identify(oud, nieuw, labels=words)