tags; .De -> . De

oud/xquery/collect.sh.oud

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+unset CDPATH
+PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
+export TZ=Europe/Amsterdam
+
+if [ "$1" = "" ]
+then
+    ds=`date -d -7days +%G-%V`
+else
+    case "$1" in
+        2[0-9][0-9][0-9]-[0-5][0-9])
+            ds=$1
+            ;;
+        *)
+            echo INVALID
+            exit 1
+            ;;
+    esac
+fi
+
+cd /net/corpora/nlnieuws/data
+
+declare -A parts
+parts[alles]='.'
+parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso'
+parts[groningen]='GG|Sikkom'
+parts[AT5]='AT5'
+parts[GG]='GG'
+parts[NOS]='NOS'
+parts[NU]='NU'
+parts[NieuwsNL]='NieuwsNL'
+parts[RO]='RO'
+parts[Sargasso]='Sargasso'
+parts[Sikkom]='Sikkom'
+parts[Tzum]='Tzum'
+parts[VRT]='VRT'
+
+for part in ${!parts[@]}
+do
+    regex=${parts[$part]}
+
+    for i in 1 4
+    do
+        files=$(find .. $(week2files $ds $i) | grep -E "$regex")
+
+        # tellingen met tags
+
+        alto tq:../xquery/nieuwe_namen.xq   $files | sort | uniq | items2count > $part-nieuwe-namen-$ds-$i
+        top20 $part-nieuwe-namen-$ds-$i
+
+        alto tq:../xquery/nieuwe_woorden.xq $files | sort | uniq | items2count > $part-nieuwe-woorden-$ds-$i
+        top20 $part-nieuwe-woorden-$ds-$i
+
+        alto tq:../xquery/locaties.xq       $files | sort | uniq | items2count > $part-locaties-$ds-$i
+
+        alto tq:../xquery/personen.xq       $files | sort | uniq | items2count > $part-personen-$ds-$i
+
+        alto tq:../xquery/organisaties.xq   $files | sort | uniq | items2count > $part-organisaties-$ds-$i
+
+        alto tq:../xquery/overige_namen.xq  $files | sort | uniq | items2count > $part-overige-namen-$ds-$i
+
+        # tellingen met postags
+
+        alto \
+            'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
+        his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
+            'tt:%w\t%l\t%P\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
+            | sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
+            | grep -v '^ *1 ' \
+            | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
+            > $part-nieuwe-woorden-extra-$ds-$i
+
+        alto \
+            'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
+            'tt:%w\t%P\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
+            | sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
+            | grep -v '^ *1 ' \
+            | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
+            > $part-nieuwe-adjww-extra-$ds-$i
+    done
+done