collect.sh

collect.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+
+unset CDPATH
+PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
+export TZ=Europe/Amsterdam
+
+if [ "$1" = "" ]
+then
+    ds=`ISOWeek -7`
+else
+    case "$1" in
+        2[0-9][0-9][0-9]-[0-5][0-9])
+            ds=$1
+            ;;
+        *)
+            echo INVALID
+            exit 1
+            ;;
+    esac
+fi
+
+files=$(find /net/corpora/nlnieuws/ $(week2files $ds))
+
+cd /net/corpora/nlnieuws/data
+
+alto \
+    'fp://node[((@cat="mwu" and node[@pt="spec"]) or (@pt and @*="eigen" and not(@rel="mwp"))) and not(@his="normal") and not(@his_1="decap" or @his_1="0")]' \
+    tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sort -nr > nieuw-namen-$ds
+
+alto \
+    'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
+his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
+    tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sort -nr > nieuw-woorden-$ds
+
+alto \
+    'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
+his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
+    'tt:%w\t%l\t%P' $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > nieuw-extra-$ds
+
+alto \
+    'fp://node[(@neclass="LOC" and @his="normal" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="LOC"] and @his="normal")]' \
+    tt:%l $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > locaties-$ds
+
+alto \
+    'fp://node[(@neclass="PER" and @his="normal" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="PER"] and @his="normal")]' \
+    tt:%l $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > personen-$ds
+
+alto \
+    'fp://node[(@neclass="ORG" and @his="normal" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="ORG"] and @his=" normal")]' \
+    tt:%l $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > organisatie-$ds
+
+alto \
+    'fp://node[(@neclass="MISC" and @his="normal" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="MISC"] and @his="normal")]' \
+    tt:%l $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > overigen-$ds
+
+alto \
+    'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
+    'tt:%w\t%P' $files | sort | uniq -c | grep -v '^ *1 '  | sort -nr > overigen-extra-$ds