stijgers

oud/rang/collect-rang.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+set -e
+
+unset CDPATH
+PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
+export TZ=Europe/Amsterdam
+
+verbose=0
+if [ "$1" = "-v" ]
+then
+    shift
+    verbose=1
+fi
+
+say () {
+    if [ "$verbose" = "1" ]
+    then
+        echo "$*"
+    fi
+}
+
+if [ "$1" = "" ]
+then
+    ds=`date -d -7days +%G.%V`
+else
+    case "$1" in
+        2[0-9][0-9][0-9].[0-5][0-9])
+            ds=$1
+            ;;
+        *)
+            echo INVALID
+            exit 1
+            ;;
+    esac
+fi
+
+year=${ds%%.*}
+
+mkdir -p /net/corpora/nlnieuws/data/$year
+mkdir -p /net/corpora/nlnieuws/data/json/$year
+cd /net/corpora/nlnieuws/data/$year
+
+declare -A parts
+#parts[alles]='.'
+parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso|Volkskrant'
+#parts[amsterdam]='AT5|BuurtAdam|Parool'
+#parts[groningen]='BuurtGrn|GG|Oog|RTVNoord|Sikkom'
+#parts[literatuur]='LitNL|Tzum'
+#parts[vlaanderen]='HLN|VRT'
+#parts[AT5]='AT5'
+#parts[BuurtAdam]='BuurtAdam'
+#parts[BuurtGrn]='BuurtGrn'
+#parts[GG]='GG'
+#parts[HLN]='HLN'
+#parts[LitNL]='LitNL'
+#parts[NOS]='NOS'
+#parts[NU]='NU'
+#parts[NieuwsNL]='NieuwsNL'
+#parts[Oog]='Oog'
+#parts[Parool]='Parool'
+#parts[RO]='RO'
+#parts[RTVNoord]='RTVNoord'
+#parts[Sargasso]='Sargasso'
+#parts[Sikkom]='Sikkom'
+#parts[Tzum]='Tzum'
+#parts[Volkskrant]='Volkskrant'
+#parts[VRT]='VRT'
+
+for part in ${!parts[@]}
+do
+    regex=${parts[$part]}
+
+    for i in 1 4
+    do
+        files=$(find ../.. $(week2files $ds $i) | grep -E "$regex") || true
+        if [ -z "$files" ]
+        then
+            continue
+        fi
+
+        say $part-rang-word-$ds-$i
+        alto \
+            'fp://node[(@pt and not(@pt="let" or @rel="mwp" or @neclass)) or (@cat="mwu" and not(.//node[@neclass]))]' \
+            'tt:%l\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
+            > $part-rang-word-$ds-$i
+
+        say $part-rang-loc-$ds-$i
+        alto \
+            'fp://node[(@neclass="LOC"  and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="LOC" ])]' \
+            'tt:%l\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
+            > $part-rang-loc-$ds-$i
+
+        say $part-rang-per-$ds-$i
+        alto \
+            'fp://node[(@neclass="PER"  and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="PER" ])]' \
+            'tt:%l\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
+            > $part-rang-per-$ds-$i
+
+        say $part-rang-org-$ds-$i
+        alto \
+            'fp://node[(@neclass="ORG"  and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="ORG" ])]' \
+            'tt:%l\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
+            > $part-rang-org-$ds-$i
+
+        say $part-rang-misc-$ds-$i
+        alto \
+            'fp://node[(@neclass="MISC"  and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="MISC" ])]' \
+            'tt:%l\t%I' $files \
+            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
+            > $part-rang-misc-$ds-$i
+
+    done
+done