nlnieuws/collect.sh

#!/bin/bash

set -e

unset CDPATH
PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
export TZ=Europe/Amsterdam

verbose=0
if [ "$1" = "-v" ]
then
    shift
    verbose=1
fi

say () {
    if [ "$verbose" = "1" ]
    then
        echo "$*"
    fi
}

if [ "$1" = "" ]
then
    ds=`ISOWeek -7`
else
    case "$1" in
        2[0-9][0-9][0-9]-[0-5][0-9])
            ds=$1
            ;;
        *)
            echo INVALID
            exit 1
            ;;
    esac
fi

cd /net/corpora/nlnieuws/data

declare -A parts
parts[alles]='.'
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso'
parts[groningen]='GG|Oog|RTVNoord|Sikkom'
parts[AT5]='AT5'
parts[GG]='GG'
parts[NOS]='NOS'
parts[NU]='NU'
parts[NieuwsNL]='NieuwsNL'
parts[RO]='RO'
parts[Sargasso]='Sargasso'
parts[Sikkom]='Sikkom'
parts[Tzum]='Tzum'
parts[VRT]='VRT'

for part in ${!parts[@]}
do
    regex=${parts[$part]}

    for i in 1 4
    do
        files=$(find .. $(week2files $ds $i) | grep -E "$regex")

        # tellingen met tags

        # pubdate soms kan verschillen voor hetzelfde bericht,
        # bijvoorbeeld wanneer een oude versie op zaterdag is verschenen
        # en een bijgewerkte versie op maandag. Dan valt de nieuwe versie in
        # een volgende week (en dus directory) dan de oude versie, en wordt het
        # bericht dus opieuw opgeslagen.
        # Daarom moet pubdate verwijderd worden uit de metadata, om dubbele
        # tellingen te voorkomen.
        # Dit speelt alleen(?) bij atom-feeds, zoals van de VRT.

        say $part-nieuwe-namen-$ds-$i
        alto \
            'fp://node[((@cat="mwu" and node[@pt="spec"]) or (@pt and @*="eigen" and not(@rel="mwp"))) and not(@his="normal") and not(@his_1="decap" or @his_1="0")]' \
            'tt:%w\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-nieuwe-namen-$ds-$i
        top20 $part-nieuwe-namen-$ds-$i

        say $part-nieuwe-woorden-$ds-$i
        alto \
            'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
            'tt:%w\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-nieuwe-woorden-$ds-$i
        top20 $part-nieuwe-woorden-$ds-$i

        say $part-locaties-$ds-$i
        alto \
            'fp://node[(@neclass="LOC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="LOC"])]' \
            'tt:%l\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-locaties-$ds-$i

        say $part-personen-$ds-$i
        alto \
            'fp://node[(@neclass="PER" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="PER"])]' \
            'tt:%l\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-personen-$ds-$i

        say $part-organisaties-$ds-$i
        alto \
            'fp://node[(@neclass="ORG" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="ORG"])]' \
            'tt:%l\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-organisaties-$ds-$i

        say $part-overige-namen-$ds-$i
        alto \
            'fp://node[(@neclass="MISC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="MISC"])]' \
            'tt:%l\t%d\t%I' $files \
            | sed -e 's/pubdate: "[-0-9]*"//' \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | items2count > $part-overige-namen-$ds-$i

        # tellingen met postags

        say $part-nieuwe-woorden-extra-$ds-$i
        alto \
            'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
        his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
            'tt:%w\t%l\t%P\t%I' $files \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
            | grep -v '^ *1 ' \
            | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
            > $part-nieuwe-woorden-extra-$ds-$i

        say $part-nieuwe-adjww-extra-$ds-$i
        alto \
            'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
            'tt:%w\t%P\t%I' $files \
            | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
            | sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
            | grep -v '^ *1 ' \
            | sed -e 's/\([0-9]\)  */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
            > $part-nieuwe-adjww-extra-$ds-$i
    done
done