p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

www per bron

Browse Source
This commit is contained in:
Peter Kleiweg
2026-04-04 16:31:12 +02:00
parent bb2e90dafe
commit 913bb8079f
16 changed files with 23058 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
collect2.sh Executable file
View File

@@ -0,0 +1,95 @@
#!/bin/bash
set -e
unset CDPATH
PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
export TZ=Europe/Amsterdam
if [ "$1" = "" ]
then
ds=`ISOWeek -7`
else
case "$1" in
2[0-9][0-9][0-9]-[0-5][0-9])
ds=$1
;;
*)
echo INVALID
exit 1
;;
esac
fi
cd /net/corpora/nlnieuws/data
for corpus in AT5 GG NOS NU NieuwsNL RO Sargasso Sikkom Tzum VRT
do
for i in 1 4
do
files=$(find ../$corpus $(week2files $ds $i))
alto \
'fp://node[((@cat="mwu" and node[@pt="spec"]) or (@pt and @*="eigen" and not(@rel="mwp"))) and not(@his="normal") and not(@his_1="decap" or @his_1="0")]' \
tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-nieuw-namen-$ds-$i
top20 $corpus-nieuw-namen-$ds-$i
# score $corpus-nieuw-namen-$ds-$i > $corpus-nieuw-namen-$ds-$i.score
alto \
'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-nieuw-woorden-$ds-$i
top20 $corpus-nieuw-woorden-$ds-$i
# score $corpus-nieuw-woorden-$ds-$i > $corpus-nieuw-woorden-$ds-$i.score
alto \
'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
'tt:%w\t%l\t%P' $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-nieuw-woorden-extra-$ds-$i
top20 $corpus-nieuw-woorden-extra-$ds-$i
# score $corpus-nieuw-woorden-extra-$ds-$i > $corpus-nieuw-woorden-extra-$ds-$i.score
alto \
'fp://node[(@neclass="LOC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="LOC"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-locaties-$ds-$i
# score $corpus-locaties-$ds-$i > $corpus-locaties-$ds-$i.score
alto \
'fp://node[(@neclass="PER" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="PER"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-personen-$ds-$i
# score $corpus-personen-$ds-$i > $corpus-personen-$ds-$i.score
alto \
'fp://node[(@neclass="ORG" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="ORG"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-organisaties-$ds-$i
# score $corpus-organisaties-$ds-$i > $corpus-organisaties-$ds-$i.score
alto \
'fp://node[(@neclass="MISC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="MISC"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-overige-namen-$ds-$i
# score $corpus-overige-namen-$ds-$i > $corpus-overige-namen-$ds-$i.score
alto \
'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
'tt:%w\t%P' $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > $corpus-nieuw-adjww-extra-$ds-$i
# score $corpus-nieuws-adjww-extra-$ds-$i > $corpus-nieuw-adjww-extra-$ds-$i.score
done
done