p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

groepering en tags

Browse Source
This commit is contained in:
Peter Kleiweg
2026-04-09 17:56:59 +02:00
parent b05ff1a4a9
commit 970e133e65
15 changed files with 308 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
collect.sh
View File

@@ -23,70 +23,64 @@ fi
cd /net/corpora/nlnieuws/data
for i in 1 4
declare -A parts
parts[alles]='.'
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso'
parts[groningen]='GG|Sikkom'
parts[AT5]='AT5'
parts[GG]='GG'
parts[NOS]='NOS'
parts[NU]='NU'
parts[NieuwsNL]='NieuwsNL'
parts[RO]='RO'
parts[Sargasso]='Sargasso'
parts[Sikkom]='Sikkom'
parts[Tzum]='Tzum'
parts[VRT]='VRT'
for part in ${!parts[@]}
do
regex=${parts[$part]}
files=$(find .. $(week2files $ds $i))
for i in 1 4
do
files=$(find .. $(week2files $ds $i) | grep -E "$regex")
alto \
'fp://node[((@cat="mwu" and node[@pt="spec"]) or (@pt and @*="eigen" and not(@rel="mwp"))) and not(@his="normal") and not(@his_1="decap" or @his_1="0")]' \
tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > nieuw-namen-$ds-$i
# tellingen met tags
top20 nieuw-namen-$ds-$i
# score nieuw-namen-$ds-$i > nieuw-namen-$ds-$i.score
alto tq:../xquery/nieuwe_namen.xq $files | sort | uniq | items2count > $part-nieuwe-namen-$ds-$i
top20 $part-nieuwe-namen-$ds-$i
alto \
'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
tt:%w $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > nieuw-woorden-$ds-$i
alto tq:../xquery/nieuwe_woorden.xq $files | sort | uniq | items2count > $part-nieuwe-woorden-$ds-$i
top20 $part-nieuwe-woorden-$ds-$i
top20 nieuw-woorden-$ds-$i
# score nieuw-woorden-$ds-$i > nieuw-woorden-$ds-$i.score
alto tq:../xquery/locaties.xq $files | sort | uniq | items2count > $part-locaties-$ds-$i
alto \
'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
'tt:%w\t%l\t%P' $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > nieuw-woorden-extra-$ds-$i
alto tq:../xquery/personen.xq $files | sort | uniq | items2count > $part-personen-$ds-$i
top20 nieuw-woorden-extra-$ds-$i
# score nieuw-woorden-extra-$ds-$i > nieuw-woorden-extra-$ds-$i.score
alto tq:../xquery/organisaties.xq $files | sort | uniq | items2count > $part-organisaties-$ds-$i
alto \
'fp://node[(@neclass="LOC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="LOC"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > locaties-$ds-$i
alto tq:../xquery/overige_namen.xq $files | sort | uniq | items2count > $part-overige-namen-$ds-$i
# score locaties-$ds-$i > locaties-$ds-$i.score
# tellingen met postags
alto \
'fp://node[(@neclass="PER" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="PER"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > personen-$ds-$i
# score personen-$ds-$i > personen-$ds-$i.score
alto \
'fp://node[(@neclass="ORG" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="ORG"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > organisaties-$ds-$i
# score organisaties-$ds-$i > organisaties-$ds-$i.score
alto \
'fp://node[(@neclass="MISC" and not(@rel="mwp")) or (@cat="mwu" and node[@pt="spec" and @neclass="MISC"])]' \
tt:%l $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > overige-namen-$ds-$i
# score overige-namen-$ds-$i > overige-namen-$ds-$i.score
alto \
'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
'tt:%w\t%P' $files | sort | uniq -c | grep -v '^ *1 ' | sed -e 's/\([0-9]\) */\1\t/' | \
sort -f -k 2 | sort -n -r -k 1,1 -s > nieuw-adjww-extra-$ds-$i
# score nieuws-adjww-extra-$ds-$i > nieuw-adjww-extra-$ds-$i.score
alto \
'fp://node[@his and not(@rel="mwp" or @cat="mwu") and not(@his="normal" or @his="name" or @his="prefix_name" or @his_1="decap" or @
his_1="0" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="within_word_conjunct")]' \
'tt:%w\t%l\t%P\t%I' $files \
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
| sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
| grep -v '^ *1 ' \
| sed -e 's/\([0-9]\) */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
> $part-nieuwe-woorden-extra-$ds-$i
alto \
'fp://node[@pt and @his and not(../@his="normal" or @rel="mwp" or ../@his="name" or ../@his_1="decap") and not(@his="normal" or @his="name" or @his="skip" or @his="robust_skip" or @his="w_dia" or @his="wo_dia" or @his="decap" or @his="within_word_conjunct") and not(@pt="n") ]' \
'tt:%w\t%P\t%I' $files \
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
| sed -e 's/\(.*\)\t.*/\1/' | uniq -c \
| grep -v '^ *1 ' \
| sed -e 's/\([0-9]\) */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
> $part-nieuwe-adjww-extra-$ds-$i
done
done