p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cats en tags

Browse Source
This commit is contained in:
Peter Kleiweg
2026-04-01 19:30:10 +02:00
parent 1fb1867550
commit e550b58889
14 changed files with 61 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
NOS/txt2corpus.sh
View File

@@ -58,6 +58,10 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 9 -s $corpus.data.dz > $corpus.cat.txt
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

12
NU/cmd/nu/nu.go
View File

@@ -211,11 +211,11 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
return false
}
cats := make([]string, 0)
tags := make([]string, 0)
var buffer bytes.Buffer
for _, i := range doc.Graph {
p(buffer.WriteString(html.UnescapeString(i.ArticleBody)))
cats = append(cats, i.ArticleSection...)
tags = append(tags, i.ArticleSection...)
}
text := buffer.String()
@@ -245,11 +245,11 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
fp, err := os.Create(filename + ".txt")
p(err)
if len(cats) == 0 {
p(fmt.Fprintln(fp, "##META text cat ="))
if len(tags) == 0 {
p(fmt.Fprintln(fp, "##META text tag ="))
} else {
for _, cat := range cats {
p(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(cat)))
for _, tag := range tags {
p(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(tag)))
}
}
p(fp.WriteString(text))

3
NU/txt2corpus.sh
View File

@@ -56,6 +56,9 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

10
NieuwsNL/cmd/nieuwsnl/nieuwsnl.go
View File

@@ -161,19 +161,19 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
root := doc.Root()
var cat string
var tag string
aa, err := root.Search(`//a[contains(@class, "articleHeader__info__category")]`)
p(err)
if len(aa) == 0 {
p(fmt.Fprintln(&buf, "##META text cat ="))
_ = w(fmt.Errorf("no cat: %s", url))
p(fmt.Fprintln(&buf, "##META text tag ="))
_ = w(fmt.Errorf("no tag: %s", url))
// geen fout, maar waarschuwing als er meer fouten zijn
fouten = append(fouten, fmt.Sprintf("no text: %s\n", url))
// dus geen return false
} else {
for _, a := range aa {
cat = strings.ReplaceAll(a.Content(), "\n", " ")
p(fmt.Fprintf(&buf, "##META text cat = %s\n", fixSpace(cat)))
tag = strings.ReplaceAll(a.Content(), "\n", " ")
p(fmt.Fprintf(&buf, "##META text tag = %s\n", fixSpace(tag)))
}
}

3
NieuwsNL/txt2corpus.sh
View File

@@ -58,6 +58,9 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

2
RO/cmd/xml2txt/xml2txt.go
View File

@@ -90,7 +90,7 @@ func main() {
fp, err := os.Create("out/" + filename[:len(filename)-4] + ".txt")
x(err)
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
}
x(fp.WriteString(text))
x(fp.Close())

3
RO/txt2corpus.sh
View File

@@ -58,6 +58,9 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

2
Sargasso/cmd/xml2txt/xml2txt.go
View File

@@ -57,7 +57,7 @@ func main() {
var item Item
x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
}
x(fp.WriteString(addEnd(fixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))

3
Sargasso/txt2corpus.sh
View File

@@ -58,6 +58,9 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

2
Tzum/cmd/xml2txt/xml2txt.go
View File

@@ -57,7 +57,7 @@ func main() {
var item Item
x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
}
x(fp.WriteString(addEnd(fixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))

3
Tzum/txt2corpus.sh
View File

@@ -58,6 +58,9 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

18
VRT/cmd/vrt/vrt.go
View File

@@ -151,7 +151,9 @@ func main() {
}
}
func doArticle(filename string, url string, title string, tags []string, labels []string, timestamp time.Time, needUpdate bool) bool {
// Nstag -> tag
// Nslabeltag -> cat
func doArticle(filename string, url string, title string, tags []string, cats []string, timestamp time.Time, needUpdate bool) bool {
if exists(filename + ".skip") {
return true
}
@@ -236,18 +238,18 @@ func doArticle(filename string, url string, title string, tags []string, labels
}
}
if len(tags) == 0 {
if len(cats) == 0 {
p(fmt.Fprintln(&buf, "##META text cat ="))
} else {
for _, tag := range tags {
p(fmt.Fprintf(&buf, "##META text cat = %s\n", fixSpace(tag)))
for _, cat := range cats {
p(fmt.Fprintf(&buf, "##META text cat = %s\n", fixSpace(cat)))
}
}
if len(labels) == 0 {
p(fmt.Fprintln(&buf, "##META text label ="))
if len(tags) == 0 {
p(fmt.Fprintln(&buf, "##META text tag ="))
} else {
for _, label := range labels {
p(fmt.Fprintf(&buf, "##META text label = %s\n", fixSpace(label)))
for _, tag := range tags {
p(fmt.Fprintf(&buf, "##META text tag = %s\n", fixSpace(tag)))
}
}

4
VRT/txt2corpus.sh
View File

@@ -56,6 +56,10 @@ Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.
cd xml
alto -o $corpus.data.dz *.xml 2> /dev/null
# telling per bericht, niet per zin
/net/corpora/nlnieuws/namen.sh -x 9 -s $corpus.data.dz > $corpus.cat.txt
/net/corpora/nlnieuws/namen.sh -x 10 -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out

14
namen.sh
View File

@@ -20,6 +20,8 @@ gebruik:
6 : bestaande organisaties
7 : bestaande andere namen
8 : nieuwe adjectieven, deelwoorden en werkwoorden
9 : categorieën
10 : tags
-i : kies interactief
@@ -114,6 +116,18 @@ case $XN in
TEMPLATE='tt:%w\t%P'
XVALID=1
;;
9)
# categorieën
EXPR='fp://meta[@name="cat"]/@value'
TEMPLATE='tt:%m'
XVALID=1
;;
10)
# tags
EXPR='fp://meta[@name="tag"]/@value'
TEMPLATE='tt:%m'
XVALID=1
;;
'')
;;
*)