p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Peter Kleiweg
2026-06-05 16:05:46 +02:00
parent efa301cc4a
commit a76fa21584
12 changed files with 645 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -38,6 +38,8 @@ Sikkom/sikkom
Tzum/metadata
Tzum/tzum
Tzum/xml2txt
Volkskrant/metadata
Volkskrant/volkskrant
VRT/metadata
VRT/vrt
bin/data2json

1
Makefile
View File

@@ -16,6 +16,7 @@ all:
make -C Sargasso
make -C Sikkom
make -C Tzum
make -C Volkskrant
make -C VRT
make bin/data2json
make bin/dates2json

8
Parool/cmd/parool/parool.go
View File

@@ -280,6 +280,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
}
header := headers[0]
isVideo := false
tags := make([]string, 0)
ell, err := header.Search(`.//*[@data-test-id="article-label"]`)
p(err)
@@ -291,6 +292,9 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
if s != "" && s != "Nieuws" {
tags = append(tags, s)
}
if strings.ToLower(s) == "video" {
isVideo = true
}
}
pars := make([]string, 0)
@@ -344,7 +348,9 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
}
}
if !found {
_ = w(fmt.Errorf("no text, skipping: %s", url))
if !isVideo {
_ = w(fmt.Errorf("no text, skipping: %s", url))
}
fp, err := os.Create(filename + ".skip")
p(fp.WriteString(url + "\n"))
p(err)

2
README.md
View File

@@ -37,6 +37,7 @@ crontab van p209327@colossus
17 * * * * /net/corpora/nlnieuws/Sikkom/sikkom
18 * * * * /net/corpora/nlnieuws/Tzum/tzum
19 * * * * /net/corpora/nlnieuws/VRT/vrt
20 * * * * /net/corpora/nlnieuws/Volkskrant/volkskrant
```
## 2. Teksten verwerken: omzetten naar zinnen, parsen, metadata toevoegen
@@ -53,6 +54,7 @@ crontab van p209327@colossus
0 1 * * * /net/corpora/nlnieuws/NU/txt2corpus.sh
0 1 * * * /net/corpora/nlnieuws/NieuwsNL/txt2corpus.sh
0 1 * * * /net/corpora/nlnieuws/VRT/txt2corpus.sh
0 1 * * * /net/corpora/nlnieuws/Volkskrant/txt2corpus.sh
# weinig data: alleen op dinsdag
0 1 * * 2 /net/corpora/nlnieuws/AT5/txt2corpus.sh
0 1 * * 2 /net/corpora/nlnieuws/BuurtAdam/txt2corpus.sh

23
Tzum/cmd/xml2txt/xml2txt.go
View File

@@ -24,8 +24,6 @@ var (
x = e.ExitErr
reYearWeek = regexp.MustCompile(`^2[0-9][0-9][0-9]\.[0-5][0-9]$`)
reEM = regexp.MustCompile(`::EM::.*?::/EM::`)
reTitle = regexp.MustCompile(`^\p{Lu}`)
)
func main() {
@@ -68,7 +66,7 @@ func main() {
x(fmt.Fprintf(fp, "##META text tag = %s\n", t))
}
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + em1(u.HtmlFixString(item.Text)) + `</body></html>`))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
x(err)
root := doc.Root()
pp, err := root.Search(`//body/p`)
@@ -76,26 +74,9 @@ func main() {
for _, p := range pp {
s := p.Content()
if !strings.Contains(s, "verscheen eerst op Tzum.") {
x(fp.WriteString(em2(u.AddEnd(u.FixSpace(p.Content())))))
x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
}
}
x(fp.Close())
}
}
func em1(s string) string {
return strings.ReplaceAll(
strings.ReplaceAll(s, "<em>", " ::EM::"),
"</em>",
"::/EM:: ")
}
func em2(s string) string {
return reEM.ReplaceAllStringFunc(s, func(s1 string) string {
s1 = s1[6 : len(s1)-7]
if reTitle.MatchString(s1) {
return `"` + s1 + `"`
}
return s1
})
}

9
Volkskrant/Makefile Normal file
View File

@@ -0,0 +1,9 @@
all: \
metadata \
volkskrant
metadata: cmd/metadata/*.go
go build -o $@ $^
volkskrant: cmd/volkskrant/*.go ../internal/util/*.go
go build -o $@ $<

131
Volkskrant/cmd/metadata/metadata.go Normal file
View File

@@ -0,0 +1,131 @@
package main
import (
e "codeberg.org/pebbe/errors"
"bufio"
"encoding/xml"
"fmt"
"html"
"os"
"strings"
"time"
)
type Item struct {
XMLName xml.Name `xml:"item"`
UnixTime int64 `xml:"unixTime"`
}
var (
x = e.ExitErr
escape = html.EscapeString
data = make(map[string][]string)
location *time.Location
)
func main() {
var err error
location, err = time.LoadLocation("Europe/Amsterdam")
x(err)
files, err := os.ReadDir(".")
x(err)
for _, file := range files {
filename := file.Name()
if strings.HasSuffix(filename, ".txt") {
doText("", filename)
} else if strings.HasSuffix(filename, ".xml") {
doXml("", filename)
}
}
files, err = os.ReadDir("..")
x(err)
for _, file := range files {
filename := file.Name()
if strings.HasSuffix(filename, ".txt") {
doText("../", filename)
} else if strings.HasSuffix(filename, ".xml") {
doXml("../", filename)
}
}
files, err = os.ReadDir("xml")
x(err)
for _, file := range files {
filename := file.Name()
if !strings.HasSuffix(filename, ".xml") {
continue
}
aa := strings.Split(filename, ".")
base := strings.Join(aa[1:len(aa)-2], ".")
b, err := os.ReadFile("xml/" + filename)
x(err)
s := string(b)
i := strings.Index(s, "<alpino") + 1
i += strings.Index(s[i:], "<")
fp, err := os.Create("xml/" + filename + ".tmp")
x(err)
x(fp.WriteString(s[:i]))
x(fp.WriteString("<metadata>\n <meta type=\"text\" name=\"source\" value=\"Volkskrant\"/>\n"))
for _, m := range data[base] {
x(fp.WriteString(" " + m + "\n"))
}
x(fp.WriteString(" </metadata>\n "))
x(fp.WriteString(stripMeta(s[i:])))
x(fp.Close())
x(os.Rename("xml/"+filename+".tmp", "xml/"+filename))
}
}
func doText(dirname, filename string) {
base := filename[:len(filename)-4]
if _, ok := data[base]; !ok {
data[base] = make([]string, 0)
}
fp, err := os.Open(dirname + filename)
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {
continue
}
aa := strings.Fields(line)
if len(aa) > 4 {
data[base] = append(data[base],
fmt.Sprintf(`<meta type="%s" name="%s" value="%s"/>`,
aa[1],
escape(aa[2]),
escape(strings.Join(aa[4:], " "))))
}
}
x(scanner.Err())
}
func doXml(dirname, filename string) {
base := filename[:len(filename)-4]
if _, ok := data[base]; !ok {
data[base] = make([]string, 0)
}
b, err := os.ReadFile(dirname + filename)
x(err)
var item Item
x(xml.Unmarshal(b, &item))
t := time.Unix(item.UnixTime, 0).In(location)
data[base] = append(data[base],
fmt.Sprintf(`<meta type="date" name="pubdate" value="%d-%02d-%02d"/>`,
t.Year(),
int(t.Month()),
t.Day()))
}
func stripMeta(s string) string {
i1 := strings.Index(s, "<metadata>")
if i1 < 0 {
return s
}
i2 := i1 + strings.Index(s[i1:], "</metadata>") + 11
return s[:i1] + strings.TrimLeft(s[i2:], " \t\r\n")
}

387
Volkskrant/cmd/volkskrant/volkskrant.go Normal file
View File

@@ -0,0 +1,387 @@
package main
import (
e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
//"encoding/json"
"encoding/xml"
"fmt"
//"html"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
type Rss struct {
XMLName xml.Name `xml:"rss"`
Items []ItemT `xml:"channel>item"`
}
type ItemT struct {
PubDate string `xml:"pubDate"`
UnixTime int64 `xml:"unixTime"`
Guid string `xml:"guid"`
Link string `xml:"link"`
Title string `xml:"title"`
Data []byte `xml:",innerxml"`
}
/*
type GraphT struct {
Graph []map[string]any `json:"@graph"`
}
*/
var (
p = e.PanicErr
w = e.WarnErr
agent = "AhrefsBot/7.0"
)
func exists(filename string) bool {
_, err := os.Stat(filename)
return err == nil
}
func fileDate(filename string) string {
b, err := os.ReadFile(filename)
if err != nil {
return ""
}
s := string(b)
i1 := strings.Index(s, "<unixTime>") + 10
i2 := strings.Index(s, "</unixTime>")
if i2 < i1 {
return ""
}
return s[i1:i2]
}
func main() {
defer func() {
if e.Panicked {
_ = recover()
os.Exit(1)
}
}()
myLock := "/net/corpora/nlnieuws/Volkskrant/lock"
u.MkLock(myLock)
defer func() {
_ = os.Remove(myLock)
}()
req, err := http.NewRequest("GET", "https://www.volkskrant.nl/rss.xml", nil)
p(err)
req.Header.Set("User-Agent", agent)
client := &http.Client{}
resp, err := client.Do(req)
p(err)
body, err := io.ReadAll(resp.Body)
p(err)
p(resp.Body.Close())
var rss Rss
p(xml.Unmarshal(body, &rss))
if len(rss.Items) == 0 {
p(fmt.Errorf("len(rss.Items) == 0"))
}
for _, item := range rss.Items {
t, err := time.Parse(time.RFC1123Z, item.PubDate)
if err != nil {
t, err = time.Parse(time.RFC1123, item.PubDate)
}
p(err)
dirname := fmt.Sprintf("/net/corpora/nlnieuws/Volkskrant/%d/%02d/%02d", t.Year(), int(t.Month()), t.Day())
if exists(dirname + "/lock") {
continue
}
basename := item.Guid
filename := dirname + "/" + url.PathEscape(basename)
ts := fmt.Sprintf("%d", t.Unix())
needUpdate := fileDate(filename+".xml") != ts
p(os.MkdirAll(dirname, 0777))
func() {
var ok bool
defer func() {
if e.Panicked {
fmt.Fprintln(os.Stderr, "----", filename)
fmt.Fprintln(os.Stderr, "----", item.Link)
}
if !ok {
_ = os.Remove(filename + ".xml")
}
}()
fp, err := os.Create(filename + ".xml")
p(err)
p(fp.WriteString("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<item>\n"))
p(fmt.Fprintf(fp, "<unixTime>%d</unixTime>", t.Unix()))
p(fp.Write(item.Data))
p(fp.WriteString("</item>\n"))
p(fp.Close())
p(os.Chtimes(filename+".xml", t, t))
ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
}()
}
}
func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) (ok bool) {
if exists(filename + ".skip") {
return true
}
if needUpdate {
_ = os.Remove(filename + ".err")
_ = os.Remove(filename + ".html")
// _ = os.Remove(filename + ".json")
_ = os.Remove(filename + ".txt")
} else {
if exists(filename + ".txt") {
return true
}
}
time.Sleep(2 * time.Second)
req, err := http.NewRequest("GET", url, nil)
p(err)
req.Header.Set("User-Agent", agent)
client := &http.Client{}
resp, err := client.Do(req)
p(err)
body, err := io.ReadAll(resp.Body)
p(err)
p(resp.Body.Close())
body = u.HtmlFix(body)
doc, err := gokogiri.ParseHtml(body)
p(err)
/*
s := string(body)
ok = true
i1 := strings.Index(s, `<script type="application/ld+json"`)
if i1 < 0 {
ok = false
} else {
i1 += strings.Index(s[i1:], `>`) + 1
i2 := i1 + strings.Index(s[i1:], `</script>`)
if i2 < i1 {
ok = false
} else {
s = html.UnescapeString(s[i1:i2])
}
}
if !ok {
_ = w(fmt.Errorf("script jsonld not found: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "script jsonld not found: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
var graph GraphT
p(json.Unmarshal([]byte(s), &graph))
for _, g := range graph.Graph {
t := g["@type"]
switch v := t.(type) {
case string:
if v == "NewsArticle" {
b, err := json.Marshal(g)
p(err)
s = string(b)
}
}
}
fp, err := os.Create(filename + ".json")
p(err)
p(fp.WriteString(s))
p(fp.Close())
p(os.Chtimes(filename+".json", timestamp, timestamp))
*/
root := doc.Root()
articles, err := root.Search(`//article[@id="article-content"]`)
p(err)
if len(articles) == 0 {
_ = w(fmt.Errorf("empty: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "empty: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
article := articles[0]
live, err := article.Search(`.//*[@data-test-id="live-blog-label"]`)
p(err)
if len(live) > 0 {
fp, err := os.Create(filename + ".skip")
p(fp.WriteString("liveblog\n"))
p(err)
p(os.Chtimes(filename+".skip", timestamp, timestamp))
return true
}
headers, err := article.Search(`.//header`)
p(err)
if len(headers) == 0 {
_ = w(fmt.Errorf("no header: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "no elements: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
header := headers[0]
isOpinie := false
tags := make([]string, 0)
ell, err := header.Search(`.//*[@data-test-id="article-label"]`)
p(err)
if len(ell) == 0 {
_ = w(fmt.Errorf("no labels: %s", url))
}
for _, el := range ell {
s := strings.TrimSpace(el.Content())
if s != "" && s != "Nieuws" {
tags = append(tags, s)
}
if strings.ToLower(s) == "opinie" {
isOpinie = true
}
}
pars := make([]string, 0)
found := false
ell, err = header.Search(`.//*[@data-test-id="header-intro"]`)
p(err)
for _, el := range ell {
s := strings.TrimSpace(el.Content())
if s != "" {
pars = append(pars, s)
found = true
}
}
if !found && !isOpinie {
_ = w(fmt.Errorf("no intro: %s", url))
}
specials, err := article.Search(`.//section//aside | .//section//figure | .//section//b`)
p(err)
for _, special := range specials {
special.Remove()
}
ell, err = article.Search(`.//section//*[@data-article-element-index]`)
p(err)
if len(ell) == 0 {
_ = w(fmt.Errorf("no elements: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "no elements: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
found = false
for _, el := range ell {
s := strings.TrimSpace(el.Content())
if s != "" {
pars = append(pars, s)
found = true
}
}
if !found {
_ = w(fmt.Errorf("no text, skipping: %s", url))
fp, err := os.Create(filename + ".skip")
p(fp.WriteString(url + "\n"))
p(err)
p(os.Chtimes(filename+".skip", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return true
}
fp, err := os.Create(filename + ".txt")
p(err)
if len(tags) == 0 {
p(fmt.Fprintln(fp, "##META text tag ="))
} else {
for _, tag := range tags {
p(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
}
}
p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
for _, par := range pars {
p(fp.WriteString(u.AddEnd(u.FixSpace(par))))
}
p(fp.Close())
p(os.Chtimes(filename+".txt", timestamp, timestamp))
return true
}

70
Volkskrant/txt2corpus.sh Executable file
View File

@@ -0,0 +1,70 @@
#!/bin/bash
set -e
BASE=/net/corpora/nlnieuws
PART=$BASE/Volkskrant
unset CDPATH
PATH=$PART:$BASE/bin:$BASE:/net/aps/bin:$PATH
export TZ=Europe/Amsterdam
. /net/aps/etc/alpino-activate.sh > /dev/null
if [ "$1" = "" ]
then
ds=`date -d -2days +%Y-%m-%d`
else
case "$1" in
2[0-9][0-9][0-9]-[01][0-9]-[0-3][0-9])
ds=$1
;;
*)
echo INVALID
exit 1
;;
esac
fi
dp=${ds//-//}
year=${ds%%-*}
corpus=$PART/corpus/$year/$ds
mkdir -p $PART/corpus/$year
cd $PART/$dp
ln -s lock.$$ lock
if [ "`readlink lock`" != lock.$$ ]
then
echo Getting lock failed
exit 1
fi
rm -fr out
mkdir out
rm -f $corpus.lines
for i in *.txt
do
b=`basename $i .txt`
perl -p -e 's/^\s*//; s/^##META.*\n//' $i | tokenize.sh \
| perl -e '$n = 0; while(<>) { $n++; print("vk.'$b'.$n|$_"); }' \
>> $corpus.lines
done
cd out
mkdir xml
Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.lines 2> $corpus.log
metadata
cd xml
rm -f $corpus.data.dz $corpus.index
alto -q -o $corpus.data.dz *.xml
# telling per bericht, niet per zin
query.sh -x T -s $corpus.data.dz > $corpus.tag.txt
cd ../..
rm -fr out
rm -f lock

3
collect.sh
View File

@@ -43,7 +43,7 @@ cd /net/corpora/nlnieuws/data/$year
declare -A parts
#parts[alles]='.'
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso'
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso|Volkskrant'
parts[amsterdam]='AT5|BuurtAdam|Parool'
parts[groningen]='BuurtGrn|GG|Oog|RTVNoord|Sikkom'
parts[literatuur]='LitNL|Tzum'
@@ -64,6 +64,7 @@ parts[vlaanderen]='HLN|VRT'
#parts[Sargasso]='Sargasso'
#parts[Sikkom]='Sikkom'
#parts[Tzum]='Tzum'
#parts[Volkskrant]='Volkskrant'
#parts[VRT]='VRT'
for part in ${!parts[@]}

24
internal/util/util.go
View File

@@ -11,19 +11,25 @@ import (
)
var (
p = e.PanicErr
reEOL = regexp.MustCompile(`[.!?]['"”’]?$`)
reNEOL = regexp.MustCompile(`[.!?]['"”’]?\p{Lu}\p{Ll}+\.?`)
reLET = regexp.MustCompile(`\p{Lu}`)
reBody = regexp.MustCompile(`<[bB][rR][ /]*>`)
p = e.PanicErr
reEOL = regexp.MustCompile(`[.!?]['"”’]?$`)
reNEOL = regexp.MustCompile(`[.!?]['"”’]?\p{Lu}\p{Ll}+\.?`)
reLET = regexp.MustCompile(`\p{Lu}`)
reBody = regexp.MustCompile(`<[bB][rR][ /]*>`)
reQuotLeft = regexp.MustCompile(`<em>|<i>`)
reQuotRight = regexp.MustCompile(`</em>|</i>`)
)
func HtmlFix(body []byte) []byte {
return reBody.ReplaceAllLiteral(body, []byte(" "))
func HtmlFix(html []byte) []byte {
html = reQuotLeft.ReplaceAllLiteral(html, []byte(" „"))
html = reQuotRight.ReplaceAllLiteral(html, []byte("” "))
return reBody.ReplaceAllLiteral(html, []byte(" "))
}
func HtmlFixString(body string) string {
return reBody.ReplaceAllLiteralString(body, " ")
func HtmlFixString(html string) string {
html = reQuotLeft.ReplaceAllLiteralString(html, " „")
html = reQuotRight.ReplaceAllLiteralString(html, "” ")
return reBody.ReplaceAllLiteralString(html, " ")
}
func AddEnd(s string) string {

23
www/app.html
View File

@@ -110,11 +110,16 @@
<td class="bar"><div id="RO"></div></td>
<td><a href="https://reportersonline.nl/">Reporters Online</a></td>
</tr>
<tr class="last">
<tr>
<td></td>
<td class="bar"><div id="Sargasso"></div></td>
<td><a href="https://sargasso.nl/">Sargasso</a></td>
</tr>
<tr class="last">
<td></td>
<td class="bar"><div id="Volkskrant"></div></td>
<td><a href="https://www.volkskrant.nl/">de Volkskrant</a></td>
</tr>
<tr class="first">
<td>Amsterdam</td>
<td class="bar"><div id="AT5"></div></td>
@@ -122,16 +127,20 @@
</tr>
<tr>
<td></td>
<td class="bar"><div id="BuurtAdam"></div></td>
<td class="bar"><div id="Parool"></div></td>
<td>
<a href="https://indebuurt.nl/amsterdam/">In de buurt Amsterdam</a>
<a href="https://www.parool.nl/amsterdam/"
>Het Parool | Amsterdam</a
>
</td>
</tr>
<tr class="last">
<td></td>
<td class="bar"><div id="Parool"></div></td>
<td class="bar"><div id="BuurtAdam"></div></td>
<td>
<a href="https://www.parool.nl/amsterdam/">Parool Amsterdam</a>
<a href="https://indebuurt.nl/amsterdam/"
>In de buurt | Amsterdam</a
>
</td>
</tr>
<tr class="first">
@@ -147,7 +156,9 @@
<td></td>
<td class="bar"><div id="BuurtGrn"></div></td>
<td>
<a href="https://indebuurt.nl/groningen/">In de buurt Groningen</a>
<a href="https://indebuurt.nl/groningen/"
>In de buurt | Groningen</a
>
</td>
</tr>
<tr>