Compare commits
4 Commits
73c21b4f8f
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
1de42635d0 | ||
|
|
16843570e2 | ||
|
|
c856a372a1 | ||
|
|
2466dbb43c |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -43,6 +43,7 @@ Volkskrant/volkskrant
|
||||
VRT/metadata
|
||||
VRT/vrt
|
||||
bin/data2json
|
||||
bin/data22json
|
||||
bin/dates2json
|
||||
bin/flush
|
||||
bin/items2count
|
||||
|
||||
4
Makefile
4
Makefile
@@ -19,6 +19,7 @@ all:
|
||||
make -C Volkskrant
|
||||
make -C VRT
|
||||
make bin/data2json
|
||||
make bin/data22json
|
||||
make bin/dates2json
|
||||
make bin/flush
|
||||
make bin/items2count
|
||||
@@ -31,6 +32,9 @@ all:
|
||||
bin/data2json: cmd/data2json/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/data22json: cmd/data22json/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/dates2json: cmd/dates2json/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"bufio"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -17,6 +19,7 @@ type Item struct {
|
||||
|
||||
var (
|
||||
x = e.ExitErr
|
||||
escape = html.EscapeString
|
||||
data = make(map[string][]string)
|
||||
location *time.Location
|
||||
)
|
||||
@@ -33,6 +36,9 @@ func main() {
|
||||
if strings.HasSuffix(filename, ".xml") {
|
||||
doXml(filename)
|
||||
}
|
||||
if strings.HasSuffix(filename, ".txt") {
|
||||
doText(filename)
|
||||
}
|
||||
}
|
||||
|
||||
files, err = os.ReadDir("xml")
|
||||
@@ -80,6 +86,32 @@ func doXml(filename string) {
|
||||
t.Day()))
|
||||
}
|
||||
|
||||
func doText(filename string) {
|
||||
base := filename[:len(filename)-4]
|
||||
if _, ok := data[base]; !ok {
|
||||
data[base] = make([]string, 0)
|
||||
}
|
||||
fp, err := os.Open("../" + filename)
|
||||
x(err)
|
||||
defer func() { x(fp.Close()) }()
|
||||
scanner := bufio.NewScanner(fp)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.HasPrefix(line, "##META") {
|
||||
continue
|
||||
}
|
||||
aa := strings.Fields(line)
|
||||
if len(aa) > 4 {
|
||||
data[base] = append(data[base],
|
||||
fmt.Sprintf(`<meta type="%s" name="%s" value="%s"/>`,
|
||||
aa[1],
|
||||
escape(aa[2]),
|
||||
escape(strings.Join(aa[4:], " "))))
|
||||
}
|
||||
}
|
||||
x(scanner.Err())
|
||||
}
|
||||
|
||||
func stripMeta(s string) string {
|
||||
i1 := strings.Index(s, "<metadata>")
|
||||
if i1 < 0 {
|
||||
|
||||
@@ -3,14 +3,18 @@ package main
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
"github.com/jbowtie/gokogiri"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
|
||||
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
|
||||
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -23,18 +27,29 @@ type Rss struct {
|
||||
}
|
||||
|
||||
type ItemT struct {
|
||||
Title string `xml:"title"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
UnixTime int64 `xml:"unixTime"`
|
||||
Guid string `xml:"guid"`
|
||||
Link string `xml:"link"`
|
||||
Data []byte `xml:",innerxml"`
|
||||
Title string `xml:"title"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
UnixTime int64 `xml:"unixTime"`
|
||||
Guid string `xml:"guid"`
|
||||
Link string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link
|
||||
Premium bool `xml:"premium"`
|
||||
Tags []string `xml:"category"`
|
||||
Data []byte `xml:",innerxml"`
|
||||
}
|
||||
|
||||
type LdJson struct {
|
||||
Type string `json:"@type"`
|
||||
IsFree string `json:"isAccessibleForFree"`
|
||||
H1 string `json:"headline"`
|
||||
H2 string `json:"description"`
|
||||
}
|
||||
|
||||
var (
|
||||
p = e.PanicErr
|
||||
w = e.WarnErr
|
||||
agent = "AhrefsBot/7.0"
|
||||
// agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
|
||||
jar *cookiejar.Jar
|
||||
)
|
||||
|
||||
func exists(filename string) bool {
|
||||
@@ -64,26 +79,36 @@ func main() {
|
||||
}
|
||||
}()
|
||||
|
||||
var err error
|
||||
jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
|
||||
p(err)
|
||||
|
||||
myLock := "/net/corpora/nlnieuws/Sikkom/lock"
|
||||
u.MkLock(myLock)
|
||||
defer func() {
|
||||
_ = os.Remove(myLock)
|
||||
}()
|
||||
|
||||
resp, err := http.Get("https://www.sikkom.nl/api/feed/rss")
|
||||
resp, err := http.Get("https://www.sikkom.nl/rss")
|
||||
p(err)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
// zet de default namespace om onderscheid te kunnen maken met overige namespaces
|
||||
b2 := bytes.Replace(body, []byte("<rss"), []byte(`<rss xmlns="https://www.rssboard.org/rss-specification"`), 1)
|
||||
|
||||
var rss Rss
|
||||
p(xml.Unmarshal(body, &rss))
|
||||
p(xml.Unmarshal(b2, &rss))
|
||||
|
||||
if len(rss.Items) == 0 {
|
||||
p(fmt.Errorf("len(rss.Items) == 0"))
|
||||
}
|
||||
|
||||
for _, item := range rss.Items {
|
||||
if item.Premium {
|
||||
continue
|
||||
}
|
||||
t, err := time.Parse(time.RFC1123Z, item.PubDate)
|
||||
if err != nil {
|
||||
t, err = time.Parse(time.RFC1123, item.PubDate)
|
||||
@@ -119,12 +144,12 @@ func main() {
|
||||
p(fp.WriteString("</item>\n"))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".xml", t, t))
|
||||
ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
|
||||
ok = doArticle(filename, item.Link, item.Title, item.Tags, t, needUpdate)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) bool {
|
||||
func doArticle(filename string, url string, title string, tags []string, timestamp time.Time, needUpdate bool) bool {
|
||||
if exists(filename + ".skip") {
|
||||
return true
|
||||
}
|
||||
@@ -145,7 +170,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
req.Header.Set("User-Agent", agent)
|
||||
|
||||
client := &http.Client{}
|
||||
client := &http.Client{Jar: jar}
|
||||
resp, err := client.Do(req)
|
||||
p(err)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
@@ -154,22 +179,31 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
s := string(body)
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
ok := true
|
||||
i1 := strings.Index(s, `"application/ld+json"`)
|
||||
if i1 < 0 {
|
||||
ok = false
|
||||
} else {
|
||||
i1 += strings.Index(s[i1:], `>`) + 1
|
||||
i2 := i1 + strings.Index(s[i1:], `</script>`)
|
||||
if i2 < i1 {
|
||||
ok = false
|
||||
} else {
|
||||
s = html.UnescapeString(s[i1:i2])
|
||||
root := doc.Root()
|
||||
|
||||
ldjsons, err := root.Search(`//script[@type="application/ld+json"]`)
|
||||
p(err)
|
||||
found := false
|
||||
var jsmain LdJson
|
||||
for _, ldjson := range ldjsons {
|
||||
s := ldjson.Content()
|
||||
var js LdJson
|
||||
p(json.Unmarshal([]byte(s), &js))
|
||||
if js.Type == "NewsArticle" {
|
||||
found = true
|
||||
fp, err := os.Create(filename + ".json")
|
||||
p(err)
|
||||
p(fp.WriteString(s))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".json", timestamp, timestamp))
|
||||
jsmain = js
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
if !found {
|
||||
_ = w(fmt.Errorf("script jsonld not found: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
@@ -183,30 +217,24 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
fp, err := os.Create(filename + ".json")
|
||||
locations, err := root.Search(`//span[starts-with(@class, "location")]`)
|
||||
p(err)
|
||||
p(fp.WriteString(s))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".json", timestamp, timestamp))
|
||||
for _, location := range locations {
|
||||
location.Remove()
|
||||
}
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
articles, err := root.Search(`//article[starts-with(@class, "article")]`)
|
||||
p(err)
|
||||
|
||||
root := doc.Root()
|
||||
|
||||
pp, err := root.Search(`//div[contains(@class,"article-page__body")]//p`)
|
||||
p(err)
|
||||
|
||||
if len(pp) == 0 {
|
||||
_ = w(fmt.Errorf("empty: %s", url))
|
||||
if len(articles) != 1 {
|
||||
msg := fmt.Sprintf("len(articles) == %d in %s", len(articles), url)
|
||||
_ = w(errors.New(msg))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "empty: %s\n", url))
|
||||
p(fmt.Fprintln(fp, msg))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
@@ -220,15 +248,70 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
return false
|
||||
}
|
||||
|
||||
fp, err = os.Create(filename + ".txt")
|
||||
p(err)
|
||||
art := articles[0]
|
||||
|
||||
p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
|
||||
|
||||
for _, p1 := range pp {
|
||||
p(fp.WriteString(u.AddEnd(u.FixSpace(p1.Content()))))
|
||||
text := make([]string, 0)
|
||||
if jsmain.H1 != "" {
|
||||
text = append(text, jsmain.H1)
|
||||
} else {
|
||||
text = append(text, title)
|
||||
}
|
||||
if jsmain.H2 != "" {
|
||||
text = append(text, jsmain.H2)
|
||||
}
|
||||
|
||||
abb, err := art.Search(`.//div[starts-with(@class, "article-body")]`)
|
||||
p(err)
|
||||
|
||||
found = false
|
||||
for _, ab := range abb {
|
||||
ell, err := ab.Search(`.//p | .//h4`)
|
||||
p(err)
|
||||
for _, el := range ell {
|
||||
if el.Name() == "p" {
|
||||
found = true
|
||||
}
|
||||
text = append(text, el.Content())
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
_ = w(fmt.Errorf("empty body: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "empty body: %s\n", url))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
fp, err := os.Create(filename + ".txt")
|
||||
p(err)
|
||||
|
||||
found = false
|
||||
for _, tag := range tags {
|
||||
if tag == "Actueel" || tag == "Nieuws" {
|
||||
continue
|
||||
}
|
||||
p(fmt.Fprintln(fp, "##META text tag =", tag))
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
p(fp.WriteString("##META text tag =\n"))
|
||||
}
|
||||
|
||||
for _, t := range text {
|
||||
p(fp.WriteString(u.AddEnd(u.FixSpace(t))))
|
||||
}
|
||||
p(fp.Close())
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -62,6 +62,9 @@ cd xml
|
||||
rm -f $corpus.data.dz $corpus.index
|
||||
alto -q -o $corpus.data.dz *.xml
|
||||
|
||||
# telling per bericht, niet per zin
|
||||
query.sh -x T -s $corpus.data.dz > $corpus.tag.txt
|
||||
|
||||
cd ../..
|
||||
rm -fr out
|
||||
|
||||
|
||||
244
cmd/data22json/data22json.go
Normal file
244
cmd/data22json/data22json.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Data struct {
|
||||
Year int `json:"year"`
|
||||
Week int `json:"week"`
|
||||
First string `json:"first"`
|
||||
Last string `json:"last"`
|
||||
Period int `json:"period"`
|
||||
Start string `json:"start"`
|
||||
Max int `json:"max"`
|
||||
Sources map[string]int `json:"sources"`
|
||||
Algemeen *Parts `json:"Algemeen"`
|
||||
Groningen *Parts `json:"Groningen"`
|
||||
Amsterdam *Parts `json:"Amsterdam"`
|
||||
Literatuur *Parts `json:"Literatuur"`
|
||||
Vlaanderen *Parts `json:"Vlaanderen"`
|
||||
}
|
||||
|
||||
type Parts struct {
|
||||
Woorden [][5]any `json:"woorden"`
|
||||
Personen [][5]any `json:"personen"`
|
||||
AndereNamen [][5]any `json:"andere namen"`
|
||||
Locaties [][5]any `json:"locaties"`
|
||||
Organisaties [][5]any `json:"organisaties"`
|
||||
}
|
||||
|
||||
type Scores struct {
|
||||
Up []Item `json:"up"`
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Extras string `json:"extras"`
|
||||
G2 float64 `json:"g2"`
|
||||
N int `json:"n"`
|
||||
PG2 float64 `json:"p_g2"`
|
||||
Word string `json:"word"`
|
||||
}
|
||||
|
||||
var (
|
||||
sources = map[string]string{
|
||||
"Algemeen": "algemeen",
|
||||
"Amsterdam": "amsterdam",
|
||||
"Groningen": "groningen",
|
||||
"Literatuur": "literatuur",
|
||||
"Vlaanderen": "vlaanderen",
|
||||
}
|
||||
|
||||
parts = map[string]struct {
|
||||
file string
|
||||
re *regexp.Regexp
|
||||
}{
|
||||
"woorden": {"allewoorden", nil},
|
||||
"personen": {"personen", nil},
|
||||
"andere namen": {"overige-namen", nil},
|
||||
"locaties": {"locaties", nil},
|
||||
"organisaties": {"organisaties", regexp.MustCompile(`^(ANP|AT5)`)},
|
||||
}
|
||||
|
||||
maanden = strings.Fields("x januari februari maart april mei juni juli augustus september oktober november december")
|
||||
dagen = strings.Fields("zondag maandag dinsdag woensdag donderdag vrijdag zaterdag")
|
||||
|
||||
year int
|
||||
week int
|
||||
size int
|
||||
|
||||
x = e.ExitErr
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
aa := strings.Split(os.Args[1], ".")
|
||||
if len(aa) != 2 {
|
||||
x(fmt.Errorf("ongeldig argument, moet in formaat yyyy.dd zijn"))
|
||||
}
|
||||
|
||||
var err error
|
||||
year, err = strconv.Atoi(aa[0])
|
||||
x(err)
|
||||
week, err = strconv.Atoi(aa[1])
|
||||
x(err)
|
||||
size, err = strconv.Atoi(os.Args[2])
|
||||
x(err)
|
||||
|
||||
if year < 1000 || year > 9999 {
|
||||
x(fmt.Errorf("ongeldig year: %d", year))
|
||||
}
|
||||
if week < 1 || week > 53 {
|
||||
x(fmt.Errorf("ongeldige week: %d", week))
|
||||
}
|
||||
|
||||
start, first, last, names := dates()
|
||||
|
||||
max, sources := makeCounts(names)
|
||||
data := &Data{
|
||||
Year: year,
|
||||
Week: week,
|
||||
First: first,
|
||||
Last: last,
|
||||
Period: size,
|
||||
Start: start,
|
||||
Max: max,
|
||||
Sources: sources,
|
||||
Algemeen: makeParts("Algemeen"),
|
||||
Groningen: makeParts("Groningen"),
|
||||
Amsterdam: makeParts("Amsterdam"),
|
||||
Literatuur: makeParts("Literatuur"),
|
||||
Vlaanderen: makeParts("Vlaanderen"),
|
||||
}
|
||||
|
||||
b, err := json.Marshal(data)
|
||||
x(err)
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
func makeParts(source string) *Parts {
|
||||
return &Parts{
|
||||
Woorden: makeValues(source, "woorden"),
|
||||
Personen: makeValues(source, "personen"),
|
||||
AndereNamen: makeValues(source, "andere namen"),
|
||||
Locaties: makeValues(source, "locaties"),
|
||||
Organisaties: makeValues(source, "organisaties"),
|
||||
}
|
||||
}
|
||||
|
||||
func makeValues(source, part string) [][5]any {
|
||||
v := make([][5]any, 0)
|
||||
|
||||
filename := fmt.Sprintf("/net/corpora/nlnieuws/data/%d/%s-%s-%d.%02d.score%d4.json",
|
||||
year,
|
||||
sources[source],
|
||||
parts[part].file,
|
||||
year,
|
||||
week,
|
||||
size)
|
||||
|
||||
b, err := os.ReadFile(filename)
|
||||
x(err)
|
||||
|
||||
var scores Scores
|
||||
x(json.Unmarshal(b, &scores))
|
||||
|
||||
lineno := 0
|
||||
for _, item := range scores.Up {
|
||||
if item.N > 0 {
|
||||
lineno++
|
||||
tags, _, _ := strings.Cut(item.Extras, "\t")
|
||||
v = append(v, [5]any{item.N, item.Word, tags, item.G2, item.PG2})
|
||||
if lineno == 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return v
|
||||
|
||||
}
|
||||
|
||||
func makeCounts(names []string) (int, map[string]int) {
|
||||
max := 0
|
||||
counts := make(map[string]int)
|
||||
x(os.Chdir("/net/corpora/nlnieuws"))
|
||||
files, err := os.ReadDir(".")
|
||||
x(err)
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
continue
|
||||
}
|
||||
filename := file.Name()
|
||||
if filename[0] < 'A' || filename[0] > 'Z' {
|
||||
continue
|
||||
}
|
||||
count := 0
|
||||
for _, name := range names {
|
||||
files2, err := os.ReadDir(filename + "/" + name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, f := range files2 {
|
||||
if n := f.Name(); strings.HasSuffix(n, ".xml") {
|
||||
count++
|
||||
} else if strings.HasSuffix(n, ".skip") {
|
||||
count--
|
||||
}
|
||||
}
|
||||
}
|
||||
counts[filename] = count
|
||||
if count > max {
|
||||
max = count
|
||||
}
|
||||
}
|
||||
return max, counts
|
||||
}
|
||||
|
||||
func dates() (start, first, last string, names []string) {
|
||||
|
||||
// 1 januari
|
||||
t := time.Date(year, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// zoek eerste donderdag
|
||||
day := int(t.Weekday()) // 0 = zondag
|
||||
donderdag := 4 - day
|
||||
if donderdag < 0 {
|
||||
donderdag = donderdag + 7
|
||||
}
|
||||
|
||||
// schuif naar maandag voor eerste donderdag: donderdag - 3
|
||||
// in de gewenste week: 7 * (week - 1)
|
||||
t = t.AddDate(0, 0, donderdag-3+7*(week-1))
|
||||
|
||||
t2 := t.AddDate(0, 0, 6)
|
||||
tStart := t.AddDate(0, 0, (1-size)*7)
|
||||
|
||||
names = make([]string, 0)
|
||||
t3 := tStart
|
||||
for range size {
|
||||
y, w := t3.ISOWeek()
|
||||
names = append(names, fmt.Sprintf("%d/w%02d", y, w))
|
||||
t3 = t3.AddDate(0, 0, 7)
|
||||
}
|
||||
t3 = tStart
|
||||
for range 7 * size {
|
||||
names = append(names, fmt.Sprintf("%d/%02d/%02d", t3.Year(), t3.Month(), t3.Day()))
|
||||
t3 = t3.AddDate(0, 0, 1)
|
||||
}
|
||||
|
||||
return makeDate(tStart), makeDate(t), makeDate(t2), names
|
||||
|
||||
}
|
||||
|
||||
func makeDate(d time.Time) string {
|
||||
return fmt.Sprintf("%s %d %s %d", dagen[d.Weekday()][:2], d.Day(), maanden[int(d.Month())][:3], d.Year())
|
||||
}
|
||||
@@ -26,6 +26,8 @@ var (
|
||||
|
||||
func main() {
|
||||
|
||||
suffix := os.Args[1]
|
||||
|
||||
dirs, err := os.ReadDir("/net/corpora/nlnieuws/data/json")
|
||||
x(err)
|
||||
for _, dir := range dirs {
|
||||
@@ -41,7 +43,7 @@ func main() {
|
||||
|
||||
for _, file := range files {
|
||||
filename := file.Name()
|
||||
if strings.HasPrefix(filename, "DATA-") && strings.HasSuffix(filename, "-4.json") {
|
||||
if strings.HasPrefix(filename, "DATA-") && strings.HasSuffix(filename, suffix) {
|
||||
addWeek(filename[5:12])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,8 +201,10 @@ done
|
||||
|
||||
data2json $ds 1 > ../json/$year/DATA-$ds-1.json
|
||||
data2json $ds 4 > ../json/$year/DATA-$ds-4.json
|
||||
dates2json > ../json/index1.json
|
||||
dates2json > ../json/index4.json
|
||||
data22json $ds 1 > ../json/$year/DATA-$ds-G2.json
|
||||
dates2json 1.json > ../json/index1.json
|
||||
dates2json 4.json > ../json/index4.json
|
||||
dates2json G2.json > ../json/indexG2.json
|
||||
|
||||
# rechten bijwerken
|
||||
chmod -R g+w /net/corpora/nlnieuws
|
||||
|
||||
8
go.mod
8
go.mod
@@ -5,9 +5,15 @@ go 1.26.1
|
||||
require (
|
||||
codeberg.org/pebbe/errors v0.4.0
|
||||
github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5
|
||||
github.com/kr/pretty v0.3.1
|
||||
github.com/pebbe/compactcorpus v1.0.3
|
||||
github.com/pebbe/textcat/v2 v2.3.0
|
||||
github.com/rug-compling/alpinods v1.18.1
|
||||
golang.org/x/net v0.57.0
|
||||
)
|
||||
|
||||
require github.com/pebbe/util v0.9.0 // indirect
|
||||
require (
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/pebbe/util v0.9.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.9.0 // indirect
|
||||
)
|
||||
|
||||
10
go.sum
10
go.sum
@@ -1,12 +1,22 @@
|
||||
codeberg.org/pebbe/errors v0.4.0 h1:G05wsXpC/LRPaL02QYDwtz0sWFWQcIWK1s+MC79LBzU=
|
||||
codeberg.org/pebbe/errors v0.4.0/go.mod h1:O7PPxUJM1bWRHq11CRK3wqVaH/3NnRaSVZvh3UhzDCY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5 h1:tQbR4RKFBFi0+Ll69dXejKKUbQVNaOAT2fjlDvSAfx4=
|
||||
github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5/go.mod h1:kQE2lxPgVKe0JsBZMFFfMm5zBDCuRhaHFKOBzZeCLiw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/pebbe/compactcorpus v1.0.3 h1:6qlfXKHTKg7oWKLPCgEgv1scplfvphg/9l9XiRT2HzQ=
|
||||
github.com/pebbe/compactcorpus v1.0.3/go.mod h1:SSpTeCZataCjjs82RJb8SOGdjkB3PlR7Z19EY4rInoQ=
|
||||
github.com/pebbe/textcat/v2 v2.3.0 h1:RB2egIQgI2a2Ls+I9No6KFQKCZBIFt8Cc/SWCnVtC7Y=
|
||||
github.com/pebbe/textcat/v2 v2.3.0/go.mod h1:WLXWuL+fOlQJqn6LmubjD+e78hCC6Y/rAWInh0wq/kg=
|
||||
github.com/pebbe/util v0.9.0 h1:PMZd+CpWb8GbWEmFGlL3qd6XPuywl6xFIbrXWi870OA=
|
||||
github.com/pebbe/util v0.9.0/go.mod h1:ynWl/SFX4+Seb9fpjVlYevr1f4TP7FrCmyZHiBCg69Q=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rug-compling/alpinods v1.18.1 h1:BvPcCnNEQ1QoVSc0RmwJd3kZmvo4iqZ52/vFzVvFS7w=
|
||||
github.com/rug-compling/alpinods v1.18.1/go.mod h1:R3BBX8RIw9InVqHZ+1W+MsX8WX8uBkoVNNGE38mqF1Q=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
|
||||
209
www/app2.html
Normal file
209
www/app2.html
Normal file
@@ -0,0 +1,209 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title id="title">Stijger van de week</title>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="icon" href="favicon.ico" type="image/ico" />
|
||||
<link rel="stylesheet" href="style.css" />
|
||||
<link rel="stylesheet" type="text/css" href="tooltip.css" />
|
||||
<script type="text/javascript" src="tooltip.js"></script>
|
||||
<script type="text/javascript" src="app2.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="title">
|
||||
<h1>Stijger van de week</h1>
|
||||
</div>
|
||||
<div class="form">
|
||||
<form name="choice">
|
||||
Wat wil je zien?
|
||||
<br />
|
||||
<ol>
|
||||
<li>
|
||||
<input type="radio" name="fWhat" id="fOpt1" value="opt1" onclick="opt(1)" />
|
||||
<label for="fOpt1">Een specifieke bron</label>
|
||||
</li>
|
||||
<li>
|
||||
<input type="radio" name="fWhat" id="fOpt2" value="opt2" onclick="opt(2)" />
|
||||
<label for="fOpt2">Een specifiek onderdeel</label>
|
||||
</li>
|
||||
<li>
|
||||
<input type="radio" name="fWhat" id="fOpt3" value="opt3" onclick="opt(3)" />
|
||||
<label for="fOpt3">Een tijdreeks voor een bron en onderdeel</label>
|
||||
</li>
|
||||
</ol>
|
||||
<div class="option" id="source">
|
||||
bron:
|
||||
<select name="source" id="fSource">
|
||||
<option>Algemeen</option>
|
||||
<option>Amsterdam</option>
|
||||
<option>Groningen</option>
|
||||
<option>Literatuur</option>
|
||||
<option>Vlaanderen</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="option" id="part">
|
||||
onderdeel:
|
||||
<select name="part" id="fPart">
|
||||
<option>woorden</option>
|
||||
<option>personen</option>
|
||||
<option>locaties</option>
|
||||
<option>organisaties</option>
|
||||
<option>andere namen</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="option" id="week">
|
||||
week:
|
||||
<input type="date" id="fDate" name="date" step="7" />
|
||||
<span class="validity"></span>
|
||||
</div>
|
||||
<button type="button" onclick="kies()" id="fSubmit" disabled>— laat maar zien —</button>
|
||||
</form>
|
||||
</div>
|
||||
<div class="title">
|
||||
<h2 id="subtitle"></h2>
|
||||
</div>
|
||||
<div class="main" id="data"></div>
|
||||
<div class="foot">
|
||||
<h2>Bronnen</h2>
|
||||
<table class="bron">
|
||||
<tr>
|
||||
<td>Algemeen</td>
|
||||
<td class="bar">
|
||||
<div id="NieuwsNL" style="width: 100%"></div>
|
||||
</td>
|
||||
<td><a href="https://nieuws.nl/">NieuwsNL</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="NOS"></div>
|
||||
</td>
|
||||
<td><a href="https://nos.nl/">NOS</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="NU"></div>
|
||||
</td>
|
||||
<td><a href="https://www.nu.nl/">NU</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="RO"></div>
|
||||
</td>
|
||||
<td><a href="https://reportersonline.nl/">Reporters Online</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Sargasso"></div>
|
||||
</td>
|
||||
<td><a href="https://sargasso.nl/">Sargasso</a></td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Volkskrant"></div>
|
||||
</td>
|
||||
<td><a href="https://www.volkskrant.nl/">de Volkskrant</a></td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
<td>Amsterdam</td>
|
||||
<td class="bar">
|
||||
<div id="AT5"></div>
|
||||
</td>
|
||||
<td><a href="https://www.at5.nl/">AT5</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Parool"></div>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.parool.nl/amsterdam/">Het Parool | Amsterdam</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="BuurtAdam"></div>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://indebuurt.nl/amsterdam/">In de buurt | Amsterdam</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
<td>Groningen</td>
|
||||
<td class="bar">
|
||||
<div id="GG"></div>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://gemeente.groningen.nl/nieuwsoverzicht">Gemeente Groningen</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="BuurtGrn"></div>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://indebuurt.nl/groningen/">In de buurt | Groningen</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Oog"></div>
|
||||
</td>
|
||||
<td><a href="https://www.oogtv.nl/">Oog</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="RTVNoord"></div>
|
||||
</td>
|
||||
<td><a href="https://www.rtvnoord.nl/">RTV Noord</a></td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Sikkom"></div>
|
||||
</td>
|
||||
<td><a href="https://sikkom.nl/">Sikkom</a></td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
<td>Literatuur</td>
|
||||
<td class="bar">
|
||||
<div id="LitNL"></div>
|
||||
</td>
|
||||
<td>
|
||||
<a href="https://www.literairnederland.nl/">Literair Nederland</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="Tzum"></div>
|
||||
</td>
|
||||
<td><a href="https://www.tzum.info/">Tzum</a></td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
<td>Vlaanderen</td>
|
||||
<td class="bar">
|
||||
<div id="HLN"></div>
|
||||
</td>
|
||||
<td><a href="https://www.hln.be/">HLN</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar">
|
||||
<div id="VRT"></div>
|
||||
</td>
|
||||
<td><a href="https://www.vrt.be/vrtnws/nl/">VRT NWS</a></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
273
www/app2.js
Normal file
273
www/app2.js
Normal file
@@ -0,0 +1,273 @@
|
||||
var dates
|
||||
var datesNr
|
||||
var countsWeek
|
||||
|
||||
var parts = ['woorden', 'personen', 'locaties', 'organisaties', 'andere namen']
|
||||
|
||||
var sources = ['Algemeen', 'Amsterdam', 'Groningen', 'Literatuur', 'Vlaanderen']
|
||||
|
||||
var data = new Map()
|
||||
|
||||
const idSource = document.getElementById('source')
|
||||
const idPart = document.getElementById('part')
|
||||
const idSubtitle = document.getElementById('subtitle')
|
||||
const idData = document.getElementById('data')
|
||||
const form1 = document.forms['choice']
|
||||
const fWhat = form1['fWhat']
|
||||
const fSource = form1['fSource']
|
||||
const fPart = form1['fPart']
|
||||
const fDate = form1['fDate']
|
||||
const fSubmit = form1['fSubmit']
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
function setCounts(week) {
|
||||
if (week != countsWeek) {
|
||||
countsWeek = week
|
||||
var max = data[week].max
|
||||
var pp = Object.entries(data[week].sources)
|
||||
for (var i in pp) {
|
||||
document.getElementById(pp[i][0]).style.width =
|
||||
(pp[i][1] / max) * 100 + '%'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getJSON(url) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
var xhr = new XMLHttpRequest()
|
||||
xhr.open('get', '/alfa/wvdm/data/' + url, true)
|
||||
xhr.responseType = 'json'
|
||||
xhr.onload = function () {
|
||||
var status = xhr.status
|
||||
if (status == 200) {
|
||||
resolve(xhr.response)
|
||||
} else {
|
||||
reject(status)
|
||||
}
|
||||
}
|
||||
xhr.send()
|
||||
})
|
||||
}
|
||||
|
||||
function escape(s) {
|
||||
return s
|
||||
.replace('&', '&')
|
||||
.replace("'", ''')
|
||||
.replace('<', '<')
|
||||
.replace('>', '>')
|
||||
}
|
||||
|
||||
function makeTD(title, values) {
|
||||
const td = document.createElement('td')
|
||||
const h3 = document.createElement('h3')
|
||||
const txt = document.createTextNode(title)
|
||||
h3.appendChild(txt)
|
||||
td.appendChild(h3)
|
||||
const tab = document.createElement('table')
|
||||
|
||||
var max
|
||||
for (var i = 0; i < 20; i++) {
|
||||
var value
|
||||
if (i < values.length) {
|
||||
value = values[i]
|
||||
} else {
|
||||
value = [0, '\xa0', '', '', '']
|
||||
}
|
||||
if (i == 0) {
|
||||
max = value[3]
|
||||
}
|
||||
const tr = document.createElement('tr')
|
||||
if (i < values.length) {
|
||||
var t2 = ''
|
||||
if (value[2]) {
|
||||
tr.classList.add('tags')
|
||||
t2 = '<br><small>' + escape(value[2]) + '</small>'
|
||||
}
|
||||
tr.setAttribute(
|
||||
'onmouseover',
|
||||
"tooltip.show('" + value[0] + ' \xa0 ' + escape(value[1]) + t2 + "')",
|
||||
)
|
||||
tr.setAttribute('onmouseout', 'tooltip.hide()')
|
||||
}
|
||||
const td1 = document.createElement('td')
|
||||
const div1 = document.createElement('div')
|
||||
pc = (value[3] / max) * 100
|
||||
div1.style.width = pc + '%'
|
||||
td1.appendChild(div1)
|
||||
tr.appendChild(td1)
|
||||
const td2 = document.createElement('td')
|
||||
const txt2 = document.createTextNode(value[1])
|
||||
td2.appendChild(txt2)
|
||||
tr.appendChild(td2)
|
||||
tab.appendChild(tr)
|
||||
}
|
||||
|
||||
td.appendChild(tab)
|
||||
return td
|
||||
}
|
||||
|
||||
async function loadSource(source, week) {
|
||||
if (!data.has(week)) {
|
||||
data[week] = await getJSON(
|
||||
week.substring(0, 4) + '/DATA-' + week + '-G2.json',
|
||||
)
|
||||
}
|
||||
|
||||
idSubtitle.innerHTML = source + ' — t/m ' + data[week].last
|
||||
|
||||
const d = document.createElement('div')
|
||||
const tab = document.createElement('table')
|
||||
tab.classList.add('outer')
|
||||
tr = document.createElement('tr')
|
||||
parts.forEach(function (part) {
|
||||
var values = data[week][source][part]
|
||||
tr.appendChild(makeTD(part, values))
|
||||
})
|
||||
tab.appendChild(tr)
|
||||
d.appendChild(tab)
|
||||
idData.innerHTML = d.innerHTML
|
||||
setCounts(week)
|
||||
}
|
||||
|
||||
async function loadPart(part, week) {
|
||||
if (!data.has(week)) {
|
||||
data[week] = await getJSON(
|
||||
week.substring(0, 4) + '/DATA-' + week + '-G2.json',
|
||||
)
|
||||
}
|
||||
|
||||
idSubtitle.innerHTML = part + ' — t/m ' + data[week].last
|
||||
|
||||
const d = document.createElement('div')
|
||||
const tab = document.createElement('table')
|
||||
tab.classList.add('outer')
|
||||
tr = document.createElement('tr')
|
||||
sources.forEach(function (source) {
|
||||
var values = data[week][source][part]
|
||||
tr.appendChild(makeTD(source, values))
|
||||
})
|
||||
tab.appendChild(tr)
|
||||
d.appendChild(tab)
|
||||
idData.innerHTML = d.innerHTML
|
||||
setCounts(week)
|
||||
}
|
||||
|
||||
async function loadWeken(source, part) {
|
||||
idSubtitle.innerHTML = source + ' — ' + part
|
||||
|
||||
const d = document.createElement('div')
|
||||
const tab = document.createElement('table')
|
||||
tab.classList.add('outer')
|
||||
tr = document.createElement('tr')
|
||||
|
||||
for (var i = datesNr; i < datesNr + 10; i++) {
|
||||
if (i < dates.length) {
|
||||
var week = dates[i].week
|
||||
if (!data.has(week)) {
|
||||
data[week] = await getJSON(
|
||||
week.substring(0, 4) + '/DATA-' + week + '-G2.json',
|
||||
)
|
||||
}
|
||||
var values = data[week][source][part]
|
||||
tr.appendChild(makeTD('t/m ' + data[week].last, values))
|
||||
}
|
||||
}
|
||||
|
||||
tab.appendChild(tr)
|
||||
d.appendChild(tab)
|
||||
idData.innerHTML = d.innerHTML
|
||||
setCounts(dates[datesNr].week)
|
||||
}
|
||||
|
||||
function locateWeek(date) {
|
||||
if (date == '') {
|
||||
datesNr = 0
|
||||
return
|
||||
}
|
||||
var n = dates.length - 1
|
||||
if (date < dates[n].first) {
|
||||
datesNr = n
|
||||
return
|
||||
}
|
||||
if (date > dates[0].last) {
|
||||
datesNr = 0
|
||||
return
|
||||
}
|
||||
var p1 = 0
|
||||
var p2 = n
|
||||
while (true) {
|
||||
// fix voor als de weken niet aansluitend zijn (er ontbreken weken)
|
||||
if (p1 > p2) {
|
||||
datesNr = p2
|
||||
return
|
||||
}
|
||||
var i = Math.floor((p1 + p2) / 2)
|
||||
if (dates[i].first > date) {
|
||||
p1 = i + 1 // dit klopt: lijst is van groot naar klein
|
||||
} else if (dates[i].last < date) {
|
||||
p2 = i - 1
|
||||
} else {
|
||||
datesNr = i
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function kies() {
|
||||
let what = fWhat.value
|
||||
let source = fSource.value
|
||||
let part = fPart.value
|
||||
let date = fDate.value
|
||||
|
||||
locateWeek(date)
|
||||
|
||||
idData.classList.add('fade')
|
||||
idSubtitle.classList.add('fade')
|
||||
await sleep(20)
|
||||
|
||||
if (what == 'opt1') {
|
||||
await loadSource(source, dates[datesNr].week)
|
||||
} else if (what == 'opt2') {
|
||||
await loadPart(part, dates[datesNr].week)
|
||||
} else {
|
||||
await loadWeken(source, part)
|
||||
}
|
||||
idSubtitle.classList.remove('fade')
|
||||
idData.classList.remove('fade')
|
||||
}
|
||||
|
||||
function opt(i) {
|
||||
fSubmit.disabled = false
|
||||
|
||||
if (i == 1) {
|
||||
idSource.classList.remove('disabled')
|
||||
fSource.disabled = false
|
||||
idPart.classList.add('disabled')
|
||||
fPart.disabled = true
|
||||
}
|
||||
if (i == 2) {
|
||||
idSource.classList.add('disabled')
|
||||
fSource.disabled = true
|
||||
idPart.classList.remove('disabled')
|
||||
fPart.disabled = false
|
||||
}
|
||||
if (i == 3) {
|
||||
idSource.classList.remove('disabled')
|
||||
fSource.disabled = false
|
||||
idPart.classList.remove('disabled')
|
||||
fPart.disabled = false
|
||||
}
|
||||
}
|
||||
|
||||
async function init() {
|
||||
dates = await getJSON('indexG2.json')
|
||||
datesNr = 0
|
||||
fDate.setAttribute('min', dates[dates.length - 1].last)
|
||||
fDate.setAttribute('max', dates[0].last)
|
||||
loadSource('Algemeen', dates[datesNr].week)
|
||||
}
|
||||
|
||||
init()
|
||||
Reference in New Issue
Block a user