Compare commits
13 Commits
ca4e7af8fa
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
c2389c65af | ||
|
|
802a84a929 | ||
|
|
8289072a52 | ||
|
|
e0550197e3 | ||
|
|
01e6d48665 | ||
|
|
a8bea0ab44 | ||
|
|
d7adc17d4b | ||
|
|
a9f9e17acf | ||
|
|
1f4a084624 | ||
|
|
9f29222909 | ||
|
|
a76fa21584 | ||
|
|
efa301cc4a | ||
|
|
14590570ba |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -38,14 +38,19 @@ Sikkom/sikkom
|
||||
Tzum/metadata
|
||||
Tzum/tzum
|
||||
Tzum/xml2txt
|
||||
Volkskrant/metadata
|
||||
Volkskrant/volkskrant
|
||||
VRT/metadata
|
||||
VRT/vrt
|
||||
bin/data2json
|
||||
bin/dates2json
|
||||
bin/flush
|
||||
bin/items2count
|
||||
bin/rang
|
||||
bin/top20
|
||||
bin/trends
|
||||
bin/week2files
|
||||
bin/weekadd
|
||||
20??
|
||||
corpus
|
||||
data
|
||||
|
||||
@@ -58,7 +58,7 @@ func main() {
|
||||
var item Item
|
||||
x(xml.Unmarshal(b, &item), filename)
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
pp, err := root.Search(`//body/p | //body/h2`)
|
||||
|
||||
@@ -159,6 +159,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
|
||||
@@ -158,6 +158,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
|
||||
@@ -155,6 +155,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
|
||||
@@ -168,6 +168,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
/*
|
||||
s := string(body)
|
||||
ok = true
|
||||
@@ -276,8 +278,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
|
||||
specials, err := article.Search(`.//*[@data-content-type="GROUP"]`)
|
||||
p(err)
|
||||
for _, special := range specials {
|
||||
special.Remove()
|
||||
for i := len(specials) - 1; i >= 0; i-- {
|
||||
specials[i].Remove()
|
||||
}
|
||||
|
||||
other, err := article.Search(`.//*[@data-content-type="PODCAST"]`)
|
||||
|
||||
@@ -63,7 +63,7 @@ func main() {
|
||||
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
|
||||
}
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
pp, err := root.Search(`//body//p`)
|
||||
|
||||
13
Makefile
13
Makefile
@@ -16,13 +16,17 @@ all:
|
||||
make -C Sargasso
|
||||
make -C Sikkom
|
||||
make -C Tzum
|
||||
make -C Volkskrant
|
||||
make -C VRT
|
||||
make bin/data2json
|
||||
make bin/dates2json
|
||||
make bin/flush
|
||||
make bin/items2count
|
||||
make bin/rang
|
||||
make bin/top20
|
||||
make bin/trends
|
||||
make bin/week2files
|
||||
make bin/weekadd
|
||||
|
||||
bin/data2json: cmd/data2json/*.go
|
||||
go build -o $@ $^
|
||||
@@ -36,9 +40,18 @@ bin/flush: cmd/flush/*.go
|
||||
bin/items2count: cmd/items2count/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/rang: cmd/rang/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/top20: cmd/top20/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/trends: cmd/trends/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/week2files: cmd/week2files/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
bin/weekadd: cmd/weekadd/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
|
||||
@@ -69,7 +69,9 @@ func main() {
|
||||
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
|
||||
}
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
x(fp.WriteString(u.FixSpace(item.Text)))
|
||||
for _, line := range strings.SplitAfter(item.Text, "\n") {
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(line, true))))
|
||||
}
|
||||
x(fp.Close())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,6 +161,8 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
s := string(body)
|
||||
ok := true
|
||||
i1 := strings.Index(s, `<script type="application/ld+json"`)
|
||||
@@ -226,22 +228,9 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
|
||||
// text bevat kopjes zonder punt aan het eind
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, line := range lines {
|
||||
line = u.FixSpace(line)
|
||||
n := len(line)
|
||||
if n > 0 {
|
||||
if strings.ContainsAny(line[n-1:], ".!?") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if n > 1 {
|
||||
s := line[n-2:]
|
||||
if s == `."` || s == `!"` || s == `?"` {
|
||||
continue
|
||||
}
|
||||
}
|
||||
lines[i] = line + "."
|
||||
lines[i] = u.AddEnd(u.FixSpace(line, true))
|
||||
}
|
||||
text = strings.Join(lines, "\n") + "\n"
|
||||
text = strings.Join(lines, "") + "\n"
|
||||
|
||||
fp, err := os.Create(filename + ".txt")
|
||||
p(err)
|
||||
|
||||
@@ -154,6 +154,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
pp, err := root.Search(`//body/p`)
|
||||
|
||||
@@ -164,6 +164,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
@@ -278,6 +280,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
}
|
||||
header := headers[0]
|
||||
|
||||
isVideo := false
|
||||
tags := make([]string, 0)
|
||||
ell, err := header.Search(`.//*[@data-test-id="article-label"]`)
|
||||
p(err)
|
||||
@@ -289,6 +292,9 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
if s != "" && s != "Nieuws" {
|
||||
tags = append(tags, s)
|
||||
}
|
||||
if strings.ToLower(s) == "video" {
|
||||
isVideo = true
|
||||
}
|
||||
}
|
||||
|
||||
pars := make([]string, 0)
|
||||
@@ -307,10 +313,10 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
_ = w(fmt.Errorf("no intro: %s", url))
|
||||
}
|
||||
|
||||
specials, err := article.Search(`.//section//aside | .//section//figure | .//section//b`)
|
||||
specials, err := article.Search(`.//aside | .//figure | .//figcaption | .//section//b`)
|
||||
p(err)
|
||||
for _, special := range specials {
|
||||
special.Remove()
|
||||
for i := len(specials) - 1; i >= 0; i-- {
|
||||
specials[i].Remove()
|
||||
}
|
||||
|
||||
ell, err = article.Search(`.//section//*[@data-article-element-index]`)
|
||||
@@ -342,7 +348,9 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
_ = w(fmt.Errorf("no text, skipping: %s", url))
|
||||
if !isVideo {
|
||||
_ = w(fmt.Errorf("no text, skipping: %s", url))
|
||||
}
|
||||
fp, err := os.Create(filename + ".skip")
|
||||
p(fp.WriteString(url + "\n"))
|
||||
p(err)
|
||||
|
||||
@@ -37,6 +37,7 @@ crontab van p209327@colossus
|
||||
17 * * * * /net/corpora/nlnieuws/Sikkom/sikkom
|
||||
18 * * * * /net/corpora/nlnieuws/Tzum/tzum
|
||||
19 * * * * /net/corpora/nlnieuws/VRT/vrt
|
||||
20 * * * * /net/corpora/nlnieuws/Volkskrant/volkskrant
|
||||
```
|
||||
|
||||
## 2. Teksten verwerken: omzetten naar zinnen, parsen, metadata toevoegen
|
||||
@@ -53,6 +54,7 @@ crontab van p209327@colossus
|
||||
0 1 * * * /net/corpora/nlnieuws/NU/txt2corpus.sh
|
||||
0 1 * * * /net/corpora/nlnieuws/NieuwsNL/txt2corpus.sh
|
||||
0 1 * * * /net/corpora/nlnieuws/VRT/txt2corpus.sh
|
||||
0 1 * * * /net/corpora/nlnieuws/Volkskrant/txt2corpus.sh
|
||||
# weinig data: alleen op dinsdag
|
||||
0 1 * * 2 /net/corpora/nlnieuws/AT5/txt2corpus.sh
|
||||
0 1 * * 2 /net/corpora/nlnieuws/BuurtAdam/txt2corpus.sh
|
||||
|
||||
@@ -29,8 +29,9 @@ type ItemT struct {
|
||||
}
|
||||
|
||||
var (
|
||||
p = e.PanicErr
|
||||
agent = "AhrefsBot/7.0"
|
||||
p = e.PanicErr
|
||||
//agent = "AhrefsBot/7.0"
|
||||
agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
func exists(filename string) bool {
|
||||
|
||||
@@ -64,13 +64,13 @@ func main() {
|
||||
var item Item
|
||||
x(xml.Unmarshal(b, &item))
|
||||
x(buf.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
divs, err := root.Search(`//div[@class="donatieformlinks"]`)
|
||||
x(err)
|
||||
for _, div := range divs {
|
||||
div.Remove()
|
||||
for i := len(divs) - 1; i >= 0; i-- {
|
||||
divs[i].Remove()
|
||||
}
|
||||
pp, err := root.Search(`//body//p[not(.//a[contains(@href,"reportersonline.nl/support")])]`)
|
||||
x(err)
|
||||
|
||||
@@ -226,7 +226,7 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
|
||||
// text bevat kopjes zonder punt aan het eind
|
||||
lines := strings.Split(doc.Text, "\n")
|
||||
for i, line := range lines {
|
||||
lines[i] = u.AddEnd(u.FixSpace(line))
|
||||
lines[i] = u.AddEnd(u.FixSpace(line, true))
|
||||
}
|
||||
text := strings.Join(lines, "") + "\n"
|
||||
|
||||
@@ -240,9 +240,7 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
|
||||
if strings.HasPrefix(t, "br_") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(t, "tr_") {
|
||||
t = t[3:]
|
||||
}
|
||||
t = strings.TrimPrefix(t, "tr_")
|
||||
p(fmt.Fprintf(fp, "##META text tag = %s\n", t))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ func main() {
|
||||
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
|
||||
}
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
pp, err := root.Search(`//body//p`)
|
||||
|
||||
@@ -152,6 +152,8 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
s := string(body)
|
||||
|
||||
ok := true
|
||||
|
||||
@@ -66,7 +66,7 @@ func main() {
|
||||
x(fmt.Fprintf(fp, "##META text tag = %s\n", t))
|
||||
}
|
||||
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
|
||||
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + u.HtmlFixString(item.Text) + `</body></html>`))
|
||||
x(err)
|
||||
root := doc.Root()
|
||||
pp, err := root.Search(`//body/p`)
|
||||
|
||||
@@ -179,6 +179,8 @@ func doArticle(filename string, url string, title string, tags []string, cats []
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
/*
|
||||
s := string(body)
|
||||
ok := true
|
||||
|
||||
9
Volkskrant/Makefile
Normal file
9
Volkskrant/Makefile
Normal file
@@ -0,0 +1,9 @@
|
||||
all: \
|
||||
metadata \
|
||||
volkskrant
|
||||
|
||||
metadata: cmd/metadata/*.go
|
||||
go build -o $@ $^
|
||||
|
||||
volkskrant: cmd/volkskrant/*.go ../internal/util/*.go
|
||||
go build -o $@ $<
|
||||
131
Volkskrant/cmd/metadata/metadata.go
Normal file
131
Volkskrant/cmd/metadata/metadata.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"bufio"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
XMLName xml.Name `xml:"item"`
|
||||
UnixTime int64 `xml:"unixTime"`
|
||||
}
|
||||
|
||||
var (
|
||||
x = e.ExitErr
|
||||
escape = html.EscapeString
|
||||
data = make(map[string][]string)
|
||||
location *time.Location
|
||||
)
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
location, err = time.LoadLocation("Europe/Amsterdam")
|
||||
x(err)
|
||||
|
||||
files, err := os.ReadDir(".")
|
||||
x(err)
|
||||
for _, file := range files {
|
||||
filename := file.Name()
|
||||
if strings.HasSuffix(filename, ".txt") {
|
||||
doText("", filename)
|
||||
} else if strings.HasSuffix(filename, ".xml") {
|
||||
doXml("", filename)
|
||||
}
|
||||
}
|
||||
files, err = os.ReadDir("..")
|
||||
x(err)
|
||||
for _, file := range files {
|
||||
filename := file.Name()
|
||||
if strings.HasSuffix(filename, ".txt") {
|
||||
doText("../", filename)
|
||||
} else if strings.HasSuffix(filename, ".xml") {
|
||||
doXml("../", filename)
|
||||
}
|
||||
}
|
||||
|
||||
files, err = os.ReadDir("xml")
|
||||
x(err)
|
||||
for _, file := range files {
|
||||
filename := file.Name()
|
||||
if !strings.HasSuffix(filename, ".xml") {
|
||||
continue
|
||||
}
|
||||
aa := strings.Split(filename, ".")
|
||||
base := strings.Join(aa[1:len(aa)-2], ".")
|
||||
b, err := os.ReadFile("xml/" + filename)
|
||||
x(err)
|
||||
s := string(b)
|
||||
i := strings.Index(s, "<alpino") + 1
|
||||
i += strings.Index(s[i:], "<")
|
||||
fp, err := os.Create("xml/" + filename + ".tmp")
|
||||
x(err)
|
||||
x(fp.WriteString(s[:i]))
|
||||
x(fp.WriteString("<metadata>\n <meta type=\"text\" name=\"source\" value=\"Volkskrant\"/>\n"))
|
||||
for _, m := range data[base] {
|
||||
x(fp.WriteString(" " + m + "\n"))
|
||||
}
|
||||
x(fp.WriteString(" </metadata>\n "))
|
||||
x(fp.WriteString(stripMeta(s[i:])))
|
||||
x(fp.Close())
|
||||
x(os.Rename("xml/"+filename+".tmp", "xml/"+filename))
|
||||
}
|
||||
}
|
||||
|
||||
func doText(dirname, filename string) {
|
||||
base := filename[:len(filename)-4]
|
||||
if _, ok := data[base]; !ok {
|
||||
data[base] = make([]string, 0)
|
||||
}
|
||||
fp, err := os.Open(dirname + filename)
|
||||
x(err)
|
||||
defer func() { x(fp.Close()) }()
|
||||
scanner := bufio.NewScanner(fp)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.HasPrefix(line, "##META") {
|
||||
continue
|
||||
}
|
||||
aa := strings.Fields(line)
|
||||
if len(aa) > 4 {
|
||||
data[base] = append(data[base],
|
||||
fmt.Sprintf(`<meta type="%s" name="%s" value="%s"/>`,
|
||||
aa[1],
|
||||
escape(aa[2]),
|
||||
escape(strings.Join(aa[4:], " "))))
|
||||
}
|
||||
}
|
||||
x(scanner.Err())
|
||||
}
|
||||
|
||||
func doXml(dirname, filename string) {
|
||||
base := filename[:len(filename)-4]
|
||||
if _, ok := data[base]; !ok {
|
||||
data[base] = make([]string, 0)
|
||||
}
|
||||
b, err := os.ReadFile(dirname + filename)
|
||||
x(err)
|
||||
var item Item
|
||||
x(xml.Unmarshal(b, &item))
|
||||
t := time.Unix(item.UnixTime, 0).In(location)
|
||||
data[base] = append(data[base],
|
||||
fmt.Sprintf(`<meta type="date" name="pubdate" value="%d-%02d-%02d"/>`,
|
||||
t.Year(),
|
||||
int(t.Month()),
|
||||
t.Day()))
|
||||
}
|
||||
|
||||
func stripMeta(s string) string {
|
||||
i1 := strings.Index(s, "<metadata>")
|
||||
if i1 < 0 {
|
||||
return s
|
||||
}
|
||||
i2 := i1 + strings.Index(s[i1:], "</metadata>") + 11
|
||||
return s[:i1] + strings.TrimLeft(s[i2:], " \t\r\n")
|
||||
}
|
||||
390
Volkskrant/cmd/volkskrant/volkskrant.go
Normal file
390
Volkskrant/cmd/volkskrant/volkskrant.go
Normal file
@@ -0,0 +1,390 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
"github.com/jbowtie/gokogiri"
|
||||
|
||||
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
|
||||
|
||||
//"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
//"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Rss struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Items []ItemT `xml:"channel>item"`
|
||||
}
|
||||
|
||||
type ItemT struct {
|
||||
PubDate string `xml:"pubDate"`
|
||||
UnixTime int64 `xml:"unixTime"`
|
||||
Guid string `xml:"guid"`
|
||||
Link string `xml:"link"`
|
||||
Title string `xml:"title"`
|
||||
Data []byte `xml:",innerxml"`
|
||||
}
|
||||
|
||||
/*
|
||||
type GraphT struct {
|
||||
Graph []map[string]any `json:"@graph"`
|
||||
}
|
||||
*/
|
||||
|
||||
var (
|
||||
p = e.PanicErr
|
||||
w = e.WarnErr
|
||||
agent = "AhrefsBot/7.0"
|
||||
)
|
||||
|
||||
func exists(filename string) bool {
|
||||
_, err := os.Stat(filename)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func fileDate(filename string) string {
|
||||
b, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
s := string(b)
|
||||
i1 := strings.Index(s, "<unixTime>") + 10
|
||||
i2 := strings.Index(s, "</unixTime>")
|
||||
if i2 < i1 {
|
||||
return ""
|
||||
}
|
||||
return s[i1:i2]
|
||||
}
|
||||
|
||||
func main() {
|
||||
defer func() {
|
||||
if e.Panicked {
|
||||
_ = recover()
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
myLock := "/net/corpora/nlnieuws/Volkskrant/lock"
|
||||
u.MkLock(myLock)
|
||||
defer func() {
|
||||
_ = os.Remove(myLock)
|
||||
}()
|
||||
|
||||
req, err := http.NewRequest("GET", "https://www.volkskrant.nl/rss.xml", nil)
|
||||
p(err)
|
||||
req.Header.Set("User-Agent", agent)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
p(err)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
var rss Rss
|
||||
p(xml.Unmarshal(body, &rss))
|
||||
|
||||
if len(rss.Items) == 0 {
|
||||
p(fmt.Errorf("len(rss.Items) == 0"))
|
||||
}
|
||||
|
||||
for _, item := range rss.Items {
|
||||
t, err := time.Parse(time.RFC1123Z, item.PubDate)
|
||||
if err != nil {
|
||||
t, err = time.Parse(time.RFC1123, item.PubDate)
|
||||
}
|
||||
p(err)
|
||||
dirname := fmt.Sprintf("/net/corpora/nlnieuws/Volkskrant/%d/%02d/%02d", t.Year(), int(t.Month()), t.Day())
|
||||
if exists(dirname + "/lock") {
|
||||
continue
|
||||
}
|
||||
basename := item.Guid
|
||||
filename := dirname + "/" + url.PathEscape(basename)
|
||||
|
||||
ts := fmt.Sprintf("%d", t.Unix())
|
||||
needUpdate := fileDate(filename+".xml") != ts
|
||||
|
||||
p(os.MkdirAll(dirname, 0777))
|
||||
func() {
|
||||
var ok bool
|
||||
defer func() {
|
||||
if e.Panicked {
|
||||
fmt.Fprintln(os.Stderr, "----", filename)
|
||||
fmt.Fprintln(os.Stderr, "----", item.Link)
|
||||
}
|
||||
if !ok {
|
||||
_ = os.Remove(filename + ".xml")
|
||||
}
|
||||
}()
|
||||
fp, err := os.Create(filename + ".xml")
|
||||
p(err)
|
||||
p(fp.WriteString("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<item>\n"))
|
||||
p(fmt.Fprintf(fp, "<unixTime>%d</unixTime>", t.Unix()))
|
||||
p(fp.Write(item.Data))
|
||||
p(fp.WriteString("</item>\n"))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".xml", t, t))
|
||||
ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) (ok bool) {
|
||||
if exists(filename + ".skip") {
|
||||
return true
|
||||
}
|
||||
if needUpdate {
|
||||
_ = os.Remove(filename + ".err")
|
||||
_ = os.Remove(filename + ".html")
|
||||
// _ = os.Remove(filename + ".json")
|
||||
_ = os.Remove(filename + ".txt")
|
||||
} else {
|
||||
if exists(filename + ".txt") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
p(err)
|
||||
req.Header.Set("User-Agent", agent)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
p(err)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
p(err)
|
||||
p(resp.Body.Close())
|
||||
|
||||
body = u.HtmlFix(body)
|
||||
|
||||
doc, err := gokogiri.ParseHtml(body)
|
||||
p(err)
|
||||
|
||||
/*
|
||||
|
||||
s := string(body)
|
||||
|
||||
ok = true
|
||||
i1 := strings.Index(s, `<script type="application/ld+json"`)
|
||||
if i1 < 0 {
|
||||
ok = false
|
||||
} else {
|
||||
i1 += strings.Index(s[i1:], `>`) + 1
|
||||
i2 := i1 + strings.Index(s[i1:], `</script>`)
|
||||
if i2 < i1 {
|
||||
ok = false
|
||||
} else {
|
||||
s = html.UnescapeString(s[i1:i2])
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
_ = w(fmt.Errorf("script jsonld not found: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "script jsonld not found: %s\n", url))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
var graph GraphT
|
||||
p(json.Unmarshal([]byte(s), &graph))
|
||||
for _, g := range graph.Graph {
|
||||
t := g["@type"]
|
||||
switch v := t.(type) {
|
||||
case string:
|
||||
if v == "NewsArticle" {
|
||||
b, err := json.Marshal(g)
|
||||
p(err)
|
||||
s = string(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fp, err := os.Create(filename + ".json")
|
||||
p(err)
|
||||
p(fp.WriteString(s))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".json", timestamp, timestamp))
|
||||
*/
|
||||
|
||||
root := doc.Root()
|
||||
|
||||
articles, err := root.Search(`//article[@id="article-content"]`)
|
||||
p(err)
|
||||
if len(articles) == 0 {
|
||||
_ = w(fmt.Errorf("empty: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "empty: %s\n", url))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
}
|
||||
article := articles[0]
|
||||
|
||||
live, err := article.Search(`.//*[@data-test-id="live-blog-label"]`)
|
||||
p(err)
|
||||
if len(live) > 0 {
|
||||
fp, err := os.Create(filename + ".skip")
|
||||
p(fp.WriteString("liveblog\n"))
|
||||
p(err)
|
||||
p(os.Chtimes(filename+".skip", timestamp, timestamp))
|
||||
return true
|
||||
}
|
||||
|
||||
headers, err := article.Search(`.//header`)
|
||||
p(err)
|
||||
if len(headers) == 0 {
|
||||
_ = w(fmt.Errorf("no header: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "no elements: %s\n", url))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
|
||||
}
|
||||
header := headers[0]
|
||||
|
||||
isOpinie := false
|
||||
isColumn := false
|
||||
tags := make([]string, 0)
|
||||
ell, err := header.Search(`.//*[@data-test-id="article-label"]`)
|
||||
p(err)
|
||||
if len(ell) == 0 {
|
||||
_ = w(fmt.Errorf("no labels: %s", url))
|
||||
}
|
||||
for _, el := range ell {
|
||||
s := strings.TrimSpace(el.Content())
|
||||
if s != "" && s != "Nieuws" {
|
||||
tags = append(tags, s)
|
||||
}
|
||||
if s1 := strings.ToLower(s); s1 == "opinie" {
|
||||
isOpinie = true
|
||||
} else if s1 == "column" {
|
||||
isColumn = true
|
||||
}
|
||||
}
|
||||
|
||||
pars := make([]string, 0)
|
||||
|
||||
found := false
|
||||
ell, err = header.Search(`.//*[@data-test-id="header-intro"]`)
|
||||
p(err)
|
||||
for _, el := range ell {
|
||||
s := strings.TrimSpace(el.Content())
|
||||
if s != "" {
|
||||
pars = append(pars, s)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found && !isOpinie && !isColumn {
|
||||
_ = w(fmt.Errorf("no intro: %s", url))
|
||||
}
|
||||
|
||||
specials, err := article.Search(`.//aside | .//figure | .//figcaption | .//section//b`)
|
||||
p(err)
|
||||
for i := len(specials) - 1; i >= 0; i-- {
|
||||
specials[i].Remove()
|
||||
}
|
||||
|
||||
ell, err = article.Search(`.//section//*[@data-article-element-index]`)
|
||||
p(err)
|
||||
if len(ell) == 0 {
|
||||
_ = w(fmt.Errorf("no elements: %s", url))
|
||||
|
||||
fp, err := os.Create(filename + ".err")
|
||||
p(err)
|
||||
p(fmt.Fprintf(fp, "no elements: %s\n", url))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".err", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
found = false
|
||||
for _, el := range ell {
|
||||
s := strings.TrimSpace(el.Content())
|
||||
if s != "" {
|
||||
pars = append(pars, s)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
_ = w(fmt.Errorf("no text, skipping: %s", url))
|
||||
fp, err := os.Create(filename + ".skip")
|
||||
p(fp.WriteString(url + "\n"))
|
||||
p(err)
|
||||
p(os.Chtimes(filename+".skip", timestamp, timestamp))
|
||||
|
||||
fp, err = os.Create(filename + ".html")
|
||||
p(err)
|
||||
p(fp.Write(body))
|
||||
p(fp.Close())
|
||||
p(os.Chtimes(filename+".html", timestamp, timestamp))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
fp, err := os.Create(filename + ".txt")
|
||||
p(err)
|
||||
|
||||
if len(tags) == 0 {
|
||||
p(fmt.Fprintln(fp, "##META text tag ="))
|
||||
} else {
|
||||
for _, tag := range tags {
|
||||
p(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
|
||||
}
|
||||
}
|
||||
|
||||
p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
|
||||
|
||||
for _, par := range pars {
|
||||
p(fp.WriteString(u.AddEnd(u.FixSpace(par))))
|
||||
}
|
||||
|
||||
p(fp.Close())
|
||||
|
||||
p(os.Chtimes(filename+".txt", timestamp, timestamp))
|
||||
|
||||
return true
|
||||
}
|
||||
70
Volkskrant/txt2corpus.sh
Executable file
70
Volkskrant/txt2corpus.sh
Executable file
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
BASE=/net/corpora/nlnieuws
|
||||
PART=$BASE/Volkskrant
|
||||
|
||||
unset CDPATH
|
||||
PATH=$PART:$BASE/bin:$BASE:/net/aps/bin:$PATH
|
||||
export TZ=Europe/Amsterdam
|
||||
. /net/aps/etc/alpino-activate.sh > /dev/null
|
||||
|
||||
if [ "$1" = "" ]
|
||||
then
|
||||
ds=`date -d -2days +%Y-%m-%d`
|
||||
else
|
||||
case "$1" in
|
||||
2[0-9][0-9][0-9]-[01][0-9]-[0-3][0-9])
|
||||
ds=$1
|
||||
;;
|
||||
*)
|
||||
echo INVALID
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
dp=${ds//-//}
|
||||
year=${ds%%-*}
|
||||
corpus=$PART/corpus/$year/$ds
|
||||
mkdir -p $PART/corpus/$year
|
||||
|
||||
cd $PART/$dp
|
||||
|
||||
ln -s lock.$$ lock
|
||||
if [ "`readlink lock`" != lock.$$ ]
|
||||
then
|
||||
echo Getting lock failed
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -fr out
|
||||
mkdir out
|
||||
|
||||
rm -f $corpus.lines
|
||||
for i in *.txt
|
||||
do
|
||||
b=`basename $i .txt`
|
||||
perl -p -e 's/^\s*//; s/^##META.*\n//' $i | tokenize.sh \
|
||||
| perl -e '$n = 0; while(<>) { $n++; print("vk.'$b'.$n|$_"); }' \
|
||||
>> $corpus.lines
|
||||
done
|
||||
|
||||
cd out
|
||||
mkdir xml
|
||||
Alpino -flag treebank xml debug=1 end_hook=xml user_max=900000 -parse < $corpus.lines 2> $corpus.log
|
||||
|
||||
metadata
|
||||
|
||||
cd xml
|
||||
rm -f $corpus.data.dz $corpus.index
|
||||
alto -q -o $corpus.data.dz *.xml
|
||||
|
||||
# telling per bericht, niet per zin
|
||||
query.sh -x T -s $corpus.data.dz > $corpus.tag.txt
|
||||
|
||||
cd ../..
|
||||
rm -fr out
|
||||
|
||||
rm -f lock
|
||||
63
cmd/rang/rang.go
Normal file
63
cmd/rang/rang.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
// alto 'fp://node[....]' 'tt:%w\t%I' $files | sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang
|
||||
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
x = e.ExitErr
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
word string
|
||||
count int
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
counts := make(map[string]int)
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
word := strings.Split(scanner.Text(), "\t")[0]
|
||||
counts[word] = counts[word] + 1
|
||||
}
|
||||
x(scanner.Err())
|
||||
|
||||
items := make([]Item, 0)
|
||||
for key, value := range counts {
|
||||
items = append(items, Item{
|
||||
word: key,
|
||||
count: value,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(items, func(a, b int) bool {
|
||||
if items[a].count == items[b].count {
|
||||
return items[a].word < items[b].word
|
||||
}
|
||||
return items[a].count > items[b].count
|
||||
})
|
||||
|
||||
rang := 0
|
||||
prev := 0
|
||||
for _, item := range items {
|
||||
if item.count < 2 {
|
||||
break
|
||||
}
|
||||
if item.count != prev {
|
||||
rang++
|
||||
prev = item.count
|
||||
}
|
||||
fmt.Printf("%d\t%s\n", rang, item.word)
|
||||
}
|
||||
|
||||
}
|
||||
108
cmd/trends/trends.go
Normal file
108
cmd/trends/trends.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
word string
|
||||
diff float64
|
||||
gone bool
|
||||
}
|
||||
|
||||
var (
|
||||
x = e.ExitErr
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
refs := make(map[string]int)
|
||||
refmax := 0
|
||||
fp, err := os.Open(os.Args[1])
|
||||
x(err)
|
||||
scanner := bufio.NewScanner(fp)
|
||||
for scanner.Scan() {
|
||||
aa := strings.Split(scanner.Text(), "\t")
|
||||
n, err := strconv.Atoi(aa[0])
|
||||
x(err)
|
||||
refs[aa[1]] = n
|
||||
if n > refmax {
|
||||
refmax = n
|
||||
}
|
||||
}
|
||||
x(scanner.Err())
|
||||
fp.Close()
|
||||
refmax++
|
||||
|
||||
lines := make([]string, 0)
|
||||
fp, err = os.Open(os.Args[2])
|
||||
x(err)
|
||||
scanner = bufio.NewScanner(fp)
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
x(scanner.Err())
|
||||
fp.Close()
|
||||
|
||||
curmax, err := strconv.Atoi(strings.Split(lines[len(lines)-1], "\t")[0])
|
||||
x(err)
|
||||
curmax++
|
||||
|
||||
items := make([]Item, 0)
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, line := range lines {
|
||||
aa := strings.Split(line, "\t")
|
||||
seen[aa[1]] = true
|
||||
n, err := strconv.Atoi(aa[0])
|
||||
x(err)
|
||||
m, ok := refs[aa[1]]
|
||||
if !ok {
|
||||
//continue
|
||||
m = refmax
|
||||
}
|
||||
diff := float64(m)/float64(refmax) - float64(n)/float64(curmax)
|
||||
if diff > 0.05 || diff < -0.05 {
|
||||
items = append(items, Item{
|
||||
word: aa[1],
|
||||
diff: diff,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for key, value := range refs {
|
||||
if !seen[key] {
|
||||
diff := float64(value)/float64(refmax) - 1.0
|
||||
if diff > 0.05 || diff < -0.05 {
|
||||
items = append(items, Item{
|
||||
word: key,
|
||||
diff: diff,
|
||||
gone: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(items, func(a, b int) bool {
|
||||
if items[a].diff == items[b].diff {
|
||||
return items[a].word < items[b].word
|
||||
}
|
||||
return items[a].diff > items[b].diff
|
||||
})
|
||||
|
||||
for _, item := range items {
|
||||
var s string
|
||||
if item.gone {
|
||||
s = "X"
|
||||
}
|
||||
fmt.Printf("%f\t%s\t%s\n", item.diff, s, item.word)
|
||||
}
|
||||
|
||||
}
|
||||
50
cmd/weekadd/weekadd.go
Normal file
50
cmd/weekadd/weekadd.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
ww "git.web.rug.nl/p209327/nlnieuws/internal/week"
|
||||
|
||||
e "codeberg.org/pebbe/errors"
|
||||
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
x = e.ExitErr
|
||||
)
|
||||
|
||||
func syntax(w io.Writer) {
|
||||
fmt.Fprintf(w, `
|
||||
Usage: %s yyyy.ww n
|
||||
|
||||
`, os.Args[0])
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 3 {
|
||||
syntax(os.Stdout)
|
||||
return
|
||||
}
|
||||
|
||||
aa := strings.Split(os.Args[1], ".")
|
||||
if len(aa) != 2 {
|
||||
syntax(os.Stderr)
|
||||
return
|
||||
}
|
||||
year, err := strconv.Atoi(aa[0])
|
||||
x(err)
|
||||
week, err := strconv.Atoi(aa[1])
|
||||
x(err)
|
||||
|
||||
n, err := strconv.Atoi(os.Args[2])
|
||||
x(err)
|
||||
|
||||
t, err := ww.WeekDate(year, week)
|
||||
x(err)
|
||||
|
||||
y, w := t.AddDate(0, 0, n*7).ISOWeek()
|
||||
fmt.Printf("%d.%02d\n", y, w)
|
||||
}
|
||||
42
collect.sh
42
collect.sh
@@ -3,8 +3,9 @@
|
||||
set -e
|
||||
|
||||
unset CDPATH
|
||||
PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
|
||||
PATH=/net/corpora/nlnieuws/bin:/net/corpora/nlnieuws/python:/net/aps/bin:$PATH
|
||||
export TZ=Europe/Amsterdam
|
||||
. /net/corpora/nlnieuws/python/env/bin/activate
|
||||
|
||||
verbose=0
|
||||
if [ "$1" = "-v" ]
|
||||
@@ -35,6 +36,8 @@ else
|
||||
esac
|
||||
fi
|
||||
|
||||
ds1=$(weekadd $ds -1)
|
||||
|
||||
year=${ds%%.*}
|
||||
|
||||
mkdir -p /net/corpora/nlnieuws/data/$year
|
||||
@@ -43,7 +46,7 @@ cd /net/corpora/nlnieuws/data/$year
|
||||
|
||||
declare -A parts
|
||||
#parts[alles]='.'
|
||||
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso'
|
||||
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso|Volkskrant'
|
||||
parts[amsterdam]='AT5|BuurtAdam|Parool'
|
||||
parts[groningen]='BuurtGrn|GG|Oog|RTVNoord|Sikkom'
|
||||
parts[literatuur]='LitNL|Tzum'
|
||||
@@ -64,6 +67,7 @@ parts[vlaanderen]='HLN|VRT'
|
||||
#parts[Sargasso]='Sargasso'
|
||||
#parts[Sikkom]='Sikkom'
|
||||
#parts[Tzum]='Tzum'
|
||||
#parts[Volkskrant]='Volkskrant'
|
||||
#parts[VRT]='VRT'
|
||||
|
||||
for part in ${!parts[@]}
|
||||
@@ -162,7 +166,41 @@ do
|
||||
| sed -e 's/\([0-9]\) */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
|
||||
> $part-nieuwe-adjww-extra-$ds-$i
|
||||
|
||||
# kale tellingen
|
||||
|
||||
say $part-allewoorden-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@pt and not(@pt="let" or @rel="mwp" or @neclass)) or (@cat="mwu" and not(.//node[@neclass]))]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq \
|
||||
| sed -e 's/\t.*//' | uniq -c \
|
||||
| grep -v '^ *1 ' \
|
||||
| sed -e 's/\([0-9]\) */\1\t/' | sort -f -k 2 | sort -n -r -k 1,1 -s \
|
||||
> $part-allewoorden-$ds-$i
|
||||
|
||||
done
|
||||
|
||||
# score
|
||||
say $part-allewoorden-$ds.score14
|
||||
stijgers.py $part-allewoorden-$ds1-4 $part-allewoorden-$ds-1 $part-allewoorden-$ds.score14
|
||||
stijgers2json.py $part-allewoorden-$ds1-4 $part-allewoorden-$ds-1 $part-allewoorden-$ds.score14.json
|
||||
|
||||
say $part-locaties-$ds.score14
|
||||
stijgers.py $part-locaties-$ds1-4 $part-locaties-$ds-1 $part-locaties-$ds.score14
|
||||
stijgers2json.py $part-locaties-$ds1-4 $part-locaties-$ds-1 $part-locaties-$ds.score14.json
|
||||
|
||||
say $part-personen-$ds.score14
|
||||
stijgers.py $part-personen-$ds1-4 $part-personen-$ds-1 $part-personen-$ds.score14
|
||||
stijgers2json.py $part-personen-$ds1-4 $part-personen-$ds-1 $part-personen-$ds.score14.json
|
||||
|
||||
say $part-organisaties-$ds.score14
|
||||
stijgers.py $part-organisaties-$ds1-4 $part-organisaties-$ds-1 $part-organisaties-$ds.score14
|
||||
stijgers2json.py $part-organisaties-$ds1-4 $part-organisaties-$ds-1 $part-organisaties-$ds.score14.json
|
||||
|
||||
say $part-overige-namen-$ds.score14
|
||||
stijgers.py $part-overige-namen-$ds1-4 $part-overige-namen-$ds-1 $part-overige-namen-$ds.score14
|
||||
stijgers2json.py $part-overige-namen-$ds1-4 $part-overige-namen-$ds-1 $part-overige-namen-$ds.score14.json
|
||||
|
||||
done
|
||||
|
||||
data2json $ds 1 > ../json/$year/DATA-$ds-1.json
|
||||
|
||||
@@ -11,12 +11,27 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
p = e.PanicErr
|
||||
reEOL = regexp.MustCompile(`[.!?]['"”’]?$`)
|
||||
reNEOL = regexp.MustCompile(`[.!?]['"”’]?\p{Lu}\p{Ll}`)
|
||||
reLET = regexp.MustCompile(`\p{Lu}`)
|
||||
p = e.PanicErr
|
||||
reEOL = regexp.MustCompile(`[.!?]['"”’]?$`)
|
||||
reNEOL = regexp.MustCompile(`[.!?]['"”’]?\p{Lu}\p{Ll}+\.?`)
|
||||
reLET = regexp.MustCompile(`\p{Lu}`)
|
||||
reBody = regexp.MustCompile(`<[bB][rR][ /]*>`)
|
||||
reQuotLeft = regexp.MustCompile(`<em>|<i>`)
|
||||
reQuotRight = regexp.MustCompile(`</em>|</i>`)
|
||||
)
|
||||
|
||||
func HtmlFix(html []byte) []byte {
|
||||
html = reQuotLeft.ReplaceAllLiteral(html, []byte(" „"))
|
||||
html = reQuotRight.ReplaceAllLiteral(html, []byte("” "))
|
||||
return reBody.ReplaceAllLiteral(html, []byte(" "))
|
||||
}
|
||||
|
||||
func HtmlFixString(html string) string {
|
||||
html = reQuotLeft.ReplaceAllLiteralString(html, " „")
|
||||
html = reQuotRight.ReplaceAllLiteralString(html, "” ")
|
||||
return reBody.ReplaceAllLiteralString(html, " ")
|
||||
}
|
||||
|
||||
func AddEnd(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
@@ -28,12 +43,20 @@ func AddEnd(s string) string {
|
||||
return s + ".\n"
|
||||
}
|
||||
|
||||
func FixSpace(s string) string {
|
||||
func FixSpace(s string, opt ...bool) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
s = reNEOL.ReplaceAllStringFunc(s, func(s1 string) string {
|
||||
i := reLET.FindStringIndex(s1)[0]
|
||||
return s1[:i] + " " + s1[i:]
|
||||
})
|
||||
|
||||
if len(opt) > 0 && opt[0] {
|
||||
s = reNEOL.ReplaceAllStringFunc(s, func(s1 string) string {
|
||||
if strings.HasSuffix(s1, ".") {
|
||||
// zoals: v.Chr.
|
||||
return s1
|
||||
}
|
||||
i := reLET.FindStringIndex(s1)[0]
|
||||
return s1[:i] + " " + s1[i:]
|
||||
})
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
30
internal/week/week.go
Normal file
30
internal/week/week.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package week
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// returns first date of week
|
||||
func WeekDate(year, week int) (time.Time, error) {
|
||||
// 1 januari
|
||||
t := time.Date(year, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// zoek eerste donderdag
|
||||
day := int(t.Weekday()) // 0 = zondag
|
||||
donderdag := 4 - day
|
||||
if donderdag < 0 {
|
||||
donderdag = donderdag + 7
|
||||
}
|
||||
|
||||
// schuif naar maandag voor eerste donderdag: donderdag - 3
|
||||
// in de gewenste week: 7 * (week - 1)
|
||||
t = t.AddDate(0, 0, donderdag-3+7*(week-1))
|
||||
|
||||
y, w := t.ISOWeek()
|
||||
if y != year || w != week {
|
||||
return t, fmt.Errorf("week number out of range")
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
118
oud/rang/collect-rang.sh
Normal file
118
oud/rang/collect-rang.sh
Normal file
@@ -0,0 +1,118 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
unset CDPATH
|
||||
PATH=/net/corpora/nlnieuws/bin:/net/aps/bin:$PATH
|
||||
export TZ=Europe/Amsterdam
|
||||
|
||||
verbose=0
|
||||
if [ "$1" = "-v" ]
|
||||
then
|
||||
shift
|
||||
verbose=1
|
||||
fi
|
||||
|
||||
say () {
|
||||
if [ "$verbose" = "1" ]
|
||||
then
|
||||
echo "$*"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$1" = "" ]
|
||||
then
|
||||
ds=`date -d -7days +%G.%V`
|
||||
else
|
||||
case "$1" in
|
||||
2[0-9][0-9][0-9].[0-5][0-9])
|
||||
ds=$1
|
||||
;;
|
||||
*)
|
||||
echo INVALID
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
year=${ds%%.*}
|
||||
|
||||
mkdir -p /net/corpora/nlnieuws/data/$year
|
||||
mkdir -p /net/corpora/nlnieuws/data/json/$year
|
||||
cd /net/corpora/nlnieuws/data/$year
|
||||
|
||||
declare -A parts
|
||||
#parts[alles]='.'
|
||||
parts[algemeen]='NOS|NU|NieuwsNL|RO|Sargasso|Volkskrant'
|
||||
#parts[amsterdam]='AT5|BuurtAdam|Parool'
|
||||
#parts[groningen]='BuurtGrn|GG|Oog|RTVNoord|Sikkom'
|
||||
#parts[literatuur]='LitNL|Tzum'
|
||||
#parts[vlaanderen]='HLN|VRT'
|
||||
#parts[AT5]='AT5'
|
||||
#parts[BuurtAdam]='BuurtAdam'
|
||||
#parts[BuurtGrn]='BuurtGrn'
|
||||
#parts[GG]='GG'
|
||||
#parts[HLN]='HLN'
|
||||
#parts[LitNL]='LitNL'
|
||||
#parts[NOS]='NOS'
|
||||
#parts[NU]='NU'
|
||||
#parts[NieuwsNL]='NieuwsNL'
|
||||
#parts[Oog]='Oog'
|
||||
#parts[Parool]='Parool'
|
||||
#parts[RO]='RO'
|
||||
#parts[RTVNoord]='RTVNoord'
|
||||
#parts[Sargasso]='Sargasso'
|
||||
#parts[Sikkom]='Sikkom'
|
||||
#parts[Tzum]='Tzum'
|
||||
#parts[Volkskrant]='Volkskrant'
|
||||
#parts[VRT]='VRT'
|
||||
|
||||
for part in ${!parts[@]}
|
||||
do
|
||||
regex=${parts[$part]}
|
||||
|
||||
for i in 1 4
|
||||
do
|
||||
files=$(find ../.. $(week2files $ds $i) | grep -E "$regex") || true
|
||||
if [ -z "$files" ]
|
||||
then
|
||||
continue
|
||||
fi
|
||||
|
||||
say $part-rang-word-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@pt and not(@pt="let" or @rel="mwp" or @neclass)) or (@cat="mwu" and not(.//node[@neclass]))]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
|
||||
> $part-rang-word-$ds-$i
|
||||
|
||||
say $part-rang-loc-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@neclass="LOC" and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="LOC" ])]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
|
||||
> $part-rang-loc-$ds-$i
|
||||
|
||||
say $part-rang-per-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@neclass="PER" and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="PER" ])]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
|
||||
> $part-rang-per-$ds-$i
|
||||
|
||||
say $part-rang-org-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@neclass="ORG" and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="ORG" ])]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
|
||||
> $part-rang-org-$ds-$i
|
||||
|
||||
say $part-rang-misc-$ds-$i
|
||||
alto \
|
||||
'fp://node[(@neclass="MISC" and not(@rel="mwp")) or (@cat="mwu" and .//node[@neclass="MISC" ])]' \
|
||||
'tt:%l\t%I' $files \
|
||||
| sed -e 's/\.[0-9][0-9]*$//' | sort | uniq | rang \
|
||||
> $part-rang-misc-$ds-$i
|
||||
|
||||
done
|
||||
done
|
||||
270
python/Claude.md
Normal file
270
python/Claude.md
Normal file
@@ -0,0 +1,270 @@
|
||||
---
|
||||
title: Claude
|
||||
updated: 2026-06-18 13:20:25Z
|
||||
created: 2026-06-18 13:15:32Z
|
||||
source: https://claude.ai/share/55700eff-d389-48d5-b1f5-cff59059ce61
|
||||
---
|
||||
|
||||
> I have collected a dataset of news articles from the past few weeks. I would like to know which words occur significantly more or less frequently in the last week compared to the four weeks before. What are good methods to do this?
|
||||
|
||||
There are several good methods for detecting statistically significant shifts in word frequency between two time periods. Here's an overview:
|
||||
|
||||
### Corpus Comparison Methods
|
||||
|
||||
**1\. Log-Likelihood Ratio (G²)**
|
||||
The most widely used method in corpus linguistics (Dunning, 1993). It compares observed vs. expected frequencies under the null hypothesis that both corpora are drawn from the same distribution. It handles low-frequency words better than chi-squared and works well even when the corpora are very different sizes — which matters here since you're comparing 1 week vs. 4 weeks.
|
||||
|
||||
**2\. Chi-Squared (χ²) Test**
|
||||
A simpler alternative, but it's unreliable for low-frequency words (expected count < 5) and can produce many false positives in sparse data. Log-likelihood is generally preferred over it for text.
|
||||
|
||||
**3\. Mann-Whitney U / Permutation Tests**
|
||||
Non-parametric tests that make no distributional assumptions. Useful if you want to compare word rates *per article* rather than pooled counts, since they respect article-level variance.
|
||||
|
||||
### Effect Size Measures (use alongside significance)
|
||||
|
||||
Significance alone doesn't tell you how *much* a word has changed. Pair it with:
|
||||
|
||||
- **%DIFF** — simple percentage difference in relative frequency, easy to interpret
|
||||
- **Bayes Factor (BIC approximation)** — penalises complexity, good for prioritising the most meaningful differences
|
||||
- **Log Ratio** — `log2(freq_week5 / freq_weeks1-4)`, gives a symmetric, interpretable effect size: +1 means doubled, −1 means halved
|
||||
|
||||
### Practical Considerations
|
||||
|
||||
| Issue | Recommendation |
|
||||
| --- | --- |
|
||||
| Unequal corpus sizes | Normalize to relative frequencies (per million words) before comparing |
|
||||
| Multiple comparisons | Apply Benjamini-Hochberg FDR correction — you're testing thousands of words simultaneously |
|
||||
| Low-frequency words | Set a minimum frequency threshold (e.g., appears ≥ 5 times total) to reduce noise |
|
||||
| Stop words | Decide whether to include them — they shift too and can be informative (e.g., rising use of "we" vs. "they") |
|
||||
| Multiword expressions | Consider bigrams/trigrams if single words feel too decontextualised |
|
||||
|
||||
### Recommended Pipeline
|
||||
|
||||
1. **Tokenize and normalize** (lowercase, optional lemmatization)
|
||||
2. **Count frequencies** in each time window
|
||||
3. **Filter** by minimum frequency
|
||||
4. **Run Log-Likelihood** for significance; compute **Log Ratio** for effect size
|
||||
5. **Apply FDR correction** across all words
|
||||
6. **Rank** by effect size among significant words
|
||||
|
||||
### Tools & Libraries
|
||||
|
||||
- **Python**: `scipy.stats` for chi-squared/G², `statsmodels` for FDR correction, `nltk`/`spacy` for tokenization
|
||||
- **R**: the `quanteda` package has `textstat_keyness()` built-in, which implements log-likelihood and several other keyness measures out of the box — very convenient for exactly this use case
|
||||
- **Ready-made**: AntConc (free GUI tool) supports keyness analysis directly
|
||||
|
||||
Would you like help implementing any of these in code?
|
||||
|
||||
> For the %DIFF, do you calculate it as "freq_week5 - freq_weeks1-4" ?
|
||||
>
|
||||
> Can you show scipy.stats code for chi-squared and G^2 ?
|
||||
|
||||
### %DIFF Formula
|
||||
|
||||
It's the relative difference in *normalised* frequencies (not raw counts, since your corpora are different sizes):
|
||||
|
||||
```
|
||||
%DIFF = ((freq_B / size_B) - (freq_A / size_A)) / (freq_A / size_A) * 100
|
||||
```
|
||||
|
||||
Where A is the reference corpus (weeks 1–4) and B is the target (week 5). So it answers: "by what percentage did the relative frequency go up or down?"
|
||||
|
||||
* * *
|
||||
|
||||
### Chi-Squared and G² in scipy
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from scipy.stats import chi2_contingency, chi2
|
||||
|
||||
def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):
|
||||
"""
|
||||
word : the word being tested
|
||||
counts_recent : raw count in week 5
|
||||
counts_reference : raw count in weeks 1-4
|
||||
total_recent : total tokens in week 5
|
||||
total_reference : total tokens in weeks 1-4
|
||||
"""
|
||||
a = counts_recent # word in recent
|
||||
b = counts_reference # word in reference
|
||||
c = total_recent - a # non-word in recent
|
||||
d = total_reference - b # non-word in reference
|
||||
|
||||
|
||||
contingency = np.array([[a, b],
|
||||
[c, d]])
|
||||
|
||||
|
||||
# --- Chi-Squared ---
|
||||
chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)
|
||||
|
||||
|
||||
# --- Log-Likelihood (G²) ---
|
||||
# G² = 2 * sum(observed * log(observed / expected))
|
||||
# scipy's chi2_contingency with lambda_="log-likelihood" computes this
|
||||
g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_="log-likelihood")
|
||||
|
||||
|
||||
# --- Effect sizes ---
|
||||
freq_recent = a / total_recent
|
||||
freq_reference = b / total_reference
|
||||
|
||||
pct_diff = (freq_recent - freq_reference) / freq_reference * 100
|
||||
|
||||
|
||||
# Avoid log(0) with a small epsilon
|
||||
eps = 1e-9
|
||||
log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))
|
||||
|
||||
|
||||
return {
|
||||
"word": word,
|
||||
"freq_recent": freq_recent,
|
||||
"freq_reference": freq_reference,
|
||||
"pct_diff": pct_diff,
|
||||
"log_ratio": log_ratio,
|
||||
"chi2": chi2_stat,
|
||||
"p_chi2": p_chi2,
|
||||
"g2": g2_stat,
|
||||
"p_g2": p_g2,
|
||||
}
|
||||
```
|
||||
|
||||
A few notes:
|
||||
|
||||
- `correction=False` disables Yates' continuity correction for chi-squared, which is standard for large corpora
|
||||
- `lambda_="log-likelihood"` is scipy's built-in way to get G²; it uses the same contingency table logic
|
||||
- Both tests have **1 degree of freedom** for a 2×2 table, so the p-values are directly comparable
|
||||
|
||||
* * *
|
||||
|
||||
### Running it over a full vocabulary
|
||||
|
||||
```python
|
||||
from statsmodels.stats.multitest import multipletests
|
||||
|
||||
results = [
|
||||
corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),
|
||||
total_recent, total_reference)
|
||||
for word in counts_recent
|
||||
]
|
||||
|
||||
|
||||
# FDR correction across all words
|
||||
|
||||
p_values = [r["p_g2"] for r in results]
|
||||
|
||||
_, p_adjusted, _, _ = multipletests(p_values, method="fdr_bh")
|
||||
|
||||
|
||||
for r, p_adj in zip(results, p_adjusted):
|
||||
r["p_g2_adjusted"] = p_adj
|
||||
```
|
||||
|
||||
Would you like help loading your articles and building the frequency counts to feed into this?
|
||||
|
||||
> And what about tf-idf?
|
||||
|
||||
TF-IDF is a slightly different tool — worth understanding the distinction before deciding whether to use it.
|
||||
|
||||
### What TF-IDF actually measures
|
||||
|
||||
TF-IDF scores a word relative to how *distinctive* it is within a document compared to a collection. The formula is:
|
||||
|
||||
```
|
||||
TF-IDF(word, doc) = TF(word, doc) × IDF(word, corpus)
|
||||
```
|
||||
|
||||
- **TF** — how often the word appears in a specific document
|
||||
- **IDF** — penalises words that appear in many documents (so "the" scores low)
|
||||
|
||||
It was designed for **ranking documents by relevance to a query**, not for comparing two time periods.
|
||||
|
||||
### The core problem for your use case
|
||||
|
||||
If you treat each time period as one big document, TF-IDF collapses to just IDF — because TF becomes the same for both "documents" and cancels out. You'd just be finding words that are rare across the whole corpus, which isn't what you want.
|
||||
|
||||
You *could* treat each article as a document and then aggregate scores by time period, but then you're measuring **within-article distinctiveness**, not **frequency shifts over time**.
|
||||
|
||||
### When TF-IDF *is* useful in your pipeline
|
||||
|
||||
| Use case | Good fit? |
|
||||
| --- | --- |
|
||||
| Detecting words that increased/decreased in frequency week-over-week | ✗ Use G² + log ratio instead |
|
||||
| Finding the most topically distinctive words *within* a single article | ✓ |
|
||||
| Building features for a classifier or clustering articles by topic | ✓ |
|
||||
| Summarising what a time period was "about" relative to a background corpus | Partially — see below |
|
||||
|
||||
### The partial exception: keyness as pseudo-TF-IDF
|
||||
|
||||
Some researchers compute a **keyness score** by treating the recent period as the "document" and the reference period as the "corpus" — which is conceptually similar to TF-IDF. But the statistically rigorous version of this is exactly the log-likelihood + log ratio approach already described. G² is essentially the principled, significance-tested version of what TF-IDF approximates intuitively.
|
||||
|
||||
### Bottom line
|
||||
|
||||
For your goal — words that shifted significantly between week 5 and weeks 1–4 — G² with log ratio is the right tool. TF-IDF would give you a rough proxy at best and a misleading ranking at worst.
|
||||
|
||||
> The corpus_stats() function contains the following comment:
|
||||
>
|
||||
> `# G² = 2 * sum(observed * log(observed / expected))`
|
||||
>
|
||||
> Is this the formula to calculate the log-likelihood ratio? What do the observed and expected variables contain?
|
||||
|
||||
Yes, that's the correct formula for G². Let me break it down.
|
||||
|
||||
### The full formula
|
||||
|
||||
```
|
||||
G² = 2 × Σ Oᵢ × ln(Oᵢ / Eᵢ)
|
||||
```
|
||||
|
||||
The sum runs over all **four cells** of the contingency table:
|
||||
|
||||
| | Recent (week 5) | Reference (weeks 1–4) |
|
||||
| --- | --- | --- |
|
||||
| **Target word** | a | b |
|
||||
| **All other words** | c | d |
|
||||
|
||||
### Observed values
|
||||
|
||||
Simply the raw counts from your data:
|
||||
|
||||
```
|
||||
O₁ = a (word in recent)
|
||||
O₂ = b (word in reference)
|
||||
O₃ = c (all other words in recent)
|
||||
O₄ = d (all other words in reference)
|
||||
```
|
||||
|
||||
### Expected values
|
||||
|
||||
The counts you would *expect* if the word's relative frequency were identical in both corpora — i.e. under the null hypothesis that nothing changed. They're computed from the row and column marginals:
|
||||
|
||||
```
|
||||
E₁ = (a + b) × (a + c) / N
|
||||
E₂ = (a + b) × (b + d) / N
|
||||
E₃ = (c + d) × (a + c) / N
|
||||
E₄ = (c + d) × (b + d) / N
|
||||
```
|
||||
|
||||
Where `N = a + b + c + d` is the total number of tokens across both corpora.
|
||||
|
||||
### Concrete example
|
||||
|
||||
Say your recent corpus has 10,000 tokens and your reference has 40,000 tokens, and the word "crisis" appears 30 times recently and 60 times in the reference:
|
||||
|
||||
```
|
||||
a = 30, b = 60
|
||||
c = 9970, d = 39940
|
||||
N = 50000
|
||||
|
||||
E₁ = (30+60) × (30+9970) / 50000 = 90 × 10000 / 50000 = 18
|
||||
E₂ = (30+60) × (60+39940) / 50000 = 90 × 40000 / 50000 = 72
|
||||
E₃ = (9970+39940) × 10000 / 50000 = 49910 × 10000 / 50000 = 9982
|
||||
E₄ = (9970+39940) × 40000 / 50000 = 49910 × 40000 / 50000 = 39928
|
||||
|
||||
G² = 2 × (30×ln(30/18) + 60×ln(60/72) + 9970×ln(9970/9982) + 39940×ln(39940/39928))
|
||||
≈ 2 × (16.2 − 9.9 − 1.2 + 1.2)
|
||||
≈ 12.7
|
||||
```
|
||||
|
||||
"crisis" appears at 3× its expected relative frequency (30/18), and G² = 12.7 exceeds the chi-squared critical value of 3.84 at p < 0.05 with 1 degree of freedom, so the increase is significant.
|
||||
BIN
python/Claude.pdf
Normal file
BIN
python/Claude.pdf
Normal file
Binary file not shown.
4762
python/data.txt
Normal file
4762
python/data.txt
Normal file
File diff suppressed because it is too large
Load Diff
9123
python/namen.html
Normal file
9123
python/namen.html
Normal file
File diff suppressed because one or more lines are too long
1563
python/namen.ipynb
Normal file
1563
python/namen.ipynb
Normal file
File diff suppressed because it is too large
Load Diff
82
python/namen.py
Executable file
82
python/namen.py
Executable file
@@ -0,0 +1,82 @@
|
||||
#!/net/corpora/nlnieuws/notebook/bin/python3
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import chi2_contingency
|
||||
from statsmodels.stats.multitest import multipletests
|
||||
import pandas as pd
|
||||
|
||||
# waarom werkt dit niet?
|
||||
pd.set_option('display.max_rows', 40)
|
||||
|
||||
def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):
|
||||
"""
|
||||
word : the word being tested
|
||||
counts_recent : raw count in week 5
|
||||
counts_reference : raw count in weeks 1-4
|
||||
total_recent : total tokens in week 5
|
||||
total_reference : total tokens in weeks 1-4
|
||||
"""
|
||||
a = counts_recent # word in recent
|
||||
b = counts_reference # word in reference
|
||||
c = total_recent - a # non-word in recent
|
||||
d = total_reference - b # non-word in reference
|
||||
|
||||
contingency = np.array([[a, b],
|
||||
[c, d]])
|
||||
|
||||
# --- Chi-Squared ---
|
||||
chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)
|
||||
|
||||
# --- Log-Likelihood (G²) ---
|
||||
# G² = 2 * sum(observed * log(observed / expected))
|
||||
# scipy's chi2_contingency with lambda_="log-likelihood" computes this
|
||||
g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_="log-likelihood")
|
||||
|
||||
# --- Effect sizes ---
|
||||
freq_recent = a / total_recent
|
||||
freq_reference = b / total_reference
|
||||
|
||||
pct_diff = (freq_recent - freq_reference) / freq_reference * 100
|
||||
|
||||
# Avoid log(0) with a small epsilon
|
||||
eps = 1e-9
|
||||
log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))
|
||||
|
||||
return {
|
||||
"word": word,
|
||||
"freq_recent": freq_recent,
|
||||
"freq_reference": freq_reference,
|
||||
"pct_diff": pct_diff,
|
||||
"log_ratio": log_ratio,
|
||||
"chi2": chi2_stat,
|
||||
"p_chi2": p_chi2,
|
||||
"g2": g2_stat,
|
||||
"p_g2": p_g2,
|
||||
}
|
||||
|
||||
counts_recent = {}
|
||||
counts_reference = {}
|
||||
with open("data.txt", "rt", encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
aa = line.split("\t")
|
||||
counts_reference[aa[0]] = max(int(aa[1]), 0.5)
|
||||
counts_recent[aa[0]] = max(int(aa[2]), 0.5)
|
||||
total_recent = sum(counts_recent.values())
|
||||
total_reference = sum(counts_reference.values())
|
||||
|
||||
results = [
|
||||
corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),
|
||||
total_recent, total_reference)
|
||||
for word in counts_recent]
|
||||
|
||||
# FDR correction across all words
|
||||
p_values = [r["p_g2"] for r in results]
|
||||
_, p_adjusted, _, _ = multipletests(p_values, method="fdr_bh")
|
||||
|
||||
for r, p_adj in zip(results, p_adjusted):
|
||||
r["p_g2_adjusted"] = p_adj
|
||||
|
||||
results = pd.DataFrame(results)
|
||||
print(results)
|
||||
print(results.sort_values('g2'))
|
||||
print(results.sort_values('pct_diff'))
|
||||
30
python/score.txt
Normal file
30
python/score.txt
Normal file
@@ -0,0 +1,30 @@
|
||||
Er zijn twee simpele formules om de "effect size" van het verschil tussen twee
|
||||
relatieve frequenties te rapporteren:
|
||||
* %DIFF = (freq_B - freq_A) / freq_A * 100
|
||||
Percentage verschil in relatieve frequenties, makkelijk te interpreteren,
|
||||
maar niet symmetrisch.
|
||||
* Log Ratio: log2(freq_A / freq_B)
|
||||
Een symmetrische en interpreteerbare effect size; +1 is een verdubbeling, -1
|
||||
een halvering
|
||||
Twee populaire methodes om de significantie van frequentieverschillen te testen
|
||||
(ook wel keyword extraction):
|
||||
* Log-Likelihood Ratio (G^2): meest gebruikte methode in Corpus Linguistics.
|
||||
Vergelijkt observed vs expected frequency.
|
||||
* Chi-Squared test (X^2): simpeler dan G^2, maar geeft meer false positives
|
||||
bij sparse data, werkt niet goed met lage frequenties.
|
||||
Je kunt dan de gebruiker alleen de woorden met significante verschillen laten
|
||||
zien (dit zijn dan de keywords). Ik heb met behulp van Claude een notebook in
|
||||
elkaar gezet met een demonstratie van deze methodes:
|
||||
→ Word freq comparison.ipynb
|
||||
|
||||
|
||||
Er zijn ook geavanceerdere methodes die me te ingewikkeld lijken om te
|
||||
implementeren, maar ik noem ze voor de volledigheid. In de stylometrie
|
||||
is Burrow's Zeta populair, deze is bijv. beschikbaar in Stylo
|
||||
https://github.com/computationalstylistics/stylo onder de oppose()
|
||||
functie
|
||||
https://cran.r-project.org/web/packages/stylo/stylo.pdf#Rfn.oppose.1 .
|
||||
Er is ook een methode die gebruik maakt van Bayesiaanse statistiek en
|
||||
frequenties uit een achtergrondcorpus, de Fightin' Words methode van
|
||||
Monroe et al: https://github.com/jmhessel/FightingWords
|
||||
|
||||
104
python/stijgers.py
Executable file
104
python/stijgers.py
Executable file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import numpy as np
|
||||
from scipy.stats import chi2_contingency
|
||||
from statsmodels.stats.multitest import multipletests
|
||||
import pandas as pd
|
||||
|
||||
# waarom werkt dit niet?
|
||||
pd.set_option('display.max_rows', 40)
|
||||
pd.set_option('display.max_columns', None)
|
||||
|
||||
def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):
|
||||
"""
|
||||
word : the word being tested
|
||||
counts_recent : raw count in week 5
|
||||
counts_reference : raw count in weeks 1-4
|
||||
total_recent : total tokens in week 5
|
||||
total_reference : total tokens in weeks 1-4
|
||||
"""
|
||||
a = counts_recent # word in recent
|
||||
b = counts_reference # word in reference
|
||||
c = total_recent - a # non-word in recent
|
||||
d = total_reference - b # non-word in reference
|
||||
|
||||
contingency = np.array([[a, b],
|
||||
[c, d]])
|
||||
|
||||
# --- Chi-Squared ---
|
||||
chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)
|
||||
|
||||
# --- Log-Likelihood (G²) ---
|
||||
# G² = 2 * sum(observed * log(observed / expected))
|
||||
# scipy's chi2_contingency with lambda_="log-likelihood" computes this
|
||||
g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_="log-likelihood")
|
||||
|
||||
# --- Effect sizes ---
|
||||
freq_recent = a / total_recent
|
||||
freq_reference = b / total_reference
|
||||
|
||||
pct_diff = (freq_recent - freq_reference) / freq_reference * 100
|
||||
|
||||
# Avoid log(0) with a small epsilon
|
||||
eps = 1e-9
|
||||
log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))
|
||||
|
||||
return {
|
||||
"word": word,
|
||||
"freq_recent": freq_recent,
|
||||
"freq_reference": freq_reference,
|
||||
"pct_diff": pct_diff,
|
||||
"log_ratio": log_ratio,
|
||||
"chi2": chi2_stat,
|
||||
"p_chi2": p_chi2,
|
||||
"g2": g2_stat,
|
||||
"p_g2": p_g2,
|
||||
}
|
||||
|
||||
oudfile = sys.argv[1]
|
||||
nieuwfile = sys.argv[2]
|
||||
textfile = sys.argv[3]
|
||||
|
||||
counts_recent = {}
|
||||
counts_reference = {}
|
||||
|
||||
with open(oudfile, "rt", encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
aa = line.split("\t")
|
||||
counts_reference[aa[1].strip()] = int(aa[0])
|
||||
with open(nieuwfile, "rt", encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
aa = line.split("\t")
|
||||
counts_recent[aa[1].strip()] = int(aa[0])
|
||||
|
||||
for key in counts_recent:
|
||||
if not key in counts_reference:
|
||||
counts_reference[key] = 0.5
|
||||
for key in counts_reference:
|
||||
if not key in counts_recent:
|
||||
counts_recent[key] = 0.5
|
||||
|
||||
total_recent = sum(counts_recent.values())
|
||||
total_reference = sum(counts_reference.values())
|
||||
|
||||
results = [
|
||||
corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),
|
||||
total_recent, total_reference)
|
||||
for word in counts_recent]
|
||||
|
||||
# FDR correction across all words
|
||||
p_values = [r["p_g2"] for r in results]
|
||||
_, p_adjusted, _, _ = multipletests(p_values, method="fdr_bh")
|
||||
|
||||
for r, p_adj in zip(results, p_adjusted):
|
||||
r["p_g2_adjusted"] = p_adj
|
||||
|
||||
results = pd.DataFrame(results)
|
||||
|
||||
with open(textfile, "wt", encoding="utf-8") as fp:
|
||||
fp.write("STIJGERS\n")
|
||||
fp.write(results[results.pct_diff > 0].sort_values('g2', ascending=False)[:40].to_string())
|
||||
fp.write("\n\nDALERS\n")
|
||||
fp.write(results[results.pct_diff < 0].sort_values('g2', ascending=False)[:40].to_string())
|
||||
fp.write("\n")
|
||||
99
python/stijgers2json.py
Executable file
99
python/stijgers2json.py
Executable file
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import sys
|
||||
import numpy as np
|
||||
from scipy.stats import chi2_contingency
|
||||
from statsmodels.stats.multitest import multipletests
|
||||
|
||||
def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):
|
||||
"""
|
||||
word : the word being tested
|
||||
counts_recent : raw count in week 5
|
||||
counts_reference : raw count in weeks 1-4
|
||||
total_recent : total tokens in week 5
|
||||
total_reference : total tokens in weeks 1-4
|
||||
"""
|
||||
a = counts_recent # word in recent
|
||||
b = counts_reference # word in reference
|
||||
c = total_recent - a # non-word in recent
|
||||
d = total_reference - b # non-word in reference
|
||||
|
||||
contingency = np.array([[a, b],
|
||||
[c, d]])
|
||||
|
||||
# --- Chi-Squared ---
|
||||
chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)
|
||||
|
||||
# --- Log-Likelihood (G²) ---
|
||||
# G² = 2 * sum(observed * log(observed / expected))
|
||||
# scipy's chi2_contingency with lambda_="log-likelihood" computes this
|
||||
g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_="log-likelihood")
|
||||
|
||||
# --- Effect sizes ---
|
||||
freq_recent = a / total_recent
|
||||
freq_reference = b / total_reference
|
||||
|
||||
pct_diff = (freq_recent - freq_reference) / freq_reference * 100
|
||||
|
||||
# Avoid log(0) with a small epsilon
|
||||
eps = 1e-9
|
||||
log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))
|
||||
|
||||
return {
|
||||
"word": word,
|
||||
"freq_recent": freq_recent,
|
||||
"freq_reference": freq_reference,
|
||||
"pct_diff": pct_diff,
|
||||
"log_ratio": log_ratio,
|
||||
"chi2": chi2_stat,
|
||||
"p_chi2": p_chi2,
|
||||
"g2": g2_stat,
|
||||
"p_g2": p_g2,
|
||||
}
|
||||
|
||||
oudfile = sys.argv[1]
|
||||
nieuwfile = sys.argv[2]
|
||||
jsonfile = sys.argv[3]
|
||||
|
||||
counts_recent = {}
|
||||
counts_reference = {}
|
||||
|
||||
with open(oudfile, "rt", encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
aa = line.split("\t")
|
||||
counts_reference[aa[1].strip()] = int(aa[0])
|
||||
with open(nieuwfile, "rt", encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
aa = line.split("\t")
|
||||
counts_recent[aa[1].strip()] = int(aa[0])
|
||||
|
||||
for key in counts_recent:
|
||||
if not key in counts_reference:
|
||||
counts_reference[key] = 0.5
|
||||
for key in counts_reference:
|
||||
if not key in counts_recent:
|
||||
counts_recent[key] = 0.5
|
||||
|
||||
total_recent = sum(counts_recent.values())
|
||||
total_reference = sum(counts_reference.values())
|
||||
|
||||
results = [
|
||||
corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),
|
||||
total_recent, total_reference)
|
||||
for word in counts_recent]
|
||||
|
||||
# FDR correction across all words
|
||||
p_values = [r["p_g2"] for r in results]
|
||||
_, p_adjusted, _, _ = multipletests(p_values, method="fdr_bh")
|
||||
|
||||
for r, p_adj in zip(results, p_adjusted):
|
||||
r["p_g2_adjusted"] = p_adj
|
||||
|
||||
o = {}
|
||||
#o['up'] = sorted([x for x in results if x['p_g2'] < .05 and x['pct_diff'] > 0], key=lambda x: x['g2'], reverse=True)[:40]
|
||||
#o['dn'] = sorted([x for x in results if x['p_g2'] < .05 and x['pct_diff'] < 0], key=lambda x: x['g2'], reverse=True)[:40]
|
||||
o['up'] = sorted([x for x in results if x['pct_diff'] > 0], key=lambda x: x['g2'], reverse=True)[:40]
|
||||
o['dn'] = sorted([x for x in results if x['pct_diff'] < 0], key=lambda x: x['g2'], reverse=True)[:40]
|
||||
with open(jsonfile, "wt", encoding="utf-8") as fp:
|
||||
json.dump(o, fp)
|
||||
8194
python/word_freq_comparison.html
Normal file
8194
python/word_freq_comparison.html
Normal file
File diff suppressed because one or more lines are too long
500
python/word_freq_comparison.ipynb
Normal file
500
python/word_freq_comparison.ipynb
Normal file
@@ -0,0 +1,500 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"executionInfo": {
|
||||
"elapsed": 1341,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100698726,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "YMifluhW2rZp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from scipy.stats import chi2_contingency\n",
|
||||
"from statsmodels.stats.multitest import multipletests\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"def corpus_stats(word, counts_recent, counts_reference, total_recent, total_reference):\n",
|
||||
" \"\"\"\n",
|
||||
" word : the word being tested\n",
|
||||
" counts_recent : raw count in week 5\n",
|
||||
" counts_reference : raw count in weeks 1-4\n",
|
||||
" total_recent : total tokens in week 5\n",
|
||||
" total_reference : total tokens in weeks 1-4\n",
|
||||
" \"\"\"\n",
|
||||
" a = counts_recent # word in recent\n",
|
||||
" b = counts_reference # word in reference\n",
|
||||
" c = total_recent - a # non-word in recent\n",
|
||||
" d = total_reference - b # non-word in reference\n",
|
||||
"\n",
|
||||
" contingency = np.array([[a, b],\n",
|
||||
" [c, d]])\n",
|
||||
"\n",
|
||||
" # --- Chi-Squared ---\n",
|
||||
" chi2_stat, p_chi2, _, _ = chi2_contingency(contingency, correction=False)\n",
|
||||
"\n",
|
||||
" # --- Log-Likelihood (G²) ---\n",
|
||||
" # G² = 2 * sum(observed * log(observed / expected))\n",
|
||||
" # scipy's chi2_contingency with lambda_=\"log-likelihood\" computes this\n",
|
||||
" g2_stat, p_g2, _, _ = chi2_contingency(contingency, lambda_=\"log-likelihood\")\n",
|
||||
"\n",
|
||||
" # --- Effect sizes ---\n",
|
||||
" freq_recent = a / total_recent\n",
|
||||
" freq_reference = b / total_reference\n",
|
||||
"\n",
|
||||
" pct_diff = (freq_recent - freq_reference) / freq_reference * 100\n",
|
||||
"\n",
|
||||
" # Avoid log(0) with a small epsilon\n",
|
||||
" eps = 1e-9\n",
|
||||
" log_ratio = np.log2((freq_recent + eps) / (freq_reference + eps))\n",
|
||||
"\n",
|
||||
" return {\n",
|
||||
" \"word\": word,\n",
|
||||
" \"freq_recent\": freq_recent,\n",
|
||||
" \"freq_reference\": freq_reference,\n",
|
||||
" \"pct_diff\": pct_diff,\n",
|
||||
" \"log_ratio\": log_ratio,\n",
|
||||
" \"chi2\": chi2_stat,\n",
|
||||
" \"p_chi2\": p_chi2,\n",
|
||||
" \"g2\": g2_stat,\n",
|
||||
" \"p_g2\": p_g2,\n",
|
||||
" }\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"executionInfo": {
|
||||
"elapsed": 38,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100880331,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "mHH718-222BM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Example data\n",
|
||||
"counts_recent = {'eend': 150, 'tafel': 101, 'fiets': 102}\n",
|
||||
"counts_reference = {'eend': 77, 'tafel': 100, 'fiets': 142}\n",
|
||||
"total_recent = sum(counts_recent.values())\n",
|
||||
"total_reference = sum(counts_reference.values())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"executionInfo": {
|
||||
"elapsed": 7,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100881153,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "urBml1212wxb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Run tests on whole vocabulary, including correction for multiple tests\n",
|
||||
"# (false discovery rate).\n",
|
||||
"\n",
|
||||
"results = [\n",
|
||||
" corpus_stats(word, counts_recent[word], counts_reference.get(word, 0),\n",
|
||||
" total_recent, total_reference)\n",
|
||||
" for word in counts_recent]\n",
|
||||
"\n",
|
||||
"# FDR correction across all words\n",
|
||||
"p_values = [r[\"p_g2\"] for r in results]\n",
|
||||
"_, p_adjusted, _, _ = multipletests(p_values, method=\"fdr_bh\")\n",
|
||||
"\n",
|
||||
"for r, p_adj in zip(results, p_adjusted):\n",
|
||||
" r[\"p_g2_adjusted\"] = p_adj"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 163
|
||||
},
|
||||
"executionInfo": {
|
||||
"elapsed": 12,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100882491,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "-y3MUOgI3PFn",
|
||||
"outputId": "f9a90951-c8f2-45a7-8d8d-fb0e279dd5b3"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>word</th>\n",
|
||||
" <th>freq_recent</th>\n",
|
||||
" <th>freq_reference</th>\n",
|
||||
" <th>pct_diff</th>\n",
|
||||
" <th>log_ratio</th>\n",
|
||||
" <th>chi2</th>\n",
|
||||
" <th>p_chi2</th>\n",
|
||||
" <th>g2</th>\n",
|
||||
" <th>p_g2</th>\n",
|
||||
" <th>p_g2_adjusted</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>eend</td>\n",
|
||||
" <td>0.424929</td>\n",
|
||||
" <td>0.241379</td>\n",
|
||||
" <td>76.042088</td>\n",
|
||||
" <td>0.815920</td>\n",
|
||||
" <td>25.238117</td>\n",
|
||||
" <td>5.067080e-07</td>\n",
|
||||
" <td>24.764140</td>\n",
|
||||
" <td>6.479173e-07</td>\n",
|
||||
" <td>0.000002</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>tafel</td>\n",
|
||||
" <td>0.286119</td>\n",
|
||||
" <td>0.313480</td>\n",
|
||||
" <td>-8.728045</td>\n",
|
||||
" <td>-0.131756</td>\n",
|
||||
" <td>0.598371</td>\n",
|
||||
" <td>4.392004e-01</td>\n",
|
||||
" <td>0.474701</td>\n",
|
||||
" <td>4.908322e-01</td>\n",
|
||||
" <td>0.490832</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>fiets</td>\n",
|
||||
" <td>0.288952</td>\n",
|
||||
" <td>0.445141</td>\n",
|
||||
" <td>-35.087579</td>\n",
|
||||
" <td>-0.623434</td>\n",
|
||||
" <td>17.676782</td>\n",
|
||||
" <td>2.618028e-05</td>\n",
|
||||
" <td>17.051468</td>\n",
|
||||
" <td>3.638025e-05</td>\n",
|
||||
" <td>0.000055</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" word freq_recent freq_reference pct_diff log_ratio chi2 \\\n",
|
||||
"0 eend 0.424929 0.241379 76.042088 0.815920 25.238117 \n",
|
||||
"1 tafel 0.286119 0.313480 -8.728045 -0.131756 0.598371 \n",
|
||||
"2 fiets 0.288952 0.445141 -35.087579 -0.623434 17.676782 \n",
|
||||
"\n",
|
||||
" p_chi2 g2 p_g2 p_g2_adjusted \n",
|
||||
"0 5.067080e-07 24.764140 6.479173e-07 0.000002 \n",
|
||||
"1 4.392004e-01 0.474701 4.908322e-01 0.490832 \n",
|
||||
"2 2.618028e-05 17.051468 3.638025e-05 0.000055 "
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results = pd.DataFrame(results)\n",
|
||||
"results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 132
|
||||
},
|
||||
"executionInfo": {
|
||||
"elapsed": 65,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100883685,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "nTpOtOka3ViF",
|
||||
"outputId": "2430f959-eeb9-4670-da76-613406cbf473"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>word</th>\n",
|
||||
" <th>freq_recent</th>\n",
|
||||
" <th>freq_reference</th>\n",
|
||||
" <th>pct_diff</th>\n",
|
||||
" <th>log_ratio</th>\n",
|
||||
" <th>chi2</th>\n",
|
||||
" <th>p_chi2</th>\n",
|
||||
" <th>g2</th>\n",
|
||||
" <th>p_g2</th>\n",
|
||||
" <th>p_g2_adjusted</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>eend</td>\n",
|
||||
" <td>0.424929</td>\n",
|
||||
" <td>0.241379</td>\n",
|
||||
" <td>76.042088</td>\n",
|
||||
" <td>0.815920</td>\n",
|
||||
" <td>25.238117</td>\n",
|
||||
" <td>5.067080e-07</td>\n",
|
||||
" <td>24.764140</td>\n",
|
||||
" <td>6.479173e-07</td>\n",
|
||||
" <td>0.000002</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>fiets</td>\n",
|
||||
" <td>0.288952</td>\n",
|
||||
" <td>0.445141</td>\n",
|
||||
" <td>-35.087579</td>\n",
|
||||
" <td>-0.623434</td>\n",
|
||||
" <td>17.676782</td>\n",
|
||||
" <td>2.618028e-05</td>\n",
|
||||
" <td>17.051468</td>\n",
|
||||
" <td>3.638025e-05</td>\n",
|
||||
" <td>0.000055</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" word freq_recent freq_reference pct_diff log_ratio chi2 \\\n",
|
||||
"0 eend 0.424929 0.241379 76.042088 0.815920 25.238117 \n",
|
||||
"2 fiets 0.288952 0.445141 -35.087579 -0.623434 17.676782 \n",
|
||||
"\n",
|
||||
" p_chi2 g2 p_g2 p_g2_adjusted \n",
|
||||
"0 5.067080e-07 24.764140 6.479173e-07 0.000002 \n",
|
||||
"2 2.618028e-05 17.051468 3.638025e-05 0.000055 "
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Significant according to Chi2\n",
|
||||
"results[results['p_chi2'] < 0.05]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 132
|
||||
},
|
||||
"executionInfo": {
|
||||
"elapsed": 166,
|
||||
"status": "ok",
|
||||
"timestamp": 1781100928540,
|
||||
"user": {
|
||||
"displayName": "Andreas van Cranenburgh",
|
||||
"userId": "13143063654677287265"
|
||||
},
|
||||
"user_tz": -120
|
||||
},
|
||||
"id": "Mz4zAphE4dBY",
|
||||
"outputId": "3b42fdd2-f451-47b6-8989-ebf4dafcbf1a"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>word</th>\n",
|
||||
" <th>freq_recent</th>\n",
|
||||
" <th>freq_reference</th>\n",
|
||||
" <th>pct_diff</th>\n",
|
||||
" <th>log_ratio</th>\n",
|
||||
" <th>chi2</th>\n",
|
||||
" <th>p_chi2</th>\n",
|
||||
" <th>g2</th>\n",
|
||||
" <th>p_g2</th>\n",
|
||||
" <th>p_g2_adjusted</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>eend</td>\n",
|
||||
" <td>0.424929</td>\n",
|
||||
" <td>0.241379</td>\n",
|
||||
" <td>76.042088</td>\n",
|
||||
" <td>0.815920</td>\n",
|
||||
" <td>25.238117</td>\n",
|
||||
" <td>5.067080e-07</td>\n",
|
||||
" <td>24.764140</td>\n",
|
||||
" <td>6.479173e-07</td>\n",
|
||||
" <td>0.000002</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>fiets</td>\n",
|
||||
" <td>0.288952</td>\n",
|
||||
" <td>0.445141</td>\n",
|
||||
" <td>-35.087579</td>\n",
|
||||
" <td>-0.623434</td>\n",
|
||||
" <td>17.676782</td>\n",
|
||||
" <td>2.618028e-05</td>\n",
|
||||
" <td>17.051468</td>\n",
|
||||
" <td>3.638025e-05</td>\n",
|
||||
" <td>0.000055</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" word freq_recent freq_reference pct_diff log_ratio chi2 \\\n",
|
||||
"0 eend 0.424929 0.241379 76.042088 0.815920 25.238117 \n",
|
||||
"2 fiets 0.288952 0.445141 -35.087579 -0.623434 17.676782 \n",
|
||||
"\n",
|
||||
" p_chi2 g2 p_g2 p_g2_adjusted \n",
|
||||
"0 5.067080e-07 24.764140 6.479173e-07 0.000002 \n",
|
||||
"2 2.618028e-05 17.051468 3.638025e-05 0.000055 "
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Significant according to G2 (LLR)\n",
|
||||
"results[results['p_g2_adjusted'] < 0.05]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "JNCCUpdC4jK5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"authorship_tag": "ABX9TyOWNAG6IZoh+ik4rqgeMAZj",
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.13.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
1
python/word_freq_comparison.ipynb.ori
Normal file
1
python/word_freq_comparison.ipynb.ori
Normal file
File diff suppressed because one or more lines are too long
28
r/test-count.R
Normal file
28
r/test-count.R
Normal file
@@ -0,0 +1,28 @@
|
||||
nw <- read.table('data/2026/algemeen-count-per-2026.23-1', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
|
||||
od <- read.table('data/2026/algemeen-count-per-2026.22-4', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
|
||||
words <- unique(c(od$word, nw$word))
|
||||
o <- order(words)
|
||||
words <- words[o]
|
||||
n <- length(words)
|
||||
d <- data.frame(word=words, od=rep(0, n), nw=rep(0, n))
|
||||
|
||||
for (word in nw$word) {
|
||||
d$nw[d$word==word] <- nw$f[nw$word==word]
|
||||
}
|
||||
for (word in od$word) {
|
||||
d$od[d$word==word] <- od$f[od$word==word]
|
||||
}
|
||||
|
||||
write.table(d, file="tmp.csv", quote=FALSE, sep="\t", row.names=FALSE, fileEncoding="utf-8")
|
||||
|
||||
####
|
||||
|
||||
oud <- d$od
|
||||
nieuw <- d$nw
|
||||
|
||||
oud[oud == 0] <- 0.5
|
||||
nieuw[nieuw == 0] <- 0.5
|
||||
|
||||
plot(log(oud), log(nieuw))
|
||||
lines(log(range(oud)), log(range(nieuw)))
|
||||
identify(log(oud), log(nieuw), labels=words)
|
||||
32
r/test-rang.R
Normal file
32
r/test-rang.R
Normal file
@@ -0,0 +1,32 @@
|
||||
nw <- read.table('data/2026/algemeen-rang-per-2026.23-1', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
|
||||
od <- read.table('data/2026/algemeen-rang-per-2026.22-4', sep="\t", quote="", encoding="utf-8", col.names=c("f", "word"))
|
||||
words <- unique(c(od$word, nw$word))
|
||||
o <- order(words)
|
||||
words <- words[o]
|
||||
n <- length(words)
|
||||
d <- data.frame(word=words, od=rep(NA, n), nw=rep(NA, n))
|
||||
|
||||
for (word in nw$word) {
|
||||
d$nw[d$word==word] <- nw$f[nw$word==word]
|
||||
}
|
||||
for (word in od$word) {
|
||||
d$od[d$word==word] <- od$f[od$word==word]
|
||||
}
|
||||
|
||||
d$nw[is.na(d$nw)] <- max(nw$f) + 2
|
||||
d$od[is.na(d$od)] <- max(od$f) + 2
|
||||
|
||||
myplot <- function(values, labels, titel="", sub ="") {
|
||||
y <- 1:length(values)
|
||||
xx <- range(values)
|
||||
plot(values, y, xlim=c(xx[1], xx[2] + (xx[2]-xx[1]) / 5), xlab="score", ylab="index", main=titel, sub=sub)
|
||||
text(values, y, labels, pos=4)
|
||||
}
|
||||
|
||||
nwn <- (d$nw - 1) / (max(nw$f) + 1)
|
||||
odn <- (d$od - 1) / (max(od$f) + 1)
|
||||
|
||||
v <- odn - nwn
|
||||
o <- order(-v)
|
||||
myplot(v[o][1:40], words[o][1:40], titel="score op basis van genormaliseerde rang", "week 23 t.o.v. week 19 t/m 22")
|
||||
|
||||
23
www/app.html
23
www/app.html
@@ -110,11 +110,16 @@
|
||||
<td class="bar"><div id="RO"></div></td>
|
||||
<td><a href="https://reportersonline.nl/">Reporters Online</a></td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar"><div id="Sargasso"></div></td>
|
||||
<td><a href="https://sargasso.nl/">Sargasso</a></td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar"><div id="Volkskrant"></div></td>
|
||||
<td><a href="https://www.volkskrant.nl/">de Volkskrant</a></td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
<td>Amsterdam</td>
|
||||
<td class="bar"><div id="AT5"></div></td>
|
||||
@@ -122,16 +127,20 @@
|
||||
</tr>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td class="bar"><div id="BuurtAdam"></div></td>
|
||||
<td class="bar"><div id="Parool"></div></td>
|
||||
<td>
|
||||
<a href="https://indebuurt.nl/amsterdam/">In de buurt Amsterdam</a>
|
||||
<a href="https://www.parool.nl/amsterdam/"
|
||||
>Het Parool | Amsterdam</a
|
||||
>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="last">
|
||||
<td></td>
|
||||
<td class="bar"><div id="Parool"></div></td>
|
||||
<td class="bar"><div id="BuurtAdam"></div></td>
|
||||
<td>
|
||||
<a href="https://www.parool.nl/amsterdam/">Parool Amsterdam</a>
|
||||
<a href="https://indebuurt.nl/amsterdam/"
|
||||
>In de buurt | Amsterdam</a
|
||||
>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="first">
|
||||
@@ -147,7 +156,9 @@
|
||||
<td></td>
|
||||
<td class="bar"><div id="BuurtGrn"></div></td>
|
||||
<td>
|
||||
<a href="https://indebuurt.nl/groningen/">In de buurt Groningen</a>
|
||||
<a href="https://indebuurt.nl/groningen/"
|
||||
>In de buurt | Groningen</a
|
||||
>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
Reference in New Issue
Block a user