p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
AT5/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
at5 at5
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

30
AT5/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"os" "os"
@@ -55,39 +57,15 @@ func main() {
x(err) x(err)
var item Item var item Item
x(xml.Unmarshal(b, &item), filename) x(xml.Unmarshal(b, &item), filename)
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
pp, err := root.Search(`//body/p | //body/h2`) pp, err := root.Search(`//body/p | //body/h2`)
x(err) x(err)
for _, p := range pp { for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content())))) x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
BuurtAdam/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
buurtadam: cmd/buurtadam/*.go buurtadam: cmd/buurtadam/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

44
BuurtAdam/cmd/buurtadam/buurtadam.go
View File

@@ -4,13 +4,14 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -64,7 +65,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/BuurtAdam/lock" myLock := "/net/corpora/nlnieuws/BuurtAdam/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -202,7 +203,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
} }
for _, div := range divs { for _, div := range divs {
p(fp.WriteString(addEnd(fixSpace(div.Content())))) p(fp.WriteString(u.AddEnd(u.FixSpace(div.Content()))))
} }
p(fp.Close()) p(fp.Close())
@@ -211,40 +212,3 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
BuurtGrn/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
buurtgrn: cmd/buurtgrn/*.go buurtgrn: cmd/buurtgrn/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

43
BuurtGrn/cmd/buurtgrn/buurtgrn.go
View File

@@ -6,11 +6,11 @@ import (
"encoding/xml" "encoding/xml"
"fmt" "fmt"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -64,7 +64,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/BuurtGrn/lock" myLock := "/net/corpora/nlnieuws/BuurtGrn/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -202,7 +202,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
} }
for _, div := range divs { for _, div := range divs {
p(fp.WriteString(addEnd(fixSpace(div.Content())))) p(fp.WriteString(u.AddEnd(u.FixSpace(div.Content()))))
} }
p(fp.Close()) p(fp.Close())
@@ -211,40 +211,3 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
GG/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
gg: cmd/gg/*.go gg: cmd/gg/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

46
GG/cmd/gg/gg.go
View File

@@ -4,13 +4,14 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -64,7 +65,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/GG/lock" myLock := "/net/corpora/nlnieuws/GG/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -191,10 +192,10 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
fp, err := os.Create(filename + ".txt") fp, err := os.Create(filename + ".txt")
p(err) p(err)
p(fp.WriteString(addEnd(fixSpace(title)))) p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
for _, el := range ell { for _, el := range ell {
p(fp.WriteString(addEnd(fixSpace(el.Content())))) p(fp.WriteString(u.AddEnd(u.FixSpace(el.Content()))))
} }
p(fp.Close()) p(fp.Close())
@@ -203,40 +204,3 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
HLN/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
hln: cmd/hln/*.go hln: cmd/hln/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

131
HLN/cmd/hln/hln.go
View File

@@ -4,13 +4,16 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
// "encoding/json"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
// "html"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -25,9 +28,16 @@ type ItemT struct {
UnixTime int64 `xml:"unixTime"` UnixTime int64 `xml:"unixTime"`
Guid string `xml:"guid"` Guid string `xml:"guid"`
Link string `xml:"link"` Link string `xml:"link"`
Title string `xml:"title"`
Data []byte `xml:",innerxml"` Data []byte `xml:",innerxml"`
} }
/*
type GraphT struct {
Graph []map[string]any `json:"@graph"`
}
*/
var ( var (
p = e.PanicErr p = e.PanicErr
w = e.WarnErr w = e.WarnErr
@@ -62,7 +72,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/HLN/lock" myLock := "/net/corpora/nlnieuws/HLN/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -127,18 +137,19 @@ func main() {
p(fp.WriteString("</item>\n")) p(fp.WriteString("</item>\n"))
p(fp.Close()) p(fp.Close())
p(os.Chtimes(filename+".xml", t, t)) p(os.Chtimes(filename+".xml", t, t))
ok = doArticle(filename, item.Link, t, needUpdate) ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
}() }()
} }
} }
func doArticle(filename string, url string, timestamp time.Time, needUpdate bool) (ok bool) { func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) (ok bool) {
if exists(filename + ".skip") { if exists(filename + ".skip") {
return true return true
} }
if needUpdate { if needUpdate {
_ = os.Remove(filename + ".err") _ = os.Remove(filename + ".err")
_ = os.Remove(filename + ".html") _ = os.Remove(filename + ".html")
// _ = os.Remove(filename + ".json")
_ = os.Remove(filename + ".txt") _ = os.Remove(filename + ".txt")
} else { } else {
if exists(filename + ".txt") { if exists(filename + ".txt") {
@@ -158,6 +169,60 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
p(err) p(err)
p(resp.Body.Close()) p(resp.Body.Close())
/*
s := string(body)
ok = true
i1 := strings.Index(s, `type="application/ld+json"`)
if i1 < 0 {
ok = false
} else {
i1 += strings.Index(s[i1:], `>`) + 1
i2 := i1 + strings.Index(s[i1:], `</script>`)
if i2 < i1 {
ok = false
} else {
s = html.UnescapeString(s[i1:i2])
}
}
if !ok {
_ = w(fmt.Errorf("script jsonld not found: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "script jsonld not found: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
var graph GraphT
p(json.Unmarshal([]byte(s), &graph))
for _, g := range graph.Graph {
t := g["@type"]
switch v := t.(type) {
case string:
if v == "NewsArticle" {
b, err := json.Marshal(g)
p(err)
s = string(b)
}
}
}
fp, err := os.Create(filename + ".json")
p(err)
p(fp.WriteString(s))
p(fp.Close())
p(os.Chtimes(filename+".json", timestamp, timestamp))
*/
doc, err := gokogiri.ParseHtml(body) doc, err := gokogiri.ParseHtml(body)
p(err) p(err)
@@ -196,18 +261,6 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
pars := make([]string, 0) pars := make([]string, 0)
ell, err = article.Search(`.//*[@data-content-type="TITLE"]`)
p(err)
if len(ell) != 1 {
_ = w(fmt.Errorf("found %d titles: %s", len(ell), url))
}
for _, el := range ell {
s := strings.TrimSpace(el.Content())
if s != "" {
pars = append(pars, s)
}
}
hasIntro := false hasIntro := false
ell, err = article.Search(`.//*[@data-content-type="INTRO"]`) ell, err = article.Search(`.//*[@data-content-type="INTRO"]`)
p(err) p(err)
@@ -285,12 +338,14 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
p(fmt.Fprintln(fp, "##META text tag =")) p(fmt.Fprintln(fp, "##META text tag ="))
} else { } else {
for _, tag := range tags { for _, tag := range tags {
p(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(tag))) p(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
} }
} }
p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
for _, par := range pars { for _, par := range pars {
p(fp.WriteString(addEnd(fixSpace(par)))) p(fp.WriteString(u.AddEnd(u.FixSpace(par))))
} }
p(fp.Close()) p(fp.Close())
@@ -299,43 +354,3 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
if strings.HasSuffix(s, `.”`) || strings.HasSuffix(s, `!”`) || strings.HasSuffix(s, `?”`) {
return s + "\n"
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

8
LitNL/Makefile
View File

@@ -3,11 +3,11 @@ all: \
metadata \ metadata \
litnl litnl
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
litnl: cmd/litnl/*.go litnl: cmd/litnl/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

18
LitNL/cmd/litnl/litnl.go
View File

@@ -3,13 +3,14 @@ package main
import ( import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -46,7 +47,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/LitNL/lock" myLock := "/net/corpora/nlnieuws/LitNL/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -108,16 +109,3 @@ func main() {
} }
} }
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

32
LitNL/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"os" "os"
@@ -58,9 +60,9 @@ func main() {
var item Item var item Item
x(xml.Unmarshal(b, &item)) x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
} }
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
@@ -74,32 +76,8 @@ func main() {
_ = w(fmt.Errorf("empty: %s", filename)) _ = w(fmt.Errorf("empty: %s", filename))
} }
for _, p := range pp { for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content())))) x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
NOS/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
nos nos
json2txt: cmd/json2txt/*.go json2txt: cmd/json2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

34
NOS/cmd/json2txt/json2txt.go
View File

@@ -3,6 +3,8 @@ package main
import ( import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
@@ -61,13 +63,13 @@ func main() {
x(err) x(err)
item := getItem(b, filename) item := getItem(b, filename)
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text cat = %s\n", u.FixSpace(cat)))
} }
for _, tag := range item.Tags { for _, tag := range item.Tags {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(tag))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
} }
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
x(fp.WriteString(fixSpace(item.Text))) x(fp.WriteString(u.FixSpace(item.Text)))
x(fp.Close()) x(fp.Close())
} }
} }
@@ -90,27 +92,3 @@ func getItem(b []byte, filename string) Item {
x(json.Unmarshal(b, &item), filename) x(json.Unmarshal(b, &item), filename)
return item return item
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

3
NOS/cmd/nos/nos.go
View File

@@ -1,12 +1,11 @@
package main package main
import ( import (
"html"
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"html"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"

4
NieuwsNL/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
nieuwsnl: cmd/nieuwsnl/*.go nieuwsnl: cmd/nieuwsnl/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

48
NieuwsNL/cmd/nieuwsnl/nieuwsnl.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"bytes" "bytes"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
@@ -11,7 +13,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -64,7 +65,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/NieuwsNL/lock" myLock := "/net/corpora/nlnieuws/NieuwsNL/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -173,11 +174,11 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
} else { } else {
for _, a := range aa { for _, a := range aa {
tag = strings.ReplaceAll(a.Content(), "\n", " ") tag = strings.ReplaceAll(a.Content(), "\n", " ")
p(fmt.Fprintf(&buf, "##META text tag = %s\n", fixSpace(tag))) p(fmt.Fprintf(&buf, "##META text tag = %s\n", u.FixSpace(tag)))
} }
} }
p(buf.WriteString(addEnd(fixSpace(title)))) p(buf.WriteString(u.AddEnd(u.FixSpace(title))))
// oud: //div[@id="article-blocks"]//p // oud: //div[@id="article-blocks"]//p
pp, err := root.Search(`//div[@id="article-blocks"]//div[contains(@class, "paragraph-content")]`) pp, err := root.Search(`//div[@id="article-blocks"]//div[contains(@class, "paragraph-content")]`)
@@ -204,7 +205,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
return false // echt fout return false // echt fout
} }
for _, p1 := range pp { for _, p1 := range pp {
p(buf.WriteString(addEnd(fixSpace(p1.Content())))) p(buf.WriteString(u.AddEnd(u.FixSpace(p1.Content()))))
} }
fp, err := os.Create(filename + ".txt") fp, err := os.Create(filename + ".txt")
@@ -216,40 +217,3 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
Oog/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
oog oog
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

32
Oog/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"os" "os"
@@ -57,41 +59,17 @@ func main() {
var item Item var item Item
x(xml.Unmarshal(b, &item)) x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
} }
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
pp, err := root.Search(`//body/p`) pp, err := root.Search(`//body/p`)
x(err) x(err)
for _, p := range pp { for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content())))) x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
Parool/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
parool: cmd/parool/*.go parool: cmd/parool/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

133
Parool/cmd/parool/parool.go
View File

@@ -4,13 +4,16 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
//"encoding/json"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
//"html"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -25,9 +28,16 @@ type ItemT struct {
UnixTime int64 `xml:"unixTime"` UnixTime int64 `xml:"unixTime"`
Guid string `xml:"guid"` Guid string `xml:"guid"`
Link string `xml:"link"` Link string `xml:"link"`
Title string `xml:"title"`
Data []byte `xml:",innerxml"` Data []byte `xml:",innerxml"`
} }
/*
type GraphT struct {
Graph []map[string]any `json:"@graph"`
}
*/
var ( var (
p = e.PanicErr p = e.PanicErr
w = e.WarnErr w = e.WarnErr
@@ -62,7 +72,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/Parool/lock" myLock := "/net/corpora/nlnieuws/Parool/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -122,18 +132,19 @@ func main() {
p(fp.WriteString("</item>\n")) p(fp.WriteString("</item>\n"))
p(fp.Close()) p(fp.Close())
p(os.Chtimes(filename+".xml", t, t)) p(os.Chtimes(filename+".xml", t, t))
ok = doArticle(filename, item.Link, t, needUpdate) ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
}() }()
} }
} }
func doArticle(filename string, url string, timestamp time.Time, needUpdate bool) (ok bool) { func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) (ok bool) {
if exists(filename + ".skip") { if exists(filename + ".skip") {
return true return true
} }
if needUpdate { if needUpdate {
_ = os.Remove(filename + ".err") _ = os.Remove(filename + ".err")
_ = os.Remove(filename + ".html") _ = os.Remove(filename + ".html")
// _ = os.Remove(filename + ".json")
_ = os.Remove(filename + ".txt") _ = os.Remove(filename + ".txt")
} else { } else {
if exists(filename + ".txt") { if exists(filename + ".txt") {
@@ -156,6 +167,62 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
doc, err := gokogiri.ParseHtml(body) doc, err := gokogiri.ParseHtml(body)
p(err) p(err)
/*
s := string(body)
ok = true
i1 := strings.Index(s, `<script type="application/ld+json"`)
if i1 < 0 {
ok = false
} else {
i1 += strings.Index(s[i1:], `>`) + 1
i2 := i1 + strings.Index(s[i1:], `</script>`)
if i2 < i1 {
ok = false
} else {
s = html.UnescapeString(s[i1:i2])
}
}
if !ok {
_ = w(fmt.Errorf("script jsonld not found: %s", url))
fp, err := os.Create(filename + ".err")
p(err)
p(fmt.Fprintf(fp, "script jsonld not found: %s\n", url))
p(fp.Close())
p(os.Chtimes(filename+".err", timestamp, timestamp))
fp, err = os.Create(filename + ".html")
p(err)
p(fp.Write(body))
p(fp.Close())
p(os.Chtimes(filename+".html", timestamp, timestamp))
return false
}
var graph GraphT
p(json.Unmarshal([]byte(s), &graph))
for _, g := range graph.Graph {
t := g["@type"]
switch v := t.(type) {
case string:
if v == "NewsArticle" {
b, err := json.Marshal(g)
p(err)
s = string(b)
}
}
}
fp, err := os.Create(filename + ".json")
p(err)
p(fp.WriteString(s))
p(fp.Close())
p(os.Chtimes(filename+".json", timestamp, timestamp))
*/
root := doc.Root() root := doc.Root()
articles, err := root.Search(`//article[@id="article-content"]`) articles, err := root.Search(`//article[@id="article-content"]`)
@@ -226,18 +293,6 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
pars := make([]string, 0) pars := make([]string, 0)
ell, err = header.Search(`.//*[@data-test-id="article-title"]`)
p(err)
if len(ell) != 1 {
_ = w(fmt.Errorf("found %d titles: %s", len(ell), url))
}
for _, el := range ell {
s := strings.TrimSpace(el.Content())
if s != "" {
pars = append(pars, s)
}
}
found := false found := false
ell, err = header.Search(`.//*[@data-test-id="header-intro"]`) ell, err = header.Search(`.//*[@data-test-id="header-intro"]`)
p(err) p(err)
@@ -309,12 +364,14 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
p(fmt.Fprintln(fp, "##META text tag =")) p(fmt.Fprintln(fp, "##META text tag ="))
} else { } else {
for _, tag := range tags { for _, tag := range tags {
p(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(tag))) p(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
} }
} }
p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
for _, par := range pars { for _, par := range pars {
p(fp.WriteString(addEnd(fixSpace(par)))) p(fp.WriteString(u.AddEnd(u.FixSpace(par))))
} }
p(fp.Close()) p(fp.Close())
@@ -323,43 +380,3 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
if strings.HasSuffix(s, `.”`) || strings.HasSuffix(s, `!”`) || strings.HasSuffix(s, `?”`) {
return s + "\n"
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
RO/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
ro ro
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

32
RO/cmd/xml2txt/xml2txt.go
View File

@@ -5,6 +5,8 @@ import (
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
"github.com/pebbe/textcat/v2" "github.com/pebbe/textcat/v2"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"bytes" "bytes"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
@@ -61,7 +63,7 @@ func main() {
var buf bytes.Buffer var buf bytes.Buffer
var item Item var item Item
x(xml.Unmarshal(b, &item)) x(xml.Unmarshal(b, &item))
x(buf.WriteString(addEnd(fixSpace(item.Title)))) x(buf.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
@@ -73,7 +75,7 @@ func main() {
pp, err := root.Search(`//body//p[not(.//a[contains(@href,"reportersonline.nl/support")])]`) pp, err := root.Search(`//body//p[not(.//a[contains(@href,"reportersonline.nl/support")])]`)
x(err) x(err)
for _, p := range pp { for _, p := range pp {
x(buf.WriteString(addEnd(fixSpace(p.Content())))) x(buf.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
text := buf.String() text := buf.String()
@@ -90,33 +92,9 @@ func main() {
fp, err := os.Create("out/" + filename[:len(filename)-4] + ".txt") fp, err := os.Create("out/" + filename[:len(filename)-4] + ".txt")
x(err) x(err)
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
} }
x(fp.WriteString(text)) x(fp.WriteString(text))
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
RTVNoord/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
rtvnoord: cmd/rtvnoord/*.go rtvnoord: cmd/rtvnoord/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

50
RTVNoord/cmd/rtvnoord/rtvnoord.go
View File

@@ -3,6 +3,8 @@ package main
import ( import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/json" "encoding/json"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
@@ -10,7 +12,6 @@ import (
"io" "io"
"net/http" "net/http"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -75,7 +76,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/RTVNoord/lock" myLock := "/net/corpora/nlnieuws/RTVNoord/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -225,7 +226,7 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
// text bevat kopjes zonder punt aan het eind // text bevat kopjes zonder punt aan het eind
lines := strings.Split(doc.Text, "\n") lines := strings.Split(doc.Text, "\n")
for i, line := range lines { for i, line := range lines {
lines[i] = addEnd(fixSpace(line)) lines[i] = u.AddEnd(u.FixSpace(line))
} }
text := strings.Join(lines, "") + "\n" text := strings.Join(lines, "") + "\n"
@@ -235,16 +236,16 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
p(fmt.Fprintln(fp, "##META text tag =")) p(fmt.Fprintln(fp, "##META text tag ="))
} else { } else {
for _, tag := range doc.Tags { for _, tag := range doc.Tags {
p(fmt.Fprintf(fp, "##META text tag = %s\n", strings.ToLower(fixSpace(tag)))) p(fmt.Fprintf(fp, "##META text tag = %s\n", strings.ToLower(u.FixSpace(tag))))
} }
} }
if doc.Cat == "" { if doc.Cat == "" {
p(fmt.Fprintln(fp, "##META text cat =")) p(fmt.Fprintln(fp, "##META text cat ="))
} else { } else {
p(fmt.Fprintf(fp, "##META text cat = %s\n", fixSpace(doc.Cat))) p(fmt.Fprintf(fp, "##META text cat = %s\n", u.FixSpace(doc.Cat)))
} }
p(fp.WriteString(addEnd(doc.Title))) p(fp.WriteString(u.AddEnd(doc.Title)))
p(fp.WriteString(text)) p(fp.WriteString(text))
p(fp.Close()) p(fp.Close())
@@ -252,40 +253,3 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
Sargasso/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
sargasso sargasso
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

32
Sargasso/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"os" "os"
@@ -57,41 +59,17 @@ func main() {
var item Item var item Item
x(xml.Unmarshal(b, &item)) x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
} }
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
pp, err := root.Search(`//body//p`) pp, err := root.Search(`//body//p`)
x(err) x(err)
for _, p := range pp { for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content())))) x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
Sikkom/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
sikkom: cmd/sikkom/*.go sikkom: cmd/sikkom/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

46
Sikkom/cmd/sikkom/sikkom.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"html" "html"
@@ -11,7 +13,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -64,7 +65,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/Sikkom/lock" myLock := "/net/corpora/nlnieuws/Sikkom/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -220,49 +221,12 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
fp, err = os.Create(filename + ".txt") fp, err = os.Create(filename + ".txt")
p(err) p(err)
p(fp.WriteString(addEnd(fixSpace(title)))) p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
for _, p1 := range pp { for _, p1 := range pp {
p(fp.WriteString(addEnd(fixSpace(p1.Content())))) p(fp.WriteString(u.AddEnd(u.FixSpace(p1.Content()))))
} }
p(fp.Close()) p(fp.Close())
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

4
Tzum/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \ metadata \
tzum tzum
xml2txt: cmd/xml2txt/*.go xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^

32
Tzum/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"os" "os"
@@ -57,9 +59,9 @@ func main() {
var item Item var item Item
x(xml.Unmarshal(b, &item)) x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats { for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat))) x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
} }
x(fp.WriteString(addEnd(fixSpace(item.Title)))) x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`)) doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err) x(err)
root := doc.Root() root := doc.Root()
@@ -68,33 +70,9 @@ func main() {
for _, p := range pp { for _, p := range pp {
s := p.Content() s := p.Content()
if !strings.Contains(s, "verscheen eerst op Tzum.") { if !strings.Contains(s, "verscheen eerst op Tzum.") {
x(fp.WriteString(addEnd(fixSpace(p.Content())))) x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
} }
} }
x(fp.Close()) x(fp.Close())
} }
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}

4
VRT/Makefile
View File

@@ -5,5 +5,5 @@ all: \
metadata: cmd/metadata/*.go metadata: cmd/metadata/*.go
go build -o $@ $^ go build -o $@ $^
vrt: cmd/vrt/*.go vrt: cmd/vrt/*.go ../internal/util/*.go
go build -o $@ $^ go build -o $@ $<

52
VRT/cmd/vrt/vrt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors" e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri" "github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"bytes" "bytes"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
@@ -11,7 +13,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
) )
@@ -78,7 +79,7 @@ func main() {
}() }()
myLock := "/net/corpora/nlnieuws/VRT/lock" myLock := "/net/corpora/nlnieuws/VRT/lock"
mkLock(myLock) u.MkLock(myLock)
defer func() { defer func() {
_ = os.Remove(myLock) _ = os.Remove(myLock)
}() }()
@@ -242,18 +243,18 @@ func doArticle(filename string, url string, title string, tags []string, cats []
p(fmt.Fprintln(&buf, "##META text cat =")) p(fmt.Fprintln(&buf, "##META text cat ="))
} else { } else {
for _, cat := range cats { for _, cat := range cats {
p(fmt.Fprintf(&buf, "##META text cat = %s\n", fixSpace(cat))) p(fmt.Fprintf(&buf, "##META text cat = %s\n", u.FixSpace(cat)))
} }
} }
if len(tags) == 0 { if len(tags) == 0 {
p(fmt.Fprintln(&buf, "##META text tag =")) p(fmt.Fprintln(&buf, "##META text tag ="))
} else { } else {
for _, tag := range tags { for _, tag := range tags {
p(fmt.Fprintf(&buf, "##META text tag = %s\n", fixSpace(tag))) p(fmt.Fprintf(&buf, "##META text tag = %s\n", u.FixSpace(tag)))
} }
} }
_, err = buf.WriteString(addEnd(fixSpace(title))) _, err = buf.WriteString(u.AddEnd(u.FixSpace(title)))
p(err) p(err)
fouten := make([]string, 0) fouten := make([]string, 0)
@@ -262,7 +263,7 @@ func doArticle(filename string, url string, title string, tags []string, cats []
pp, err := root.Search(`//div[@data-sentry-component="ArticleHeading"]//*[contains(@class,"prose-article-body-r")]`) pp, err := root.Search(`//div[@data-sentry-component="ArticleHeading"]//*[contains(@class,"prose-article-body-r")]`)
p(err) p(err)
for _, p1 := range pp { for _, p1 := range pp {
p(fmt.Fprint(&buf, addEnd(fixSpace(p1.Content())))) p(fmt.Fprint(&buf, u.AddEnd(u.FixSpace(p1.Content()))))
found = true found = true
} }
if !found { if !found {
@@ -277,7 +278,7 @@ func doArticle(filename string, url string, title string, tags []string, cats []
`//div[@data-sentry-component="ArticleTitle"]//h2`) `//div[@data-sentry-component="ArticleTitle"]//h2`)
p(err) p(err)
for _, p1 := range pp { for _, p1 := range pp {
p(fmt.Fprint(&buf, addEnd(fixSpace(p1.Content())))) p(fmt.Fprint(&buf, u.AddEnd(u.FixSpace(p1.Content()))))
found = true found = true
} }
if !found { if !found {
@@ -311,40 +312,3 @@ func doArticle(filename string, url string, title string, tags []string, cats []
return true return true
} }
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

2
go.mod
View File

@@ -1,4 +1,4 @@
module nlnieuws module git.web.rug.nl/p209327/nlnieuws
go 1.26.1 go 1.26.1

41
internal/util/util.go Normal file
View File

@@ -0,0 +1,41 @@
package util
import (
e "codeberg.org/pebbe/errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
var (
p = e.PanicErr
reEOL = regexp.MustCompile(`[.!?]['"”’]?$`)
)
func AddEnd(s string) string {
s = strings.TrimSpace(s)
if s == "" || reEOL.MatchString(s) {
return s
}
return s + ".\n"
}
func FixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func MkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}