From c856a372a10a42c58be9c22c221bbb769e13e6f3 Mon Sep 17 00:00:00 2001 From: Peter Kleiweg Date: Thu, 23 Jul 2026 18:47:57 +0200 Subject: [PATCH] Nieuw formaat van Sikkom --- Sikkom/cmd/sikkom/sikkom.go | 146 ++++++++++++++++++++++++++---------- go.mod | 8 +- go.sum | 10 +++ www/app2.html | 4 +- 4 files changed, 127 insertions(+), 41 deletions(-) diff --git a/Sikkom/cmd/sikkom/sikkom.go b/Sikkom/cmd/sikkom/sikkom.go index 53bb5c6..3d83161 100644 --- a/Sikkom/cmd/sikkom/sikkom.go +++ b/Sikkom/cmd/sikkom/sikkom.go @@ -3,14 +3,18 @@ package main import ( e "codeberg.org/pebbe/errors" "github.com/jbowtie/gokogiri" + "golang.org/x/net/publicsuffix" u "git.web.rug.nl/p209327/nlnieuws/internal/util" + "bytes" + "encoding/json" "encoding/xml" + "errors" "fmt" - "html" "io" "net/http" + "net/http/cookiejar" "net/url" "os" "strings" @@ -28,13 +32,23 @@ type ItemT struct { UnixTime int64 `xml:"unixTime"` Guid string `xml:"guid"` Link string `xml:"link"` + Premium bool `xml:"premium"` Data []byte `xml:",innerxml"` } +type LdJson struct { + Type string `json:"@type"` + IsFree string `json:"isAccessibleForFree"` + H1 string `json:"headline"` + H2 string `json:"description"` +} + var ( p = e.PanicErr w = e.WarnErr agent = "AhrefsBot/7.0" + // agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" + jar *cookiejar.Jar ) func exists(filename string) bool { @@ -64,26 +78,37 @@ func main() { } }() + var err error + jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + p(err) + myLock := "/net/corpora/nlnieuws/Sikkom/lock" u.MkLock(myLock) defer func() { _ = os.Remove(myLock) }() - resp, err := http.Get("https://www.sikkom.nl/api/feed/rss") + resp, err := http.Get("https://www.sikkom.nl/rss") p(err) body, err := io.ReadAll(resp.Body) p(err) p(resp.Body.Close()) + b2 := bytes.ReplaceAll( + bytes.ReplaceAll(body, []byte("`) + 1 - i2 := i1 + strings.Index(s[i1:], ``) - if i2 < i1 { - ok = false - } else { - s = html.UnescapeString(s[i1:i2]) + root := doc.Root() + + ldjsons, err := root.Search(`//script[@type="application/ld+json"]`) + p(err) + found := false + var jsmain LdJson + for _, ldjson := range ldjsons { + s := ldjson.Content() + var js LdJson + p(json.Unmarshal([]byte(s), &js)) + if js.Type == "NewsArticle" { + found = true + fp, err := os.Create(filename + ".json") + p(err) + p(fp.WriteString(s)) + p(fp.Close()) + p(os.Chtimes(filename+".json", timestamp, timestamp)) + jsmain = js + break } } - if !ok { + if !found { _ = w(fmt.Errorf("script jsonld not found: %s", url)) fp, err := os.Create(filename + ".err") @@ -183,30 +217,24 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n p(fp.Write(body)) p(fp.Close()) p(os.Chtimes(filename+".html", timestamp, timestamp)) - return false } - fp, err := os.Create(filename + ".json") + locations, err := root.Search(`//span[starts-with(@class, "location")]`) p(err) - p(fp.WriteString(s)) - p(fp.Close()) - p(os.Chtimes(filename+".json", timestamp, timestamp)) + for _, location := range locations { + location.Remove() + } - doc, err := gokogiri.ParseHtml(body) + articles, err := root.Search(`//article[starts-with(@class, "article")]`) p(err) - - root := doc.Root() - - pp, err := root.Search(`//div[contains(@class,"article-page__body")]//p`) - p(err) - - if len(pp) == 0 { - _ = w(fmt.Errorf("empty: %s", url)) + if len(articles) != 1 { + msg := fmt.Sprintf("len(articles) == %d in %s", len(articles), url) + _ = w(errors.New(msg)) fp, err := os.Create(filename + ".err") p(err) - p(fmt.Fprintf(fp, "empty: %s\n", url)) + p(fmt.Fprintln(fp, msg)) p(fp.Close()) p(os.Chtimes(filename+".err", timestamp, timestamp)) @@ -220,15 +248,57 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n return false } - fp, err = os.Create(filename + ".txt") - p(err) + art := articles[0] - p(fp.WriteString(u.AddEnd(u.FixSpace(title)))) - - for _, p1 := range pp { - p(fp.WriteString(u.AddEnd(u.FixSpace(p1.Content())))) + text := make([]string, 0) + if jsmain.H1 != "" { + text = append(text, jsmain.H1) + } else { + text = append(text, title) + } + if jsmain.H2 != "" { + text = append(text, jsmain.H2) } + abb, err := art.Search(`.//div[starts-with(@class, "article-body")]`) + p(err) + + found = false + for _, ab := range abb { + ell, err := ab.Search(`.//p | .//h4`) + p(err) + for _, el := range ell { + if el.Name() == "p" { + found = true + } + text = append(text, el.Content()) + } + } + + if !found { + _ = w(fmt.Errorf("empty body: %s", url)) + + fp, err := os.Create(filename + ".err") + p(err) + p(fmt.Fprintf(fp, "empty body: %s\n", url)) + p(fp.Close()) + p(os.Chtimes(filename+".err", timestamp, timestamp)) + + fp, err = os.Create(filename + ".html") + p(err) + p(fp.Write(body)) + p(fp.Close()) + p(os.Chtimes(filename+".html", timestamp, timestamp)) + + return false + + } + + fp, err := os.Create(filename + ".txt") + p(err) + for _, t := range text { + p(fp.WriteString(u.AddEnd(u.FixSpace(t)))) + } p(fp.Close()) return true } diff --git a/go.mod b/go.mod index eadde75..fa49861 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,15 @@ go 1.26.1 require ( codeberg.org/pebbe/errors v0.4.0 github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5 + github.com/kr/pretty v0.3.1 github.com/pebbe/compactcorpus v1.0.3 github.com/pebbe/textcat/v2 v2.3.0 github.com/rug-compling/alpinods v1.18.1 + golang.org/x/net v0.57.0 ) -require github.com/pebbe/util v0.9.0 // indirect +require ( + github.com/kr/text v0.2.0 // indirect + github.com/pebbe/util v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect +) diff --git a/go.sum b/go.sum index a45ccdc..366cb09 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,22 @@ codeberg.org/pebbe/errors v0.4.0 h1:G05wsXpC/LRPaL02QYDwtz0sWFWQcIWK1s+MC79LBzU= codeberg.org/pebbe/errors v0.4.0/go.mod h1:O7PPxUJM1bWRHq11CRK3wqVaH/3NnRaSVZvh3UhzDCY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5 h1:tQbR4RKFBFi0+Ll69dXejKKUbQVNaOAT2fjlDvSAfx4= github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5/go.mod h1:kQE2lxPgVKe0JsBZMFFfMm5zBDCuRhaHFKOBzZeCLiw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pebbe/compactcorpus v1.0.3 h1:6qlfXKHTKg7oWKLPCgEgv1scplfvphg/9l9XiRT2HzQ= github.com/pebbe/compactcorpus v1.0.3/go.mod h1:SSpTeCZataCjjs82RJb8SOGdjkB3PlR7Z19EY4rInoQ= github.com/pebbe/textcat/v2 v2.3.0 h1:RB2egIQgI2a2Ls+I9No6KFQKCZBIFt8Cc/SWCnVtC7Y= github.com/pebbe/textcat/v2 v2.3.0/go.mod h1:WLXWuL+fOlQJqn6LmubjD+e78hCC6Y/rAWInh0wq/kg= github.com/pebbe/util v0.9.0 h1:PMZd+CpWb8GbWEmFGlL3qd6XPuywl6xFIbrXWi870OA= github.com/pebbe/util v0.9.0/go.mod h1:ynWl/SFX4+Seb9fpjVlYevr1f4TP7FrCmyZHiBCg69Q= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rug-compling/alpinods v1.18.1 h1:BvPcCnNEQ1QoVSc0RmwJd3kZmvo4iqZ52/vFzVvFS7w= github.com/rug-compling/alpinods v1.18.1/go.mod h1:R3BBX8RIw9InVqHZ+1W+MsX8WX8uBkoVNNGE38mqF1Q= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= diff --git a/www/app2.html b/www/app2.html index 24b60f2..159e62d 100644 --- a/www/app2.html +++ b/www/app2.html @@ -1,7 +1,7 @@ - Woord van de maand + Stijger van de week @@ -12,7 +12,7 @@
-

Woord van de maand

+

Stijger van de week