Nieuw formaat van Sikkom

Sikkom/cmd/sikkom/sikkom.go

@@ -3,14 +3,18 @@ package main
 import (
 	e "codeberg.org/pebbe/errors"
 	"github.com/jbowtie/gokogiri"
+	"golang.org/x/net/publicsuffix"
 
 	u "git.web.rug.nl/p209327/nlnieuws/internal/util"
 
+	"bytes"
+	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
-	"html"
 	"io"
 	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 	"os"
 	"strings"
@@ -28,13 +32,23 @@ type ItemT struct {
 	UnixTime int64  `xml:"unixTime"`
 	Guid     string `xml:"guid"`
 	Link     string `xml:"link"`
+	Premium  bool   `xml:"premium"`
 	Data     []byte `xml:",innerxml"`
 }
 
+type LdJson struct {
+	Type   string `json:"@type"`
+	IsFree string `json:"isAccessibleForFree"`
+	H1     string `json:"headline"`
+	H2     string `json:"description"`
+}
+
 var (
 	p     = e.PanicErr
 	w     = e.WarnErr
 	agent = "AhrefsBot/7.0"
+	// agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+	jar *cookiejar.Jar
 )
 
 func exists(filename string) bool {
@@ -64,26 +78,37 @@ func main() {
 		}
 	}()
 
+	var err error
+	jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	p(err)
+
 	myLock := "/net/corpora/nlnieuws/Sikkom/lock"
 	u.MkLock(myLock)
 	defer func() {
 		_ = os.Remove(myLock)
 	}()
 
-	resp, err := http.Get("https://www.sikkom.nl/api/feed/rss")
+	resp, err := http.Get("https://www.sikkom.nl/rss")
 	p(err)
 	body, err := io.ReadAll(resp.Body)
 	p(err)
 	p(resp.Body.Close())
 
+	b2 := bytes.ReplaceAll(
+		bytes.ReplaceAll(body, []byte("<atom:"), []byte("<atom_")),
+		[]byte("</atom:"), []byte("</atom_"))
+
 	var rss Rss
-	p(xml.Unmarshal(body, &rss))
+	p(xml.Unmarshal(b2, &rss))
 
 	if len(rss.Items) == 0 {
 		p(fmt.Errorf("len(rss.Items) == 0"))
 	}
 
 	for _, item := range rss.Items {
+		if item.Premium {
+			continue
+		}
 		t, err := time.Parse(time.RFC1123Z, item.PubDate)
 		if err != nil {
 			t, err = time.Parse(time.RFC1123, item.PubDate)
@@ -145,7 +170,7 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 	p(err)
 	req.Header.Set("User-Agent", agent)
 
-	client := &http.Client{}
+	client := &http.Client{Jar: jar}
 	resp, err := client.Do(req)
 	p(err)
 	body, err := io.ReadAll(resp.Body)
@@ -154,22 +179,31 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 
 	body = u.HtmlFix(body)
 
-	s := string(body)
+	doc, err := gokogiri.ParseHtml(body)
+	p(err)
 
-	ok := true
+	root := doc.Root()
-	i1 := strings.Index(s, `"application/ld+json"`)
+
-	if i1 < 0 {
+	ldjsons, err := root.Search(`//script[@type="application/ld+json"]`)
-		ok = false
+	p(err)
-	} else {
+	found := false
-		i1 += strings.Index(s[i1:], `>`) + 1
+	var jsmain LdJson
-		i2 := i1 + strings.Index(s[i1:], `</script>`)
+	for _, ldjson := range ldjsons {
-		if i2 < i1 {
+		s := ldjson.Content()
-			ok = false
+		var js LdJson
-		} else {
+		p(json.Unmarshal([]byte(s), &js))
-			s = html.UnescapeString(s[i1:i2])
+		if js.Type == "NewsArticle" {
+			found = true
+			fp, err := os.Create(filename + ".json")
+			p(err)
+			p(fp.WriteString(s))
+			p(fp.Close())
+			p(os.Chtimes(filename+".json", timestamp, timestamp))
+			jsmain = js
+			break
 		}
 	}
-	if !ok {
+	if !found {
 		_ = w(fmt.Errorf("script jsonld not found: %s", url))
 
 		fp, err := os.Create(filename + ".err")
@@ -183,30 +217,24 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 		p(fp.Write(body))
 		p(fp.Close())
 		p(os.Chtimes(filename+".html", timestamp, timestamp))
-
 		return false
 	}
 
-	fp, err := os.Create(filename + ".json")
+	locations, err := root.Search(`//span[starts-with(@class, "location")]`)
 	p(err)
-	p(fp.WriteString(s))
+	for _, location := range locations {
-	p(fp.Close())
+		location.Remove()
-	p(os.Chtimes(filename+".json", timestamp, timestamp))
+	}
 
-	doc, err := gokogiri.ParseHtml(body)
+	articles, err := root.Search(`//article[starts-with(@class, "article")]`)
 	p(err)
-
+	if len(articles) != 1 {
-	root := doc.Root()
+		msg := fmt.Sprintf("len(articles) == %d in %s", len(articles), url)
-
+		_ = w(errors.New(msg))
-	pp, err := root.Search(`//div[contains(@class,"article-page__body")]//p`)
-	p(err)
-
-	if len(pp) == 0 {
-		_ = w(fmt.Errorf("empty: %s", url))
 
 		fp, err := os.Create(filename + ".err")
 		p(err)
-		p(fmt.Fprintf(fp, "empty: %s\n", url))
+		p(fmt.Fprintln(fp, msg))
 		p(fp.Close())
 		p(os.Chtimes(filename+".err", timestamp, timestamp))
 
@@ -220,15 +248,57 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 		return false
 	}
 
-	fp, err = os.Create(filename + ".txt")
+	art := articles[0]
-	p(err)
 
-	p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
+	text := make([]string, 0)
-
+	if jsmain.H1 != "" {
-	for _, p1 := range pp {
+		text = append(text, jsmain.H1)
-		p(fp.WriteString(u.AddEnd(u.FixSpace(p1.Content()))))
+	} else {
+		text = append(text, title)
+	}
+	if jsmain.H2 != "" {
+		text = append(text, jsmain.H2)
 	}
 
+	abb, err := art.Search(`.//div[starts-with(@class, "article-body")]`)
+	p(err)
+
+	found = false
+	for _, ab := range abb {
+		ell, err := ab.Search(`.//p | .//h4`)
+		p(err)
+		for _, el := range ell {
+			if el.Name() == "p" {
+				found = true
+			}
+			text = append(text, el.Content())
+		}
+	}
+
+	if !found {
+		_ = w(fmt.Errorf("empty body: %s", url))
+
+		fp, err := os.Create(filename + ".err")
+		p(err)
+		p(fmt.Fprintf(fp, "empty body: %s\n", url))
+		p(fp.Close())
+		p(os.Chtimes(filename+".err", timestamp, timestamp))
+
+		fp, err = os.Create(filename + ".html")
+		p(err)
+		p(fp.Write(body))
+		p(fp.Close())
+		p(os.Chtimes(filename+".html", timestamp, timestamp))
+
+		return false
+
+	}
+
+	fp, err := os.Create(filename + ".txt")
+	p(err)
+	for _, t := range text {
+		p(fp.WriteString(u.AddEnd(u.FixSpace(t))))
+	}
 	p(fp.Close())
 	return true
 }

go.mod

@@ -5,9 +5,15 @@ go 1.26.1
 require (
 	codeberg.org/pebbe/errors v0.4.0
 	github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5
+	github.com/kr/pretty v0.3.1
 	github.com/pebbe/compactcorpus v1.0.3
 	github.com/pebbe/textcat/v2 v2.3.0
 	github.com/rug-compling/alpinods v1.18.1
+	golang.org/x/net v0.57.0
 )
 
-require github.com/pebbe/util v0.9.0 // indirect
+require (
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pebbe/util v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+)

go.sum

@@ -1,12 +1,22 @@
 codeberg.org/pebbe/errors v0.4.0 h1:G05wsXpC/LRPaL02QYDwtz0sWFWQcIWK1s+MC79LBzU=
 codeberg.org/pebbe/errors v0.4.0/go.mod h1:O7PPxUJM1bWRHq11CRK3wqVaH/3NnRaSVZvh3UhzDCY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5 h1:tQbR4RKFBFi0+Ll69dXejKKUbQVNaOAT2fjlDvSAfx4=
 github.com/jbowtie/gokogiri v0.0.0-20250107075044-de0f9d4877a5/go.mod h1:kQE2lxPgVKe0JsBZMFFfMm5zBDCuRhaHFKOBzZeCLiw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/pebbe/compactcorpus v1.0.3 h1:6qlfXKHTKg7oWKLPCgEgv1scplfvphg/9l9XiRT2HzQ=
 github.com/pebbe/compactcorpus v1.0.3/go.mod h1:SSpTeCZataCjjs82RJb8SOGdjkB3PlR7Z19EY4rInoQ=
 github.com/pebbe/textcat/v2 v2.3.0 h1:RB2egIQgI2a2Ls+I9No6KFQKCZBIFt8Cc/SWCnVtC7Y=
 github.com/pebbe/textcat/v2 v2.3.0/go.mod h1:WLXWuL+fOlQJqn6LmubjD+e78hCC6Y/rAWInh0wq/kg=
 github.com/pebbe/util v0.9.0 h1:PMZd+CpWb8GbWEmFGlL3qd6XPuywl6xFIbrXWi870OA=
 github.com/pebbe/util v0.9.0/go.mod h1:ynWl/SFX4+Seb9fpjVlYevr1f4TP7FrCmyZHiBCg69Q=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rug-compling/alpinods v1.18.1 h1:BvPcCnNEQ1QoVSc0RmwJd3kZmvo4iqZ52/vFzVvFS7w=
 github.com/rug-compling/alpinods v1.18.1/go.mod h1:R3BBX8RIw9InVqHZ+1W+MsX8WX8uBkoVNNGE38mqF1Q=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=

www/app2.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
     <head>
-        <title id="title">Woord van de maand</title>
+        <title id="title">Stijger van de week</title>
         <meta charset="utf-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <link rel="icon" href="favicon.ico" type="image/ico" />
@@ -12,7 +12,7 @@
     </head>
     <body>
         <div class="title">
-            <h1>Woord van de maand</h1>
+            <h1>Stijger van de week</h1>
         </div>
         <div class="form">
             <form name="choice">