fix title voor HLN en Parool; helperfuncties naar internal/util

Parool/Makefile

@@ -5,5 +5,5 @@ all: \
 metadata: cmd/metadata/*.go
 	go build -o $@ $^
 
-parool: cmd/parool/*.go
-	go build -o $@ $^
+parool: cmd/parool/*.go ../internal/util/*.go
+	go build -o $@ $<

Parool/cmd/parool/parool.go

@@ -4,13 +4,16 @@ import (
 	e "codeberg.org/pebbe/errors"
 	"github.com/jbowtie/gokogiri"
 
+	u "git.web.rug.nl/p209327/nlnieuws/internal/util"
+
+	//"encoding/json"
 	"encoding/xml"
 	"fmt"
+	//"html"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -25,9 +28,16 @@ type ItemT struct {
 	UnixTime int64  `xml:"unixTime"`
 	Guid     string `xml:"guid"`
 	Link     string `xml:"link"`
+	Title    string `xml:"title"`
 	Data     []byte `xml:",innerxml"`
 }
 
+/*
+type GraphT struct {
+	Graph []map[string]any `json:"@graph"`
+}
+*/
+
 var (
 	p     = e.PanicErr
 	w     = e.WarnErr
@@ -62,7 +72,7 @@ func main() {
 	}()
 
 	myLock := "/net/corpora/nlnieuws/Parool/lock"
-	mkLock(myLock)
+	u.MkLock(myLock)
 	defer func() {
 		_ = os.Remove(myLock)
 	}()
@@ -122,18 +132,19 @@ func main() {
 			p(fp.WriteString("</item>\n"))
 			p(fp.Close())
 			p(os.Chtimes(filename+".xml", t, t))
-			ok = doArticle(filename, item.Link, t, needUpdate)
+			ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
 		}()
 	}
 }
 
-func doArticle(filename string, url string, timestamp time.Time, needUpdate bool) (ok bool) {
+func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) (ok bool) {
 	if exists(filename + ".skip") {
 		return true
 	}
 	if needUpdate {
 		_ = os.Remove(filename + ".err")
 		_ = os.Remove(filename + ".html")
+		// _ = os.Remove(filename + ".json")
 		_ = os.Remove(filename + ".txt")
 	} else {
 		if exists(filename + ".txt") {
@@ -156,6 +167,62 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
 	doc, err := gokogiri.ParseHtml(body)
 	p(err)
 
+	/*
+
+		s := string(body)
+
+		ok = true
+		i1 := strings.Index(s, `<script type="application/ld+json"`)
+		if i1 < 0 {
+			ok = false
+		} else {
+			i1 += strings.Index(s[i1:], `>`) + 1
+			i2 := i1 + strings.Index(s[i1:], `</script>`)
+			if i2 < i1 {
+				ok = false
+			} else {
+				s = html.UnescapeString(s[i1:i2])
+			}
+		}
+		if !ok {
+			_ = w(fmt.Errorf("script jsonld not found: %s", url))
+
+			fp, err := os.Create(filename + ".err")
+			p(err)
+			p(fmt.Fprintf(fp, "script jsonld not found: %s\n", url))
+			p(fp.Close())
+			p(os.Chtimes(filename+".err", timestamp, timestamp))
+
+			fp, err = os.Create(filename + ".html")
+			p(err)
+			p(fp.Write(body))
+			p(fp.Close())
+			p(os.Chtimes(filename+".html", timestamp, timestamp))
+
+			return false
+		}
+
+		var graph GraphT
+		p(json.Unmarshal([]byte(s), &graph))
+		for _, g := range graph.Graph {
+			t := g["@type"]
+			switch v := t.(type) {
+			case string:
+				if v == "NewsArticle" {
+					b, err := json.Marshal(g)
+					p(err)
+					s = string(b)
+				}
+			}
+		}
+
+		fp, err := os.Create(filename + ".json")
+		p(err)
+		p(fp.WriteString(s))
+		p(fp.Close())
+		p(os.Chtimes(filename+".json", timestamp, timestamp))
+	*/
+
 	root := doc.Root()
 
 	articles, err := root.Search(`//article[@id="article-content"]`)
@@ -226,18 +293,6 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
 
 	pars := make([]string, 0)
 
-	ell, err = header.Search(`.//*[@data-test-id="article-title"]`)
-	p(err)
-	if len(ell) != 1 {
-		_ = w(fmt.Errorf("found %d titles: %s", len(ell), url))
-	}
-	for _, el := range ell {
-		s := strings.TrimSpace(el.Content())
-		if s != "" {
-			pars = append(pars, s)
-		}
-	}
-
 	found := false
 	ell, err = header.Search(`.//*[@data-test-id="header-intro"]`)
 	p(err)
@@ -309,12 +364,14 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
 		p(fmt.Fprintln(fp, "##META text tag ="))
 	} else {
 		for _, tag := range tags {
-			p(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(tag)))
+			p(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(tag)))
 		}
 	}
 
+	p(fp.WriteString(u.AddEnd(u.FixSpace(title))))
+
 	for _, par := range pars {
-		p(fp.WriteString(addEnd(fixSpace(par))))
+		p(fp.WriteString(u.AddEnd(u.FixSpace(par))))
 	}
 
 	p(fp.Close())
@@ -323,43 +380,3 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool
 
 	return true
 }
-
-func addEnd(s string) string {
-	s = strings.TrimSpace(s)
-	n := len(s)
-	if n == 0 {
-		return ""
-	}
-	if n > 0 {
-		if strings.ContainsAny(s[n-1:], ".!?") {
-			return s + "\n"
-		}
-	}
-	if n > 1 {
-		s2 := s[n-2:]
-		if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
-			return s + "\n"
-		}
-	}
-	if strings.HasSuffix(s, `.”`) || strings.HasSuffix(s, `!”`) || strings.HasSuffix(s, `?”`) {
-		return s + "\n"
-	}
-	return s + ".\n"
-}
-
-func fixSpace(s string) string {
-	return strings.Join(strings.Fields(s), " ")
-}
-
-func mkLock(filename string) {
-	pid := os.Getpid()
-	link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
-	p(os.Symlink(link, filename))
-
-	name, err := os.Readlink(filename)
-	p(err)
-
-	if name != link {
-		p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
-	}
-}