p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
RO/cmd/xml2txt/xml2txt.go
View File

@@ -5,6 +5,8 @@ import (
"github.com/jbowtie/gokogiri"
"github.com/pebbe/textcat/v2"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"bytes"
"encoding/xml"
"fmt"
@@ -61,7 +63,7 @@ func main() {
var buf bytes.Buffer
var item Item
x(xml.Unmarshal(b, &item))
x(buf.WriteString(addEnd(fixSpace(item.Title))))
x(buf.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err)
root := doc.Root()
@@ -73,7 +75,7 @@ func main() {
pp, err := root.Search(`//body//p[not(.//a[contains(@href,"reportersonline.nl/support")])]`)
x(err)
for _, p := range pp {
x(buf.WriteString(addEnd(fixSpace(p.Content()))))
x(buf.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
}
text := buf.String()
@@ -90,33 +92,9 @@ func main() {
fp, err := os.Create("out/" + filename[:len(filename)-4] + ".txt")
x(err)
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
}
x(fp.WriteString(text))
x(fp.Close())
}
}
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}