p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
AT5/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml"
"fmt"
"os"
@@ -55,39 +57,15 @@ func main() {
x(err)
var item Item
x(xml.Unmarshal(b, &item), filename)
x(fp.WriteString(addEnd(fixSpace(item.Title))))
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err)
root := doc.Root()
pp, err := root.Search(`//body/p | //body/h2`)
x(err)
for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content()))))
x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
}
x(fp.Close())
}
}
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}