p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Oog/Makefile
View File

@@ -3,8 +3,8 @@ all: \
metadata \
oog
xml2txt: cmd/xml2txt/*.go
go build -o $@ $^
xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $<
metadata: cmd/metadata/*.go
go build -o $@ $^

32
Oog/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml"
"fmt"
"os"
@@ -57,41 +59,17 @@ func main() {
var item Item
x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
}
x(fp.WriteString(addEnd(fixSpace(item.Title))))
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err)
root := doc.Root()
pp, err := root.Search(`//body/p`)
x(err)
for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content()))))
x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
}
x(fp.Close())
}
}
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}