p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
VRT/cmd/vrt/vrt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"bytes"
"encoding/xml"
"fmt"
@@ -11,7 +13,6 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
@@ -78,7 +79,7 @@ func main() {
}()
myLock := "/net/corpora/nlnieuws/VRT/lock"
mkLock(myLock)
u.MkLock(myLock)
defer func() {
_ = os.Remove(myLock)
}()
@@ -242,18 +243,18 @@ func doArticle(filename string, url string, title string, tags []string, cats []
p(fmt.Fprintln(&buf, "##META text cat ="))
} else {
for _, cat := range cats {
p(fmt.Fprintf(&buf, "##META text cat = %s\n", fixSpace(cat)))
p(fmt.Fprintf(&buf, "##META text cat = %s\n", u.FixSpace(cat)))
}
}
if len(tags) == 0 {
p(fmt.Fprintln(&buf, "##META text tag ="))
} else {
for _, tag := range tags {
p(fmt.Fprintf(&buf, "##META text tag = %s\n", fixSpace(tag)))
p(fmt.Fprintf(&buf, "##META text tag = %s\n", u.FixSpace(tag)))
}
}
_, err = buf.WriteString(addEnd(fixSpace(title)))
_, err = buf.WriteString(u.AddEnd(u.FixSpace(title)))
p(err)
fouten := make([]string, 0)
@@ -262,7 +263,7 @@ func doArticle(filename string, url string, title string, tags []string, cats []
pp, err := root.Search(`//div[@data-sentry-component="ArticleHeading"]//*[contains(@class,"prose-article-body-r")]`)
p(err)
for _, p1 := range pp {
p(fmt.Fprint(&buf, addEnd(fixSpace(p1.Content()))))
p(fmt.Fprint(&buf, u.AddEnd(u.FixSpace(p1.Content()))))
found = true
}
if !found {
@@ -277,7 +278,7 @@ func doArticle(filename string, url string, title string, tags []string, cats []
`//div[@data-sentry-component="ArticleTitle"]//h2`)
p(err)
for _, p1 := range pp {
p(fmt.Fprint(&buf, addEnd(fixSpace(p1.Content()))))
p(fmt.Fprint(&buf, u.AddEnd(u.FixSpace(p1.Content()))))
found = true
}
if !found {
@@ -311,40 +312,3 @@ func doArticle(filename string, url string, title string, tags []string, cats []
return true
}
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}