p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix title voor HLN en Parool; helperfuncties naar internal/util

Browse Source
This commit is contained in:
Peter Kleiweg
2026-05-24 16:16:21 +02:00
parent 4b56c0cd70
commit 75832c3132
36 changed files with 301 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
LitNL/Makefile
View File

@@ -3,11 +3,11 @@ all: \
metadata \
litnl
xml2txt: cmd/xml2txt/*.go
go build -o $@ $^
xml2txt: cmd/xml2txt/*.go ../internal/util/*.go
go build -o $@ $<
metadata: cmd/metadata/*.go
go build -o $@ $^
litnl: cmd/litnl/*.go
go build -o $@ $^
litnl: cmd/litnl/*.go ../internal/util/*.go
go build -o $@ $<

18
LitNL/cmd/litnl/litnl.go
View File

@@ -3,13 +3,14 @@ package main
import (
e "codeberg.org/pebbe/errors"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
@@ -46,7 +47,7 @@ func main() {
}()
myLock := "/net/corpora/nlnieuws/LitNL/lock"
mkLock(myLock)
u.MkLock(myLock)
defer func() {
_ = os.Remove(myLock)
}()
@@ -108,16 +109,3 @@ func main() {
}
}
func mkLock(filename string) {
pid := os.Getpid()
link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid)
p(os.Symlink(link, filename))
name, err := os.Readlink(filename)
p(err)
if name != link {
p(fmt.Errorf("wrong lock name %q, should be %q", name, link))
}
}

32
LitNL/cmd/xml2txt/xml2txt.go
View File

@@ -4,6 +4,8 @@ import (
e "codeberg.org/pebbe/errors"
"github.com/jbowtie/gokogiri"
u "git.web.rug.nl/p209327/nlnieuws/internal/util"
"encoding/xml"
"fmt"
"os"
@@ -58,9 +60,9 @@ func main() {
var item Item
x(xml.Unmarshal(b, &item))
for _, cat := range item.Cats {
x(fmt.Fprintf(fp, "##META text tag = %s\n", fixSpace(cat)))
x(fmt.Fprintf(fp, "##META text tag = %s\n", u.FixSpace(cat)))
}
x(fp.WriteString(addEnd(fixSpace(item.Title))))
x(fp.WriteString(u.AddEnd(u.FixSpace(item.Title))))
doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
x(err)
root := doc.Root()
@@ -74,32 +76,8 @@ func main() {
_ = w(fmt.Errorf("empty: %s", filename))
}
for _, p := range pp {
x(fp.WriteString(addEnd(fixSpace(p.Content()))))
x(fp.WriteString(u.AddEnd(u.FixSpace(p.Content()))))
}
x(fp.Close())
}
}
func addEnd(s string) string {
s = strings.TrimSpace(s)
n := len(s)
if n == 0 {
return ""
}
if n > 0 {
if strings.ContainsAny(s[n-1:], ".!?") {
return s + "\n"
}
}
if n > 1 {
s2 := s[n-2:]
if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
return s + "\n"
}
}
return s + ".\n"
}
func fixSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}