first commit

2026-03-02 15:34:37 +01:00
commit 36f051a8a9
50 changed files with 4241 additions and 0 deletions
--- a/AT5/cmd/xml2txt/xml2txt.go
+++ b/AT5/cmd/xml2txt/xml2txt.go
@@ -0,0 +1,92 @@
+package main
+
+import (
+	"github.com/jbowtie/gokogiri"
+	"github.com/pebbe/util"
+
+	"encoding/xml"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type Item struct {
+	Title string `xml:"title"`
+	Text  string `xml:"description"`
+}
+
+var (
+	x = util.CheckErr
+
+	reYearWeek = regexp.MustCompile(`^2[0-9][0-9][0-9]-[0-5][0-9]$`)
+)
+
+func main() {
+
+	var ds string
+	switch len(os.Args) {
+	case 1:
+		year, week := time.Now().AddDate(0, 0, -7).ISOWeek()
+		ds = fmt.Sprintf("%d-%02d", year, week)
+	case 2:
+		if !reYearWeek.MatchString(os.Args[1]) {
+			x(fmt.Errorf("arg must be yyyy-ww"))
+		}
+		ds = os.Args[1]
+	default:
+		x(fmt.Errorf("too many arguments"))
+	}
+	dp := ds[:4] + "/" + ds[5:]
+
+	x(os.Chdir("/net/corpora/nlnieuws/AT5/" + dp))
+	x(os.MkdirAll("out", 0777))
+	files, err := os.ReadDir(".")
+	x(err)
+	for _, file := range files {
+		filename := file.Name()
+		if !strings.HasSuffix(filename, ".xml") {
+			continue
+		}
+		b, err := os.ReadFile(filename)
+		x(err)
+		fp, err := os.Create("out/" + filename[:len(filename)-4] + ".txt")
+		x(err)
+		var item Item
+		x(xml.Unmarshal(b, &item), filename)
+		_, err = fp.WriteString(addEnd(item.Title))
+		x(err)
+		doc, err := gokogiri.ParseHtml([]byte(`<html><body>` + item.Text + `</body></html>`))
+		x(err)
+		root := doc.Root()
+		pp, err := root.Search(`//body/p | //body/h2`)
+		x(err)
+		for _, p := range pp {
+			_, err = fp.WriteString(addEnd(p.Content()))
+			x(err)
+		}
+		x(err)
+		x(fp.Close())
+	}
+}
+
+func addEnd(s string) string {
+	s = strings.TrimSpace(s)
+	n := len(s)
+	if n == 0 {
+		return ""
+	}
+	if n > 0 {
+		if strings.ContainsAny(s[n-1:], ".!?") {
+			return s + "\n"
+		}
+	}
+	if n > 1 {
+		s2 := s[n-2:]
+		if s2 == `."` || s2 == `!"` || s2 == `?"` || s2 == `.'` || s2 == `!'` || s2 == `?'` {
+			return s + "\n"
+		}
+	}
+	return s + ".\n"
+}