sikkom: tags

Sikkom/cmd/metadata/metadata.go

@@ -3,8 +3,10 @@ package main
 import (
 	e "codeberg.org/pebbe/errors"
 
+	"bufio"
 	"encoding/xml"
 	"fmt"
+	"html"
 	"os"
 	"strings"
 	"time"
@@ -17,6 +19,7 @@ type Item struct {
 
 var (
 	x        = e.ExitErr
+	escape   = html.EscapeString
 	data     = make(map[string][]string)
 	location *time.Location
 )
@@ -33,6 +36,9 @@ func main() {
 		if strings.HasSuffix(filename, ".xml") {
 			doXml(filename)
 		}
+		if strings.HasSuffix(filename, ".txt") {
+			doText(filename)
+		}
 	}
 
 	files, err = os.ReadDir("xml")
@@ -80,6 +86,32 @@ func doXml(filename string) {
 			t.Day()))
 }
 
+func doText(filename string) {
+	base := filename[:len(filename)-4]
+	if _, ok := data[base]; !ok {
+		data[base] = make([]string, 0)
+	}
+	fp, err := os.Open("../" + filename)
+	x(err)
+	defer func() { x(fp.Close()) }()
+	scanner := bufio.NewScanner(fp)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "##META") {
+			continue
+		}
+		aa := strings.Fields(line)
+		if len(aa) > 4 {
+			data[base] = append(data[base],
+				fmt.Sprintf(`<meta type="%s" name="%s" value="%s"/>`,
+					aa[1],
+					escape(aa[2]),
+					escape(strings.Join(aa[4:], " "))))
+		}
+	}
+	x(scanner.Err())
+}
+
 func stripMeta(s string) string {
 	i1 := strings.Index(s, "<metadata>")
 	if i1 < 0 {

Sikkom/cmd/sikkom/sikkom.go

@@ -27,13 +27,14 @@ type Rss struct {
 }
 
 type ItemT struct {
-	Title    string `xml:"title"`
+	Title    string   `xml:"title"`
-	PubDate  string `xml:"pubDate"`
+	PubDate  string   `xml:"pubDate"`
-	UnixTime int64  `xml:"unixTime"`
+	UnixTime int64    `xml:"unixTime"`
-	Guid     string `xml:"guid"`
+	Guid     string   `xml:"guid"`
-	Link     string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link
+	Link     string   `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link
-	Premium  bool   `xml:"premium"`
+	Premium  bool     `xml:"premium"`
-	Data     []byte `xml:",innerxml"`
+	Tags     []string `xml:"category"`
+	Data     []byte   `xml:",innerxml"`
 }
 
 type LdJson struct {
@@ -143,12 +144,12 @@ func main() {
 			p(fp.WriteString("</item>\n"))
 			p(fp.Close())
 			p(os.Chtimes(filename+".xml", t, t))
-			ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
+			ok = doArticle(filename, item.Link, item.Title, item.Tags, t, needUpdate)
 		}()
 	}
 }
 
-func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) bool {
+func doArticle(filename string, url string, title string, tags []string, timestamp time.Time, needUpdate bool) bool {
 	if exists(filename + ".skip") {
 		return true
 	}
@@ -295,6 +296,19 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 
 	fp, err := os.Create(filename + ".txt")
 	p(err)
+
+	found = false
+	for _, tag := range tags {
+		if tag == "Actueel" || tag == "Nieuws" {
+			continue
+		}
+		p(fmt.Fprintln(fp, "##META text tag =", tag))
+		found = true
+	}
+	if !found {
+		p(fp.WriteString("##META text tag =\n"))
+	}
+
 	for _, t := range text {
 		p(fp.WriteString(u.AddEnd(u.FixSpace(t))))
 	}

Sikkom/txt2corpus.sh

@@ -62,6 +62,9 @@ cd xml
 rm -f $corpus.data.dz $corpus.index
 alto -q -o $corpus.data.dz *.xml
 
+# telling per bericht, niet per zin
+query.sh -x T -s $corpus.data.dz > $corpus.tag.txt
+
 cd ../..
 rm -fr out