diff --git a/Sikkom/cmd/metadata/metadata.go b/Sikkom/cmd/metadata/metadata.go
index 5051d6c..b1830ee 100644
--- a/Sikkom/cmd/metadata/metadata.go
+++ b/Sikkom/cmd/metadata/metadata.go
@@ -3,8 +3,10 @@ package main
import (
e "codeberg.org/pebbe/errors"
+ "bufio"
"encoding/xml"
"fmt"
+ "html"
"os"
"strings"
"time"
@@ -17,6 +19,7 @@ type Item struct {
var (
x = e.ExitErr
+ escape = html.EscapeString
data = make(map[string][]string)
location *time.Location
)
@@ -33,6 +36,9 @@ func main() {
if strings.HasSuffix(filename, ".xml") {
doXml(filename)
}
+ if strings.HasSuffix(filename, ".txt") {
+ doText(filename)
+ }
}
files, err = os.ReadDir("xml")
@@ -80,6 +86,32 @@ func doXml(filename string) {
t.Day()))
}
+func doText(filename string) {
+ base := filename[:len(filename)-4]
+ if _, ok := data[base]; !ok {
+ data[base] = make([]string, 0)
+ }
+ fp, err := os.Open("../" + filename)
+ x(err)
+ defer func() { x(fp.Close()) }()
+ scanner := bufio.NewScanner(fp)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if !strings.HasPrefix(line, "##META") {
+ continue
+ }
+ aa := strings.Fields(line)
+ if len(aa) > 4 {
+ data[base] = append(data[base],
+ fmt.Sprintf(``,
+ aa[1],
+ escape(aa[2]),
+ escape(strings.Join(aa[4:], " "))))
+ }
+ }
+ x(scanner.Err())
+}
+
func stripMeta(s string) string {
i1 := strings.Index(s, "")
if i1 < 0 {
diff --git a/Sikkom/cmd/sikkom/sikkom.go b/Sikkom/cmd/sikkom/sikkom.go
index b079935..39432c6 100644
--- a/Sikkom/cmd/sikkom/sikkom.go
+++ b/Sikkom/cmd/sikkom/sikkom.go
@@ -27,13 +27,14 @@ type Rss struct {
}
type ItemT struct {
- Title string `xml:"title"`
- PubDate string `xml:"pubDate"`
- UnixTime int64 `xml:"unixTime"`
- Guid string `xml:"guid"`
- Link string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link
- Premium bool `xml:"premium"`
- Data []byte `xml:",innerxml"`
+ Title string `xml:"title"`
+ PubDate string `xml:"pubDate"`
+ UnixTime int64 `xml:"unixTime"`
+ Guid string `xml:"guid"`
+ Link string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link
+ Premium bool `xml:"premium"`
+ Tags []string `xml:"category"`
+ Data []byte `xml:",innerxml"`
}
type LdJson struct {
@@ -143,12 +144,12 @@ func main() {
p(fp.WriteString("\n"))
p(fp.Close())
p(os.Chtimes(filename+".xml", t, t))
- ok = doArticle(filename, item.Link, item.Title, t, needUpdate)
+ ok = doArticle(filename, item.Link, item.Title, item.Tags, t, needUpdate)
}()
}
}
-func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) bool {
+func doArticle(filename string, url string, title string, tags []string, timestamp time.Time, needUpdate bool) bool {
if exists(filename + ".skip") {
return true
}
@@ -295,6 +296,19 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
fp, err := os.Create(filename + ".txt")
p(err)
+
+ found = false
+ for _, tag := range tags {
+ if tag == "Actueel" || tag == "Nieuws" {
+ continue
+ }
+ p(fmt.Fprintln(fp, "##META text tag =", tag))
+ found = true
+ }
+ if !found {
+ p(fp.WriteString("##META text tag =\n"))
+ }
+
for _, t := range text {
p(fp.WriteString(u.AddEnd(u.FixSpace(t))))
}
diff --git a/Sikkom/txt2corpus.sh b/Sikkom/txt2corpus.sh
index 3627c89..47d6fe4 100755
--- a/Sikkom/txt2corpus.sh
+++ b/Sikkom/txt2corpus.sh
@@ -62,6 +62,9 @@ cd xml
rm -f $corpus.data.dz $corpus.index
alto -q -o $corpus.data.dz *.xml
+# telling per bericht, niet per zin
+query.sh -x T -s $corpus.data.dz > $corpus.tag.txt
+
cd ../..
rm -fr out