diff --git a/Sikkom/cmd/metadata/metadata.go b/Sikkom/cmd/metadata/metadata.go index 5051d6c..b1830ee 100644 --- a/Sikkom/cmd/metadata/metadata.go +++ b/Sikkom/cmd/metadata/metadata.go @@ -3,8 +3,10 @@ package main import ( e "codeberg.org/pebbe/errors" + "bufio" "encoding/xml" "fmt" + "html" "os" "strings" "time" @@ -17,6 +19,7 @@ type Item struct { var ( x = e.ExitErr + escape = html.EscapeString data = make(map[string][]string) location *time.Location ) @@ -33,6 +36,9 @@ func main() { if strings.HasSuffix(filename, ".xml") { doXml(filename) } + if strings.HasSuffix(filename, ".txt") { + doText(filename) + } } files, err = os.ReadDir("xml") @@ -80,6 +86,32 @@ func doXml(filename string) { t.Day())) } +func doText(filename string) { + base := filename[:len(filename)-4] + if _, ok := data[base]; !ok { + data[base] = make([]string, 0) + } + fp, err := os.Open("../" + filename) + x(err) + defer func() { x(fp.Close()) }() + scanner := bufio.NewScanner(fp) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "##META") { + continue + } + aa := strings.Fields(line) + if len(aa) > 4 { + data[base] = append(data[base], + fmt.Sprintf(``, + aa[1], + escape(aa[2]), + escape(strings.Join(aa[4:], " ")))) + } + } + x(scanner.Err()) +} + func stripMeta(s string) string { i1 := strings.Index(s, "") if i1 < 0 { diff --git a/Sikkom/cmd/sikkom/sikkom.go b/Sikkom/cmd/sikkom/sikkom.go index b079935..39432c6 100644 --- a/Sikkom/cmd/sikkom/sikkom.go +++ b/Sikkom/cmd/sikkom/sikkom.go @@ -27,13 +27,14 @@ type Rss struct { } type ItemT struct { - Title string `xml:"title"` - PubDate string `xml:"pubDate"` - UnixTime int64 `xml:"unixTime"` - Guid string `xml:"guid"` - Link string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link - Premium bool `xml:"premium"` - Data []byte `xml:",innerxml"` + Title string `xml:"title"` + PubDate string `xml:"pubDate"` + UnixTime int64 `xml:"unixTime"` + Guid string `xml:"guid"` + Link string `xml:"https://www.rssboard.org/rss-specification link"` // i.p.v. atom:link + Premium bool `xml:"premium"` + Tags []string `xml:"category"` + Data []byte `xml:",innerxml"` } type LdJson struct { @@ -143,12 +144,12 @@ func main() { p(fp.WriteString("\n")) p(fp.Close()) p(os.Chtimes(filename+".xml", t, t)) - ok = doArticle(filename, item.Link, item.Title, t, needUpdate) + ok = doArticle(filename, item.Link, item.Title, item.Tags, t, needUpdate) }() } } -func doArticle(filename string, url string, title string, timestamp time.Time, needUpdate bool) bool { +func doArticle(filename string, url string, title string, tags []string, timestamp time.Time, needUpdate bool) bool { if exists(filename + ".skip") { return true } @@ -295,6 +296,19 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n fp, err := os.Create(filename + ".txt") p(err) + + found = false + for _, tag := range tags { + if tag == "Actueel" || tag == "Nieuws" { + continue + } + p(fmt.Fprintln(fp, "##META text tag =", tag)) + found = true + } + if !found { + p(fp.WriteString("##META text tag =\n")) + } + for _, t := range text { p(fp.WriteString(u.AddEnd(u.FixSpace(t)))) } diff --git a/Sikkom/txt2corpus.sh b/Sikkom/txt2corpus.sh index 3627c89..47d6fe4 100755 --- a/Sikkom/txt2corpus.sh +++ b/Sikkom/txt2corpus.sh @@ -62,6 +62,9 @@ cd xml rm -f $corpus.data.dz $corpus.index alto -q -o $corpus.data.dz *.xml +# telling per bericht, niet per zin +query.sh -x T -s $corpus.data.dz > $corpus.tag.txt + cd ../.. rm -fr out