diff --git a/BuurtAdam/cmd/metadata/metadata.go b/BuurtAdam/cmd/metadata/metadata.go index f87bcc2..91d4602 100644 --- a/BuurtAdam/cmd/metadata/metadata.go +++ b/BuurtAdam/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/BuurtGrn/cmd/metadata/metadata.go b/BuurtGrn/cmd/metadata/metadata.go index 4f68304..17af13b 100644 --- a/BuurtGrn/cmd/metadata/metadata.go +++ b/BuurtGrn/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/HLN/cmd/hln/hln.go b/HLN/cmd/hln/hln.go index 6209cad..275c5c0 100644 --- a/HLN/cmd/hln/hln.go +++ b/HLN/cmd/hln/hln.go @@ -250,6 +250,18 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n } article := articles[0] + hh, err := article.Search(`//h2`) + p(err) + for _, h := range hh { + if h.Content() == "liveblog" { + fp, err := os.Create(filename + ".skip") + p(fp.WriteString("liveblog\n" + url + "\n")) + p(err) + p(os.Chtimes(filename+".skip", timestamp, timestamp)) + return true + } + } + tags := make([]string, 0) ell, err := article.Search(`.//*[@data-content-type="LABEL"]`) p(err) diff --git a/HLN/cmd/metadata/metadata.go b/HLN/cmd/metadata/metadata.go index 18ff292..8321c36 100644 --- a/HLN/cmd/metadata/metadata.go +++ b/HLN/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/LitNL/cmd/metadata/metadata.go b/LitNL/cmd/metadata/metadata.go index 95e3d9d..a5ac415 100644 --- a/LitNL/cmd/metadata/metadata.go +++ b/LitNL/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/NOS/cmd/metadata/metadata.go b/NOS/cmd/metadata/metadata.go index f072704..27bde39 100644 --- a/NOS/cmd/metadata/metadata.go +++ b/NOS/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/NU/cmd/metadata/metadata.go b/NU/cmd/metadata/metadata.go index cad5eaa..fa26c49 100644 --- a/NU/cmd/metadata/metadata.go +++ b/NU/cmd/metadata/metadata.go @@ -77,6 +77,7 @@ func doText(filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/NieuwsNL/cmd/metadata/metadata.go b/NieuwsNL/cmd/metadata/metadata.go index bbb6eba..2a57fb6 100644 --- a/NieuwsNL/cmd/metadata/metadata.go +++ b/NieuwsNL/cmd/metadata/metadata.go @@ -77,6 +77,7 @@ func doText(filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Oog/cmd/metadata/metadata.go b/Oog/cmd/metadata/metadata.go index 0c4d11f..38cbc33 100644 --- a/Oog/cmd/metadata/metadata.go +++ b/Oog/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Parool/cmd/metadata/metadata.go b/Parool/cmd/metadata/metadata.go index 6427f23..6070533 100644 --- a/Parool/cmd/metadata/metadata.go +++ b/Parool/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/RO/cmd/metadata/metadata.go b/RO/cmd/metadata/metadata.go index 8f603c9..8fa55c9 100644 --- a/RO/cmd/metadata/metadata.go +++ b/RO/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/RTVNoord/cmd/metadata/metadata.go b/RTVNoord/cmd/metadata/metadata.go index 0fbeadf..332a11e 100644 --- a/RTVNoord/cmd/metadata/metadata.go +++ b/RTVNoord/cmd/metadata/metadata.go @@ -77,6 +77,7 @@ func doText(filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Sargasso/cmd/metadata/metadata.go b/Sargasso/cmd/metadata/metadata.go index fe344f5..9a38277 100644 --- a/Sargasso/cmd/metadata/metadata.go +++ b/Sargasso/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Sikkom/cmd/metadata/metadata.go b/Sikkom/cmd/metadata/metadata.go index b1830ee..a7c00bf 100644 --- a/Sikkom/cmd/metadata/metadata.go +++ b/Sikkom/cmd/metadata/metadata.go @@ -95,6 +95,7 @@ func doText(filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Sikkom/cmd/sikkom/sikkom.go b/Sikkom/cmd/sikkom/sikkom.go index 39432c6..1eac6ce 100644 --- a/Sikkom/cmd/sikkom/sikkom.go +++ b/Sikkom/cmd/sikkom/sikkom.go @@ -45,11 +45,11 @@ type LdJson struct { } var ( - p = e.PanicErr - w = e.WarnErr - agent = "AhrefsBot/7.0" - // agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" - jar *cookiejar.Jar + p = e.PanicErr + w = e.WarnErr + //agent = "AhrefsBot/7.0" + agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36" + jar *cookiejar.Jar ) func exists(filename string) bool { diff --git a/Tzum/cmd/metadata/metadata.go b/Tzum/cmd/metadata/metadata.go index 46d704d..ab75523 100644 --- a/Tzum/cmd/metadata/metadata.go +++ b/Tzum/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { @@ -101,7 +102,7 @@ func doText(dirname, filename string) { escape(strings.Join(aa[4:], " ")))) } } - x(scanner.Err()) + x(scanner.Err(), dirname, filename) } func doXml(dirname, filename string) { diff --git a/VRT/cmd/metadata/metadata.go b/VRT/cmd/metadata/metadata.go index 43f29e8..ea2c64d 100644 --- a/VRT/cmd/metadata/metadata.go +++ b/VRT/cmd/metadata/metadata.go @@ -77,6 +77,7 @@ func doText(filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/Volkskrant/cmd/metadata/metadata.go b/Volkskrant/cmd/metadata/metadata.go index 49f82c7..47ad61f 100644 --- a/Volkskrant/cmd/metadata/metadata.go +++ b/Volkskrant/cmd/metadata/metadata.go @@ -87,6 +87,7 @@ func doText(dirname, filename string) { x(err) defer func() { x(fp.Close()) }() scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "##META") { diff --git a/cmd/data2json/data2json.go b/cmd/data2json/data2json.go index 8dc6c76..004c634 100644 --- a/cmd/data2json/data2json.go +++ b/cmd/data2json/data2json.go @@ -142,6 +142,7 @@ func makeValues(source, part string) [][5]any { fp, err := os.Open(filename) x(err) scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) lineno := 0 for scanner.Scan() { line := scanner.Text() diff --git a/cmd/flush/flush.go b/cmd/flush/flush.go index 4fbd27f..289b4e1 100644 --- a/cmd/flush/flush.go +++ b/cmd/flush/flush.go @@ -12,6 +12,7 @@ func main() { chSignal := make(chan os.Signal, 1) signal.Notify(chSignal, syscall.SIGPIPE) scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() select { diff --git a/cmd/items2count/items2count.go b/cmd/items2count/items2count.go index 77d138a..0bb74ea 100644 --- a/cmd/items2count/items2count.go +++ b/cmd/items2count/items2count.go @@ -42,6 +42,7 @@ var ( func main() { scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() aa := strings.Split(line, "\t") diff --git a/cmd/rang/rang.go b/cmd/rang/rang.go index c07cd8e..1db2802 100644 --- a/cmd/rang/rang.go +++ b/cmd/rang/rang.go @@ -26,6 +26,7 @@ func main() { counts := make(map[string]int) scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(nil, 1000000) for scanner.Scan() { word := strings.Split(scanner.Text(), "\t")[0] counts[word] = counts[word] + 1 diff --git a/cmd/top20/top20.go b/cmd/top20/top20.go index 47de59b..6e81888 100644 --- a/cmd/top20/top20.go +++ b/cmd/top20/top20.go @@ -41,6 +41,7 @@ func main() { fp, err := os.Open("../" + dirname + "/" + name) x(err) scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { seen[strings.Split(scanner.Text(), "\t")[1]] = true } @@ -55,6 +56,7 @@ func main() { fpout, err := os.Create(target) x(err) scanner := bufio.NewScanner(fpin) + scanner.Buffer(nil, 1000000) n := 0 for scanner.Scan() && n < 20 { line := scanner.Text() diff --git a/cmd/trends/trends.go b/cmd/trends/trends.go index 5e2fa52..0731eaa 100644 --- a/cmd/trends/trends.go +++ b/cmd/trends/trends.go @@ -28,6 +28,7 @@ func main() { fp, err := os.Open(os.Args[1]) x(err) scanner := bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { aa := strings.Split(scanner.Text(), "\t") n, err := strconv.Atoi(aa[0]) @@ -45,6 +46,7 @@ func main() { fp, err = os.Open(os.Args[2]) x(err) scanner = bufio.NewScanner(fp) + scanner.Buffer(nil, 1000000) for scanner.Scan() { lines = append(lines, scanner.Text()) } diff --git a/oud/xquery/items2count.go.oud b/oud/xquery/items2count.go.oud index 552565a..73b63eb 100644 --- a/oud/xquery/items2count.go.oud +++ b/oud/xquery/items2count.go.oud @@ -47,6 +47,7 @@ var ( func main() { scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(nil, 1000000) for scanner.Scan() { var item Item line := scanner.Text() diff --git a/oud/xquery/new2old.go b/oud/xquery/new2old.go index 728383a..2a321d6 100644 --- a/oud/xquery/new2old.go +++ b/oud/xquery/new2old.go @@ -27,6 +27,7 @@ var ( func main() { scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(nil, 1000000) for scanner.Scan() { line := scanner.Text() aa := strings.Split(line, "\t")