grotere buffer voor scanner

BuurtAdam/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

BuurtGrn/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

HLN/cmd/hln/hln.go

@@ -250,6 +250,18 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
 	}
 	article := articles[0]
 
+	hh, err := article.Search(`//h2`)
+	p(err)
+	for _, h := range hh {
+		if h.Content() == "liveblog" {
+			fp, err := os.Create(filename + ".skip")
+			p(fp.WriteString("liveblog\n" + url + "\n"))
+			p(err)
+			p(os.Chtimes(filename+".skip", timestamp, timestamp))
+			return true
+		}
+	}
+
 	tags := make([]string, 0)
 	ell, err := article.Search(`.//*[@data-content-type="LABEL"]`)
 	p(err)

HLN/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

LitNL/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

NOS/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

NU/cmd/metadata/metadata.go

@@ -77,6 +77,7 @@ func doText(filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

NieuwsNL/cmd/metadata/metadata.go

@@ -77,6 +77,7 @@ func doText(filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Oog/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Parool/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

RO/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

RTVNoord/cmd/metadata/metadata.go

@@ -77,6 +77,7 @@ func doText(filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Sargasso/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Sikkom/cmd/metadata/metadata.go

@@ -95,6 +95,7 @@ func doText(filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Sikkom/cmd/sikkom/sikkom.go

@@ -47,8 +47,8 @@ type LdJson struct {
 var (
 	p = e.PanicErr
 	w = e.WarnErr
-	agent = "AhrefsBot/7.0"
+	//agent = "AhrefsBot/7.0"
-	// agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+	agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"
 	jar   *cookiejar.Jar
 )
 

Tzum/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {
@@ -101,7 +102,7 @@ func doText(dirname, filename string) {
 					escape(strings.Join(aa[4:], " "))))
 		}
 	}
-	x(scanner.Err())
+	x(scanner.Err(), dirname, filename)
 }
 
 func doXml(dirname, filename string) {

VRT/cmd/metadata/metadata.go

@@ -77,6 +77,7 @@ func doText(filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

Volkskrant/cmd/metadata/metadata.go

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
 	x(err)
 	defer func() { x(fp.Close()) }()
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !strings.HasPrefix(line, "##META") {

cmd/data2json/data2json.go

@@ -142,6 +142,7 @@ func makeValues(source, part string) [][5]any {
 	fp, err := os.Open(filename)
 	x(err)
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	lineno := 0
 	for scanner.Scan() {
 		line := scanner.Text()

cmd/flush/flush.go

@@ -12,6 +12,7 @@ func main() {
 	chSignal := make(chan os.Signal, 1)
 	signal.Notify(chSignal, syscall.SIGPIPE)
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		select {

cmd/items2count/items2count.go

@@ -42,6 +42,7 @@ var (
 func main() {
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		aa := strings.Split(line, "\t")

cmd/rang/rang.go

@@ -26,6 +26,7 @@ func main() {
 	counts := make(map[string]int)
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		word := strings.Split(scanner.Text(), "\t")[0]
 		counts[word] = counts[word] + 1

cmd/top20/top20.go

@@ -41,6 +41,7 @@ func main() {
 				fp, err := os.Open("../" + dirname + "/" + name)
 				x(err)
 				scanner := bufio.NewScanner(fp)
+				scanner.Buffer(nil, 1000000)
 				for scanner.Scan() {
 					seen[strings.Split(scanner.Text(), "\t")[1]] = true
 				}
@@ -55,6 +56,7 @@ func main() {
 	fpout, err := os.Create(target)
 	x(err)
 	scanner := bufio.NewScanner(fpin)
+	scanner.Buffer(nil, 1000000)
 	n := 0
 	for scanner.Scan() && n < 20 {
 		line := scanner.Text()

cmd/trends/trends.go

@@ -28,6 +28,7 @@ func main() {
 	fp, err := os.Open(os.Args[1])
 	x(err)
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		aa := strings.Split(scanner.Text(), "\t")
 		n, err := strconv.Atoi(aa[0])
@@ -45,6 +46,7 @@ func main() {
 	fp, err = os.Open(os.Args[2])
 	x(err)
 	scanner = bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		lines = append(lines, scanner.Text())
 	}

oud/xquery/items2count.go.oud

@@ -47,6 +47,7 @@ var (
 func main() {
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		var item Item
 		line := scanner.Text()

oud/xquery/new2old.go

@@ -27,6 +27,7 @@ var (
 func main() {
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		aa := strings.Split(line, "\t")