grotere buffer voor scanner

cmd/data2json/data2json.go

@@ -142,6 +142,7 @@ func makeValues(source, part string) [][5]any {
 	fp, err := os.Open(filename)
 	x(err)
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	lineno := 0
 	for scanner.Scan() {
 		line := scanner.Text()

cmd/flush/flush.go

@@ -12,6 +12,7 @@ func main() {
 	chSignal := make(chan os.Signal, 1)
 	signal.Notify(chSignal, syscall.SIGPIPE)
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		select {

cmd/items2count/items2count.go

@@ -42,6 +42,7 @@ var (
 func main() {
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		line := scanner.Text()
 		aa := strings.Split(line, "\t")

cmd/rang/rang.go

@@ -26,6 +26,7 @@ func main() {
 	counts := make(map[string]int)
 
 	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		word := strings.Split(scanner.Text(), "\t")[0]
 		counts[word] = counts[word] + 1

cmd/top20/top20.go

@@ -41,6 +41,7 @@ func main() {
 				fp, err := os.Open("../" + dirname + "/" + name)
 				x(err)
 				scanner := bufio.NewScanner(fp)
+				scanner.Buffer(nil, 1000000)
 				for scanner.Scan() {
 					seen[strings.Split(scanner.Text(), "\t")[1]] = true
 				}
@@ -55,6 +56,7 @@ func main() {
 	fpout, err := os.Create(target)
 	x(err)
 	scanner := bufio.NewScanner(fpin)
+	scanner.Buffer(nil, 1000000)
 	n := 0
 	for scanner.Scan() && n < 20 {
 		line := scanner.Text()

cmd/trends/trends.go

@@ -28,6 +28,7 @@ func main() {
 	fp, err := os.Open(os.Args[1])
 	x(err)
 	scanner := bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		aa := strings.Split(scanner.Text(), "\t")
 		n, err := strconv.Atoi(aa[0])
@@ -45,6 +46,7 @@ func main() {
 	fp, err = os.Open(os.Args[2])
 	x(err)
 	scanner = bufio.NewScanner(fp)
+	scanner.Buffer(nil, 1000000)
 	for scanner.Scan() {
 		lines = append(lines, scanner.Text())
 	}