p209327/nlnieuws
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

grotere buffer voor scanner

Browse Source
This commit is contained in:
Peter Kleiweg
2026-08-04 13:32:12 +02:00
parent 1de42635d0
commit c37c54148e
26 changed files with 44 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
BuurtAdam/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
BuurtGrn/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

12
HLN/cmd/hln/hln.go
View File

@@ -250,6 +250,18 @@ func doArticle(filename string, url string, title string, timestamp time.Time, n
}
article := articles[0]
hh, err := article.Search(`//h2`)
p(err)
for _, h := range hh {
if h.Content() == "liveblog" {
fp, err := os.Create(filename + ".skip")
p(fp.WriteString("liveblog\n" + url + "\n"))
p(err)
p(os.Chtimes(filename+".skip", timestamp, timestamp))
return true
}
}
tags := make([]string, 0)
ell, err := article.Search(`.//*[@data-content-type="LABEL"]`)
p(err)

1
HLN/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
LitNL/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
NOS/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
NU/cmd/metadata/metadata.go
View File

@@ -77,6 +77,7 @@ func doText(filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
NieuwsNL/cmd/metadata/metadata.go
View File

@@ -77,6 +77,7 @@ func doText(filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
Oog/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
Parool/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
RO/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
RTVNoord/cmd/metadata/metadata.go
View File

@@ -77,6 +77,7 @@ func doText(filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
Sargasso/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
Sikkom/cmd/metadata/metadata.go
View File

@@ -95,6 +95,7 @@ func doText(filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

4
Sikkom/cmd/sikkom/sikkom.go
View File

@@ -47,8 +47,8 @@ type LdJson struct {
var (
p = e.PanicErr
w = e.WarnErr
agent = "AhrefsBot/7.0"
// agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
//agent = "AhrefsBot/7.0"
agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"
jar *cookiejar.Jar
)

3
Tzum/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {
@@ -101,7 +102,7 @@ func doText(dirname, filename string) {
escape(strings.Join(aa[4:], " "))))
}
}
x(scanner.Err())
x(scanner.Err(), dirname, filename)
}
func doXml(dirname, filename string) {

1
VRT/cmd/metadata/metadata.go
View File

@@ -77,6 +77,7 @@ func doText(filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
Volkskrant/cmd/metadata/metadata.go
View File

@@ -87,6 +87,7 @@ func doText(dirname, filename string) {
x(err)
defer func() { x(fp.Close()) }()
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "##META") {

1
cmd/data2json/data2json.go
View File

@@ -142,6 +142,7 @@ func makeValues(source, part string) [][5]any {
fp, err := os.Open(filename)
x(err)
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
lineno := 0
for scanner.Scan() {
line := scanner.Text()

1
cmd/flush/flush.go
View File

@@ -12,6 +12,7 @@ func main() {
chSignal := make(chan os.Signal, 1)
signal.Notify(chSignal, syscall.SIGPIPE)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
select {

1
cmd/items2count/items2count.go
View File

@@ -42,6 +42,7 @@ var (
func main() {
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
aa := strings.Split(line, "\t")

1
cmd/rang/rang.go
View File

@@ -26,6 +26,7 @@ func main() {
counts := make(map[string]int)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
word := strings.Split(scanner.Text(), "\t")[0]
counts[word] = counts[word] + 1

2
cmd/top20/top20.go
View File

@@ -41,6 +41,7 @@ func main() {
fp, err := os.Open("../" + dirname + "/" + name)
x(err)
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
seen[strings.Split(scanner.Text(), "\t")[1]] = true
}
@@ -55,6 +56,7 @@ func main() {
fpout, err := os.Create(target)
x(err)
scanner := bufio.NewScanner(fpin)
scanner.Buffer(nil, 1000000)
n := 0
for scanner.Scan() && n < 20 {
line := scanner.Text()

2
cmd/trends/trends.go
View File

@@ -28,6 +28,7 @@ func main() {
fp, err := os.Open(os.Args[1])
x(err)
scanner := bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
aa := strings.Split(scanner.Text(), "\t")
n, err := strconv.Atoi(aa[0])
@@ -45,6 +46,7 @@ func main() {
fp, err = os.Open(os.Args[2])
x(err)
scanner = bufio.NewScanner(fp)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}

1
oud/xquery/items2count.go.oud
View File

@@ -47,6 +47,7 @@ var (
func main() {
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
var item Item
line := scanner.Text()

1
oud/xquery/new2old.go
View File

@@ -27,6 +27,7 @@ var (
func main() {
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(nil, 1000000)
for scanner.Scan() {
line := scanner.Text()
aa := strings.Split(line, "\t")