From 6d15441a5a671d5dd4679a1b9651ac7b4da65a27 Mon Sep 17 00:00:00 2001 From: Peter Kleiweg Date: Thu, 19 Mar 2026 14:42:35 +0100 Subject: [PATCH] lockfiles --- AT5/cmd/at5/at5.go | 20 ++++++++++++++++++++ Amsterdam/amsterdam.go | 22 +++++++++++++++++++++- GG/cmd/gg/gg.go | 20 ++++++++++++++++++++ NOS/cmd/nos/nos.go | 20 ++++++++++++++++++++ NU/cmd/nu/nu.go | 20 ++++++++++++++++++++ NieuwsNL/cmd/nieuwsnl/nieuwsnl.go | 20 ++++++++++++++++++++ RO/cmd/ro/ro.go | 20 ++++++++++++++++++++ Sargasso/cmd/sargasso/sargasso.go | 20 ++++++++++++++++++++ Sikkom/cmd/sikkom/sikkom.go | 20 ++++++++++++++++++++ Tzum/cmd/tzum/tzum.go | 20 ++++++++++++++++++++ VRT/cmd/vrt/vrt.go | 20 ++++++++++++++++++++ 11 files changed, 221 insertions(+), 1 deletion(-) diff --git a/AT5/cmd/at5/at5.go b/AT5/cmd/at5/at5.go index f3c026a..38a2bd7 100644 --- a/AT5/cmd/at5/at5.go +++ b/AT5/cmd/at5/at5.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -43,6 +44,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/AT5/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://rss.at5.nl/rss", nil) p(err) req.Header.Set("User-Agent", agent) @@ -101,3 +108,16 @@ func main() { }() } } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/Amsterdam/amsterdam.go b/Amsterdam/amsterdam.go index 47c308b..063d8ea 100644 --- a/Amsterdam/amsterdam.go +++ b/Amsterdam/amsterdam.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -47,6 +48,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/Amsterdam/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://www.amsterdam.nl/nieuws/nieuwsoverzicht/?rss=true", nil) p(err) req.Header.Set("User-Agent", agent) @@ -71,7 +78,7 @@ func main() { t, err = time.Parse(time.RFC1123, item.PubDate) } p(err) - dirname := fmt.Sprintf("/net/corpora/nlnieuws/amsterdam/%d/%02d", t.Year(), int(t.Month())) + dirname := fmt.Sprintf("/net/corpora/nlnieuws/Amsterdam/%d/%02d", t.Year(), int(t.Month())) if exists(dirname + "/lock") { continue } @@ -178,3 +185,16 @@ func addEnd(s string) string { } return s + ".\n" } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/GG/cmd/gg/gg.go b/GG/cmd/gg/gg.go index 9b37aa4..f711775 100644 --- a/GG/cmd/gg/gg.go +++ b/GG/cmd/gg/gg.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -62,6 +63,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/GG/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://gemeente.groningen.nl/feed/rss/nieuws", nil) p(err) req.Header.Set("User-Agent", agent) @@ -211,3 +218,16 @@ func addEnd(s string) string { func fixSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/NOS/cmd/nos/nos.go b/NOS/cmd/nos/nos.go index 481979f..1ad228e 100644 --- a/NOS/cmd/nos/nos.go +++ b/NOS/cmd/nos/nos.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -59,6 +60,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/NOS/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + resp, err := http.Get("https://feeds.nos.nl/nosnieuwsalgemeen") p(err) body, err := io.ReadAll(resp.Body) @@ -183,3 +190,16 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool p(os.Chtimes(filename+".json", timestamp, timestamp)) return true } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/NU/cmd/nu/nu.go b/NU/cmd/nu/nu.go index 2e6d38a..5dd5960 100644 --- a/NU/cmd/nu/nu.go +++ b/NU/cmd/nu/nu.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -71,6 +72,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/NU/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + resp, err := http.Get("https://www.nu.nl/rss") p(err) body, err := io.ReadAll(resp.Body) @@ -253,3 +260,16 @@ func doArticle(filename string, url string, timestamp time.Time, needUpdate bool func fixSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/NieuwsNL/cmd/nieuwsnl/nieuwsnl.go b/NieuwsNL/cmd/nieuwsnl/nieuwsnl.go index 710148d..3db5d4a 100644 --- a/NieuwsNL/cmd/nieuwsnl/nieuwsnl.go +++ b/NieuwsNL/cmd/nieuwsnl/nieuwsnl.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -62,6 +63,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/NieuwsNL/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://nieuws.nl/sitemap/news.xml", nil) p(err) req.Header.Set("User-Agent", agent) @@ -230,3 +237,16 @@ func addEnd(s string) string { func fixSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/RO/cmd/ro/ro.go b/RO/cmd/ro/ro.go index 2681236..9bcbdf4 100644 --- a/RO/cmd/ro/ro.go +++ b/RO/cmd/ro/ro.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -43,6 +44,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/RO/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://reportersonline.nl/feed/", nil) p(err) req.Header.Set("User-Agent", agent) @@ -102,3 +109,16 @@ func main() { } } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/Sargasso/cmd/sargasso/sargasso.go b/Sargasso/cmd/sargasso/sargasso.go index a69e6a5..39a4d31 100644 --- a/Sargasso/cmd/sargasso/sargasso.go +++ b/Sargasso/cmd/sargasso/sargasso.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -43,6 +44,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/Sargasso/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://sargasso.nl/feed/", nil) p(err) req.Header.Set("User-Agent", agent) @@ -102,3 +109,16 @@ func main() { } } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/Sikkom/cmd/sikkom/sikkom.go b/Sikkom/cmd/sikkom/sikkom.go index 7d539b6..0cdbdf5 100644 --- a/Sikkom/cmd/sikkom/sikkom.go +++ b/Sikkom/cmd/sikkom/sikkom.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -61,6 +62,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/Sikkom/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + resp, err := http.Get("https://www.sikkom.nl/api/feed/rss") p(err) body, err := io.ReadAll(resp.Body) @@ -242,3 +249,16 @@ func addEnd(s string) string { func fixSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/Tzum/cmd/tzum/tzum.go b/Tzum/cmd/tzum/tzum.go index 5bf16bc..6ced178 100644 --- a/Tzum/cmd/tzum/tzum.go +++ b/Tzum/cmd/tzum/tzum.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -43,6 +44,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/Tzum/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + req, err := http.NewRequest("GET", "https://www.tzum.info/feed/", nil) p(err) req.Header.Set("User-Agent", agent) @@ -102,3 +109,16 @@ func main() { } } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +} diff --git a/VRT/cmd/vrt/vrt.go b/VRT/cmd/vrt/vrt.go index 0e03e5d..42c4001 100644 --- a/VRT/cmd/vrt/vrt.go +++ b/VRT/cmd/vrt/vrt.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "time" ) @@ -76,6 +77,12 @@ func main() { } }() + myLock := "/net/corpora/nlnieuws/VRT/lock" + mkLock(myLock) + defer func() { + _ = os.Remove(myLock) + }() + resp, err := http.Get("https://www.vrt.be/vrtnws/nl.rss.headlines.xml") p(err) body, err := io.ReadAll(resp.Body) @@ -323,3 +330,16 @@ func addEnd(s string) string { func fixSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +func mkLock(filename string) { + pid := os.Getpid() + link := fmt.Sprintf("%s.%d", filepath.Base(filename), pid) + p(os.Symlink(link, filename)) + + name, err := os.Readlink(filename) + p(err) + + if name != link { + p(fmt.Errorf("wrong lock name %q, should be %q", name, link)) + } +}