// Copyright (C) 2021 nytpu // SPDX-License-Identifier: AGPL-3.0-only // For more license details, see LICENSE or . // package fetch implements routines for fetching gemini, gopher, and http // resources and parsing feed formats package fetch import ( "fmt" "net/url" "sync" "golang.nytpu.com/comitium/localizations" "golang.nytpu.com/comitium/core" ) // Feed will fetch and insert a new core.Feed into a core.FullData given any url // (or update if it's preexisting) func Feed(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": return geminiFeed(data, remote.String(), title) case "http", "https": return httpFeed(data, remote.String(), title) case "gopher", "gophers": return gopherFeed(data, remote.String(), title) default: return fmt.Errorf(core.Localizer.Get("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme})) } } // Page will fetch and insert a new core.Page into a core.FullData given any url // (or update if it's preexisting) func Page(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": return geminiPage(data, remote.String(), title) case "http", "https": return httpPage(data, remote.String(), title) case "gopher", "gophers": return gopherPage(data, remote.String(), title) default: return fmt.Errorf(core.Localizer.Get("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme})) } } type refreshJob struct { typ string // "feed" or "page" url *url.URL } // RefreshAll will check all feeds and pages in a core.FullData for updates func RefreshAll(data *core.FullData, numWorkers int) error { var wg sync.WaitGroup data.RLock() numJobs := len(data.Feeds) + len(data.Pages) if numJobs == 0 { data.RUnlock() return nil } if numWorkers < 1 { numWorkers = 1 } jobs := make(chan refreshJob, numJobs) returns := make(chan error, numJobs) // start workers but jobs is blocking for right now for w := 0; w < numWorkers; w++ { wg.Add(1) go refreshWorker(&wg, jobs, returns, data) } // get all keys in maps feedKeys := make([]*url.URL, 0, len(data.Feeds)) for k := range data.Feeds { // we know that the keys are already validated uris, so no err u, _ := url.ParseRequestURI(k) feedKeys = append(feedKeys, u) } pageKeys := make([]*url.URL, 0, len(data.Pages)) for k := range data.Pages { u, _ := url.ParseRequestURI(k) pageKeys = append(pageKeys, u) } data.RUnlock() for _, v := range feedKeys { jobs <- refreshJob{"feed", v} } for _, v := range pageKeys { jobs <- refreshJob{"page", v} } close(jobs) wg.Wait() for i := 0; i < numJobs; i++ { if err := <-returns; err != nil { return err } } return nil } func refreshWorker(wg *sync.WaitGroup, jobs <-chan refreshJob, returns chan<- error, data *core.FullData) { defer wg.Done() for j := range jobs { if j.typ == "feed" { returns <- Feed(data, j.url, "") } else if j.typ == "page" { returns <- Page(data, j.url, "") } } }