nytpu
120 lines
3.0 KiB
Go
120 lines
3.0 KiB
Go
// Copyright (C) 2021 nytpu
|
|
// SPDX-License-Identifier: AGPL-3.0-only
|
|
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
// package fetch implements routines for fetching gemini, gopher, and http
|
|
// resources and parsing feed formats
|
|
package fetch
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
"sync"
|
|
|
|
"golang.nytpu.com/comitium/localizations"
|
|
|
|
"golang.nytpu.com/comitium/core"
|
|
)
|
|
|
|
// Feed will fetch and insert a new core.Feed into a core.FullData given any url
|
|
// (or update if it's preexisting)
|
|
func Feed(data *core.FullData, remote *url.URL, title string) error {
|
|
switch remote.Scheme {
|
|
case "gemini":
|
|
return geminiFeed(data, remote.String(), title)
|
|
case "http", "https":
|
|
return httpFeed(data, remote.String(), title)
|
|
case "gopher", "gophers":
|
|
return gopherFeed(data, remote.String(), title)
|
|
default:
|
|
return fmt.Errorf(core.Localizer.Get("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme}))
|
|
}
|
|
}
|
|
|
|
// Page will fetch and insert a new core.Page into a core.FullData given any url
|
|
// (or update if it's preexisting)
|
|
func Page(data *core.FullData, remote *url.URL, title string) error {
|
|
switch remote.Scheme {
|
|
case "gemini":
|
|
return geminiPage(data, remote.String(), title)
|
|
case "http", "https":
|
|
return httpPage(data, remote.String(), title)
|
|
case "gopher", "gophers":
|
|
return gopherPage(data, remote.String(), title)
|
|
default:
|
|
return fmt.Errorf(core.Localizer.Get("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme}))
|
|
}
|
|
}
|
|
|
|
type refreshJob struct {
|
|
typ string // "feed" or "page"
|
|
url *url.URL
|
|
}
|
|
|
|
// RefreshAll will check all feeds and pages in a core.FullData for updates
|
|
func RefreshAll(data *core.FullData, numWorkers int) error {
|
|
var wg sync.WaitGroup
|
|
|
|
data.RLock()
|
|
numJobs := len(data.Feeds) + len(data.Pages)
|
|
if numJobs == 0 {
|
|
data.RUnlock()
|
|
return nil
|
|
}
|
|
|
|
if numWorkers < 1 {
|
|
numWorkers = 1
|
|
}
|
|
|
|
jobs := make(chan refreshJob, numJobs)
|
|
returns := make(chan error, numJobs)
|
|
|
|
// start workers but jobs is blocking for right now
|
|
for w := 0; w < numWorkers; w++ {
|
|
wg.Add(1)
|
|
go refreshWorker(&wg, jobs, returns, data)
|
|
}
|
|
|
|
// get all keys in maps
|
|
feedKeys := make([]*url.URL, 0, len(data.Feeds))
|
|
for k := range data.Feeds {
|
|
// we know that the keys are already validated uris, so no err
|
|
u, _ := url.ParseRequestURI(k)
|
|
feedKeys = append(feedKeys, u)
|
|
}
|
|
pageKeys := make([]*url.URL, 0, len(data.Pages))
|
|
for k := range data.Pages {
|
|
u, _ := url.ParseRequestURI(k)
|
|
pageKeys = append(pageKeys, u)
|
|
}
|
|
data.RUnlock()
|
|
|
|
for _, v := range feedKeys {
|
|
jobs <- refreshJob{"feed", v}
|
|
}
|
|
for _, v := range pageKeys {
|
|
jobs <- refreshJob{"page", v}
|
|
}
|
|
close(jobs)
|
|
|
|
wg.Wait()
|
|
|
|
for i := 0; i < numJobs; i++ {
|
|
if err := <-returns; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func refreshWorker(wg *sync.WaitGroup, jobs <-chan refreshJob, returns chan<- error, data *core.FullData) {
|
|
defer wg.Done()
|
|
for j := range jobs {
|
|
if j.typ == "feed" {
|
|
returns <- Feed(data, j.url, "")
|
|
} else if j.typ == "page" {
|
|
returns <- Page(data, j.url, "")
|
|
}
|
|
}
|
|
}
|