1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
// Copyright (C) 2021 nytpu
// SPDX-License-Identifier: AGPL-3.0-only
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
// package fetch implements routines for fetching gemini, gopher, and http
// resources and parsing feed formats
package fetch
import (
"fmt"
"net/url"
"sync"
"golang.nytpu.com/comitium/localizations"
"golang.nytpu.com/comitium/core"
)
// Feed will fetch and insert a new core.Feed into a core.FullData given any url
// (or update if it's preexisting)
func Feed(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return geminiFeed(data, remote.String(), title)
case "http", "https":
return httpFeed(data, remote.String(), title)
case "gopher", "gophers":
return gopherFeed(data, remote.String(), title)
default:
return fmt.Errorf(core.Localizer("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme})
}
}
// Page will fetch and insert a new core.Page into a core.FullData given any url
// (or update if it's preexisting)
func Page(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return geminiPage(data, remote.String(), title)
case "http", "https":
return httpPage(data, remote.String(), title)
case "gopher", "gophers":
return gopherPage(data, remote.String(), title)
default:
return fmt.Errorf(core.Localizer("errors.unsupported_protocol", &localizations.Replacements{"scheme": remote.Scheme})
}
}
type refreshJob struct {
typ string // "feed" or "page"
url *url.URL
}
// RefreshAll will check all feeds and pages in a core.FullData for updates
func RefreshAll(data *core.FullData, numWorkers int) error {
var wg sync.WaitGroup
data.RLock()
numJobs := len(data.Feeds) + len(data.Pages)
if numJobs == 0 {
data.RUnlock()
return nil
}
if numWorkers < 1 {
numWorkers = 1
}
jobs := make(chan refreshJob, numJobs)
returns := make(chan error, numJobs)
// start workers but jobs is blocking for right now
for w := 0; w < numWorkers; w++ {
wg.Add(1)
go refreshWorker(&wg, jobs, returns, data)
}
// get all keys in maps
feedKeys := make([]*url.URL, 0, len(data.Feeds))
for k := range data.Feeds {
// we know that the keys are already validated uris, so no err
u, _ := url.ParseRequestURI(k)
feedKeys = append(feedKeys, u)
}
pageKeys := make([]*url.URL, 0, len(data.Pages))
for k := range data.Pages {
u, _ := url.ParseRequestURI(k)
pageKeys = append(pageKeys, u)
}
data.RUnlock()
for _, v := range feedKeys {
jobs <- refreshJob{"feed", v}
}
for _, v := range pageKeys {
jobs <- refreshJob{"page", v}
}
close(jobs)
wg.Wait()
for i := 0; i < numJobs; i++ {
if err := <-returns; err != nil {
return err
}
}
return nil
}
func refreshWorker(wg *sync.WaitGroup, jobs <-chan refreshJob, returns chan<- error, data *core.FullData) {
defer wg.Done()
for j := range jobs {
if j.typ == "feed" {
returns <- Feed(data, j.url, "")
} else if j.typ == "page" {
returns <- Page(data, j.url, "")
}
}
}
|
