1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
// Copyright (C) 2021 nytpu
// SPDX-License-Identifier: AGPL-3.0-or-later
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
// This file incorporates code from Drew DeVault's gemreader:
// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
// Original source is Copyright 2021 Drew DeVault and is licensed under the
// terms of the GNU Affero General Public License Version 3.0.
// https://www.gnu.org/licenses/agpl-3.0-standalone.html
package fetch
import (
"context"
"crypto/sha256"
"fmt"
"io"
"mime"
"net/url"
"strings"
"time"
"git.sr.ht/~adnano/go-gemini"
"github.com/mmcdole/gofeed"
"golang.nytpu.com/comitium/core"
)
// fetchGemini will make a request for a gemini resource
func fetchGemini(remote *url.URL) (*gemini.Response, error) {
client := &gemini.Client{}
resp, err := client.Get(context.Background(), remote.String())
if err != nil {
return nil, err
}
if resp.Status != gemini.StatusSuccess {
return nil, fmt.Errorf("Unexpected Gemini response: %d %s", resp.Status, resp.Meta)
}
return resp, nil
}
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
}
defer resp.Body.Close()
mimetype, _, err := mime.ParseMediaType(resp.Meta)
if err != nil {
return fmt.Errorf("Unintelligible content type: %s", resp.Meta)
}
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
switch mimetype {
case "text/gemini":
var feed core.Feed
feed.Title = title // will be set later if given title is ""
feed.Link = remote.String()
feed.FeedLink = remote.String()
feed.Updated = time.Now()
text, err := gemini.ParseText(resp.Body)
if err != nil {
return err
}
for _, line := range text {
switch line := line.(type) {
case gemini.LineHeading1:
if feed.Title == "" {
feed.Title = strings.TrimLeft(line.String(), "# ")
}
case gemini.LineLink:
if line.Name == "" || len(line.Name) < 10 {
continue
}
name := strings.TrimLeft(line.Name[:10], " -—")
date, err := time.Parse("2006-01-02", name)
if err != nil {
continue
}
link, err := url.Parse(line.URL)
if err != nil {
continue
}
link = remote.ResolveReference(link)
item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], " :-–—")
item.Published = date
item.Link = link.String()
feed.Items = append(feed.Items, item)
}
}
data.InsertFeed(&feed, remote)
return nil
case "text/xml",
"application/rss+xml",
"application/atom+xml",
"application/xml",
"application/json":
gofeed, err := gofeed.NewParser().Parse(reader)
if err != nil {
return err
}
feed, err := core.GofeedConvert(gofeed, title)
if err != nil {
return err
}
data.InsertFeed(feed, remote)
return nil
default:
return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
}
}
// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
}
defer resp.Body.Close()
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
var page core.Page
page.Title = title
page.Link = remote.String()
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
page.Hash = newHash
page.Updated = time.Now()
data.InsertPage(&page, remote)
return nil
}
|
