joeac/comitium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor fetch functions to insert into FullData

Browse Source
This commit is contained in:
nytpu
2021-03-18 10:19:07 -06:00
parent e1a5c68ec7
commit 5ba676ed5d
4 changed files with 104 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fetch/gemini.go
+31 -31
View File
@@ -27,7 +27,7 @@ import (
)
// fetchGemini will make a request for a gemini resource
func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
func fetchGemini(remote *url.URL) (*gemini.Response, error) {
client := &gemini.Client{}
resp, err := client.Get(context.Background(), remoteURL.String())
if err != nil {
@@ -40,17 +40,18 @@ func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
return resp, nil
}
// newGeminiFeed will fetch and create a new core.Feed given a gemini url
func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
resp, err := fetchGemini(remoteURL)
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return nil, err
return err
}
defer resp.Body.Close()
mimetype, _, err := mime.ParseMediaType(resp.Meta)
if err != nil {
return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta)
return fmt.Errorf("Unintelligible content type: %s", resp.Meta)
}
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
@@ -59,11 +60,12 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
case "text/gemini":
var feed core.Feed
feed.Title = title // will be set later if given title is ""
feed.Link = remoteURL
feed.FeedLink = remoteURL
feed.Link = remote
feed.FeedLink = remote
text, err := gemini.ParseText(resp.Body)
if err != nil {
return nil, err
return err
}
for _, line := range text {
switch line := line.(type) {
@@ -84,7 +86,7 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
if err != nil {
continue
}
link = remoteURL.ResolveReference(link)
link = remote.ResolveReference(link)
item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], ": ")
item.Published = date
@@ -92,7 +94,8 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
feed.Items = append(feed.Items, item)
}
}
return &feed, nil
InsertFeed(data, &feed, remote)
return nil
case "text/xml",
"application/rss+xml",
"application/atom+xml",
@@ -100,30 +103,22 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
"application/json":
gofeed, err := gofeed.NewParser().Parse(reader)
if err != nil {
return nil, err
return err
}
feed, err := core.GofeedConvert(gofeed, title)
return feed, err
if err != nil {
return err
}
InsertFeed(data, feed, remote)
return nil
default:
return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
}
}
// newGeminiPage will create a new core.Page given a gemini url
func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) {
var page core.Page
page.Title = title
page.Link = remoteURL
err := updateGeminiPage(&page)
if err != nil {
return nil, err
}
return &page, nil
}
// updateGeminiPage will check an existing core.Page for changes
func updateGeminiPage(p *core.Page) error {
// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(p.Link)
if err != nil {
return err
@@ -132,14 +127,19 @@ func updateGeminiPage(p *core.Page) error {
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
var page core.Page
page.Title = title
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
p.Hash = newHash
p.Updated = time.Now()
page.Hash = newHash
page.Updated = time.Now()
}
InsertPage(data, &page, remote)
return nil
}