joeac/comitium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor fetch functions to insert into FullData

Browse Source
This commit is contained in:
nytpu
2021-03-18 10:19:07 -06:00
parent e1a5c68ec7
commit 5ba676ed5d
4 changed files with 104 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fetch/gopher.go
+14 -29
View File
@@ -12,7 +12,6 @@ import (
"net/url"
"time"
//"github.com/mmcdole/gofeed"
"github.com/prologic/go-gopher"
"golang.nytpu.com/comitium/core"
@@ -27,34 +26,17 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
return resp, nil
}
// newGopherFeed will fetch and create a new core.Feed given a gopher url
func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
// TODO
return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// updateGopherFeed will check an existing core.Feed for updates
func updateGopherFeed(*core.Feed) error {
// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherFeed(data *core.FullData, remote *url.URL, title string) error {
// TODO
return fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// newGopherPage will create a new core.Page given a gopher url
func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
var page core.Page
page.Title = title
page.Link = remoteURL
err := updateGopherPage(&page)
if err != nil {
return nil, err
}
return &page, nil
}
// updateGopherPage wll check an existing core.Page for changes
func updateGopherPage(p *core.Page) error {
resp, err := fetchGopher(p.Link)
// gopherPage will fetch and insert a new core.Page into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGopher(remote)
if err != nil {
return err
}
@@ -71,14 +53,17 @@ func updateGopherPage(p *core.Page) error {
reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
}
var page core.Page
page.Title = title
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
p.Hash = newHash
p.Updated = time.Now()
}
page.Hash = newHash
page.Updated = time.Now()
InsertPage(data, &page, remote)
return nil
}