add updateGopherPage function

This commit is contained in:
nytpu
2021-03-17 08:57:00 -06:00
parent e68ff691b8
commit 795979e3bb
2 changed files with 23 additions and 14 deletions
+1 -3
View File
@@ -70,9 +70,7 @@ func UpdatePage(p *core.Page) error {
case "http", "https": case "http", "https":
return updateHTTPPage(p) return updateHTTPPage(p)
case "gopher", "gophers": case "gopher", "gophers":
// return updateGopherPage(p) return updateGopherPage(p)
// TODO
fallthrough
default: default:
return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme) return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme)
} }
+22 -11
View File
@@ -35,33 +35,44 @@ func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
// newGopherPage will create a new core.Page given a gopher url // newGopherPage will create a new core.Page given a gopher url
func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) { func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
resp, err := fetchGopher(remoteURL) var page core.Page
page.Title = title
page.Link = remoteURL
err := updateGopherPage(&page)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &page, nil
}
// updateGopherPage wll check an existing core.Page for changes
func updateGopherPage(p *core.Page) error {
resp, err := fetchGopher(p.Link)
if err != nil {
return err
}
defer resp.Body.Close() defer resp.Body.Close()
var reader io.Reader var reader io.Reader
if len(resp.Dir.Items) > 0 { if len(resp.Dir.Items) > 0 {
text, err := resp.Dir.ToText() text, err := resp.Dir.ToText()
if err != nil { if err != nil {
return nil, err return err
} }
reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
} else { } else {
reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
} }
var page core.Page
page.Title = title
page.Link = remoteURL
page.Updated = time.Now()
h := sha256.New() h := sha256.New()
if _, err := io.Copy(h, reader); err != nil { if _, err := io.Copy(h, reader); err != nil {
return nil, err return err
} }
page.Hash = fmt.Sprintf("%x", h.Sum(nil)) newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
return &page, nil p.Hash = newHash
p.Updated = time.Now()
}
return nil
} }