add updateGopherPage function
This commit is contained in:
+1
-3
@@ -70,9 +70,7 @@ func UpdatePage(p *core.Page) error {
|
|||||||
case "http", "https":
|
case "http", "https":
|
||||||
return updateHTTPPage(p)
|
return updateHTTPPage(p)
|
||||||
case "gopher", "gophers":
|
case "gopher", "gophers":
|
||||||
// return updateGopherPage(p)
|
return updateGopherPage(p)
|
||||||
// TODO
|
|
||||||
fallthrough
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme)
|
return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme)
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-11
@@ -35,33 +35,44 @@ func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
|
|||||||
|
|
||||||
// newGopherPage will create a new core.Page given a gopher url
|
// newGopherPage will create a new core.Page given a gopher url
|
||||||
func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
|
func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
|
||||||
resp, err := fetchGopher(remoteURL)
|
var page core.Page
|
||||||
|
page.Title = title
|
||||||
|
page.Link = remoteURL
|
||||||
|
|
||||||
|
err := updateGopherPage(&page)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return &page, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateGopherPage wll check an existing core.Page for changes
|
||||||
|
func updateGopherPage(p *core.Page) error {
|
||||||
|
resp, err := fetchGopher(p.Link)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
var reader io.Reader
|
var reader io.Reader
|
||||||
if len(resp.Dir.Items) > 0 {
|
if len(resp.Dir.Items) > 0 {
|
||||||
text, err := resp.Dir.ToText()
|
text, err := resp.Dir.ToText()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
|
reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
|
||||||
} else {
|
} else {
|
||||||
reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
||||||
}
|
}
|
||||||
|
|
||||||
var page core.Page
|
|
||||||
page.Title = title
|
|
||||||
page.Link = remoteURL
|
|
||||||
page.Updated = time.Now()
|
|
||||||
|
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
if _, err := io.Copy(h, reader); err != nil {
|
if _, err := io.Copy(h, reader); err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
page.Hash = fmt.Sprintf("%x", h.Sum(nil))
|
newHash := fmt.Sprintf("%x", h.Sum(nil))
|
||||||
|
if newHash != p.Hash {
|
||||||
return &page, nil
|
p.Hash = newHash
|
||||||
|
p.Updated = time.Now()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user