joeac/comitium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor fetch functions to insert into FullData

Browse Source
This commit is contained in:
nytpu
2021-03-18 10:19:07 -06:00
parent e1a5c68ec7
commit 5ba676ed5d
4 changed files with 104 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fetch/fetch.go
+37 -42
View File
@@ -9,68 +9,63 @@ package fetch
import (
"fmt"
"net/url"
"reflect"
"golang.nytpu.com/comitium/core"
)
// NewFeed will fetch and create a new core.Feed given any url
func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
switch remoteURL.Scheme {
// Feed will fetch and insert a new core.Feed into a core.FullData given any url
// (or update if it's preexisting)
func Feed(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return newGeminiFeed(remoteURL, title)
return geminiFeed(data, remote, title)
case "http", "https":
return newHTTPFeed(remoteURL, title)
return httpFeed(data, remote, title)
case "gopher", "gophers":
return newGopherFeed(remoteURL, title)
return gopherFeed(data, remote, title)
default:
return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
}
// NewPage will fetch and create a new core.Page given any url
func NewPage(remoteURL *url.URL, title string) (*core.Page, error) {
switch remoteURL.Scheme {
// Page will fetch and insert a new core.Page into a core.FullData given any url
// (or update if it's preexisting)
func Page(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return newGeminiPage(remoteURL, title)
return geminiPage(data, remote, title)
case "http", "https":
return newHTTPPage(remoteURL, title)
return httpPage(data, remote, title)
case "gopher", "gophers":
return newGopherPage(remoteURL, title)
return gopherPage(data, remote, title)
default:
return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
}
// UpdateFeed will check an existing core.Feed for updates
func UpdateFeed(f *core.Feed) error {
switch f.FeedLink.Scheme {
case "gemini":
// return updateGeminiFeed(f)
// TODO
fallthrough
case "http", "https":
// return updateHTTPFeed(f)
// TODO
fallthrough
case "gopher", "gophers":
return updateGopherFeed(f)
default:
return fmt.Errorf("Unsupported protocol '%s'", f.FeedLink.Scheme)
// InsertFeed will insert or update a core.Feed in a core.FullData
func InsertFeed(data *core.FullData, new *core.Feed, remote *url.URL) {
data.FeedsMu.Lock()
oldFeed, ok := data.Feeds[*remote]
if !ok || !reflect.DeepEqual(new, oldFeed) {
// Feeds are different, or there was never an old one
data.Feeds[*remote] = new
data.FeedsMu.Unlock()
} else {
data.FeedsMu.Unlock()
}
return nil
}
// UpdatePage will check an existing core.Page for updages
func UpdatePage(p *core.Page) error {
switch p.Link.Scheme {
case "gemini":
return updateGeminiPage(p)
case "http", "https":
return updateHTTPPage(p)
case "gopher", "gophers":
return updateGopherPage(p)
default:
return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme)
// InsertFeed will insert or update a core.Page in a core.FullData
func InsertPage(data *core.FullData, new *core.Page, remote *url.URL) {
data.PagesMu.Lock()
_, ok := data.Pages[*remote]
if !ok || data.Pages[*remote].Hash != new.Hash {
// pages are different, or there was never an old one
data.Pages[*remote] = new
data.PagesMu.Unlock()
} else {
data.PagesMu.Unlock()
}
return nil
}
fetch/gemini.go
+31 -31
View File
@@ -27,7 +27,7 @@ import (
)
// fetchGemini will make a request for a gemini resource
func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
func fetchGemini(remote *url.URL) (*gemini.Response, error) {
client := &gemini.Client{}
resp, err := client.Get(context.Background(), remoteURL.String())
if err != nil {
@@ -40,17 +40,18 @@ func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
return resp, nil
}
// newGeminiFeed will fetch and create a new core.Feed given a gemini url
func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
resp, err := fetchGemini(remoteURL)
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return nil, err
return err
}
defer resp.Body.Close()
mimetype, _, err := mime.ParseMediaType(resp.Meta)
if err != nil {
return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta)
return fmt.Errorf("Unintelligible content type: %s", resp.Meta)
}
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
@@ -59,11 +60,12 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
case "text/gemini":
var feed core.Feed
feed.Title = title // will be set later if given title is ""
feed.Link = remoteURL
feed.FeedLink = remoteURL
feed.Link = remote
feed.FeedLink = remote
text, err := gemini.ParseText(resp.Body)
if err != nil {
return nil, err
return err
}
for _, line := range text {
switch line := line.(type) {
@@ -84,7 +86,7 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
if err != nil {
continue
}
link = remoteURL.ResolveReference(link)
link = remote.ResolveReference(link)
item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], ": ")
item.Published = date
@@ -92,7 +94,8 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
feed.Items = append(feed.Items, item)
}
}
return &feed, nil
InsertFeed(data, &feed, remote)
return nil
case "text/xml",
"application/rss+xml",
"application/atom+xml",
@@ -100,30 +103,22 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
"application/json":
gofeed, err := gofeed.NewParser().Parse(reader)
if err != nil {
return nil, err
return err
}
feed, err := core.GofeedConvert(gofeed, title)
return feed, err
if err != nil {
return err
}
InsertFeed(data, feed, remote)
return nil
default:
return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
}
}
// newGeminiPage will create a new core.Page given a gemini url
func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) {
var page core.Page
page.Title = title
page.Link = remoteURL
err := updateGeminiPage(&page)
if err != nil {
return nil, err
}
return &page, nil
}
// updateGeminiPage will check an existing core.Page for changes
func updateGeminiPage(p *core.Page) error {
// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGemini(p.Link)
if err != nil {
return err
@@ -132,14 +127,19 @@ func updateGeminiPage(p *core.Page) error {
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
var page core.Page
page.Title = title
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
p.Hash = newHash
p.Updated = time.Now()
page.Hash = newHash
page.Updated = time.Now()
}
InsertPage(data, &page, remote)
return nil
}
fetch/gopher.go
+14 -29
View File
@@ -12,7 +12,6 @@ import (
"net/url"
"time"
//"github.com/mmcdole/gofeed"
"github.com/prologic/go-gopher"
"golang.nytpu.com/comitium/core"
@@ -27,34 +26,17 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
return resp, nil
}
// newGopherFeed will fetch and create a new core.Feed given a gopher url
func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
// TODO
return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// updateGopherFeed will check an existing core.Feed for updates
func updateGopherFeed(*core.Feed) error {
// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherFeed(data *core.FullData, remote *url.URL, title string) error {
// TODO
return fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// newGopherPage will create a new core.Page given a gopher url
func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
var page core.Page
page.Title = title
page.Link = remoteURL
err := updateGopherPage(&page)
if err != nil {
return nil, err
}
return &page, nil
}
// updateGopherPage wll check an existing core.Page for changes
func updateGopherPage(p *core.Page) error {
resp, err := fetchGopher(p.Link)
// gopherPage will fetch and insert a new core.Page into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchGopher(remote)
if err != nil {
return err
}
@@ -71,14 +53,17 @@ func updateGopherPage(p *core.Page) error {
reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
}
var page core.Page
page.Title = title
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
p.Hash = newHash
p.Updated = time.Now()
}
page.Hash = newHash
page.Updated = time.Now()
InsertPage(data, &page, remote)
return nil
}
fetch/http.go
+22 -25
View File
@@ -48,44 +48,38 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
return resp, nil
}
// newHTTPFeed will fetch and create a new core.Feed given an http url
func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
resp, err := fetchHTTP(remoteURL)
// httpFeed will fetch and insert a new core.Feed into a core.FullData given an
// http url (or update if it's preexisting)
func httpFeed(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return nil, err
return err
}
defer resp.Body.Close()
if resp.Header.Get("Content-Type") == "text/html" {
// TODO
return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented")
return fmt.Errorf("Extracting feed from HTML pages is unimplemented")
}
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
gofeed, err := gofeed.NewParser().Parse(reader)
if err != nil {
return nil, err
return err
}
feed, err := core.GofeedConvert(gofeed, title)
return feed, err
}
// newHTTPPage will create a new core.Page given an http url
func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) {
var page core.Page
page.Title = title
page.Link = remoteURL
err := updateHTTPPage(&page)
if err != nil {
return nil, err
return err
}
return &page, nil
InsertFeed(data, feed, remote)
return nil
}
// updateHTTPPage will check an existing core.Page for changes
func updateHTTPPage(p *core.Page) error {
resp, err := fetchHTTP(p.Link)
// httpPage will fetch and insert a new core.Page into a core.FullData given an
// http url (or update if it's preexisting)
func httpPage(data *core.FullData, remote *url.URL, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return err
}
@@ -93,14 +87,17 @@ func updateHTTPPage(p *core.Page) error {
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
var page core.Page
page.Title = title
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
}
newHash := fmt.Sprintf("%x", h.Sum(nil))
if newHash != p.Hash {
p.Hash = newHash
p.Updated = time.Now()
}
page.Hash = newHash
page.Updated = time.Now()
InsertPage(data, &page, remote)
return nil
}