refactor fetch functions to insert into FullData

fetch/fetch.go

@@ -9,68 +9,63 @@ package fetch
 import (
 	"fmt"
 	"net/url"
+	"reflect"
 
 	"golang.nytpu.com/comitium/core"
 )
 
-// NewFeed will fetch and create a new core.Feed given any url
+// Feed will fetch and insert a new core.Feed into a core.FullData given any url
-func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+// (or update if it's preexisting)
-	switch remoteURL.Scheme {
+func Feed(data *core.FullData, remote *url.URL, title string) error {
+	switch remote.Scheme {
 	case "gemini":
-		return newGeminiFeed(remoteURL, title)
+		return geminiFeed(data, remote, title)
 	case "http", "https":
-		return newHTTPFeed(remoteURL, title)
+		return httpFeed(data, remote, title)
 	case "gopher", "gophers":
-		return newGopherFeed(remoteURL, title)
+		return gopherFeed(data, remote, title)
 	default:
-		return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+		return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
 	}
 }
 
-// NewPage will fetch and create a new core.Page given any url
+// Page will fetch and insert a new core.Page into a core.FullData given any url
-func NewPage(remoteURL *url.URL, title string) (*core.Page, error) {
+// (or update if it's preexisting)
-	switch remoteURL.Scheme {
+func Page(data *core.FullData, remote *url.URL, title string) error {
+	switch remote.Scheme {
 	case "gemini":
-		return newGeminiPage(remoteURL, title)
+		return geminiPage(data, remote, title)
 	case "http", "https":
-		return newHTTPPage(remoteURL, title)
+		return httpPage(data, remote, title)
 	case "gopher", "gophers":
-		return newGopherPage(remoteURL, title)
+		return gopherPage(data, remote, title)
 	default:
-		return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+		return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
 	}
 }
 
-// UpdateFeed will check an existing core.Feed for updates
+// InsertFeed will insert or update a core.Feed in a core.FullData
-func UpdateFeed(f *core.Feed) error {
+func InsertFeed(data *core.FullData, new *core.Feed, remote *url.URL) {
-	switch f.FeedLink.Scheme {
+	data.FeedsMu.Lock()
-	case "gemini":
+	oldFeed, ok := data.Feeds[*remote]
-		// return updateGeminiFeed(f)
+	if !ok || !reflect.DeepEqual(new, oldFeed) {
-		// TODO
+		// Feeds are different, or there was never an old one
-		fallthrough
+		data.Feeds[*remote] = new
-	case "http", "https":
+		data.FeedsMu.Unlock()
-		// return updateHTTPFeed(f)
+	} else {
-		// TODO
+		data.FeedsMu.Unlock()
-		fallthrough
-	case "gopher", "gophers":
-		return updateGopherFeed(f)
-	default:
-		return fmt.Errorf("Unsupported protocol '%s'", f.FeedLink.Scheme)
 	}
-	return nil
 }
 
-// UpdatePage will check an existing core.Page for updages
+// InsertFeed will insert or update a core.Page in a core.FullData
-func UpdatePage(p *core.Page) error {
+func InsertPage(data *core.FullData, new *core.Page, remote *url.URL) {
-	switch p.Link.Scheme {
+	data.PagesMu.Lock()
-	case "gemini":
+	_, ok := data.Pages[*remote]
-		return updateGeminiPage(p)
+	if !ok || data.Pages[*remote].Hash != new.Hash {
-	case "http", "https":
+		// pages are different, or there was never an old one
-		return updateHTTPPage(p)
+		data.Pages[*remote] = new
-	case "gopher", "gophers":
+		data.PagesMu.Unlock()
-		return updateGopherPage(p)
+	} else {
-	default:
+		data.PagesMu.Unlock()
-		return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme)
 	}
-	return nil
 }

fetch/gemini.go

@@ -27,7 +27,7 @@ import (
 )
 
 // fetchGemini will make a request for a gemini resource
-func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
+func fetchGemini(remote *url.URL) (*gemini.Response, error) {
 	client := &gemini.Client{}
 	resp, err := client.Get(context.Background(), remoteURL.String())
 	if err != nil {
@@ -40,17 +40,18 @@ func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
 	return resp, nil
 }
 
-// newGeminiFeed will fetch and create a new core.Feed given a gemini url
+// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
-func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+// a gemini url (or update if it's preexisting)
-	resp, err := fetchGemini(remoteURL)
+func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
+	resp, err := fetchGemini(remote)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer resp.Body.Close()
 
 	mimetype, _, err := mime.ParseMediaType(resp.Meta)
 	if err != nil {
-		return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta)
+		return fmt.Errorf("Unintelligible content type: %s", resp.Meta)
 	}
 
 	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
@@ -59,11 +60,12 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
 	case "text/gemini":
 		var feed core.Feed
 		feed.Title = title // will be set later if given title is ""
-		feed.Link = remoteURL
+		feed.Link = remote
-		feed.FeedLink = remoteURL
+		feed.FeedLink = remote
+
 		text, err := gemini.ParseText(resp.Body)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		for _, line := range text {
 			switch line := line.(type) {
@@ -84,7 +86,7 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
 				if err != nil {
 					continue
 				}
-				link = remoteURL.ResolveReference(link)
+				link = remote.ResolveReference(link)
 				item := &core.Item{}
 				item.Title = strings.TrimLeft(line.Name[10:], ": ")
 				item.Published = date
@@ -92,7 +94,8 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
 				feed.Items = append(feed.Items, item)
 			}
 		}
-		return &feed, nil
+		InsertFeed(data, &feed, remote)
+		return nil
 	case "text/xml",
 		"application/rss+xml",
 		"application/atom+xml",
@@ -100,30 +103,22 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
 		"application/json":
 		gofeed, err := gofeed.NewParser().Parse(reader)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		feed, err := core.GofeedConvert(gofeed, title)
-		return feed, err
+		if err != nil {
+			return err
+		}
+		InsertFeed(data, feed, remote)
+		return nil
 	default:
-		return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
+		return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
 	}
 }
 
-// newGeminiPage will create a new core.Page given a gemini url
+// geminiPage will fetch and insert a new core.Page into a core.FullData given
-func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) {
+// a gemini url (or update if it's preexisting)
-	var page core.Page
+func geminiPage(data *core.FullData, remote *url.URL, title string) error {
-	page.Title = title
-	page.Link = remoteURL
-
-	err := updateGeminiPage(&page)
-	if err != nil {
-		return nil, err
-	}
-	return &page, nil
-}
-
-// updateGeminiPage will check an existing core.Page for changes
-func updateGeminiPage(p *core.Page) error {
 	resp, err := fetchGemini(p.Link)
 	if err != nil {
 		return err
@@ -132,14 +127,19 @@ func updateGeminiPage(p *core.Page) error {
 
 	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
 
+	var page core.Page
+	page.Title = title
+	page.Link = remote
 	h := sha256.New()
 	if _, err := io.Copy(h, reader); err != nil {
 		return err
 	}
 	newHash := fmt.Sprintf("%x", h.Sum(nil))
 	if newHash != p.Hash {
-		p.Hash = newHash
+		page.Hash = newHash
-		p.Updated = time.Now()
+		page.Updated = time.Now()
 	}
+
+	InsertPage(data, &page, remote)
 	return nil
 }

fetch/gopher.go

@@ -12,7 +12,6 @@ import (
 	"net/url"
 	"time"
 
-	//"github.com/mmcdole/gofeed"
 	"github.com/prologic/go-gopher"
 
 	"golang.nytpu.com/comitium/core"
@@ -27,34 +26,17 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
 	return resp, nil
 }
 
-// newGopherFeed will fetch and create a new core.Feed given a gopher url
+// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
-func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+// gopher url (or update if it's preexisting)
-	// TODO
+func gopherFeed(data *core.FullData, remote *url.URL, title string) error {
-	return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
-}
-
-// updateGopherFeed will check an existing core.Feed for updates
-func updateGopherFeed(*core.Feed) error {
 	// TODO
 	return fmt.Errorf("Feeds served over gopher are currently unsupported.")
 }
 
-// newGopherPage will create a new core.Page given a gopher url
+// gopherPage will fetch and insert a new core.Page into a core.FullData given a
-func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
+// gopher url (or update if it's preexisting)
-	var page core.Page
+func gopherPage(data *core.FullData, remote *url.URL, title string) error {
-	page.Title = title
+	resp, err := fetchGopher(remote)
-	page.Link = remoteURL
-
-	err := updateGopherPage(&page)
-	if err != nil {
-		return nil, err
-	}
-	return &page, nil
-}
-
-// updateGopherPage wll check an existing core.Page for changes
-func updateGopherPage(p *core.Page) error {
-	resp, err := fetchGopher(p.Link)
 	if err != nil {
 		return err
 	}
@@ -71,14 +53,17 @@ func updateGopherPage(p *core.Page) error {
 		reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
 	}
 
+	var page core.Page
+	page.Title = title
+	page.Link = remote
 	h := sha256.New()
 	if _, err := io.Copy(h, reader); err != nil {
 		return err
 	}
 	newHash := fmt.Sprintf("%x", h.Sum(nil))
-	if newHash != p.Hash {
+	page.Hash = newHash
-		p.Hash = newHash
+	page.Updated = time.Now()
-		p.Updated = time.Now()
+
-	}
+	InsertPage(data, &page, remote)
 	return nil
 }

fetch/http.go

@@ -48,44 +48,38 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
 	return resp, nil
 }
 
-// newHTTPFeed will fetch and create a new core.Feed given an http url
+// httpFeed will fetch and insert a new core.Feed into a core.FullData given an
-func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+// http url (or update if it's preexisting)
-	resp, err := fetchHTTP(remoteURL)
+func httpFeed(data *core.FullData, remote *url.URL, title string) error {
+	resp, err := fetchHTTP(remote)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer resp.Body.Close()
 
 	if resp.Header.Get("Content-Type") == "text/html" {
 		// TODO
-		return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented")
+		return fmt.Errorf("Extracting feed from HTML pages is unimplemented")
 	}
 
 	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
 	gofeed, err := gofeed.NewParser().Parse(reader)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	feed, err := core.GofeedConvert(gofeed, title)
-	return feed, err
-}
-
-// newHTTPPage will create a new core.Page given an http url
-func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) {
-	var page core.Page
-	page.Title = title
-	page.Link = remoteURL
-
-	err := updateHTTPPage(&page)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	return &page, nil
+
+	InsertFeed(data, feed, remote)
+	return nil
 }
 
-// updateHTTPPage will check an existing core.Page for changes
+// httpPage will fetch and insert a new core.Page into a core.FullData given an
-func updateHTTPPage(p *core.Page) error {
+// http url (or update if it's preexisting)
-	resp, err := fetchHTTP(p.Link)
+func httpPage(data *core.FullData, remote *url.URL, title string) error {
+	resp, err := fetchHTTP(remote)
 	if err != nil {
 		return err
 	}
@@ -93,14 +87,17 @@ func updateHTTPPage(p *core.Page) error {
 
 	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
 
+	var page core.Page
+	page.Title = title
+	page.Link = remote
 	h := sha256.New()
 	if _, err := io.Copy(h, reader); err != nil {
 		return err
 	}
 	newHash := fmt.Sprintf("%x", h.Sum(nil))
-	if newHash != p.Hash {
+	page.Hash = newHash
-		p.Hash = newHash
+	page.Updated = time.Now()
-		p.Updated = time.Now()
+
-	}
+	InsertPage(data, &page, remote)
 	return nil
 }