From 5ba676ed5d2f152eda30fd0fcccafced3ff9bf10 Mon Sep 17 00:00:00 2001 From: nytpu Date: Thu, 18 Mar 2021 10:19:07 -0600 Subject: [PATCH] refactor fetch functions to insert into FullData --- fetch/fetch.go | 79 +++++++++++++++++++++++-------------------------- fetch/gemini.go | 62 +++++++++++++++++++------------------- fetch/gopher.go | 43 +++++++++------------------ fetch/http.go | 47 ++++++++++++++--------------- 4 files changed, 104 insertions(+), 127 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index 37d0b67..231c7dc 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -9,68 +9,63 @@ package fetch import ( "fmt" "net/url" + "reflect" "golang.nytpu.com/comitium/core" ) -// NewFeed will fetch and create a new core.Feed given any url -func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) { - switch remoteURL.Scheme { +// Feed will fetch and insert a new core.Feed into a core.FullData given any url +// (or update if it's preexisting) +func Feed(data *core.FullData, remote *url.URL, title string) error { + switch remote.Scheme { case "gemini": - return newGeminiFeed(remoteURL, title) + return geminiFeed(data, remote, title) case "http", "https": - return newHTTPFeed(remoteURL, title) + return httpFeed(data, remote, title) case "gopher", "gophers": - return newGopherFeed(remoteURL, title) + return gopherFeed(data, remote, title) default: - return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme) + return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } } -// NewPage will fetch and create a new core.Page given any url -func NewPage(remoteURL *url.URL, title string) (*core.Page, error) { - switch remoteURL.Scheme { +// Page will fetch and insert a new core.Page into a core.FullData given any url +// (or update if it's preexisting) +func Page(data *core.FullData, remote *url.URL, title string) error { + switch remote.Scheme { case "gemini": - return newGeminiPage(remoteURL, title) + return geminiPage(data, remote, title) case "http", "https": - return newHTTPPage(remoteURL, title) + return httpPage(data, remote, title) case "gopher", "gophers": - return newGopherPage(remoteURL, title) + return gopherPage(data, remote, title) default: - return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme) + return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } } -// UpdateFeed will check an existing core.Feed for updates -func UpdateFeed(f *core.Feed) error { - switch f.FeedLink.Scheme { - case "gemini": - // return updateGeminiFeed(f) - // TODO - fallthrough - case "http", "https": - // return updateHTTPFeed(f) - // TODO - fallthrough - case "gopher", "gophers": - return updateGopherFeed(f) - default: - return fmt.Errorf("Unsupported protocol '%s'", f.FeedLink.Scheme) +// InsertFeed will insert or update a core.Feed in a core.FullData +func InsertFeed(data *core.FullData, new *core.Feed, remote *url.URL) { + data.FeedsMu.Lock() + oldFeed, ok := data.Feeds[*remote] + if !ok || !reflect.DeepEqual(new, oldFeed) { + // Feeds are different, or there was never an old one + data.Feeds[*remote] = new + data.FeedsMu.Unlock() + } else { + data.FeedsMu.Unlock() } - return nil } -// UpdatePage will check an existing core.Page for updages -func UpdatePage(p *core.Page) error { - switch p.Link.Scheme { - case "gemini": - return updateGeminiPage(p) - case "http", "https": - return updateHTTPPage(p) - case "gopher", "gophers": - return updateGopherPage(p) - default: - return fmt.Errorf("Unsupported protocol '%s'", p.Link.Scheme) +// InsertFeed will insert or update a core.Page in a core.FullData +func InsertPage(data *core.FullData, new *core.Page, remote *url.URL) { + data.PagesMu.Lock() + _, ok := data.Pages[*remote] + if !ok || data.Pages[*remote].Hash != new.Hash { + // pages are different, or there was never an old one + data.Pages[*remote] = new + data.PagesMu.Unlock() + } else { + data.PagesMu.Unlock() } - return nil } diff --git a/fetch/gemini.go b/fetch/gemini.go index eb06780..516358a 100644 --- a/fetch/gemini.go +++ b/fetch/gemini.go @@ -27,7 +27,7 @@ import ( ) // fetchGemini will make a request for a gemini resource -func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) { +func fetchGemini(remote *url.URL) (*gemini.Response, error) { client := &gemini.Client{} resp, err := client.Get(context.Background(), remoteURL.String()) if err != nil { @@ -40,17 +40,18 @@ func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) { return resp, nil } -// newGeminiFeed will fetch and create a new core.Feed given a gemini url -func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { - resp, err := fetchGemini(remoteURL) +// geminiFeed will fetch and insert a new core.Feed into a core.FullData given +// a gemini url (or update if it's preexisting) +func geminiFeed(data *core.FullData, remote *url.URL, title string) error { + resp, err := fetchGemini(remote) if err != nil { - return nil, err + return err } defer resp.Body.Close() mimetype, _, err := mime.ParseMediaType(resp.Meta) if err != nil { - return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta) + return fmt.Errorf("Unintelligible content type: %s", resp.Meta) } reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB @@ -59,11 +60,12 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { case "text/gemini": var feed core.Feed feed.Title = title // will be set later if given title is "" - feed.Link = remoteURL - feed.FeedLink = remoteURL + feed.Link = remote + feed.FeedLink = remote + text, err := gemini.ParseText(resp.Body) if err != nil { - return nil, err + return err } for _, line := range text { switch line := line.(type) { @@ -84,7 +86,7 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { if err != nil { continue } - link = remoteURL.ResolveReference(link) + link = remote.ResolveReference(link) item := &core.Item{} item.Title = strings.TrimLeft(line.Name[10:], ": ") item.Published = date @@ -92,7 +94,8 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { feed.Items = append(feed.Items, item) } } - return &feed, nil + InsertFeed(data, &feed, remote) + return nil case "text/xml", "application/rss+xml", "application/atom+xml", @@ -100,30 +103,22 @@ func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { "application/json": gofeed, err := gofeed.NewParser().Parse(reader) if err != nil { - return nil, err + return err } feed, err := core.GofeedConvert(gofeed, title) - return feed, err + if err != nil { + return err + } + InsertFeed(data, feed, remote) + return nil default: - return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta) + return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta) } } -// newGeminiPage will create a new core.Page given a gemini url -func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) { - var page core.Page - page.Title = title - page.Link = remoteURL - - err := updateGeminiPage(&page) - if err != nil { - return nil, err - } - return &page, nil -} - -// updateGeminiPage will check an existing core.Page for changes -func updateGeminiPage(p *core.Page) error { +// geminiPage will fetch and insert a new core.Page into a core.FullData given +// a gemini url (or update if it's preexisting) +func geminiPage(data *core.FullData, remote *url.URL, title string) error { resp, err := fetchGemini(p.Link) if err != nil { return err @@ -132,14 +127,19 @@ func updateGeminiPage(p *core.Page) error { reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + var page core.Page + page.Title = title + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) if newHash != p.Hash { - p.Hash = newHash - p.Updated = time.Now() + page.Hash = newHash + page.Updated = time.Now() } + + InsertPage(data, &page, remote) return nil } diff --git a/fetch/gopher.go b/fetch/gopher.go index d42d5eb..c0d515b 100644 --- a/fetch/gopher.go +++ b/fetch/gopher.go @@ -12,7 +12,6 @@ import ( "net/url" "time" - //"github.com/mmcdole/gofeed" "github.com/prologic/go-gopher" "golang.nytpu.com/comitium/core" @@ -27,34 +26,17 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { return resp, nil } -// newGopherFeed will fetch and create a new core.Feed given a gopher url -func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) { - // TODO - return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.") -} - -// updateGopherFeed will check an existing core.Feed for updates -func updateGopherFeed(*core.Feed) error { +// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a +// gopher url (or update if it's preexisting) +func gopherFeed(data *core.FullData, remote *url.URL, title string) error { // TODO return fmt.Errorf("Feeds served over gopher are currently unsupported.") } -// newGopherPage will create a new core.Page given a gopher url -func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) { - var page core.Page - page.Title = title - page.Link = remoteURL - - err := updateGopherPage(&page) - if err != nil { - return nil, err - } - return &page, nil -} - -// updateGopherPage wll check an existing core.Page for changes -func updateGopherPage(p *core.Page) error { - resp, err := fetchGopher(p.Link) +// gopherPage will fetch and insert a new core.Page into a core.FullData given a +// gopher url (or update if it's preexisting) +func gopherPage(data *core.FullData, remote *url.URL, title string) error { + resp, err := fetchGopher(remote) if err != nil { return err } @@ -71,14 +53,17 @@ func updateGopherPage(p *core.Page) error { reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB } + var page core.Page + page.Title = title + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) - if newHash != p.Hash { - p.Hash = newHash - p.Updated = time.Now() - } + page.Hash = newHash + page.Updated = time.Now() + + InsertPage(data, &page, remote) return nil } diff --git a/fetch/http.go b/fetch/http.go index e9cc777..d214381 100644 --- a/fetch/http.go +++ b/fetch/http.go @@ -48,44 +48,38 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { return resp, nil } -// newHTTPFeed will fetch and create a new core.Feed given an http url -func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) { - resp, err := fetchHTTP(remoteURL) +// httpFeed will fetch and insert a new core.Feed into a core.FullData given an +// http url (or update if it's preexisting) +func httpFeed(data *core.FullData, remote *url.URL, title string) error { + resp, err := fetchHTTP(remote) if err != nil { - return nil, err + return err } defer resp.Body.Close() if resp.Header.Get("Content-Type") == "text/html" { // TODO - return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented") + return fmt.Errorf("Extracting feed from HTML pages is unimplemented") } reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB gofeed, err := gofeed.NewParser().Parse(reader) if err != nil { - return nil, err + return err } feed, err := core.GofeedConvert(gofeed, title) - return feed, err -} - -// newHTTPPage will create a new core.Page given an http url -func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) { - var page core.Page - page.Title = title - page.Link = remoteURL - - err := updateHTTPPage(&page) if err != nil { - return nil, err + return err } - return &page, nil + + InsertFeed(data, feed, remote) + return nil } -// updateHTTPPage will check an existing core.Page for changes -func updateHTTPPage(p *core.Page) error { - resp, err := fetchHTTP(p.Link) +// httpPage will fetch and insert a new core.Page into a core.FullData given an +// http url (or update if it's preexisting) +func httpPage(data *core.FullData, remote *url.URL, title string) error { + resp, err := fetchHTTP(remote) if err != nil { return err } @@ -93,14 +87,17 @@ func updateHTTPPage(p *core.Page) error { reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + var page core.Page + page.Title = title + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) - if newHash != p.Hash { - p.Hash = newHash - p.Updated = time.Now() - } + page.Hash = newHash + page.Updated = time.Now() + + InsertPage(data, &page, remote) return nil }