diff --git a/fetch/fetch.go b/fetch/fetch.go index 231c7dc..6e9e7a0 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -9,7 +9,6 @@ package fetch import ( "fmt" "net/url" - "reflect" "golang.nytpu.com/comitium/core" ) @@ -43,29 +42,3 @@ func Page(data *core.FullData, remote *url.URL, title string) error { return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } } - -// InsertFeed will insert or update a core.Feed in a core.FullData -func InsertFeed(data *core.FullData, new *core.Feed, remote *url.URL) { - data.FeedsMu.Lock() - oldFeed, ok := data.Feeds[*remote] - if !ok || !reflect.DeepEqual(new, oldFeed) { - // Feeds are different, or there was never an old one - data.Feeds[*remote] = new - data.FeedsMu.Unlock() - } else { - data.FeedsMu.Unlock() - } -} - -// InsertFeed will insert or update a core.Page in a core.FullData -func InsertPage(data *core.FullData, new *core.Page, remote *url.URL) { - data.PagesMu.Lock() - _, ok := data.Pages[*remote] - if !ok || data.Pages[*remote].Hash != new.Hash { - // pages are different, or there was never an old one - data.Pages[*remote] = new - data.PagesMu.Unlock() - } else { - data.PagesMu.Unlock() - } -} diff --git a/fetch/gemini.go b/fetch/gemini.go index 516358a..3c81eaa 100644 --- a/fetch/gemini.go +++ b/fetch/gemini.go @@ -29,7 +29,7 @@ import ( // fetchGemini will make a request for a gemini resource func fetchGemini(remote *url.URL) (*gemini.Response, error) { client := &gemini.Client{} - resp, err := client.Get(context.Background(), remoteURL.String()) + resp, err := client.Get(context.Background(), remote.String()) if err != nil { return nil, err } @@ -94,7 +94,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { feed.Items = append(feed.Items, item) } } - InsertFeed(data, &feed, remote) + data.InsertFeed(&feed, remote) return nil case "text/xml", "application/rss+xml", @@ -109,7 +109,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { if err != nil { return err } - InsertFeed(data, feed, remote) + data.InsertFeed(feed, remote) return nil default: return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta) @@ -119,7 +119,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { // geminiPage will fetch and insert a new core.Page into a core.FullData given // a gemini url (or update if it's preexisting) func geminiPage(data *core.FullData, remote *url.URL, title string) error { - resp, err := fetchGemini(p.Link) + resp, err := fetchGemini(remote) if err != nil { return err } @@ -135,11 +135,9 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) - if newHash != p.Hash { - page.Hash = newHash - page.Updated = time.Now() - } + page.Hash = newHash + page.Updated = time.Now() - InsertPage(data, &page, remote) + data.InsertPage(&page, remote) return nil } diff --git a/fetch/gopher.go b/fetch/gopher.go index c0d515b..5db1208 100644 --- a/fetch/gopher.go +++ b/fetch/gopher.go @@ -64,6 +64,6 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error { page.Hash = newHash page.Updated = time.Now() - InsertPage(data, &page, remote) + data.InsertPage(&page, remote) return nil } diff --git a/fetch/http.go b/fetch/http.go index d214381..393630d 100644 --- a/fetch/http.go +++ b/fetch/http.go @@ -72,7 +72,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error { return err } - InsertFeed(data, feed, remote) + data.InsertFeed(feed, remote) return nil } @@ -98,6 +98,6 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error { page.Hash = newHash page.Updated = time.Now() - InsertPage(data, &page, remote) + data.InsertPage(&page, remote) return nil }