From f8601b82abe77e19fbe0c97a21f00540da39ac9f Mon Sep 17 00:00:00 2001 From: nytpu Date: Thu, 18 Mar 2021 13:02:17 -0600 Subject: [PATCH] Use strings for URLs internally for consistency. --- core/structs.go | 15 +++++++-------- fetch/fetch.go | 12 ++++++------ fetch/gemini.go | 18 ++++++++++-------- fetch/gopher.go | 11 +++++------ fetch/http.go | 11 +++++------ 5 files changed, 33 insertions(+), 34 deletions(-) diff --git a/core/structs.go b/core/structs.go index dc22087..8df60ac 100644 --- a/core/structs.go +++ b/core/structs.go @@ -11,7 +11,6 @@ package core import ( - "net/url" "reflect" "sync" "time" @@ -90,12 +89,12 @@ func GofeedConvert(f *gofeed.Feed, title string) (*Feed, error) { } // InsertFeed will insert or update a core.Feed in a core.FullData -func (d *FullData) InsertFeed(new *Feed, remote *url.URL) { +func (d *FullData) InsertFeed(new *Feed, remote string) { d.FeedsMu.Lock() - oldFeed, ok := d.Feeds[remote.String()] + oldFeed, ok := d.Feeds[remote] if !ok || !reflect.DeepEqual(new, oldFeed) { // Feeds are different, or there was never an old one - d.Feeds[remote.String()] = new + d.Feeds[remote] = new d.FeedsMu.Unlock() } else { d.FeedsMu.Unlock() @@ -103,12 +102,12 @@ func (d *FullData) InsertFeed(new *Feed, remote *url.URL) { } // InsertPage will insert or update a core.Page in a core.FullData -func (d *FullData) InsertPage(new *Page, remote *url.URL) { +func (d *FullData) InsertPage(new *Page, remote string) { d.PagesMu.Lock() - _, ok := d.Pages[remote.String()] - if !ok || d.Pages[remote.String()].Hash != new.Hash { + _, ok := d.Pages[remote] + if !ok || d.Pages[remote].Hash != new.Hash { // pages are different, or there was never an old one - d.Pages[remote.String()] = new + d.Pages[remote] = new d.PagesMu.Unlock() } else { d.PagesMu.Unlock() diff --git a/fetch/fetch.go b/fetch/fetch.go index 6e9e7a0..5537339 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -18,11 +18,11 @@ import ( func Feed(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": - return geminiFeed(data, remote, title) + return geminiFeed(data, remote.String(), title) case "http", "https": - return httpFeed(data, remote, title) + return httpFeed(data, remote.String(), title) case "gopher", "gophers": - return gopherFeed(data, remote, title) + return gopherFeed(data, remote.String(), title) default: return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } @@ -33,11 +33,11 @@ func Feed(data *core.FullData, remote *url.URL, title string) error { func Page(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": - return geminiPage(data, remote, title) + return geminiPage(data, remote.String(), title) case "http", "https": - return httpPage(data, remote, title) + return httpPage(data, remote.String(), title) case "gopher", "gophers": - return gopherPage(data, remote, title) + return gopherPage(data, remote.String(), title) default: return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } diff --git a/fetch/gemini.go b/fetch/gemini.go index 6aa1231..b50505a 100644 --- a/fetch/gemini.go +++ b/fetch/gemini.go @@ -27,9 +27,9 @@ import ( ) // fetchGemini will make a request for a gemini resource -func fetchGemini(remote *url.URL) (*gemini.Response, error) { +func fetchGemini(remote string) (*gemini.Response, error) { client := &gemini.Client{} - resp, err := client.Get(context.Background(), remote.String()) + resp, err := client.Get(context.Background(), remote) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func fetchGemini(remote *url.URL) (*gemini.Response, error) { // geminiFeed will fetch and insert a new core.Feed into a core.FullData given // a gemini url (or update if it's preexisting) -func geminiFeed(data *core.FullData, remote *url.URL, title string) error { +func geminiFeed(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err @@ -60,9 +60,11 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { case "text/gemini": var feed core.Feed feed.Title = title // will be set later if given title is "" - feed.Link = remote.String() - feed.FeedLink = remote.String() + feed.Link = remote + feed.FeedLink = remote feed.Updated = time.Now() + // we know that remote is a good url because it's been parsed previously + parsedRemote, _ := url.ParseRequestURI(remote) text, err := gemini.ParseText(resp.Body) if err != nil { @@ -87,7 +89,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { if err != nil { continue } - link = remote.ResolveReference(link) + link = parsedRemote.ResolveReference(link) item := &core.Item{} item.Title = strings.TrimLeft(line.Name[10:], " :-–—") item.Published = date @@ -119,7 +121,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { // geminiPage will fetch and insert a new core.Page into a core.FullData given // a gemini url (or update if it's preexisting) -func geminiPage(data *core.FullData, remote *url.URL, title string) error { +func geminiPage(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err @@ -130,7 +132,7 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err diff --git a/fetch/gopher.go b/fetch/gopher.go index 79b5a5b..5fe24ad 100644 --- a/fetch/gopher.go +++ b/fetch/gopher.go @@ -9,7 +9,6 @@ import ( "crypto/sha256" "fmt" "io" - "net/url" "time" "github.com/prologic/go-gopher" @@ -18,8 +17,8 @@ import ( ) // fetchGopher will make a request for a gopher resource -func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { - resp, err := gopher.Get(remoteURL.String()) +func fetchGopher(remote string) (*gopher.Response, error) { + resp, err := gopher.Get(remote) if err != nil { return nil, err } @@ -28,14 +27,14 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { // gopherFeed will fetch and insert a new core.Feed into a core.FullData given a // gopher url (or update if it's preexisting) -func gopherFeed(data *core.FullData, remote *url.URL, title string) error { +func gopherFeed(data *core.FullData, remote string, title string) error { // TODO return fmt.Errorf("Feeds served over gopher are currently unsupported.") } // gopherPage will fetch and insert a new core.Page into a core.FullData given a // gopher url (or update if it's preexisting) -func gopherPage(data *core.FullData, remote *url.URL, title string) error { +func gopherPage(data *core.FullData, remote string, title string) error { resp, err := fetchGopher(remote) if err != nil { return err @@ -55,7 +54,7 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err diff --git a/fetch/http.go b/fetch/http.go index 09d5216..2661438 100644 --- a/fetch/http.go +++ b/fetch/http.go @@ -16,7 +16,6 @@ import ( "fmt" "io" "net/http" - "net/url" "time" "github.com/mmcdole/gofeed" @@ -25,12 +24,12 @@ import ( ) // fetchHTTP will make a request for an http resource -func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { +func fetchHTTP(remoteURL string) (*http.Response, error) { client := &http.Client{ Timeout: 10 * time.Second, } - req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil) req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)") if err != nil { return nil, err @@ -50,7 +49,7 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { // httpFeed will fetch and insert a new core.Feed into a core.FullData given an // http url (or update if it's preexisting) -func httpFeed(data *core.FullData, remote *url.URL, title string) error { +func httpFeed(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err @@ -78,7 +77,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error { // httpPage will fetch and insert a new core.Page into a core.FullData given an // http url (or update if it's preexisting) -func httpPage(data *core.FullData, remote *url.URL, title string) error { +func httpPage(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err @@ -89,7 +88,7 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err