diff options
| author | nytpu <alex@nytpu.com> | 2021-03-18 13:02:17 -0600 |
|---|---|---|
| committer | nytpu <alex@nytpu.com> | 2021-03-18 13:02:17 -0600 |
| commit | f8601b82abe77e19fbe0c97a21f00540da39ac9f (patch) | |
| tree | 0855e50f31d8997e10d8e6580939a902648d6ea6 /fetch | |
| parent | 794b1fc04f350c4977c873e42752ccc0ecd0a98c (diff) | |
Use strings for URLs internally for consistency.
Diffstat (limited to 'fetch')
| -rw-r--r-- | fetch/fetch.go | 12 | ||||
| -rw-r--r-- | fetch/gemini.go | 18 | ||||
| -rw-r--r-- | fetch/gopher.go | 11 | ||||
| -rw-r--r-- | fetch/http.go | 11 |
4 files changed, 26 insertions, 26 deletions
diff --git a/fetch/fetch.go b/fetch/fetch.go index 6e9e7a0..5537339 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -18,11 +18,11 @@ import ( func Feed(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": - return geminiFeed(data, remote, title) + return geminiFeed(data, remote.String(), title) case "http", "https": - return httpFeed(data, remote, title) + return httpFeed(data, remote.String(), title) case "gopher", "gophers": - return gopherFeed(data, remote, title) + return gopherFeed(data, remote.String(), title) default: return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } @@ -33,11 +33,11 @@ func Feed(data *core.FullData, remote *url.URL, title string) error { func Page(data *core.FullData, remote *url.URL, title string) error { switch remote.Scheme { case "gemini": - return geminiPage(data, remote, title) + return geminiPage(data, remote.String(), title) case "http", "https": - return httpPage(data, remote, title) + return httpPage(data, remote.String(), title) case "gopher", "gophers": - return gopherPage(data, remote, title) + return gopherPage(data, remote.String(), title) default: return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) } diff --git a/fetch/gemini.go b/fetch/gemini.go index 6aa1231..b50505a 100644 --- a/fetch/gemini.go +++ b/fetch/gemini.go @@ -27,9 +27,9 @@ import ( ) // fetchGemini will make a request for a gemini resource -func fetchGemini(remote *url.URL) (*gemini.Response, error) { +func fetchGemini(remote string) (*gemini.Response, error) { client := &gemini.Client{} - resp, err := client.Get(context.Background(), remote.String()) + resp, err := client.Get(context.Background(), remote) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func fetchGemini(remote *url.URL) (*gemini.Response, error) { // geminiFeed will fetch and insert a new core.Feed into a core.FullData given // a gemini url (or update if it's preexisting) -func geminiFeed(data *core.FullData, remote *url.URL, title string) error { +func geminiFeed(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err @@ -60,9 +60,11 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { case "text/gemini": var feed core.Feed feed.Title = title // will be set later if given title is "" - feed.Link = remote.String() - feed.FeedLink = remote.String() + feed.Link = remote + feed.FeedLink = remote feed.Updated = time.Now() + // we know that remote is a good url because it's been parsed previously + parsedRemote, _ := url.ParseRequestURI(remote) text, err := gemini.ParseText(resp.Body) if err != nil { @@ -87,7 +89,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { if err != nil { continue } - link = remote.ResolveReference(link) + link = parsedRemote.ResolveReference(link) item := &core.Item{} item.Title = strings.TrimLeft(line.Name[10:], " :-–—") item.Published = date @@ -119,7 +121,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error { // geminiPage will fetch and insert a new core.Page into a core.FullData given // a gemini url (or update if it's preexisting) -func geminiPage(data *core.FullData, remote *url.URL, title string) error { +func geminiPage(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err @@ -130,7 +132,7 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err diff --git a/fetch/gopher.go b/fetch/gopher.go index 79b5a5b..5fe24ad 100644 --- a/fetch/gopher.go +++ b/fetch/gopher.go @@ -9,7 +9,6 @@ import ( "crypto/sha256" "fmt" "io" - "net/url" "time" "github.com/prologic/go-gopher" @@ -18,8 +17,8 @@ import ( ) // fetchGopher will make a request for a gopher resource -func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { - resp, err := gopher.Get(remoteURL.String()) +func fetchGopher(remote string) (*gopher.Response, error) { + resp, err := gopher.Get(remote) if err != nil { return nil, err } @@ -28,14 +27,14 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { // gopherFeed will fetch and insert a new core.Feed into a core.FullData given a // gopher url (or update if it's preexisting) -func gopherFeed(data *core.FullData, remote *url.URL, title string) error { +func gopherFeed(data *core.FullData, remote string, title string) error { // TODO return fmt.Errorf("Feeds served over gopher are currently unsupported.") } // gopherPage will fetch and insert a new core.Page into a core.FullData given a // gopher url (or update if it's preexisting) -func gopherPage(data *core.FullData, remote *url.URL, title string) error { +func gopherPage(data *core.FullData, remote string, title string) error { resp, err := fetchGopher(remote) if err != nil { return err @@ -55,7 +54,7 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err diff --git a/fetch/http.go b/fetch/http.go index 09d5216..2661438 100644 --- a/fetch/http.go +++ b/fetch/http.go @@ -16,7 +16,6 @@ import ( "fmt" "io" "net/http" - "net/url" "time" "github.com/mmcdole/gofeed" @@ -25,12 +24,12 @@ import ( ) // fetchHTTP will make a request for an http resource -func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { +func fetchHTTP(remoteURL string) (*http.Response, error) { client := &http.Client{ Timeout: 10 * time.Second, } - req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil) req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)") if err != nil { return nil, err @@ -50,7 +49,7 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { // httpFeed will fetch and insert a new core.Feed into a core.FullData given an // http url (or update if it's preexisting) -func httpFeed(data *core.FullData, remote *url.URL, title string) error { +func httpFeed(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err @@ -78,7 +77,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error { // httpPage will fetch and insert a new core.Page into a core.FullData given an // http url (or update if it's preexisting) -func httpPage(data *core.FullData, remote *url.URL, title string) error { +func httpPage(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err @@ -89,7 +88,7 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error { var page core.Page page.Title = title - page.Link = remote.String() + page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err |
