summaryrefslogtreecommitdiff
path: root/fetch
diff options
context:
space:
mode:
authornytpu <alex@nytpu.com>2021-03-18 13:02:17 -0600
committernytpu <alex@nytpu.com>2021-03-18 13:02:17 -0600
commitf8601b82abe77e19fbe0c97a21f00540da39ac9f (patch)
tree0855e50f31d8997e10d8e6580939a902648d6ea6 /fetch
parent794b1fc04f350c4977c873e42752ccc0ecd0a98c (diff)
Use strings for URLs internally for consistency.
Diffstat (limited to 'fetch')
-rw-r--r--fetch/fetch.go12
-rw-r--r--fetch/gemini.go18
-rw-r--r--fetch/gopher.go11
-rw-r--r--fetch/http.go11
4 files changed, 26 insertions, 26 deletions
diff --git a/fetch/fetch.go b/fetch/fetch.go
index 6e9e7a0..5537339 100644
--- a/fetch/fetch.go
+++ b/fetch/fetch.go
@@ -18,11 +18,11 @@ import (
func Feed(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
- return geminiFeed(data, remote, title)
+ return geminiFeed(data, remote.String(), title)
case "http", "https":
- return httpFeed(data, remote, title)
+ return httpFeed(data, remote.String(), title)
case "gopher", "gophers":
- return gopherFeed(data, remote, title)
+ return gopherFeed(data, remote.String(), title)
default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
@@ -33,11 +33,11 @@ func Feed(data *core.FullData, remote *url.URL, title string) error {
func Page(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
- return geminiPage(data, remote, title)
+ return geminiPage(data, remote.String(), title)
case "http", "https":
- return httpPage(data, remote, title)
+ return httpPage(data, remote.String(), title)
case "gopher", "gophers":
- return gopherPage(data, remote, title)
+ return gopherPage(data, remote.String(), title)
default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
diff --git a/fetch/gemini.go b/fetch/gemini.go
index 6aa1231..b50505a 100644
--- a/fetch/gemini.go
+++ b/fetch/gemini.go
@@ -27,9 +27,9 @@ import (
)
// fetchGemini will make a request for a gemini resource
-func fetchGemini(remote *url.URL) (*gemini.Response, error) {
+func fetchGemini(remote string) (*gemini.Response, error) {
client := &gemini.Client{}
- resp, err := client.Get(context.Background(), remote.String())
+ resp, err := client.Get(context.Background(), remote)
if err != nil {
return nil, err
}
@@ -42,7 +42,7 @@ func fetchGemini(remote *url.URL) (*gemini.Response, error) {
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
-func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
+func geminiFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
@@ -60,9 +60,11 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
case "text/gemini":
var feed core.Feed
feed.Title = title // will be set later if given title is ""
- feed.Link = remote.String()
- feed.FeedLink = remote.String()
+ feed.Link = remote
+ feed.FeedLink = remote
feed.Updated = time.Now()
+ // we know that remote is a good url because it's been parsed previously
+ parsedRemote, _ := url.ParseRequestURI(remote)
text, err := gemini.ParseText(resp.Body)
if err != nil {
@@ -87,7 +89,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
if err != nil {
continue
}
- link = remote.ResolveReference(link)
+ link = parsedRemote.ResolveReference(link)
item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], " :-–—")
item.Published = date
@@ -119,7 +121,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
-func geminiPage(data *core.FullData, remote *url.URL, title string) error {
+func geminiPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
@@ -130,7 +132,7 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
- page.Link = remote.String()
+ page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
diff --git a/fetch/gopher.go b/fetch/gopher.go
index 79b5a5b..5fe24ad 100644
--- a/fetch/gopher.go
+++ b/fetch/gopher.go
@@ -9,7 +9,6 @@ import (
"crypto/sha256"
"fmt"
"io"
- "net/url"
"time"
"github.com/prologic/go-gopher"
@@ -18,8 +17,8 @@ import (
)
// fetchGopher will make a request for a gopher resource
-func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
- resp, err := gopher.Get(remoteURL.String())
+func fetchGopher(remote string) (*gopher.Response, error) {
+ resp, err := gopher.Get(remote)
if err != nil {
return nil, err
}
@@ -28,14 +27,14 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
// gopher url (or update if it's preexisting)
-func gopherFeed(data *core.FullData, remote *url.URL, title string) error {
+func gopherFeed(data *core.FullData, remote string, title string) error {
// TODO
return fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// gopherPage will fetch and insert a new core.Page into a core.FullData given a
// gopher url (or update if it's preexisting)
-func gopherPage(data *core.FullData, remote *url.URL, title string) error {
+func gopherPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGopher(remote)
if err != nil {
return err
@@ -55,7 +54,7 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
- page.Link = remote.String()
+ page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
diff --git a/fetch/http.go b/fetch/http.go
index 09d5216..2661438 100644
--- a/fetch/http.go
+++ b/fetch/http.go
@@ -16,7 +16,6 @@ import (
"fmt"
"io"
"net/http"
- "net/url"
"time"
"github.com/mmcdole/gofeed"
@@ -25,12 +24,12 @@ import (
)
// fetchHTTP will make a request for an http resource
-func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
+func fetchHTTP(remoteURL string) (*http.Response, error) {
client := &http.Client{
Timeout: 10 * time.Second,
}
- req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
+ req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil)
req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
if err != nil {
return nil, err
@@ -50,7 +49,7 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
// httpFeed will fetch and insert a new core.Feed into a core.FullData given an
// http url (or update if it's preexisting)
-func httpFeed(data *core.FullData, remote *url.URL, title string) error {
+func httpFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return err
@@ -78,7 +77,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error {
// httpPage will fetch and insert a new core.Page into a core.FullData given an
// http url (or update if it's preexisting)
-func httpPage(data *core.FullData, remote *url.URL, title string) error {
+func httpPage(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return err
@@ -89,7 +88,7 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
- page.Link = remote.String()
+ page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err