joeac/comitium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use strings for URLs internally for consistency.

Browse Source
This commit is contained in:
nytpu
2021-03-18 13:02:17 -06:00
parent 794b1fc04f
commit f8601b82ab
5 changed files with 33 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/structs.go
+7 -8
View File
@@ -11,7 +11,6 @@
package core
import (
"net/url"
"reflect"
"sync"
"time"
@@ -90,12 +89,12 @@ func GofeedConvert(f *gofeed.Feed, title string) (*Feed, error) {
}
// InsertFeed will insert or update a core.Feed in a core.FullData
func (d *FullData) InsertFeed(new *Feed, remote *url.URL) {
func (d *FullData) InsertFeed(new *Feed, remote string) {
d.FeedsMu.Lock()
oldFeed, ok := d.Feeds[remote.String()]
oldFeed, ok := d.Feeds[remote]
if !ok || !reflect.DeepEqual(new, oldFeed) {
// Feeds are different, or there was never an old one
d.Feeds[remote.String()] = new
d.Feeds[remote] = new
d.FeedsMu.Unlock()
} else {
d.FeedsMu.Unlock()
@@ -103,12 +102,12 @@ func (d *FullData) InsertFeed(new *Feed, remote *url.URL) {
}
// InsertPage will insert or update a core.Page in a core.FullData
func (d *FullData) InsertPage(new *Page, remote *url.URL) {
func (d *FullData) InsertPage(new *Page, remote string) {
d.PagesMu.Lock()
_, ok := d.Pages[remote.String()]
if !ok || d.Pages[remote.String()].Hash != new.Hash {
_, ok := d.Pages[remote]
if !ok || d.Pages[remote].Hash != new.Hash {
// pages are different, or there was never an old one
d.Pages[remote.String()] = new
d.Pages[remote] = new
d.PagesMu.Unlock()
} else {
d.PagesMu.Unlock()
fetch/fetch.go
+6 -6
View File
@@ -18,11 +18,11 @@ import (
func Feed(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return geminiFeed(data, remote, title)
return geminiFeed(data, remote.String(), title)
case "http", "https":
return httpFeed(data, remote, title)
return httpFeed(data, remote.String(), title)
case "gopher", "gophers":
return gopherFeed(data, remote, title)
return gopherFeed(data, remote.String(), title)
default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
@@ -33,11 +33,11 @@ func Feed(data *core.FullData, remote *url.URL, title string) error {
func Page(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme {
case "gemini":
return geminiPage(data, remote, title)
return geminiPage(data, remote.String(), title)
case "http", "https":
return httpPage(data, remote, title)
return httpPage(data, remote.String(), title)
case "gopher", "gophers":
return gopherPage(data, remote, title)
return gopherPage(data, remote.String(), title)
default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
}
fetch/gemini.go
+10 -8
View File
@@ -27,9 +27,9 @@ import (
)
// fetchGemini will make a request for a gemini resource
func fetchGemini(remote *url.URL) (*gemini.Response, error) {
func fetchGemini(remote string) (*gemini.Response, error) {
client := &gemini.Client{}
resp, err := client.Get(context.Background(), remote.String())
resp, err := client.Get(context.Background(), remote)
if err != nil {
return nil, err
}
@@ -42,7 +42,7 @@ func fetchGemini(remote *url.URL) (*gemini.Response, error) {
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
func geminiFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
@@ -60,9 +60,11 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
case "text/gemini":
var feed core.Feed
feed.Title = title // will be set later if given title is ""
feed.Link = remote.String()
feed.FeedLink = remote.String()
feed.Link = remote
feed.FeedLink = remote
feed.Updated = time.Now()
// we know that remote is a good url because it's been parsed previously
parsedRemote, _ := url.ParseRequestURI(remote)
text, err := gemini.ParseText(resp.Body)
if err != nil {
@@ -87,7 +89,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
if err != nil {
continue
}
link = remote.ResolveReference(link)
link = parsedRemote.ResolveReference(link)
item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], " :-–—")
item.Published = date
@@ -119,7 +121,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote *url.URL, title string) error {
func geminiPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote)
if err != nil {
return err
@@ -130,7 +132,7 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
page.Link = remote.String()
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
fetch/gopher.go
+5 -6
View File
@@ -9,7 +9,6 @@ import (
"crypto/sha256"
"fmt"
"io"
"net/url"
"time"
"github.com/prologic/go-gopher"
@@ -18,8 +17,8 @@ import (
)
// fetchGopher will make a request for a gopher resource
func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
resp, err := gopher.Get(remoteURL.String())
func fetchGopher(remote string) (*gopher.Response, error) {
resp, err := gopher.Get(remote)
if err != nil {
return nil, err
}
@@ -28,14 +27,14 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherFeed(data *core.FullData, remote *url.URL, title string) error {
func gopherFeed(data *core.FullData, remote string, title string) error {
// TODO
return fmt.Errorf("Feeds served over gopher are currently unsupported.")
}
// gopherPage will fetch and insert a new core.Page into a core.FullData given a
// gopher url (or update if it's preexisting)
func gopherPage(data *core.FullData, remote *url.URL, title string) error {
func gopherPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGopher(remote)
if err != nil {
return err
@@ -55,7 +54,7 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
page.Link = remote.String()
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err
fetch/http.go
+5 -6
View File
@@ -16,7 +16,6 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/mmcdole/gofeed"
@@ -25,12 +24,12 @@ import (
)
// fetchHTTP will make a request for an http resource
func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
func fetchHTTP(remoteURL string) (*http.Response, error) {
client := &http.Client{
Timeout: 10 * time.Second,
}
req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil)
req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
if err != nil {
return nil, err
@@ -50,7 +49,7 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
// httpFeed will fetch and insert a new core.Feed into a core.FullData given an
// http url (or update if it's preexisting)
func httpFeed(data *core.FullData, remote *url.URL, title string) error {
func httpFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return err
@@ -78,7 +77,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error {
// httpPage will fetch and insert a new core.Page into a core.FullData given an
// http url (or update if it's preexisting)
func httpPage(data *core.FullData, remote *url.URL, title string) error {
func httpPage(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote)
if err != nil {
return err
@@ -89,7 +88,7 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page
page.Title = title
page.Link = remote.String()
page.Link = remote
h := sha256.New()
if _, err := io.Copy(h, reader); err != nil {
return err