joeac/comitium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use strings for URLs internally for consistency.

Browse Source
This commit is contained in:
nytpu
2021-03-18 13:02:17 -06:00
parent 794b1fc04f
commit f8601b82ab
5 changed files with 33 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/structs.go
+7 -8
View File
@@ -11,7 +11,6 @@
package core package core
import ( import (
"net/url"
"reflect" "reflect"
"sync" "sync"
"time" "time"
@@ -90,12 +89,12 @@ func GofeedConvert(f *gofeed.Feed, title string) (*Feed, error) {
} }
// InsertFeed will insert or update a core.Feed in a core.FullData // InsertFeed will insert or update a core.Feed in a core.FullData
func (d *FullData) InsertFeed(new *Feed, remote *url.URL) { func (d *FullData) InsertFeed(new *Feed, remote string) {
d.FeedsMu.Lock() d.FeedsMu.Lock()
oldFeed, ok := d.Feeds[remote.String()] oldFeed, ok := d.Feeds[remote]
if !ok || !reflect.DeepEqual(new, oldFeed) { if !ok || !reflect.DeepEqual(new, oldFeed) {
// Feeds are different, or there was never an old one // Feeds are different, or there was never an old one
d.Feeds[remote.String()] = new d.Feeds[remote] = new
d.FeedsMu.Unlock() d.FeedsMu.Unlock()
} else { } else {
d.FeedsMu.Unlock() d.FeedsMu.Unlock()
@@ -103,12 +102,12 @@ func (d *FullData) InsertFeed(new *Feed, remote *url.URL) {
} }
// InsertPage will insert or update a core.Page in a core.FullData // InsertPage will insert or update a core.Page in a core.FullData
func (d *FullData) InsertPage(new *Page, remote *url.URL) { func (d *FullData) InsertPage(new *Page, remote string) {
d.PagesMu.Lock() d.PagesMu.Lock()
_, ok := d.Pages[remote.String()] _, ok := d.Pages[remote]
if !ok || d.Pages[remote.String()].Hash != new.Hash { if !ok || d.Pages[remote].Hash != new.Hash {
// pages are different, or there was never an old one // pages are different, or there was never an old one
d.Pages[remote.String()] = new d.Pages[remote] = new
d.PagesMu.Unlock() d.PagesMu.Unlock()
} else { } else {
d.PagesMu.Unlock() d.PagesMu.Unlock()
fetch/fetch.go
+6 -6
View File
@@ -18,11 +18,11 @@ import (
func Feed(data *core.FullData, remote *url.URL, title string) error { func Feed(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme { switch remote.Scheme {
case "gemini": case "gemini":
return geminiFeed(data, remote, title) return geminiFeed(data, remote.String(), title)
case "http", "https": case "http", "https":
return httpFeed(data, remote, title) return httpFeed(data, remote.String(), title)
case "gopher", "gophers": case "gopher", "gophers":
return gopherFeed(data, remote, title) return gopherFeed(data, remote.String(), title)
default: default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
} }
@@ -33,11 +33,11 @@ func Feed(data *core.FullData, remote *url.URL, title string) error {
func Page(data *core.FullData, remote *url.URL, title string) error { func Page(data *core.FullData, remote *url.URL, title string) error {
switch remote.Scheme { switch remote.Scheme {
case "gemini": case "gemini":
return geminiPage(data, remote, title) return geminiPage(data, remote.String(), title)
case "http", "https": case "http", "https":
return httpPage(data, remote, title) return httpPage(data, remote.String(), title)
case "gopher", "gophers": case "gopher", "gophers":
return gopherPage(data, remote, title) return gopherPage(data, remote.String(), title)
default: default:
return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme) return fmt.Errorf("Unsupported protocol '%s'", remote.Scheme)
} }
fetch/gemini.go
+10 -8
View File
@@ -27,9 +27,9 @@ import (
) )
// fetchGemini will make a request for a gemini resource // fetchGemini will make a request for a gemini resource
func fetchGemini(remote *url.URL) (*gemini.Response, error) { func fetchGemini(remote string) (*gemini.Response, error) {
client := &gemini.Client{} client := &gemini.Client{}
resp, err := client.Get(context.Background(), remote.String()) resp, err := client.Get(context.Background(), remote)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -42,7 +42,7 @@ func fetchGemini(remote *url.URL) (*gemini.Response, error) {
// geminiFeed will fetch and insert a new core.Feed into a core.FullData given // geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting) // a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote *url.URL, title string) error { func geminiFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote) resp, err := fetchGemini(remote)
if err != nil { if err != nil {
return err return err
@@ -60,9 +60,11 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
case "text/gemini": case "text/gemini":
var feed core.Feed var feed core.Feed
feed.Title = title // will be set later if given title is "" feed.Title = title // will be set later if given title is ""
feed.Link = remote.String() feed.Link = remote
feed.FeedLink = remote.String() feed.FeedLink = remote
feed.Updated = time.Now() feed.Updated = time.Now()
// we know that remote is a good url because it's been parsed previously
parsedRemote, _ := url.ParseRequestURI(remote)
text, err := gemini.ParseText(resp.Body) text, err := gemini.ParseText(resp.Body)
if err != nil { if err != nil {
@@ -87,7 +89,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
if err != nil { if err != nil {
continue continue
} }
link = remote.ResolveReference(link) link = parsedRemote.ResolveReference(link)
item := &core.Item{} item := &core.Item{}
item.Title = strings.TrimLeft(line.Name[10:], " :-–—") item.Title = strings.TrimLeft(line.Name[10:], " :-–—")
item.Published = date item.Published = date
@@ -119,7 +121,7 @@ func geminiFeed(data *core.FullData, remote *url.URL, title string) error {
// geminiPage will fetch and insert a new core.Page into a core.FullData given // geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting) // a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote *url.URL, title string) error { func geminiPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGemini(remote) resp, err := fetchGemini(remote)
if err != nil { if err != nil {
return err return err
@@ -130,7 +132,7 @@ func geminiPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page var page core.Page
page.Title = title page.Title = title
page.Link = remote.String() page.Link = remote
h := sha256.New() h := sha256.New()
if _, err := io.Copy(h, reader); err != nil { if _, err := io.Copy(h, reader); err != nil {
return err return err
fetch/gopher.go
+5 -6
View File
@@ -9,7 +9,6 @@ import (
"crypto/sha256" "crypto/sha256"
"fmt" "fmt"
"io" "io"
"net/url"
"time" "time"
"github.com/prologic/go-gopher" "github.com/prologic/go-gopher"
@@ -18,8 +17,8 @@ import (
) )
// fetchGopher will make a request for a gopher resource // fetchGopher will make a request for a gopher resource
func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { func fetchGopher(remote string) (*gopher.Response, error) {
resp, err := gopher.Get(remoteURL.String()) resp, err := gopher.Get(remote)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -28,14 +27,14 @@ func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
// gopherFeed will fetch and insert a new core.Feed into a core.FullData given a // gopherFeed will fetch and insert a new core.Feed into a core.FullData given a
// gopher url (or update if it's preexisting) // gopher url (or update if it's preexisting)
func gopherFeed(data *core.FullData, remote *url.URL, title string) error { func gopherFeed(data *core.FullData, remote string, title string) error {
// TODO // TODO
return fmt.Errorf("Feeds served over gopher are currently unsupported.") return fmt.Errorf("Feeds served over gopher are currently unsupported.")
} }
// gopherPage will fetch and insert a new core.Page into a core.FullData given a // gopherPage will fetch and insert a new core.Page into a core.FullData given a
// gopher url (or update if it's preexisting) // gopher url (or update if it's preexisting)
func gopherPage(data *core.FullData, remote *url.URL, title string) error { func gopherPage(data *core.FullData, remote string, title string) error {
resp, err := fetchGopher(remote) resp, err := fetchGopher(remote)
if err != nil { if err != nil {
return err return err
@@ -55,7 +54,7 @@ func gopherPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page var page core.Page
page.Title = title page.Title = title
page.Link = remote.String() page.Link = remote
h := sha256.New() h := sha256.New()
if _, err := io.Copy(h, reader); err != nil { if _, err := io.Copy(h, reader); err != nil {
return err return err
fetch/http.go
+5 -6
View File
@@ -16,7 +16,6 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url"
"time" "time"
"github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed"
@@ -25,12 +24,12 @@ import (
) )
// fetchHTTP will make a request for an http resource // fetchHTTP will make a request for an http resource
func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { func fetchHTTP(remoteURL string) (*http.Response, error) {
client := &http.Client{ client := &http.Client{
Timeout: 10 * time.Second, Timeout: 10 * time.Second,
} }
req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil) req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil)
req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)") req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
if err != nil { if err != nil {
return nil, err return nil, err
@@ -50,7 +49,7 @@ func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
// httpFeed will fetch and insert a new core.Feed into a core.FullData given an // httpFeed will fetch and insert a new core.Feed into a core.FullData given an
// http url (or update if it's preexisting) // http url (or update if it's preexisting)
func httpFeed(data *core.FullData, remote *url.URL, title string) error { func httpFeed(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote) resp, err := fetchHTTP(remote)
if err != nil { if err != nil {
return err return err
@@ -78,7 +77,7 @@ func httpFeed(data *core.FullData, remote *url.URL, title string) error {
// httpPage will fetch and insert a new core.Page into a core.FullData given an // httpPage will fetch and insert a new core.Page into a core.FullData given an
// http url (or update if it's preexisting) // http url (or update if it's preexisting)
func httpPage(data *core.FullData, remote *url.URL, title string) error { func httpPage(data *core.FullData, remote string, title string) error {
resp, err := fetchHTTP(remote) resp, err := fetchHTTP(remote)
if err != nil { if err != nil {
return err return err
@@ -89,7 +88,7 @@ func httpPage(data *core.FullData, remote *url.URL, title string) error {
var page core.Page var page core.Page
page.Title = title page.Title = title
page.Link = remote.String() page.Link = remote
h := sha256.New() h := sha256.New()
if _, err := io.Copy(h, reader); err != nil { if _, err := io.Copy(h, reader); err != nil {
return err return err