summaryrefslogtreecommitdiff
path: root/fetch
diff options
context:
space:
mode:
Diffstat (limited to 'fetch')
-rw-r--r--fetch/fetch.go54
-rw-r--r--fetch/gemini.go133
-rw-r--r--fetch/gopher.go67
-rw-r--r--fetch/http.go99
4 files changed, 353 insertions, 0 deletions
diff --git a/fetch/fetch.go b/fetch/fetch.go
new file mode 100644
index 0000000..5c6809a
--- /dev/null
+++ b/fetch/fetch.go
@@ -0,0 +1,54 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// package fetch implements routines for fetching gemini, gopher, and http
+// resources and parsing feed formats
+package fetch
+
+import (
+ "fmt"
+ "net/url"
+
+ "golang.nytpu.com/comitium/core"
+)
+
+// NewFeed will fetch and create a new core.Feed given any url
+func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+ switch remoteURL.Scheme {
+ case "gemini":
+ return newGeminiFeed(remoteURL, title)
+ case "http", "https":
+ return newHTTPFeed(remoteURL, title)
+ case "gopher", "gophers":
+ return newGopherFeed(remoteURL, title)
+ default:
+ return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+ }
+}
+
+// NewPage will fetch and create a new core.Page given any url
+func NewPage(remoteURL *url.URL, title string) (*core.Page, error) {
+ switch remoteURL.Scheme {
+ case "gemini":
+ return newGeminiPage(remoteURL, title)
+ case "http", "https":
+ return newHTTPPage(remoteURL, title)
+ case "gopher", "gophers":
+ return newGopherPage(remoteURL, title)
+ default:
+ return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+ }
+}
+
+// UpdateFeed will check an existing core.Feed for updates
+func UpdateFeed(feed *core.Feed) error {
+ // TODO
+ return nil
+}
+
+// UpdatePage will check an existing core.Page for updages
+func UpdatePage(page *core.Page) error {
+ // TODO
+ return nil
+}
diff --git a/fetch/gemini.go b/fetch/gemini.go
new file mode 100644
index 0000000..d32d093
--- /dev/null
+++ b/fetch/gemini.go
@@ -0,0 +1,133 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// This file incorporates code from Drew DeVault's gemreader:
+// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
+// Original source is Copyright 2021 Drew DeVault and is licensed under the
+// terms of the GNU Affero General Public License Version 3.0.
+// https://www.gnu.org/licenses/agpl-3.0-standalone.html
+
+package fetch
+
+import (
+ "context"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "mime"
+ "net/url"
+ "strings"
+ "time"
+
+ "git.sr.ht/~adnano/go-gemini"
+ "github.com/mmcdole/gofeed"
+
+ "golang.nytpu.com/comitium/core"
+)
+
+// fetchGemini will make a request for a gemini resource
+func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
+ client := &gemini.Client{}
+ resp, err := client.Get(context.Background(), remoteURL.String())
+ if err != nil {
+ return nil, err
+ }
+ if resp.Status != gemini.StatusSuccess {
+ return nil, fmt.Errorf("Unexpected Gemini response: %d %s", resp.Status, resp.Meta)
+ }
+
+ return resp, nil
+}
+
+// newGeminiFeed will fetch and create a new core.Feed given a gemini url
+func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+ resp, err := fetchGemini(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ mimetype, _, err := mime.ParseMediaType(resp.Meta)
+ if err != nil {
+ return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta)
+ }
+
+ reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+ switch mimetype {
+ case "text/gemini":
+ var feed core.Feed
+ feed.Title = title // will be set later if given title is ""
+ feed.Link = remoteURL
+ text, err := gemini.ParseText(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ for _, line := range text {
+ switch line := line.(type) {
+ case gemini.LineHeading1:
+ if feed.Title == "" {
+ feed.Title = strings.TrimLeft(line.String(), "# ")
+ }
+ case gemini.LineLink:
+ if line.Name == "" || len(line.Name) < 10 {
+ continue
+ }
+ name := strings.TrimLeft(line.Name[:10], " -—")
+ date, err := time.Parse("2006-01-02", name)
+ if err != nil {
+ continue
+ }
+ link, err := url.Parse(line.URL)
+ if err != nil {
+ continue
+ }
+ link = remoteURL.ResolveReference(link)
+ item := &core.Item{}
+ item.Title = strings.TrimLeft(line.Name[10:], ": ")
+ item.Published = date
+ item.Link = link
+ feed.Items = append(feed.Items, item)
+ }
+ }
+ return &feed, nil
+ case "text/xml",
+ "application/rss+xml",
+ "application/atom+xml",
+ "application/xml",
+ "application/json":
+ gofeed, err := gofeed.NewParser().Parse(reader)
+ if err != nil {
+ return nil, err
+ }
+ feed, err := core.GofeedConvert(gofeed, title)
+ return feed, err
+ default:
+ return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
+ }
+}
+
+// newGeminiPage will create a new core.Page given a gemini url
+func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) {
+ resp, err := fetchGemini(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+ var page core.Page
+ page.Title = title
+ page.Link = remoteURL
+ page.Updated = time.Now()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, reader); err != nil {
+ return nil, err
+ }
+ page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+ return &page, nil
+}
diff --git a/fetch/gopher.go b/fetch/gopher.go
new file mode 100644
index 0000000..5750fad
--- /dev/null
+++ b/fetch/gopher.go
@@ -0,0 +1,67 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+package fetch
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "net/url"
+ "time"
+
+ //"github.com/mmcdole/gofeed"
+ "github.com/prologic/go-gopher"
+
+ "golang.nytpu.com/comitium/core"
+)
+
+// fetchGopher will make a request for a gopher resource
+func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
+ resp, err := gopher.Get(remoteURL.String())
+ if err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+// newGopherFeed will fetch and create a new core.Feed given a gopher url
+func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+ // TODO
+ return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
+}
+
+// newGopherPage will create a new core.Page given a gopher url
+func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
+ resp, err := fetchGopher(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ var reader io.Reader
+ if len(resp.Dir.Items) > 0 {
+ text, err := resp.Dir.ToText()
+ if err != nil {
+ return nil, err
+ }
+ reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
+ } else {
+ reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
+ }
+
+ var page core.Page
+ page.Title = title
+ page.Link = remoteURL
+ page.Updated = time.Now()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, reader); err != nil {
+ return nil, err
+ }
+ page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+ return &page, nil
+}
diff --git a/fetch/http.go b/fetch/http.go
new file mode 100644
index 0000000..fb373e6
--- /dev/null
+++ b/fetch/http.go
@@ -0,0 +1,99 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// This file incorporates code from Drew DeVault's gemreader:
+// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
+// Original source is Copyright 2021 Drew DeVault and is licensed under the
+// terms of the GNU Affero General Public License Version 3.0.
+// https://www.gnu.org/licenses/agpl-3.0-standalone.html
+
+package fetch
+
+import (
+ "context"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/mmcdole/gofeed"
+
+ "golang.nytpu.com/comitium/core"
+)
+
+// fetchHTTP will make a request for an http resource
+func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
+ client := &http.Client{
+ Timeout: 10 * time.Second,
+ }
+
+ req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
+ req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != 200 {
+ resp.Body.Close()
+ return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status)
+ }
+ return resp, nil
+}
+
+// newHTTPFeed will fetch and create a new core.Feed given an http url
+func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+ resp, err := fetchHTTP(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if resp.Header.Get("Content-Type") == "text/html" {
+ // TODO
+ return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented")
+ }
+
+ reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+ data, err := io.ReadAll(reader)
+ if err != nil {
+ return nil, err
+ }
+ gofeed, err := gofeed.NewParser().Parse(reader)
+ if err != nil {
+ return nil, err
+ }
+ feed, err := core.GofeedConvert(gofeed, title)
+ return feed, err
+}
+
+// newHTTPPage will create a new core.Page given an http url
+func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) {
+ resp, err := fetchHTTP(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+ var page core.Page
+ page.Title = title
+ page.Link = remoteURL
+ page.Updated = time.Now()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, reader); err != nil {
+ return nil, err
+ }
+ page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+ return &page, nil
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-08 16:25:27 +0000