implement New* routines for fetching and creating new feeds/pages.

Also includes routines for fetching gemini, http, and gopher resources. TODO: add Update* routines to update existing feeds/pages.
2021-03-16 22:43:18 -06:00
parent f6313f4465
commit 07d85c0bfc
6 changed files with 408 additions and 0 deletions
@@ -0,0 +1,54 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// package fetch implements routines for fetching gemini, gopher, and http
+// resources and parsing feed formats
+package fetch
+
+import (
+	"fmt"
+	"net/url"
+
+	"golang.nytpu.com/comitium/core"
+)
+
+// NewFeed will fetch and create a new core.Feed given any url
+func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+	switch remoteURL.Scheme {
+	case "gemini":
+		return newGeminiFeed(remoteURL, title)
+	case "http", "https":
+		return newHTTPFeed(remoteURL, title)
+	case "gopher", "gophers":
+		return newGopherFeed(remoteURL, title)
+	default:
+		return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+	}
+}
+
+// NewPage will fetch and create a new core.Page given any url
+func NewPage(remoteURL *url.URL, title string) (*core.Page, error) {
+	switch remoteURL.Scheme {
+	case "gemini":
+		return newGeminiPage(remoteURL, title)
+	case "http", "https":
+		return newHTTPPage(remoteURL, title)
+	case "gopher", "gophers":
+		return newGopherPage(remoteURL, title)
+	default:
+		return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme)
+	}
+}
+
+// UpdateFeed will check an existing core.Feed for updates
+func UpdateFeed(feed *core.Feed) error {
+	// TODO
+	return nil
+}
+
+// UpdatePage will check an existing core.Page for updages
+func UpdatePage(page *core.Page) error {
+	// TODO
+	return nil
+}
@@ -0,0 +1,133 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// This file incorporates code from Drew DeVault's gemreader:
+// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
+// Original source is Copyright 2021 Drew DeVault and is licensed under the
+// terms of the GNU Affero General Public License Version 3.0.
+// https://www.gnu.org/licenses/agpl-3.0-standalone.html
+
+package fetch
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"mime"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.sr.ht/~adnano/go-gemini"
+	"github.com/mmcdole/gofeed"
+
+	"golang.nytpu.com/comitium/core"
+)
+
+// fetchGemini will make a request for a gemini resource
+func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) {
+	client := &gemini.Client{}
+	resp, err := client.Get(context.Background(), remoteURL.String())
+	if err != nil {
+		return nil, err
+	}
+	if resp.Status != gemini.StatusSuccess {
+		return nil, fmt.Errorf("Unexpected Gemini response: %d %s", resp.Status, resp.Meta)
+	}
+
+	return resp, nil
+}
+
+// newGeminiFeed will fetch and create a new core.Feed given a gemini url
+func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+	resp, err := fetchGemini(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	mimetype, _, err := mime.ParseMediaType(resp.Meta)
+	if err != nil {
+		return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta)
+	}
+
+	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+	switch mimetype {
+	case "text/gemini":
+		var feed core.Feed
+		feed.Title = title // will be set later if given title is ""
+		feed.Link = remoteURL
+		text, err := gemini.ParseText(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		for _, line := range text {
+			switch line := line.(type) {
+			case gemini.LineHeading1:
+				if feed.Title == "" {
+					feed.Title = strings.TrimLeft(line.String(), "# ")
+				}
+			case gemini.LineLink:
+				if line.Name == "" || len(line.Name) < 10 {
+					continue
+				}
+				name := strings.TrimLeft(line.Name[:10], " -—")
+				date, err := time.Parse("2006-01-02", name)
+				if err != nil {
+					continue
+				}
+				link, err := url.Parse(line.URL)
+				if err != nil {
+					continue
+				}
+				link = remoteURL.ResolveReference(link)
+				item := &core.Item{}
+				item.Title = strings.TrimLeft(line.Name[10:], ": ")
+				item.Published = date
+				item.Link = link
+				feed.Items = append(feed.Items, item)
+			}
+		}
+		return &feed, nil
+	case "text/xml",
+		"application/rss+xml",
+		"application/atom+xml",
+		"application/xml",
+		"application/json":
+		gofeed, err := gofeed.NewParser().Parse(reader)
+		if err != nil {
+			return nil, err
+		}
+		feed, err := core.GofeedConvert(gofeed, title)
+		return feed, err
+	default:
+		return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
+	}
+}
+
+// newGeminiPage will create a new core.Page given a gemini url
+func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) {
+	resp, err := fetchGemini(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+	var page core.Page
+	page.Title = title
+	page.Link = remoteURL
+	page.Updated = time.Now()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, reader); err != nil {
+		return nil, err
+	}
+	page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+	return &page, nil
+}
@@ -0,0 +1,67 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+package fetch
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"net/url"
+	"time"
+
+	//"github.com/mmcdole/gofeed"
+	"github.com/prologic/go-gopher"
+
+	"golang.nytpu.com/comitium/core"
+)
+
+// fetchGopher will make a request for a gopher resource
+func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
+	resp, err := gopher.Get(remoteURL.String())
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+// newGopherFeed will fetch and create a new core.Feed given a gopher url
+func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+	// TODO
+	return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
+}
+
+// newGopherPage will create a new core.Page given a gopher url
+func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
+	resp, err := fetchGopher(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var reader io.Reader
+	if len(resp.Dir.Items) > 0 {
+		text, err := resp.Dir.ToText()
+		if err != nil {
+			return nil, err
+		}
+		reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
+	} else {
+		reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
+	}
+
+	var page core.Page
+	page.Title = title
+	page.Link = remoteURL
+	page.Updated = time.Now()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, reader); err != nil {
+		return nil, err
+	}
+	page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+	return &page, nil
+}
@@ -0,0 +1,99 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+// This file incorporates code from Drew DeVault's gemreader:
+// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
+// Original source is Copyright 2021 Drew DeVault and is licensed under the
+// terms of the GNU Affero General Public License Version 3.0.
+// https://www.gnu.org/licenses/agpl-3.0-standalone.html
+
+package fetch
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+
+	"golang.nytpu.com/comitium/core"
+)
+
+// fetchHTTP will make a request for an http resource
+func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
+	req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		resp.Body.Close()
+		return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status)
+	}
+	return resp, nil
+}
+
+// newHTTPFeed will fetch and create a new core.Feed given an http url
+func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+	resp, err := fetchHTTP(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.Header.Get("Content-Type") == "text/html" {
+		// TODO
+		return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented")
+	}
+
+	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	gofeed, err := gofeed.NewParser().Parse(reader)
+	if err != nil {
+		return nil, err
+	}
+	feed, err := core.GofeedConvert(gofeed, title)
+	return feed, err
+}
+
+// newHTTPPage will create a new core.Page given an http url
+func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) {
+	resp, err := fetchHTTP(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
+
+	var page core.Page
+	page.Title = title
+	page.Link = remoteURL
+	page.Updated = time.Now()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, reader); err != nil {
+		return nil, err
+	}
+	page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+	return &page, nil
+}