implement New* routines for fetching and creating new feeds/pages.

Also includes routines for fetching gemini, http, and gopher resources.

TODO: add Update* routines to update existing feeds/pages.

fetch/gopher.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2021 nytpu
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+package fetch
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"net/url"
+	"time"
+
+	//"github.com/mmcdole/gofeed"
+	"github.com/prologic/go-gopher"
+
+	"golang.nytpu.com/comitium/core"
+)
+
+// fetchGopher will make a request for a gopher resource
+func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) {
+	resp, err := gopher.Get(remoteURL.String())
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+// newGopherFeed will fetch and create a new core.Feed given a gopher url
+func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
+	// TODO
+	return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.")
+}
+
+// newGopherPage will create a new core.Page given a gopher url
+func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) {
+	resp, err := fetchGopher(remoteURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var reader io.Reader
+	if len(resp.Dir.Items) > 0 {
+		text, err := resp.Dir.ToText()
+		if err != nil {
+			return nil, err
+		}
+		reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB
+	} else {
+		reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB
+	}
+
+	var page core.Page
+	page.Title = title
+	page.Link = remoteURL
+	page.Updated = time.Now()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, reader); err != nil {
+		return nil, err
+	}
+	page.Hash = fmt.Sprintf("%x", h.Sum(nil))
+
+	return &page, nil
+}