From 07d85c0bfc82b5df7366d3810c95aa8b959837a4 Mon Sep 17 00:00:00 2001 From: nytpu Date: Tue, 16 Mar 2021 22:43:18 -0600 Subject: implement New* routines for fetching and creating new feeds/pages. Also includes routines for fetching gemini, http, and gopher resources. TODO: add Update* routines to update existing feeds/pages. --- fetch/gopher.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 fetch/gopher.go (limited to 'fetch/gopher.go') diff --git a/fetch/gopher.go b/fetch/gopher.go new file mode 100644 index 0000000..5750fad --- /dev/null +++ b/fetch/gopher.go @@ -0,0 +1,67 @@ +// Copyright (C) 2021 nytpu +// SPDX-License-Identifier: AGPL-3.0-or-later +// For more license details, see LICENSE or . + +package fetch + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io" + "net/url" + "time" + + //"github.com/mmcdole/gofeed" + "github.com/prologic/go-gopher" + + "golang.nytpu.com/comitium/core" +) + +// fetchGopher will make a request for a gopher resource +func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { + resp, err := gopher.Get(remoteURL.String()) + if err != nil { + return nil, err + } + return resp, nil +} + +// newGopherFeed will fetch and create a new core.Feed given a gopher url +func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) { + // TODO + return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.") +} + +// newGopherPage will create a new core.Page given a gopher url +func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) { + resp, err := fetchGopher(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var reader io.Reader + if len(resp.Dir.Items) > 0 { + text, err := resp.Dir.ToText() + if err != nil { + return nil, err + } + reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB + } else { + reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB + } + + var page core.Page + page.Title = title + page.Link = remoteURL + page.Updated = time.Now() + + h := sha256.New() + if _, err := io.Copy(h, reader); err != nil { + return nil, err + } + page.Hash = fmt.Sprintf("%x", h.Sum(nil)) + + return &page, nil +} -- cgit v1.2.3