comitium/fetch/gemini.go

// Copyright (C) 2021 nytpu
// SPDX-License-Identifier: AGPL-3.0-or-later
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.

// This file incorporates code from Drew DeVault's gemreader:
// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
// Original source is Copyright 2021 Drew DeVault and is licensed under the
// terms of the GNU Affero General Public License Version 3.0.
// https://www.gnu.org/licenses/agpl-3.0-standalone.html

package fetch

import (
	"context"
	"crypto/sha256"
	"fmt"
	"io"
	"mime"
	"net/url"
	"strings"
	"time"

	"git.sr.ht/~adnano/go-gemini"
	"github.com/mmcdole/gofeed"

	"golang.nytpu.com/comitium/core"
)

// fetchGemini will make a request for a gemini resource
func fetchGemini(remote string) (*gemini.Response, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(core.Timeout)*time.Second)
	defer cancel()
	client := &gemini.Client{}
	resp, err := client.Get(ctx, remote)
	if err != nil {
		return nil, err
	}
	if resp.Status != gemini.StatusSuccess {
		return nil, fmt.Errorf("Unexpected Gemini response: %d %s", resp.Status, resp.Meta)
	}

	return resp, nil
}

// geminiFeed will fetch and insert a new core.Feed into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiFeed(data *core.FullData, remote string, title string) error {
	resp, err := fetchGemini(remote)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	mimetype, _, err := mime.ParseMediaType(resp.Meta)
	if err != nil {
		return fmt.Errorf("Unintelligible content type: %s", resp.Meta)
	}

	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB

	switch mimetype {
	case "text/gemini":
		var feed core.Feed
		feed.Title = title // will be set later if given title is ""
		feed.Link = remote
		feed.FeedLink = remote
		feed.Updated = time.Now()
		// we know that remote is a good url because it's been parsed previously
		parsedRemote, _ := url.ParseRequestURI(remote)

		text, err := gemini.ParseText(resp.Body)
		if err != nil {
			return err
		}
		for _, line := range text {
			switch line := line.(type) {
			case gemini.LineHeading1:
				if feed.Title == "" {
					feed.Title = strings.TrimLeft(line.String(), "# ")
				}
			case gemini.LineLink:
				if line.Name == "" || len(line.Name) < 10 {
					continue
				}
				name := strings.TrimLeft(line.Name[:10], " -—")
				date, err := time.Parse("2006-01-02", name)
				if err != nil {
					continue
				}
				link, err := url.Parse(line.URL)
				if err != nil {
					continue
				}
				link = parsedRemote.ResolveReference(link)
				item := &core.Item{}
				item.Title = strings.TrimLeft(line.Name[10:], " :-–—")
				item.Published = date
				item.Link = link.String()
				feed.Items = append(feed.Items, item)
			}
		}
		data.InsertFeed(&feed, remote)
		return nil
	case "text/xml",
		"application/rss+xml",
		"application/atom+xml",
		"application/xml",
		"application/json":
		gofeed, err := gofeed.NewParser().Parse(reader)
		if err != nil {
			return err
		}
		feed, err := core.GofeedConvert(gofeed, title)
		if err != nil {
			return err
		}
		data.InsertFeed(feed, remote)
		return nil
	default:
		return fmt.Errorf("Cannot interpret %s as a feed", resp.Meta)
	}
}

// geminiPage will fetch and insert a new core.Page into a core.FullData given
// a gemini url (or update if it's preexisting)
func geminiPage(data *core.FullData, remote string, title string) error {
	resp, err := fetchGemini(remote)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB

	var page core.Page
	page.Title = title
	page.Link = remote
	h := sha256.New()
	if _, err := io.Copy(h, reader); err != nil {
		return err
	}
	newHash := fmt.Sprintf("%x", h.Sum(nil))
	page.Hash = newHash
	page.Updated = time.Now()

	data.InsertPage(&page, remote)
	return nil
}