// Copyright (C) 2021 nytpu // SPDX-License-Identifier: AGPL-3.0-only // For more license details, see LICENSE or . // This file incorporates code from Drew DeVault's gemreader: // https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go // Original source is Copyright 2021 Drew DeVault and is licensed under the // terms of the GNU Affero General Public License Version 3.0. // https://www.gnu.org/licenses/agpl-3.0-standalone.html package fetch import ( "context" "crypto/sha256" "fmt" "io" "mime" "net/url" "strings" "time" "git.sr.ht/~adnano/go-gemini" "github.com/mmcdole/gofeed" "golang.nytpu.com/comitium/localizations" "golang.nytpu.com/comitium/core" ) // fetchGemini will make a request for a gemini resource func fetchGemini(remote string) (*gemini.Response, error) { ctx, _ := context.WithTimeout(context.Background(), time.Duration(core.Timeout)*time.Second) client := &gemini.Client{} resp, err := client.Get(ctx, remote) if err != nil { return nil, err } if resp.Status != gemini.StatusSuccess { return nil, fmt.Errorf(core.Localizer.Get("errors.gem_bad_response", &localizations.Replacements{"code": resp.Status, "meta": resp.Meta})) } return resp, nil } // geminiFeed will fetch and insert a new core.Feed into a core.FullData given // a gemini url (or update if it's preexisting) func geminiFeed(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err } defer resp.Body.Close() mimetype, _, err := mime.ParseMediaType(resp.Meta) if err != nil { return fmt.Errorf(core.Localizer.Get("errors.bad_content_type", &localizations.Replacements{"mime": resp.Meta})) } reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB switch mimetype { case "text/gemini": var feed core.Feed feed.Title = title // will be set later if given title is "" feed.Link = remote feed.FeedLink = remote feed.Updated = time.Now() // we know that remote is a good url because it's been parsed previously parsedRemote, _ := url.ParseRequestURI(remote) text, err := gemini.ParseText(resp.Body) if err != nil { return err } for _, line := range text { switch line := line.(type) { case gemini.LineHeading1: if feed.Title == "" { feed.Title = strings.TrimLeft(line.String(), "# ") } case gemini.LineLink: if line.Name == "" || len(line.Name) < 10 { continue } name := strings.TrimLeft(line.Name[:10], " -—") date, err := time.ParseInLocation("2006-01-02", name, time.Local) if err != nil { continue } link, err := url.Parse(line.URL) if err != nil { continue } link = parsedRemote.ResolveReference(link) item := &core.Item{} item.Title = strings.TrimLeft(line.Name[10:], " :-–—") item.Published = date item.Link = link.String() // don't insert if older than 1 year if item.Published.After(time.Now().AddDate(-1, 0, 0)) { feed.Items = append(feed.Items, item) } } } data.InsertFeed(&feed, remote) return nil case "text/xml", "application/rss+xml", "application/atom+xml", "application/xml", "application/json": gofeed, err := gofeed.NewParser().Parse(reader) if err != nil { return err } feed, err := core.GofeedConvert(gofeed, title) if err != nil { return err } data.InsertFeed(feed, remote) return nil default: return fmt.Errorf(core.Localizer.Get("errors.bad_feed", &localizations.Replacements{"meta": resp.Meta})) } } // geminiPage will fetch and insert a new core.Page into a core.FullData given // a gemini url (or update if it's preexisting) func geminiPage(data *core.FullData, remote string, title string) error { resp, err := fetchGemini(remote) if err != nil { return err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB var page core.Page page.Title = title page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) page.Hash = newHash page.Updated = time.Now() data.InsertPage(&page, remote) return nil }