// Copyright (C) 2021 nytpu // SPDX-License-Identifier: AGPL-3.0-or-later // For more license details, see LICENSE or . // This file incorporates code from Drew DeVault's gemreader: // https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go // Original source is Copyright 2021 Drew DeVault and is licensed under the // terms of the GNU Affero General Public License Version 3.0. // https://www.gnu.org/licenses/agpl-3.0-standalone.html package fetch import ( "context" "crypto/sha256" "fmt" "io" "net/http" "time" "github.com/mmcdole/gofeed" "golang.nytpu.com/comitium/core" ) // fetchHTTP will make a request for an http resource func fetchHTTP(remoteURL string) (*http.Response, error) { client := &http.Client{ Timeout: 10 * time.Second, } req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL, nil) req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)") if err != nil { return nil, err } resp, err := client.Do(req) if err != nil { return nil, err } if resp.StatusCode != 200 { resp.Body.Close() return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status) } return resp, nil } // httpFeed will fetch and insert a new core.Feed into a core.FullData given an // http url (or update if it's preexisting) func httpFeed(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err } defer resp.Body.Close() if resp.Header.Get("Content-Type") == "text/html" { // TODO return fmt.Errorf("Extracting feed from HTML pages is unimplemented") } reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB gofeed, err := gofeed.NewParser().Parse(reader) if err != nil { return err } feed, err := core.GofeedConvert(gofeed, title) if err != nil { return err } data.InsertFeed(feed, remote) return nil } // httpPage will fetch and insert a new core.Page into a core.FullData given an // http url (or update if it's preexisting) func httpPage(data *core.FullData, remote string, title string) error { resp, err := fetchHTTP(remote) if err != nil { return err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB var page core.Page page.Title = title page.Link = remote h := sha256.New() if _, err := io.Copy(h, reader); err != nil { return err } newHash := fmt.Sprintf("%x", h.Sum(nil)) page.Hash = newHash page.Updated = time.Now() data.InsertPage(&page, remote) return nil }