nytpu
07d85c0bfc
Also includes routines for fetching gemini, http, and gopher resources. TODO: add Update* routines to update existing feeds/pages.
100 lines
2.4 KiB
Go
100 lines
2.4 KiB
Go
// Copyright (C) 2021 nytpu
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
// This file incorporates code from Drew DeVault's gemreader:
|
|
// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
|
|
// Original source is Copyright 2021 Drew DeVault and is licensed under the
|
|
// terms of the GNU Affero General Public License Version 3.0.
|
|
// https://www.gnu.org/licenses/agpl-3.0-standalone.html
|
|
|
|
package fetch
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
|
|
"golang.nytpu.com/comitium/core"
|
|
)
|
|
|
|
// fetchHTTP will make a request for an http resource
|
|
func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
|
|
client := &http.Client{
|
|
Timeout: 10 * time.Second,
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
|
|
req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
|
resp.Body.Close()
|
|
return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status)
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
// newHTTPFeed will fetch and create a new core.Feed given an http url
|
|
func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) {
|
|
resp, err := fetchHTTP(remoteURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.Header.Get("Content-Type") == "text/html" {
|
|
// TODO
|
|
return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented")
|
|
}
|
|
|
|
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
|
data, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
gofeed, err := gofeed.NewParser().Parse(reader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
feed, err := core.GofeedConvert(gofeed, title)
|
|
return feed, err
|
|
}
|
|
|
|
// newHTTPPage will create a new core.Page given an http url
|
|
func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) {
|
|
resp, err := fetchHTTP(remoteURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
|
|
|
var page core.Page
|
|
page.Title = title
|
|
page.Link = remoteURL
|
|
page.Updated = time.Now()
|
|
|
|
h := sha256.New()
|
|
if _, err := io.Copy(h, reader); err != nil {
|
|
return nil, err
|
|
}
|
|
page.Hash = fmt.Sprintf("%x", h.Sum(nil))
|
|
|
|
return &page, nil
|
|
}
|