nytpu
104 lines
2.5 KiB
Go
104 lines
2.5 KiB
Go
// Copyright (C) 2021 nytpu
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// For more license details, see LICENSE or <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
// This file incorporates code from Drew DeVault's gemreader:
|
|
// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go
|
|
// Original source is Copyright 2021 Drew DeVault and is licensed under the
|
|
// terms of the GNU Affero General Public License Version 3.0.
|
|
// https://www.gnu.org/licenses/agpl-3.0-standalone.html
|
|
|
|
package fetch
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
|
|
"golang.nytpu.com/comitium/core"
|
|
)
|
|
|
|
// fetchHTTP will make a request for an http resource
|
|
func fetchHTTP(remoteURL *url.URL) (*http.Response, error) {
|
|
client := &http.Client{
|
|
Timeout: 10 * time.Second,
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil)
|
|
req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
|
resp.Body.Close()
|
|
return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status)
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
// httpFeed will fetch and insert a new core.Feed into a core.FullData given an
|
|
// http url (or update if it's preexisting)
|
|
func httpFeed(data *core.FullData, remote *url.URL, title string) error {
|
|
resp, err := fetchHTTP(remote)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.Header.Get("Content-Type") == "text/html" {
|
|
// TODO
|
|
return fmt.Errorf("Extracting feed from HTML pages is unimplemented")
|
|
}
|
|
|
|
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
|
gofeed, err := gofeed.NewParser().Parse(reader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
feed, err := core.GofeedConvert(gofeed, title)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
InsertFeed(data, feed, remote)
|
|
return nil
|
|
}
|
|
|
|
// httpPage will fetch and insert a new core.Page into a core.FullData given an
|
|
// http url (or update if it's preexisting)
|
|
func httpPage(data *core.FullData, remote *url.URL, title string) error {
|
|
resp, err := fetchHTTP(remote)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB
|
|
|
|
var page core.Page
|
|
page.Title = title
|
|
page.Link = remote
|
|
h := sha256.New()
|
|
if _, err := io.Copy(h, reader); err != nil {
|
|
return err
|
|
}
|
|
newHash := fmt.Sprintf("%x", h.Sum(nil))
|
|
page.Hash = newHash
|
|
page.Updated = time.Now()
|
|
|
|
InsertPage(data, &page, remote)
|
|
return nil
|
|
}
|