From 07d85c0bfc82b5df7366d3810c95aa8b959837a4 Mon Sep 17 00:00:00 2001 From: nytpu Date: Tue, 16 Mar 2021 22:43:18 -0600 Subject: [PATCH] implement New* routines for fetching and creating new feeds/pages. Also includes routines for fetching gemini, http, and gopher resources. TODO: add Update* routines to update existing feeds/pages. --- fetch/fetch.go | 54 ++++++++++++++++++++ fetch/gemini.go | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ fetch/gopher.go | 67 ++++++++++++++++++++++++ fetch/http.go | 99 +++++++++++++++++++++++++++++++++++ go.mod | 3 ++ go.sum | 52 +++++++++++++++++++ 6 files changed, 408 insertions(+) create mode 100644 fetch/fetch.go create mode 100644 fetch/gemini.go create mode 100644 fetch/gopher.go create mode 100644 fetch/http.go diff --git a/fetch/fetch.go b/fetch/fetch.go new file mode 100644 index 0000000..5c6809a --- /dev/null +++ b/fetch/fetch.go @@ -0,0 +1,54 @@ +// Copyright (C) 2021 nytpu +// SPDX-License-Identifier: AGPL-3.0-or-later +// For more license details, see LICENSE or . + +// package fetch implements routines for fetching gemini, gopher, and http +// resources and parsing feed formats +package fetch + +import ( + "fmt" + "net/url" + + "golang.nytpu.com/comitium/core" +) + +// NewFeed will fetch and create a new core.Feed given any url +func NewFeed(remoteURL *url.URL, title string) (*core.Feed, error) { + switch remoteURL.Scheme { + case "gemini": + return newGeminiFeed(remoteURL, title) + case "http", "https": + return newHTTPFeed(remoteURL, title) + case "gopher", "gophers": + return newGopherFeed(remoteURL, title) + default: + return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme) + } +} + +// NewPage will fetch and create a new core.Page given any url +func NewPage(remoteURL *url.URL, title string) (*core.Page, error) { + switch remoteURL.Scheme { + case "gemini": + return newGeminiPage(remoteURL, title) + case "http", "https": + return newHTTPPage(remoteURL, title) + case "gopher", "gophers": + return newGopherPage(remoteURL, title) + default: + return nil, fmt.Errorf("Unsupported protocol '%s'", remoteURL.Scheme) + } +} + +// UpdateFeed will check an existing core.Feed for updates +func UpdateFeed(feed *core.Feed) error { + // TODO + return nil +} + +// UpdatePage will check an existing core.Page for updages +func UpdatePage(page *core.Page) error { + // TODO + return nil +} diff --git a/fetch/gemini.go b/fetch/gemini.go new file mode 100644 index 0000000..d32d093 --- /dev/null +++ b/fetch/gemini.go @@ -0,0 +1,133 @@ +// Copyright (C) 2021 nytpu +// SPDX-License-Identifier: AGPL-3.0-or-later +// For more license details, see LICENSE or . + +// This file incorporates code from Drew DeVault's gemreader: +// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go +// Original source is Copyright 2021 Drew DeVault and is licensed under the +// terms of the GNU Affero General Public License Version 3.0. +// https://www.gnu.org/licenses/agpl-3.0-standalone.html + +package fetch + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "mime" + "net/url" + "strings" + "time" + + "git.sr.ht/~adnano/go-gemini" + "github.com/mmcdole/gofeed" + + "golang.nytpu.com/comitium/core" +) + +// fetchGemini will make a request for a gemini resource +func fetchGemini(remoteURL *url.URL) (*gemini.Response, error) { + client := &gemini.Client{} + resp, err := client.Get(context.Background(), remoteURL.String()) + if err != nil { + return nil, err + } + if resp.Status != gemini.StatusSuccess { + return nil, fmt.Errorf("Unexpected Gemini response: %d %s", resp.Status, resp.Meta) + } + + return resp, nil +} + +// newGeminiFeed will fetch and create a new core.Feed given a gemini url +func newGeminiFeed(remoteURL *url.URL, title string) (*core.Feed, error) { + resp, err := fetchGemini(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + mimetype, _, err := mime.ParseMediaType(resp.Meta) + if err != nil { + return nil, fmt.Errorf("Unintelligible content type: %s", resp.Meta) + } + + reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + + switch mimetype { + case "text/gemini": + var feed core.Feed + feed.Title = title // will be set later if given title is "" + feed.Link = remoteURL + text, err := gemini.ParseText(resp.Body) + if err != nil { + return nil, err + } + for _, line := range text { + switch line := line.(type) { + case gemini.LineHeading1: + if feed.Title == "" { + feed.Title = strings.TrimLeft(line.String(), "# ") + } + case gemini.LineLink: + if line.Name == "" || len(line.Name) < 10 { + continue + } + name := strings.TrimLeft(line.Name[:10], " -—") + date, err := time.Parse("2006-01-02", name) + if err != nil { + continue + } + link, err := url.Parse(line.URL) + if err != nil { + continue + } + link = remoteURL.ResolveReference(link) + item := &core.Item{} + item.Title = strings.TrimLeft(line.Name[10:], ": ") + item.Published = date + item.Link = link + feed.Items = append(feed.Items, item) + } + } + return &feed, nil + case "text/xml", + "application/rss+xml", + "application/atom+xml", + "application/xml", + "application/json": + gofeed, err := gofeed.NewParser().Parse(reader) + if err != nil { + return nil, err + } + feed, err := core.GofeedConvert(gofeed, title) + return feed, err + default: + return nil, fmt.Errorf("Cannot interpret %s as a feed", resp.Meta) + } +} + +// newGeminiPage will create a new core.Page given a gemini url +func newGeminiPage(remoteURL *url.URL, title string) (*core.Page, error) { + resp, err := fetchGemini(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + + var page core.Page + page.Title = title + page.Link = remoteURL + page.Updated = time.Now() + + h := sha256.New() + if _, err := io.Copy(h, reader); err != nil { + return nil, err + } + page.Hash = fmt.Sprintf("%x", h.Sum(nil)) + + return &page, nil +} diff --git a/fetch/gopher.go b/fetch/gopher.go new file mode 100644 index 0000000..5750fad --- /dev/null +++ b/fetch/gopher.go @@ -0,0 +1,67 @@ +// Copyright (C) 2021 nytpu +// SPDX-License-Identifier: AGPL-3.0-or-later +// For more license details, see LICENSE or . + +package fetch + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io" + "net/url" + "time" + + //"github.com/mmcdole/gofeed" + "github.com/prologic/go-gopher" + + "golang.nytpu.com/comitium/core" +) + +// fetchGopher will make a request for a gopher resource +func fetchGopher(remoteURL *url.URL) (*gopher.Response, error) { + resp, err := gopher.Get(remoteURL.String()) + if err != nil { + return nil, err + } + return resp, nil +} + +// newGopherFeed will fetch and create a new core.Feed given a gopher url +func newGopherFeed(remoteURL *url.URL, title string) (*core.Feed, error) { + // TODO + return nil, fmt.Errorf("Feeds served over gopher are currently unsupported.") +} + +// newGopherPage will create a new core.Page given a gopher url +func newGopherPage(remoteURL *url.URL, title string) (*core.Page, error) { + resp, err := fetchGopher(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var reader io.Reader + if len(resp.Dir.Items) > 0 { + text, err := resp.Dir.ToText() + if err != nil { + return nil, err + } + reader = io.LimitReader(bytes.NewReader(text), 1073741824) // 1 GiB + } else { + reader = io.LimitReader(resp.Body, 1073741824) // 1 GiB + } + + var page core.Page + page.Title = title + page.Link = remoteURL + page.Updated = time.Now() + + h := sha256.New() + if _, err := io.Copy(h, reader); err != nil { + return nil, err + } + page.Hash = fmt.Sprintf("%x", h.Sum(nil)) + + return &page, nil +} diff --git a/fetch/http.go b/fetch/http.go new file mode 100644 index 0000000..fb373e6 --- /dev/null +++ b/fetch/http.go @@ -0,0 +1,99 @@ +// Copyright (C) 2021 nytpu +// SPDX-License-Identifier: AGPL-3.0-or-later +// For more license details, see LICENSE or . + +// This file incorporates code from Drew DeVault's gemreader: +// https://git.sr.ht/~sircmpwn/gemreader/tree/master/item/feeds/fetch.go +// Original source is Copyright 2021 Drew DeVault and is licensed under the +// terms of the GNU Affero General Public License Version 3.0. +// https://www.gnu.org/licenses/agpl-3.0-standalone.html + +package fetch + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/mmcdole/gofeed" + + "golang.nytpu.com/comitium/core" +) + +// fetchHTTP will make a request for an http resource +func fetchHTTP(remoteURL *url.URL) (*http.Response, error) { + client := &http.Client{ + Timeout: 10 * time.Second, + } + + req, err := http.NewRequestWithContext(context.Background(), "GET", remoteURL.String(), nil) + req.Header.Add("User-Agent", "comitium (https://git.nytpu.com/comitium/)") + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + resp.Body.Close() + return nil, fmt.Errorf("Unexpected HTTP response %s", resp.Status) + } + return resp, nil +} + +// newHTTPFeed will fetch and create a new core.Feed given an http url +func newHTTPFeed(remoteURL *url.URL, title string) (*core.Feed, error) { + resp, err := fetchHTTP(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.Header.Get("Content-Type") == "text/html" { + // TODO + return nil, fmt.Errorf("Extracting feed from HTML pages is unimplemented") + } + + reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + gofeed, err := gofeed.NewParser().Parse(reader) + if err != nil { + return nil, err + } + feed, err := core.GofeedConvert(gofeed, title) + return feed, err +} + +// newHTTPPage will create a new core.Page given an http url +func newHTTPPage(remoteURL *url.URL, title string) (*core.Page, error) { + resp, err := fetchHTTP(remoteURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + reader := io.LimitReader(resp.Body, 1073741824) // 1 GiB + + var page core.Page + page.Title = title + page.Link = remoteURL + page.Updated = time.Now() + + h := sha256.New() + if _, err := io.Copy(h, reader); err != nil { + return nil, err + } + page.Hash = fmt.Sprintf("%x", h.Sum(nil)) + + return &page, nil +} diff --git a/go.mod b/go.mod index 337d9c1..8940067 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module golang.nytpu.com/comitium go 1.16 require ( + git.sr.ht/~adnano/go-gemini v0.1.21 github.com/mitchellh/go-homedir v1.1.0 + github.com/mmcdole/gofeed v1.1.0 + github.com/prologic/go-gopher v0.0.0-20210114215941-1859ade2470b github.com/spf13/pflag v1.0.5 ) diff --git a/go.sum b/go.sum index 8635e51..a22438e 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,56 @@ +git.sr.ht/~adnano/go-gemini v0.1.21 h1:qpCtFYgHGeW0Jf+0A5MeUtZ4/hAnfSeS7LkOTJnpKwQ= +git.sr.ht/~adnano/go-gemini v0.1.21/go.mod h1:kmWT0aLnjkuzAMouxNT6Bqv756HYHSe56HE7yoF5P7Y= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mmcdole/gofeed v1.1.0 h1:T2WrGLVJRV04PY2qwhEJLHCt9JiCtBhb6SmC8ZvJH08= +github.com/mmcdole/gofeed v1.1.0/go.mod h1:PPiVwgDXLlz2N83KB4TrIim2lyYM5Zn7ZWH9Pi4oHUk= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prologic/go-gopher v0.0.0-20210114215941-1859ade2470b h1:bOYTIzro9GwU+OV5RJnFXCUW7fVnzY+wGDaL2ceFvn4= +github.com/prologic/go-gopher v0.0.0-20210114215941-1859ade2470b/go.mod h1:ypKOIAVSaX4/TBkLcXhjcpSZKrBOmOd/sufqKUyM6Xc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=