diff options
| -rw-r--r-- | Cargo.lock | 154 | ||||
| -rw-r--r-- | Cargo.toml | 3 | ||||
| -rw-r--r-- | src/channel.rs | 34 | ||||
| -rw-r--r-- | src/config.rs | 34 | ||||
| -rw-r--r-- | src/main.rs | 47 | ||||
| -rw-r--r-- | src/retrieve.rs | 3 |
6 files changed, 259 insertions, 16 deletions
@@ -3,6 +3,15 @@ version = 4 [[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] name = "anstream" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -124,6 +133,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -162,7 +177,11 @@ version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ + "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -633,6 +652,30 @@ dependencies = [ ] [[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] name = "icu_collections" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -851,6 +894,17 @@ dependencies = [ ] [[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1116,6 +1170,7 @@ dependencies = [ "atom_syndication", "derive_builder", "quick-xml", + "serde", ] [[package]] @@ -1123,6 +1178,7 @@ name = "rss-aggregator" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", "dirs", "futures", @@ -1616,6 +1672,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] name = "webpki-root-certs" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1634,12 +1735,65 @@ dependencies = [ ] [[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -8,7 +8,8 @@ anyhow = "1.0.104" clap = { version = "4.6.6", features = ["derive"] } dirs = "6.0.0" futures = "0.3.34" -rss = "2.1.0" +rss = { version = "2.1.0", features = ["serde"] } serde = { version = "1.0.229", features = ["derive"] } toml = "1.1.5" isahc = "2.0.1" +chrono = "0.4.45" diff --git a/src/channel.rs b/src/channel.rs new file mode 100644 index 0000000..f4d4941 --- /dev/null +++ b/src/channel.rs @@ -0,0 +1,34 @@ +use chrono::Utc; +use rss::Channel; + +use crate::config::ChannelConfig; + +pub fn make_new_empty_channel(config: ChannelConfig) -> Channel { + Channel { + title: config.title, + link: config.link, + description: config.description, + language: config.language, + copyright: config.copyright, + managing_editor: config.managing_editor, + webmaster: config.webmaster, + pub_date: Some(Utc::now().to_rfc2822()), + last_build_date: Some(Utc::now().to_rfc2822()), + categories: config.categories, + generator: Some(String::from("https://git.joeac.net/rss-aggregator.git")), + docs: Some(String::from("https://git.joeac.net/rss-aggregator.git/tree/README.md")), + cloud: config.cloud, + ttl: config.ttl, + image: config.image, + text_input: config.text_input, + skip_hours: config.skip_hours, + skip_days: config.skip_days, + items: Vec::new(), + extensions: config.extensions, + itunes_ext: config.itunes_ext, + dublin_core_ext: config.dublin_core_ext, + syndication_ext: config.syndication_ext, + namespaces: config.namespaces, + rating: None, + } +} diff --git a/src/config.rs b/src/config.rs index 089d3e2..a5652c1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,11 +1,41 @@ -use std::{fs::File, io::{read_to_string}, path::Path}; +use std::{collections::BTreeMap, fs::File, io::read_to_string, path::Path}; +use rss::{Category, Cloud, Image, TextInput, extension::{Extension, dublincore::DublinCoreExtension, itunes::ITunesChannelExtension, syndication::SyndicationExtension}}; use serde::Deserialize; #[derive(Deserialize)] pub struct Config { #[serde(default)] - pub urls: Vec<String> + pub urls: Vec<String>, + pub channel: ChannelConfig, +} + +#[derive(Deserialize)] +pub struct ChannelConfig { + pub title: String, + pub link: String, + pub description: String, + pub language: Option<String>, + pub copyright: Option<String>, + pub managing_editor: Option<String>, + pub webmaster: Option<String>, + #[serde(default)] + pub categories: Vec<Category>, + pub cloud: Option<Cloud>, + pub ttl: Option<String>, + pub image: Option<Image>, + pub text_input: Option<TextInput>, + #[serde(default)] + pub skip_hours: Vec<String>, + #[serde(default)] + pub skip_days: Vec<String>, + #[serde(default)] + pub extensions: BTreeMap<String, BTreeMap<String, Vec<Extension>>>, + pub itunes_ext: Option<ITunesChannelExtension>, + pub dublin_core_ext: Option<DublinCoreExtension>, + pub syndication_ext: Option<SyndicationExtension>, + #[serde(default)] + pub namespaces: BTreeMap<String, String>, } pub fn default_config_file_locations() -> Vec<String> { diff --git a/src/main.rs b/src/main.rs index ae68477..61701ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,15 @@ +mod channel; mod config; mod retrieve; +use std::future; + +use anyhow::anyhow; use clap::Parser; -use futures::executor::block_on; +use futures::{StreamExt, executor::block_on}; +use rss::Channel; -use crate::{config::Config, retrieve::retrieve}; +use crate::{channel::make_new_empty_channel, config::{ChannelConfig, Config}, retrieve::retrieve}; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -16,17 +21,37 @@ struct Args { fn main() { let args = Args::parse(); + let config = load_config(&args.config_file).unwrap(); + let channel = generate_channel(config.urls, config.channel); + write(&channel); +} - let config = match args.config_file.as_str() { - "" => Config::find_config().unwrap_or_else(|| - panic!("Failed to find config. Paths searched: {:#?}", config::default_config_file_locations())), - path => Config::parse_config_file_at_path(&path).unwrap_or_else(|err| - panic!("Failed to parse config at {}. Error: {}", path, err)), - }; +fn load_config(config_file_arg: &str) -> anyhow::Result<Config> { + match config_file_arg { + "" => Config::find_config().ok_or_else(|| + anyhow!("Failed to find config. Paths searched: {:#?}", config::default_config_file_locations())), + path => Config::parse_config_file_at_path(path).map_err(|err| + anyhow!("Failed to parse config at {}. Error: {}", path, err)), + } +} +fn generate_channel(urls: Vec<String>, channel_config: ChannelConfig) -> Channel { + let mut channel = make_new_empty_channel(channel_config); block_on(async { - for channel in retrieve(config.urls.iter().map(AsRef::as_ref)).await { - println!("{:#?}", channel); - } + retrieve(urls.iter().map(AsRef::as_ref)).await.for_each(|c| { + channel.items.append(&mut c.into_items()); + future::ready(()) + }).await; }); + channel +} + +fn write(channel: &Channel) { + let items_summary: Vec<(String, String)> = channel + .items().iter() + .map(|item| ( + item.title().map(String::from).unwrap_or_else(String::new), + item.pub_date().map(String::from).unwrap_or_else(String::new))) + .collect(); + println!("{:#?}", items_summary); } diff --git a/src/retrieve.rs b/src/retrieve.rs index ee904ab..3b89138 100644 --- a/src/retrieve.rs +++ b/src/retrieve.rs @@ -2,13 +2,12 @@ use futures::{StreamExt, stream}; use isahc::AsyncReadResponseExt; use rss::Channel; -pub async fn retrieve<'a, I>(urls: I) -> Vec<Channel> +pub async fn retrieve<'a, I>(urls: I) -> impl StreamExt<Item = Channel> where I : IntoIterator<Item = &'a str> { stream::iter(urls) .flat_map(|url| stream::once(retrieve_one_or_warn(url))) .filter_map(async |result| result.ok()) - .collect().await } async fn retrieve_one_or_warn(url: &str) -> anyhow::Result<Channel> |