aggregate feeds into one

6 files changed, 259 insertions, 16 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 640f231..75fe398 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,6 +3,15 @@
 version = 4
 
 [[package]]
+name = "android_system_properties"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
+dependencies = [
+ "libc",
+]
+
+[[package]]
 name = "anstream"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -124,6 +133,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
 
 [[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
 name = "bytes"
 version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -162,7 +177,11 @@ version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
+ "iana-time-zone",
+ "js-sys",
  "num-traits",
+ "wasm-bindgen",
+ "windows-link",
 ]
 
 [[package]]
@@ -633,6 +652,30 @@ dependencies = [
 ]
 
 [[package]]
+name = "iana-time-zone"
+version = "0.1.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
 name = "icu_collections"
 version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -851,6 +894,17 @@ dependencies = [
 ]
 
 [[package]]
+name = "js-sys"
+version = "0.3.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
 name = "libc"
 version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1116,6 +1170,7 @@ dependencies = [
  "atom_syndication",
  "derive_builder",
  "quick-xml",
+ "serde",
 ]
 
 [[package]]
@@ -1123,6 +1178,7 @@ name = "rss-aggregator"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "chrono",
  "clap",
  "dirs",
  "futures",
@@ -1616,6 +1672,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
+name = "wasm-bindgen"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.127"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
 name = "webpki-root-certs"
 version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1634,12 +1735,65 @@ dependencies = [
 ]
 
 [[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
 name = "windows-link"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
 
 [[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
 name = "windows-sys"
 version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 1f1d497..80d1078 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,7 +8,8 @@ anyhow = "1.0.104"
 clap = { version = "4.6.6", features = ["derive"] }
 dirs = "6.0.0"
 futures = "0.3.34"
-rss = "2.1.0"
+rss = { version = "2.1.0", features = ["serde"] }
 serde = { version = "1.0.229", features = ["derive"] }
 toml = "1.1.5"
 isahc = "2.0.1"
+chrono = "0.4.45"
diff --git a/src/channel.rs b/src/channel.rs
new file mode 100644
index 0000000..f4d4941
--- /dev/null
+++ b/src/channel.rs
@@ -0,0 +1,34 @@
+use chrono::Utc;
+use rss::Channel;
+
+use crate::config::ChannelConfig;
+
+pub fn make_new_empty_channel(config: ChannelConfig) -> Channel {
+    Channel {
+        title: config.title,
+        link: config.link,
+        description: config.description,
+        language: config.language,
+        copyright: config.copyright,
+        managing_editor: config.managing_editor,
+        webmaster: config.webmaster,
+        pub_date: Some(Utc::now().to_rfc2822()),
+        last_build_date: Some(Utc::now().to_rfc2822()),
+        categories: config.categories,
+        generator: Some(String::from("https://git.joeac.net/rss-aggregator.git")),
+        docs: Some(String::from("https://git.joeac.net/rss-aggregator.git/tree/README.md")),
+        cloud: config.cloud,
+        ttl: config.ttl,
+        image: config.image,
+        text_input: config.text_input,
+        skip_hours: config.skip_hours,
+        skip_days: config.skip_days,
+        items: Vec::new(),
+        extensions: config.extensions,
+        itunes_ext: config.itunes_ext,
+        dublin_core_ext: config.dublin_core_ext,
+        syndication_ext: config.syndication_ext,
+        namespaces: config.namespaces,
+        rating: None,
+    }
+}
diff --git a/src/config.rs b/src/config.rs
index 089d3e2..a5652c1 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,11 +1,41 @@
-use std::{fs::File, io::{read_to_string}, path::Path};
+use std::{collections::BTreeMap, fs::File, io::read_to_string, path::Path};
 
+use rss::{Category, Cloud, Image, TextInput, extension::{Extension, dublincore::DublinCoreExtension, itunes::ITunesChannelExtension, syndication::SyndicationExtension}};
 use serde::Deserialize;
 
 #[derive(Deserialize)]
 pub struct Config {
     #[serde(default)]
-    pub urls: Vec<String>
+    pub urls: Vec<String>,
+    pub channel: ChannelConfig,
+}
+
+#[derive(Deserialize)]
+pub struct ChannelConfig {
+    pub title: String,
+    pub link: String,
+    pub description: String,
+    pub language: Option<String>,
+    pub copyright: Option<String>,
+    pub managing_editor: Option<String>,
+    pub webmaster: Option<String>,
+    #[serde(default)]
+    pub categories: Vec<Category>,
+    pub cloud: Option<Cloud>,
+    pub ttl: Option<String>,
+    pub image: Option<Image>,
+    pub text_input: Option<TextInput>,
+    #[serde(default)]
+    pub skip_hours: Vec<String>,
+    #[serde(default)]
+    pub skip_days: Vec<String>,
+    #[serde(default)]
+    pub extensions: BTreeMap<String, BTreeMap<String, Vec<Extension>>>,
+    pub itunes_ext: Option<ITunesChannelExtension>,
+    pub dublin_core_ext: Option<DublinCoreExtension>,
+    pub syndication_ext: Option<SyndicationExtension>,
+    #[serde(default)]
+    pub namespaces: BTreeMap<String, String>,
 }
 
 pub fn default_config_file_locations() -> Vec<String> {
diff --git a/src/main.rs b/src/main.rs
index ae68477..61701ab 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,10 +1,15 @@
+mod channel;
 mod config;
 mod retrieve;
 
+use std::future;
+
+use anyhow::anyhow;
 use clap::Parser;
-use futures::executor::block_on;
+use futures::{StreamExt, executor::block_on};
+use rss::Channel;
 
-use crate::{config::Config, retrieve::retrieve};
+use crate::{channel::make_new_empty_channel, config::{ChannelConfig, Config}, retrieve::retrieve};
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
@@ -16,17 +21,37 @@ struct Args {
 
 fn main() {
     let args = Args::parse();
+    let config = load_config(&args.config_file).unwrap();
+    let channel = generate_channel(config.urls, config.channel);
+    write(&channel);
+}
 
-    let config = match args.config_file.as_str() {
-        "" => Config::find_config().unwrap_or_else(||
-            panic!("Failed to find config. Paths searched: {:#?}", config::default_config_file_locations())),
-        path => Config::parse_config_file_at_path(&path).unwrap_or_else(|err|
-            panic!("Failed to parse config at {}. Error: {}", path, err)),
-    };
+fn load_config(config_file_arg: &str) -> anyhow::Result<Config> {
+    match config_file_arg {
+        "" => Config::find_config().ok_or_else(||
+            anyhow!("Failed to find config. Paths searched: {:#?}", config::default_config_file_locations())),
+        path => Config::parse_config_file_at_path(path).map_err(|err|
+            anyhow!("Failed to parse config at {}. Error: {}", path, err)),
+    }
+}
 
+fn generate_channel(urls: Vec<String>, channel_config: ChannelConfig) -> Channel {
+    let mut channel = make_new_empty_channel(channel_config);
     block_on(async {
-        for channel in retrieve(config.urls.iter().map(AsRef::as_ref)).await {
-            println!("{:#?}", channel);
-        }
+        retrieve(urls.iter().map(AsRef::as_ref)).await.for_each(|c| {
+            channel.items.append(&mut c.into_items());
+            future::ready(())
+        }).await;
     });
+    channel
+}
+
+fn write(channel: &Channel) {
+    let items_summary: Vec<(String, String)> = channel
+            .items().iter()
+            .map(|item| (
+                item.title().map(String::from).unwrap_or_else(String::new),
+                item.pub_date().map(String::from).unwrap_or_else(String::new)))
+            .collect();
+    println!("{:#?}", items_summary);
 }
diff --git a/src/retrieve.rs b/src/retrieve.rs
index ee904ab..3b89138 100644
--- a/src/retrieve.rs
+++ b/src/retrieve.rs
@@ -2,13 +2,12 @@ use futures::{StreamExt, stream};
 use isahc::AsyncReadResponseExt;
 use rss::Channel;
 
-pub async fn retrieve<'a, I>(urls: I) -> Vec<Channel>
+pub async fn retrieve<'a, I>(urls: I) -> impl StreamExt<Item = Channel>
     where I : IntoIterator<Item = &'a str>
 {
     stream::iter(urls)
         .flat_map(|url| stream::once(retrieve_one_or_warn(url)))
         .filter_map(async |result| result.ok())
-        .collect().await
 }
 
 async fn retrieve_one_or_warn(url: &str) -> anyhow::Result<Channel>