gourami

[UNMAINTAINED] Activitypub server in Rust
Log | Files | Refs | README | LICENSE

commit 64c643f96501f0ef52092b19e51386ceb4cc81e3
parent 0ade28150f6314a735613409391250ca7612524b
Author: alex wennerberg <alex@alexwennerberg.com>
Date:   Wed, 29 Apr 2020 16:21:20 -0500

Add

Start on activitypub work

Diffstat:
MTODO | 2++
Amigrations/2020-04-29-162311_ap-restructure/down.sql | 2++
Amigrations/2020-04-29-162311_ap-restructure/up.sql | 6++++++
Msrc/ap.rs | 68++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
Msrc/db/note.rs | 59+++++++++++++++++++++++++++++++++++++++++++++--------------
Msrc/db/schema.rs | 4++++
Msrc/lib.rs | 1+
Msrc/routes.rs | 26+++++++++++++-------------
8 files changed, 127 insertions(+), 41 deletions(-)

diff --git a/TODO b/TODO @@ -16,6 +16,8 @@ text cant exist after URLs REFACTORING -- +Add limit of 255 characters for usernames and other limits to get urls to render + askama stuff MIDDLEWARE diff --git a/migrations/2020-04-29-162311_ap-restructure/down.sql b/migrations/2020-04-29-162311_ap-restructure/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` +\ No newline at end of file diff --git a/migrations/2020-04-29-162311_ap-restructure/up.sql b/migrations/2020-04-29-162311_ap-restructure/up.sql @@ -0,0 +1,6 @@ +-- Your SQL goes here + +alter table notes add is_remote boolean default false; +alter table notes add remote_url varchar(1000); -- semi-arbitrary +alter table notes add remote_creator varchar(1000); -- semi-arbitrary +alter table notes add remote_id varchar(1000); -- semi-arbitrary; diff --git a/src/ap.rs b/src/ap.rs @@ -1,21 +1,64 @@ +/// Users don't follow users in Gourami. Instead the server does hte following +/// There are a number of reasons for this: +/// Gives it a more 'community' feel -- everyone shares the same timeline +/// Much simpler from an engineering and user perspective -- I think its difficult for +/// non-engineering people to properly separate different audience +/// +/// This is a somewhat eccentric activitypub implementation, but it is as consistent with the spec +/// as I can make it! + use activitystreams::activity::{Accept, Activity, Announce, Create, Delete, Follow, Reject}; +use activitystreams::BaseBox; use log::debug; -use serde_json::Value; +use serde_json::{Value, Error}; +use serde_json::from_str; +use crate::db::note::{RemoteNoteInput}; // gonna be big -fn process_unstructured_ap(message: &str) { +// +// TODO -- use serde json here +fn process_unstructured_ap(message: &str) -> Result<(), Box<dyn std::error::Error>>{ // Actions usually associated with notes - use serde_json::from_str; // maybe there's a cleaner way to do this. cant iterate over types // TODO inbox forwarding https://www.w3.org/TR/activitypub/#inbox-forwarding - if let Some(create) = from_str::<Create>(message).ok() { - // create note database object - } else if let Some(delete) = from_str::<Delete>(message).ok() { - // delete note database object + let v: Value = serde_json::from_str(message)?; + let _type = v.get("type").ok_or("No type found")?; + if _type == "Create" { + let object = v.get("object").ok_or("No object found")?; + let _type = object.get("type").ok_or("No object type found")?; + if _type == "Note" { + let content = object.get("content").ok_or("No content found")?.as_str().ok_or("Not a string")?; + // clean content + // let in_reply_to = match object.get("inReplyTo") { + // Some(v) => Some(v.as_str().ok_or("Not a string")?), // TODO -- get reply from database + // None => None + // }; + let remote_creator = object.get("attributedTo").ok_or("No attributedTo found")?.as_str().ok_or("Not a string")?; + let remote_url = object.get("url").ok_or("No url Found")?.as_str().ok_or("Not a string")?; + let remote_id = object.get("id").ok_or("No ID found")?.as_str().ok_or("Not a string")?; + let new_remote_note = RemoteNoteInput { + content: content, + in_reply_to: None, + neighborhood: true, + is_remote: true, + user_id: -1, // for remote. placeholder. not sure what to do with this ultimately + remote_creator: remote_creator, + remote_id: remote_id, + remote_url: remote_url } ; + println!("{:?}", new_remote_note); + } } debug!("Unrecognized or invalid activity"); + Ok(()) +} + +fn new_note_to_ap_message() { } +// /// used to send to others +// fn generate_ap(activity: Activity) { +// } + #[cfg(test)] mod tests { use super::*; @@ -77,15 +120,12 @@ mod tests { } } -pub fn post_user_inbox(user_name: String, message: Value) {} - -pub fn post_user_outbox(user_name: String, message: Value) {} - -pub fn get_user_outbox(user_name: String) {} +pub fn post_inbox(user_name: String, message: Value) {} -// requires authentication -pub fn get_user_inbox(user_name: String) {} +pub fn post_outbox(user_name: String, message: Value) {} +// TODO figure out how to follow mastodon +// pub fn user_followers(user_name: String) {} pub fn user_following(user_name: String) {} diff --git a/src/db/note.rs b/src/db/note.rs @@ -14,10 +14,14 @@ pub struct Note { // rename RenderedNote pub id: i32, pub user_id: i32, pub in_reply_to: Option<i32>, - // deserialize wiht + #[serde(deserialize_with="render_content")] pub content: String, pub created_time: String, pub neighborhood: bool, + pub is_remote: bool, + pub remote_url: Option<String>, + pub remote_creator: Option<String>, + pub remote_id: Option<String> } /// Content in the DB is stored in plaintext (WILL BE) @@ -30,16 +34,43 @@ where D: Deserializer<'de> { return Ok(parse_note_text(s)); } -#[derive(Insertable, Clone)] +/// Run on both write to db and read from db, for redundancy +/// Prevents malicious content from being rendered +/// See the mastodon page for inspiration: https://docs.joinmastodon.org/spec/activitypub/ +/// This is currently very aggressive -- maybe we could loosen it a bit +/// We probably want to allow microformats and some accessibiltiy tags +fn remove_unnacceptable_html(input_text: &str) -> String { + let ok_tags = hashset!["br", "p", "span"]; + let html_clean = ammonia::Builder::default() + .tags(ok_tags) + .clean(input_text) + .to_string(); + return html_clean +} + +#[derive(Insertable, Clone, Debug)] #[table_name = "notes"] pub struct NoteInput { //pub id: i32, //unsigned? pub user_id: i32, - pub content: String, // can we make this a slice? + pub content: String, // TODO make slice pub in_reply_to: Option<i32>, pub neighborhood: bool, } +#[derive(Insertable, Clone, Debug)] +#[table_name = "notes"] +pub struct RemoteNoteInput<'a> { + pub user_id: i32, + pub content: &'a str, + pub in_reply_to: Option<i32>, + pub neighborhood: bool, + pub is_remote: bool, + pub remote_creator: &'a str, + pub remote_url: &'a str, + pub remote_id: &'a str, +} + /// We render the first >>[num] or note emoji as a reply, for threading. pub fn get_reply(note_text: &str) -> Option<i32> { let re = Regex::new(r"\B(📝|>>)(\d+)").unwrap(); @@ -55,14 +86,14 @@ pub fn get_reply(note_text: &str) -> Option<i32> { pub fn parse_note_text(text: &str) -> String { // There shouldn't be any html tags in the db, but // Let's strip it out just in case - let html_clean = ammonia::clean_text(text); + let html_clean = remove_unnacceptable_html(text); if text.len() == 0 { return String::new(); } // this regex has to function after html parsing has happened. very weird. let re = Regex::new( r"(?ix) - \b(([\w-]+:&\#47;&\#47;?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|&\#47))) + \b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))) ", ) .unwrap(); @@ -78,6 +109,7 @@ pub fn parse_note_text(text: &str) -> String { let replace_str = "<a href=\"/user/$2\">$0</a>"; let people_parsed = person_regex.replace_all(&notes_parsed, &replace_str as &str).to_string(); // TODO get mentions too + println!("{}", people_parsed); return people_parsed; } @@ -93,20 +125,19 @@ mod tests { #[test] fn test_escape_html() { let example = "<script>haxxor</script>hi>"; - assert!(parse_note_text(example) == ammonia::clean_text(example)); + assert!(parse_note_text(example) == "hi&gt;"); } #[test] fn test_string_without_urls() { let src = "<p>Some HTML</p>"; - assert!(parse_note_text(src) == ammonia::clean_text(src)); + assert!(parse_note_text(src) == src); } #[test] - fn test_string_with_http_urls() { - let src = "Check this out: https://doc.rust-lang.org/\n - https://fr.wikipedia.org/wiki/Caf%C3%A9ine"; - let linked = "Check&#32;this&#32;out:&#32;<a href=\"https:&#47;&#47;doc.rust-lang.org&#47;&#10;&#10;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;https:&#47;&#47;fr.wikipedia.org&#47;wiki&#47;Caf%C3%A9ine\">https:&#47;&#47;doc.rust-lang.org&#47;&#10;&#10;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;&#32;https:&#47;&#47;fr.wikipedia.org&#47;wiki&#47;Caf%C3%A9ine</a>"; + fn test_string_with_http_urls() { // TODO fix test + let src = "Check this out: https://doc.rust-lang.org"; + let linked = "Check this out: <a href=\"https://doc.rust-lang.org\">https://doc.rust-lang.org</a>"; assert!(parse_note_text(src) == linked) } @@ -115,21 +146,21 @@ mod tests { let src = "Send spam to mailto://oz@cypr.io"; assert!( parse_note_text(src) - == "Send&#32;spam&#32;to&#32;<a href=\"mailto:&#47;&#47;oz@cypr.io\">mailto:&#47;&#47;oz@cypr.io</a>" + == "Send spam to <a href=\"mailto://oz@cypr.io\">mailto://oz@cypr.io</a>" ) } #[test] fn test_user_replace() { let src = "@joe whats up @sally"; - let linked = "<a href=\"/user/joe\">@joe</a>&#32;whats&#32;up&#32;<a href=\"/user/sally\">@sally</a>"; + let linked = "<a href=\"/user/joe\">@joe</a> whats up <a href=\"/user/sally\">@sally</a>"; assert!(parse_note_text(src) == linked) } #[test] fn test_note_replace() { let src = "📝123 cool post >>456"; - let linked = "<a href=\"/note/123\">📝123</a>&#32;cool&#32;post&#32;<a href=\"/note/456\">&gt;&gt;456</a>"; + let linked = "<a href=\"/note/123\">📝123</a> cool post <a href=\"/note/456\">&gt;&gt;456</a>"; assert!(parse_note_text(src) == linked) } diff --git a/src/db/schema.rs b/src/db/schema.rs @@ -6,6 +6,10 @@ table! { content -> Text, created_time -> Timestamp, neighborhood -> Bool, + is_remote -> Bool, + remote_url -> Nullable<Varchar>, + remote_creator -> Nullable<Varchar>, + remote_id -> Nullable<Varchar>, } } diff --git a/src/lib.rs b/src/lib.rs @@ -181,6 +181,7 @@ fn new_note(auth_user: User, note_input: &str, neighborhood: bool,) -> Result<() insert_into(nv::notification_viewers).values(new_nv).execute(conn)?; } + // ap::generate_ap(ap::Activity::create_note); // generate activitypub object from post request // send to outbox // add notification diff --git a/src/routes.rs b/src/routes.rs @@ -101,28 +101,28 @@ pub async fn run_server() { .and(warp::fs::dir("./static")); // activityPub stuff - // + // This stuff should filter based on the application headers // POST - let post_user_inbox = path!("user" / String / "inbox.json" ) - .and(json()) - .map(ap::post_user_inbox); + // let post_server_inbox = path!("user" / String / "inbox.json" ) + // .and(json()) + // .map(ap::post_user_inbox); - let post_user_outbox = path!("user" / String / "outbox.json" ) - .and(json()) - .map(ap::post_user_outbox); +// let post_user_outbox = path!("user" / String / "outbox.json" ) +// .and(json()) +// .map(ap::post_user_outbox); - let get_user_outbox = path!("user" / String / "outbox.json" ) - .map(ap::get_user_outbox); +// let get_user_outbox = path!("user" / String / "outbox.json" ) +// .map(ap::get_user_outbox); // let get_user_inbox = path!("user" / String / "outbox.json" ) // .and(json()) // .map(ap::post_user_outbox); - let user_followers = path!("user" / String / "followers.json" ) - .map(ap::user_followers); +// let user_followers = path!("user" / String / "followers.json" ) +// .map(ap::user_followers); - let user_following = path!("user" / String / "following.json" ) - .map(ap::user_following); +// let user_following = path!("user" / String / "following.json" ) +// .map(ap::user_following); // https://github.com/seanmonstar/warp/issues/42 -- how to set up diesel // TODO set content length limit