backups/grapevine
1
0
Fork 0
mirror of https://gitlab.computer.surgery/matrix/grapevine.git synced 2025-12-18 08:11:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

change rustfmt configuration

Browse source 
This change is fully automated, except the `rustfmt.toml` changes and
a few clippy directives to allow specific functions with too many lines
because they are longer now.
This commit is contained in:
Charles Hall 2024-05-16 01:19:04 -07:00
parent 40d6ce230d
commit 0afc1d2f50
No known key found for this signature in database
GPG key ID: 7B8E0645816E07CF
123 changed files with 7881 additions and 4687 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/database/abstraction.rs
View file

@ -1,8 +1,8 @@
use std::{future::Future, pin::Pin, sync::Arc};
use super::Config;
use crate::Result;
use std::{future::Future, pin::Pin, sync::Arc};
#[cfg(feature = "sqlite")]
pub(crate) mod sqlite;
@ -22,7 +22,8 @@ pub(crate) trait KeyValueDatabaseEngine: Send + Sync {
Ok(())
}
fn memory_usage(&self) -> Result<String> {
Ok("Current database engine does not support memory usage reporting.".to_owned())
Ok("Current database engine does not support memory usage reporting."
.to_owned())
}
fn clear_caches(&self) {}
}
@ -31,7 +32,10 @@ pub(crate) trait KvTree: Send + Sync {
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>>;
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>;
fn insert_batch(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()>;
fn insert_batch(
&self,
iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>,
) -> Result<()>;
fn remove(&self, key: &[u8]) -> Result<()>;
@ -44,14 +48,20 @@ pub(crate) trait KvTree: Send + Sync {
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
fn increment_batch(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;
fn increment_batch(
&self,
iter: &mut dyn Iterator<Item = Vec<u8>>,
) -> Result<()>;
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
fn watch_prefix<'a>(
&'a self,
prefix: &[u8],
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
fn clear(&self) -> Result<()> {
for (key, _) in self.iter() {

86
src/database/abstraction/rocksdb.rs
View file

@ -1,17 +1,21 @@
use rocksdb::{
perf::get_memory_usage_stats, BlockBasedOptions, BoundColumnFamily, Cache,
ColumnFamilyDescriptor, DBCompactionStyle, DBCompressionType, DBRecoveryMode, DBWithThreadMode,
Direction, IteratorMode, MultiThreaded, Options, ReadOptions, WriteOptions,
};
use super::{super::Config, watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{utils, Result};
use std::{
future::Future,
pin::Pin,
sync::{Arc, RwLock},
};
use rocksdb::{
perf::get_memory_usage_stats, BlockBasedOptions, BoundColumnFamily, Cache,
ColumnFamilyDescriptor, DBCompactionStyle, DBCompressionType,
DBRecoveryMode, DBWithThreadMode, Direction, IteratorMode, MultiThreaded,
Options, ReadOptions, WriteOptions,
};
use super::{
super::Config, watchers::Watchers, KeyValueDatabaseEngine, KvTree,
};
use crate::{utils, Result};
pub(crate) struct Engine {
rocks: DBWithThreadMode<MultiThreaded>,
max_open_files: i32,
@ -38,7 +42,8 @@ fn db_options(max_open_files: i32, rocksdb_cache: &Cache) -> Options {
let mut db_opts = Options::default();
db_opts.set_block_based_table_factory(&block_based_options);
db_opts.create_if_missing(true);
db_opts.increase_parallelism(num_cpus::get().try_into().unwrap_or(i32::MAX));
db_opts
.increase_parallelism(num_cpus::get().try_into().unwrap_or(i32::MAX));
db_opts.set_max_open_files(max_open_files);
db_opts.set_compression_type(DBCompressionType::Lz4);
db_opts.set_bottommost_compression_type(DBCompressionType::Zstd);
@ -69,13 +74,17 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
clippy::cast_sign_loss,
clippy::cast_possible_truncation
)]
let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
let cache_capacity_bytes =
(config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
let rocksdb_cache = Cache::new_lru_cache(cache_capacity_bytes);
let db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache);
let cfs = DBWithThreadMode::<MultiThreaded>::list_cf(&db_opts, &config.database_path)
.unwrap_or_default();
let cfs = DBWithThreadMode::<MultiThreaded>::list_cf(
&db_opts,
&config.database_path,
)
.unwrap_or_default();
let db = DBWithThreadMode::<MultiThreaded>::open_cf_descriptors(
&db_opts,
@ -119,14 +128,14 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
#[allow(clippy::as_conversions, clippy::cast_precision_loss)]
fn memory_usage(&self) -> Result<String> {
let stats = get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?;
let stats =
get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?;
Ok(format!(
"Approximate memory usage of all the mem-tables: {:.3} MB\n\
Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\
Approximate memory usage of all the table readers: {:.3} MB\n\
Approximate memory usage by cache: {:.3} MB\n\
Approximate memory usage by cache pinned: {:.3} MB\n\
",
"Approximate memory usage of all the mem-tables: {:.3} \
MB\nApproximate memory usage of un-flushed mem-tables: {:.3} \
MB\nApproximate memory usage of all the table readers: {:.3} \
MB\nApproximate memory usage by cache: {:.3} MB\nApproximate \
memory usage by cache pinned: {:.3} MB\n",
stats.mem_table_total as f64 / 1024.0 / 1024.0,
stats.mem_table_unflushed as f64 / 1024.0 / 1024.0,
stats.mem_table_readers_total as f64 / 1024.0 / 1024.0,
@ -154,9 +163,7 @@ impl KvTree for RocksDbEngineTree<'_> {
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
let writeoptions = WriteOptions::default();
let lock = self.write_lock.read().unwrap();
self.db
.rocks
.put_cf_opt(&self.cf(), key, value, &writeoptions)?;
self.db.rocks.put_cf_opt(&self.cf(), key, value, &writeoptions)?;
drop(lock);
self.watchers.wake(key);
@ -164,12 +171,13 @@ impl KvTree for RocksDbEngineTree<'_> {
Ok(())
}
fn insert_batch(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
fn insert_batch(
&self,
iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>,
) -> Result<()> {
let writeoptions = WriteOptions::default();
for (key, value) in iter {
self.db
.rocks
.put_cf_opt(&self.cf(), key, value, &writeoptions)?;
self.db.rocks.put_cf_opt(&self.cf(), key, value, &writeoptions)?;
}
Ok(())
@ -177,10 +185,7 @@ impl KvTree for RocksDbEngineTree<'_> {
fn remove(&self, key: &[u8]) -> Result<()> {
let writeoptions = WriteOptions::default();
Ok(self
.db
.rocks
.delete_cf_opt(&self.cf(), key, &writeoptions)?)
Ok(self.db.rocks.delete_cf_opt(&self.cf(), key, &writeoptions)?)
}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
@ -230,26 +235,26 @@ impl KvTree for RocksDbEngineTree<'_> {
let old = self.db.rocks.get_cf_opt(&self.cf(), key, &readoptions)?;
let new = utils::increment(old.as_deref());
self.db
.rocks
.put_cf_opt(&self.cf(), key, &new, &writeoptions)?;
self.db.rocks.put_cf_opt(&self.cf(), key, &new, &writeoptions)?;
drop(lock);
Ok(new)
}
fn increment_batch(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
fn increment_batch(
&self,
iter: &mut dyn Iterator<Item = Vec<u8>>,
) -> Result<()> {
let readoptions = ReadOptions::default();
let writeoptions = WriteOptions::default();
let lock = self.write_lock.write().unwrap();
for key in iter {
let old = self.db.rocks.get_cf_opt(&self.cf(), &key, &readoptions)?;
let old =
self.db.rocks.get_cf_opt(&self.cf(), &key, &readoptions)?;
let new = utils::increment(old.as_deref());
self.db
.rocks
.put_cf_opt(&self.cf(), key, new, &writeoptions)?;
self.db.rocks.put_cf_opt(&self.cf(), key, new, &writeoptions)?;
}
drop(lock);
@ -277,7 +282,10 @@ impl KvTree for RocksDbEngineTree<'_> {
)
}
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
fn watch_prefix<'a>(
&'a self,
prefix: &[u8],
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
self.watchers.watch(prefix)
}
}

102
src/database/abstraction/sqlite.rs
View file

@ -1,7 +1,3 @@
use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{database::Config, Result};
use parking_lot::{Mutex, MutexGuard};
use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
use std::{
cell::RefCell,
future::Future,
@ -9,9 +5,15 @@ use std::{
pin::Pin,
sync::Arc,
};
use parking_lot::{Mutex, MutexGuard};
use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
use thread_local::ThreadLocal;
use tracing::debug;
use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{database::Config, Result};
thread_local! {
static READ_CONNECTION: RefCell<Option<&'static Connection>> = RefCell::new(None);
static READ_CONNECTION_ITERATOR: RefCell<Option<&'static Connection>> = RefCell::new(None);
@ -68,7 +70,11 @@ impl Engine {
conn.pragma_update(Some(Main), "page_size", 2048)?;
conn.pragma_update(Some(Main), "journal_mode", "WAL")?;
conn.pragma_update(Some(Main), "synchronous", "NORMAL")?;
conn.pragma_update(Some(Main), "cache_size", -i64::from(cache_size_kb))?;
conn.pragma_update(
Some(Main),
"cache_size",
-i64::from(cache_size_kb),
)?;
conn.pragma_update(Some(Main), "wal_autocheckpoint", 0)?;
Ok(conn)
@ -79,18 +85,23 @@ impl Engine {
}
fn read_lock(&self) -> &Connection {
self.read_conn_tls
.get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
self.read_conn_tls.get_or(|| {
Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap()
})
}
fn read_lock_iterator(&self) -> &Connection {
self.read_iterator_conn_tls
.get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
self.read_iterator_conn_tls.get_or(|| {
Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap()
})
}
pub(crate) fn flush_wal(self: &Arc<Self>) -> Result<()> {
self.write_lock()
.pragma_update(Some(Main), "wal_checkpoint", "RESTART")?;
self.write_lock().pragma_update(
Some(Main),
"wal_checkpoint",
"RESTART",
)?;
Ok(())
}
}
@ -108,7 +119,8 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
// calculates cache-size per permanent connection
// 1. convert MB to KiB
// 2. divide by permanent connections + permanent iter connections + write connection
// 2. divide by permanent connections + permanent iter connections +
// write connection
// 3. round down to nearest integer
#[allow(
clippy::as_conversions,
@ -117,9 +129,11 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
clippy::cast_sign_loss
)]
let cache_size_per_thread = ((config.db_cache_capacity_mb * 1024.0)
/ ((num_cpus::get() as f64 * 2.0) + 1.0)) as u32;
/ ((num_cpus::get() as f64 * 2.0) + 1.0))
as u32;
let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?);
let writer =
Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?);
let arc = Arc::new(Engine {
writer,
@ -133,7 +147,13 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
}
fn open_tree(&self, name: &str) -> Result<Arc<dyn KvTree>> {
self.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {name} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )"), [])?;
self.write_lock().execute(
&format!(
"CREATE TABLE IF NOT EXISTS {name} ( \"key\" BLOB PRIMARY \
KEY, \"value\" BLOB NOT NULL )"
),
[],
)?;
Ok(Arc::new(SqliteTable {
engine: Arc::clone(self),
@ -161,14 +181,26 @@ pub(crate) struct SqliteTable {
type TupleOfBytes = (Vec<u8>, Vec<u8>);
impl SqliteTable {
fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
fn get_with_guard(
&self,
guard: &Connection,
key: &[u8],
) -> Result<Option<Vec<u8>>> {
Ok(guard
.prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
.prepare(
format!("SELECT value FROM {} WHERE key = ?", self.name)
.as_str(),
)?
.query_row([key], |row| row.get(0))
.optional()?)
}
fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> {
fn insert_with_guard(
&self,
guard: &Connection,
key: &[u8],
value: &[u8],
) -> Result<()> {
guard.execute(
format!(
"INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)",
@ -222,7 +254,10 @@ impl KvTree for SqliteTable {
Ok(())
}
fn insert_batch(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
fn insert_batch(
&self,
iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>,
) -> Result<()> {
let guard = self.engine.write_lock();
guard.execute("BEGIN", [])?;
@ -236,7 +271,10 @@ impl KvTree for SqliteTable {
Ok(())
}
fn increment_batch(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
fn increment_batch(
&self,
iter: &mut dyn Iterator<Item = Vec<u8>>,
) -> Result<()> {
let guard = self.engine.write_lock();
guard.execute("BEGIN", [])?;
@ -282,7 +320,8 @@ impl KvTree for SqliteTable {
let statement = Box::leak(Box::new(
guard
.prepare(&format!(
"SELECT key, value FROM {} WHERE key <= ? ORDER BY key DESC",
"SELECT key, value FROM {} WHERE key <= ? ORDER BY \
key DESC",
&self.name
))
.unwrap(),
@ -292,7 +331,9 @@ impl KvTree for SqliteTable {
let iterator = Box::new(
statement
.query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
.query_map([from], |row| {
Ok((row.get_unwrap(0), row.get_unwrap(1)))
})
.unwrap()
.map(Result::unwrap),
);
@ -304,7 +345,8 @@ impl KvTree for SqliteTable {
let statement = Box::leak(Box::new(
guard
.prepare(&format!(
"SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC",
"SELECT key, value FROM {} WHERE key >= ? ORDER BY \
key ASC",
&self.name
))
.unwrap(),
@ -314,7 +356,9 @@ impl KvTree for SqliteTable {
let iterator = Box::new(
statement
.query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
.query_map([from], |row| {
Ok((row.get_unwrap(0), row.get_unwrap(1)))
})
.unwrap()
.map(Result::unwrap),
);
@ -338,14 +382,20 @@ impl KvTree for SqliteTable {
Ok(new)
}
fn scan_prefix<'a>(&'a self, prefix: Vec<u8>) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
Box::new(
self.iter_from(&prefix, false)
.take_while(move |(key, _)| key.starts_with(&prefix)),
)
}
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
fn watch_prefix<'a>(
&'a self,
prefix: &[u8],
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
self.watchers.watch(prefix)
}

8
src/database/abstraction/watchers.rs
View file

@ -4,12 +4,14 @@ use std::{
pin::Pin,
sync::RwLock,
};
use tokio::sync::watch;
#[derive(Default)]
pub(super) struct Watchers {
#[allow(clippy::type_complexity)]
watchers: RwLock<HashMap<Vec<u8>, (watch::Sender<()>, watch::Receiver<()>)>>,
watchers:
RwLock<HashMap<Vec<u8>, (watch::Sender<()>, watch::Receiver<()>)>>,
}
impl Watchers {
@ -17,7 +19,8 @@ impl Watchers {
&'a self,
prefix: &[u8],
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec()) {
let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec())
{
hash_map::Entry::Occupied(o) => o.get().1.clone(),
hash_map::Entry::Vacant(v) => {
let (tx, rx) = tokio::sync::watch::channel(());
@ -31,6 +34,7 @@ impl Watchers {
rx.changed().await.unwrap();
})
}
pub(super) fn wake(&self, key: &[u8]) {
let watchers = self.watchers.read().unwrap();
let mut triggered = Vec::new();

64
src/database/key_value/account_data.rs
View file

@ -7,10 +7,13 @@ use ruma::{
RoomId, UserId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::account_data::Data for KeyValueDatabase {
/// Places one event in the account data of the user and removes the previous entry.
/// Places one event in the account data of the user and removes the
/// previous entry.
#[tracing::instrument(skip(self, room_id, user_id, event_type, data))]
fn update(
&self,
@ -24,13 +27,14 @@ impl service::account_data::Data for KeyValueDatabase {
.unwrap_or_default()
.as_bytes()
.to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(user_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
let mut roomuserdataid = prefix.clone();
roomuserdataid.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
roomuserdataid.push(0xff);
roomuserdataid
.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
roomuserdataid.push(0xFF);
roomuserdataid.extend_from_slice(event_type.to_string().as_bytes());
let mut key = prefix;
@ -45,13 +49,13 @@ impl service::account_data::Data for KeyValueDatabase {
self.roomuserdataid_accountdata.insert(
&roomuserdataid,
&serde_json::to_vec(&data).expect("to_vec always works on json values"),
&serde_json::to_vec(&data)
.expect("to_vec always works on json values"),
)?;
let prev = self.roomusertype_roomuserdataid.get(&key)?;
self.roomusertype_roomuserdataid
.insert(&key, &roomuserdataid)?;
self.roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?;
// Remove old entry
if let Some(prev) = prev {
@ -74,17 +78,15 @@ impl service::account_data::Data for KeyValueDatabase {
.unwrap_or_default()
.as_bytes()
.to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(kind.to_string().as_bytes());
self.roomusertype_roomuserdataid
.get(&key)?
.and_then(|roomuserdataid| {
self.roomuserdataid_accountdata
.get(&roomuserdataid)
.transpose()
self.roomuserdataid_accountdata.get(&roomuserdataid).transpose()
})
.transpose()?
.map(|data| {
@ -101,7 +103,8 @@ impl service::account_data::Data for KeyValueDatabase {
room_id: Option<&RoomId>,
user_id: &UserId,
since: u64,
) -> Result<HashMap<RoomAccountDataEventType, Raw<AnyEphemeralRoomEvent>>> {
) -> Result<HashMap<RoomAccountDataEventType, Raw<AnyEphemeralRoomEvent>>>
{
let mut userdata = HashMap::new();
let mut prefix = room_id
@ -109,9 +112,9 @@ impl service::account_data::Data for KeyValueDatabase {
.unwrap_or_default()
.as_bytes()
.to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(user_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
// Skip the data that's exactly at since, because we sent that last time
let mut first_possible = prefix.clone();
@ -124,14 +127,27 @@ impl service::account_data::Data for KeyValueDatabase {
.map(|(k, v)| {
Ok::<_, Error>((
RoomAccountDataEventType::from(
utils::string_from_bytes(k.rsplit(|&b| b == 0xff).next().ok_or_else(
|| Error::bad_database("RoomUserData ID in db is invalid."),
)?)
.map_err(|_| Error::bad_database("RoomUserData ID in db is invalid."))?,
utils::string_from_bytes(
k.rsplit(|&b| b == 0xFF).next().ok_or_else(
|| {
Error::bad_database(
"RoomUserData ID in db is invalid.",
)
},
)?,
)
.map_err(|_| {
Error::bad_database(
"RoomUserData ID in db is invalid.",
)
})?,
),
serde_json::from_slice::<Raw<AnyEphemeralRoomEvent>>(&v).map_err(|_| {
Error::bad_database("Database contains invalid account data.")
})?,
serde_json::from_slice::<Raw<AnyEphemeralRoomEvent>>(&v)
.map_err(|_| {
Error::bad_database(
"Database contains invalid account data.",
)
})?,
))
})
{

26
src/database/key_value/appservice.rs
View file

@ -20,8 +20,7 @@ impl service::appservice::Data for KeyValueDatabase {
///
/// * `service_name` - the name you send to register the service previously
fn unregister_appservice(&self, service_name: &str) -> Result<()> {
self.id_appserviceregistrations
.remove(service_name.as_bytes())?;
self.id_appserviceregistrations.remove(service_name.as_bytes())?;
Ok(())
}
@ -30,20 +29,25 @@ impl service::appservice::Data for KeyValueDatabase {
.get(id.as_bytes())?
.map(|bytes| {
serde_yaml::from_slice(&bytes).map_err(|_| {
Error::bad_database("Invalid registration bytes in id_appserviceregistrations.")
Error::bad_database(
"Invalid registration bytes in \
id_appserviceregistrations.",
)
})
})
.transpose()
}
fn iter_ids<'a>(&'a self) -> Result<Box<dyn Iterator<Item = Result<String>> + 'a>> {
Ok(Box::new(self.id_appserviceregistrations.iter().map(
|(id, _)| {
utils::string_from_bytes(&id).map_err(|_| {
Error::bad_database("Invalid id bytes in id_appserviceregistrations.")
})
},
)))
fn iter_ids<'a>(
&'a self,
) -> Result<Box<dyn Iterator<Item = Result<String>> + 'a>> {
Ok(Box::new(self.id_appserviceregistrations.iter().map(|(id, _)| {
utils::string_from_bytes(&id).map_err(|_| {
Error::bad_database(
"Invalid id bytes in id_appserviceregistrations.",
)
})
})))
}
fn all(&self) -> Result<Vec<(String, Registration)>> {

88
src/database/key_value/globals.rs
View file

@ -6,10 +6,13 @@ use lru_cache::LruCache;
use ruma::{
api::federation::discovery::{ServerSigningKeys, VerifyKey},
signatures::Ed25519KeyPair,
DeviceId, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, UserId,
DeviceId, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName,
UserId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
pub(crate) const COUNTER: &[u8] = b"c";
@ -27,14 +30,18 @@ impl service::globals::Data for KeyValueDatabase {
})
}
async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> {
async fn watch(
&self,
user_id: &UserId,
device_id: &DeviceId,
) -> Result<()> {
let userid_bytes = user_id.as_bytes().to_vec();
let mut userid_prefix = userid_bytes.clone();
userid_prefix.push(0xff);
userid_prefix.push(0xFF);
let mut userdeviceid_prefix = userid_prefix.clone();
userdeviceid_prefix.extend_from_slice(device_id.as_bytes());
userdeviceid_prefix.push(0xff);
userdeviceid_prefix.push(0xFF);
let mut futures = FuturesUnordered::new();
@ -46,10 +53,10 @@ impl service::globals::Data for KeyValueDatabase {
futures.push(self.userroomid_invitestate.watch_prefix(&userid_prefix));
futures.push(self.userroomid_leftstate.watch_prefix(&userid_prefix));
futures.push(
self.userroomid_notificationcount
.watch_prefix(&userid_prefix),
self.userroomid_notificationcount.watch_prefix(&userid_prefix),
);
futures.push(self.userroomid_highlightcount.watch_prefix(&userid_prefix));
futures
.push(self.userroomid_highlightcount.watch_prefix(&userid_prefix));
// Events for rooms we are in
for room_id in services()
@ -70,17 +77,24 @@ impl service::globals::Data for KeyValueDatabase {
let roomid_bytes = room_id.as_bytes().to_vec();
let mut roomid_prefix = roomid_bytes.clone();
roomid_prefix.push(0xff);
roomid_prefix.push(0xFF);
// PDUs
futures.push(self.pduid_pdu.watch_prefix(&short_roomid));
// EDUs
futures.push(Box::pin(async move {
let _result = services().rooms.edus.typing.wait_for_update(&room_id).await;
let _result = services()
.rooms
.edus
.typing
.wait_for_update(&room_id)
.await;
}));
futures.push(self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix));
futures.push(
self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix),
);
// Key changes
futures.push(self.keychangeid_userid.watch_prefix(&roomid_prefix));
@ -90,12 +104,11 @@ impl service::globals::Data for KeyValueDatabase {
roomuser_prefix.extend_from_slice(&userid_prefix);
futures.push(
self.roomusertype_roomuserdataid
.watch_prefix(&roomuser_prefix),
self.roomusertype_roomuserdataid.watch_prefix(&roomuser_prefix),
);
}
let mut globaluserdata_prefix = vec![0xff];
let mut globaluserdata_prefix = vec![0xFF];
globaluserdata_prefix.extend_from_slice(&userid_prefix);
futures.push(
@ -107,7 +120,8 @@ impl service::globals::Data for KeyValueDatabase {
futures.push(self.keychangeid_userid.watch_prefix(&userid_prefix));
// One time keys
futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes));
futures
.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes));
futures.push(Box::pin(services().globals.rotate.watch()));
@ -126,10 +140,14 @@ impl service::globals::Data for KeyValueDatabase {
let shorteventid_cache = self.shorteventid_cache.lock().unwrap().len();
let auth_chain_cache = self.auth_chain_cache.lock().unwrap().len();
let eventidshort_cache = self.eventidshort_cache.lock().unwrap().len();
let statekeyshort_cache = self.statekeyshort_cache.lock().unwrap().len();
let our_real_users_cache = self.our_real_users_cache.read().unwrap().len();
let appservice_in_room_cache = self.appservice_in_room_cache.read().unwrap().len();
let lasttimelinecount_cache = self.lasttimelinecount_cache.lock().unwrap().len();
let statekeyshort_cache =
self.statekeyshort_cache.lock().unwrap().len();
let our_real_users_cache =
self.our_real_users_cache.read().unwrap().len();
let appservice_in_room_cache =
self.appservice_in_room_cache.read().unwrap().len();
let lasttimelinecount_cache =
self.lasttimelinecount_cache.lock().unwrap().len();
let mut response = format!(
"\
@ -194,27 +212,29 @@ lasttimelinecount_cache: {lasttimelinecount_cache}\n"
|s| Ok(s.clone()),
)?;
let mut parts = keypair_bytes.splitn(2, |&b| b == 0xff);
let mut parts = keypair_bytes.splitn(2, |&b| b == 0xFF);
utils::string_from_bytes(
// 1. version
parts
.next()
.expect("splitn always returns at least one element"),
parts.next().expect("splitn always returns at least one element"),
)
.map_err(|_| Error::bad_database("Invalid version bytes in keypair."))
.and_then(|version| {
// 2. key
parts
.next()
.ok_or_else(|| Error::bad_database("Invalid keypair format in database."))
.ok_or_else(|| {
Error::bad_database("Invalid keypair format in database.")
})
.map(|key| (version, key))
})
.and_then(|(version, key)| {
Ed25519KeyPair::from_der(key, version)
.map_err(|_| Error::bad_database("Private or public keys are invalid."))
Ed25519KeyPair::from_der(key, version).map_err(|_| {
Error::bad_database("Private or public keys are invalid.")
})
})
}
fn remove_keypair(&self) -> Result<()> {
self.global.remove(b"keypair")
}
@ -231,7 +251,10 @@ lasttimelinecount_cache: {lasttimelinecount_cache}\n"
.and_then(|keys| serde_json::from_slice(&keys).ok())
.unwrap_or_else(|| {
// Just insert "now", it doesn't matter
ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now())
ServerSigningKeys::new(
origin.to_owned(),
MilliSecondsSinceUnixEpoch::now(),
)
});
let ServerSigningKeys {
@ -245,7 +268,8 @@ lasttimelinecount_cache: {lasttimelinecount_cache}\n"
self.server_signingkeys.insert(
origin.as_bytes(),
&serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"),
&serde_json::to_vec(&keys)
.expect("serversigningkeys can be serialized"),
)?;
let mut tree = keys.verify_keys;
@ -258,7 +282,8 @@ lasttimelinecount_cache: {lasttimelinecount_cache}\n"
Ok(tree)
}
/// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server.
/// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found
/// for the server.
fn signing_keys_for(
&self,
origin: &ServerName,
@ -283,8 +308,9 @@ lasttimelinecount_cache: {lasttimelinecount_cache}\n"
fn database_version(&self) -> Result<u64> {
self.global.get(b"version")?.map_or(Ok(0), |version| {
utils::u64_from_bytes(&version)
.map_err(|_| Error::bad_database("Database version id is invalid."))
utils::u64_from_bytes(&version).map_err(|_| {
Error::bad_database("Database version id is invalid.")
})
})
}

200
src/database/key_value/key_backups.rs
View file

@ -9,7 +9,9 @@ use ruma::{
OwnedRoomId, RoomId, UserId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::key_backups::Data for KeyValueDatabase {
fn create_backup(
@ -20,12 +22,13 @@ impl service::key_backups::Data for KeyValueDatabase {
let version = services().globals.next_count()?.to_string();
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
self.backupid_algorithm.insert(
&key,
&serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"),
&serde_json::to_vec(backup_metadata)
.expect("BackupAlgorithm::to_vec always works"),
)?;
self.backupid_etag
.insert(&key, &services().globals.next_count()?.to_be_bytes())?;
@ -34,13 +37,13 @@ impl service::key_backups::Data for KeyValueDatabase {
fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
self.backupid_algorithm.remove(&key)?;
self.backupid_etag.remove(&key)?;
key.push(0xff);
key.push(0xFF);
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
self.backupkeyid_backup.remove(&outdated_key)?;
@ -56,7 +59,7 @@ impl service::key_backups::Data for KeyValueDatabase {
backup_metadata: &Raw<BackupAlgorithm>,
) -> Result<String> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
if self.backupid_algorithm.get(&key)?.is_none() {
@ -73,9 +76,12 @@ impl service::key_backups::Data for KeyValueDatabase {
Ok(version.to_owned())
}
fn get_latest_backup_version(&self, user_id: &UserId) -> Result<Option<String>> {
fn get_latest_backup_version(
&self,
user_id: &UserId,
) -> Result<Option<String>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let mut last_possible_key = prefix.clone();
last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
@ -85,11 +91,13 @@ impl service::key_backups::Data for KeyValueDatabase {
.next()
.map(|(key, _)| {
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))
.map_err(|_| {
Error::bad_database("backupid_algorithm key is invalid.")
})
})
.transpose()
}
@ -99,7 +107,7 @@ impl service::key_backups::Data for KeyValueDatabase {
user_id: &UserId,
) -> Result<Option<(String, Raw<BackupAlgorithm>)>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let mut last_possible_key = prefix.clone();
last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
@ -109,33 +117,42 @@ impl service::key_backups::Data for KeyValueDatabase {
.next()
.map(|(key, value)| {
let version = utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?;
.map_err(|_| {
Error::bad_database("backupid_algorithm key is invalid.")
})?;
Ok((
version,
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("Algorithm in backupid_algorithm is invalid.")
Error::bad_database(
"Algorithm in backupid_algorithm is invalid.",
)
})?,
))
})
.transpose()
}
fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<Raw<BackupAlgorithm>>> {
fn get_backup(
&self,
user_id: &UserId,
version: &str,
) -> Result<Option<Raw<BackupAlgorithm>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
self.backupid_algorithm
.get(&key)?
.map_or(Ok(None), |bytes| {
serde_json::from_slice(&bytes)
.map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))
self.backupid_algorithm.get(&key)?.map_or(Ok(None), |bytes| {
serde_json::from_slice(&bytes).map_err(|_| {
Error::bad_database(
"Algorithm in backupid_algorithm is invalid.",
)
})
})
}
fn add_key(
@ -147,7 +164,7 @@ impl service::key_backups::Data for KeyValueDatabase {
key_data: &Raw<KeyBackupData>,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
if self.backupid_algorithm.get(&key)?.is_none() {
@ -160,9 +177,9 @@ impl service::key_backups::Data for KeyValueDatabase {
self.backupid_etag
.insert(&key, &services().globals.next_count()?.to_be_bytes())?;
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(session_id.as_bytes());
self.backupkeyid_backup
@ -173,7 +190,7 @@ impl service::key_backups::Data for KeyValueDatabase {
fn count_keys(&self, user_id: &UserId, version: &str) -> Result<usize> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(version.as_bytes());
Ok(self.backupkeyid_backup.scan_prefix(prefix).count())
@ -181,7 +198,7 @@ impl service::key_backups::Data for KeyValueDatabase {
fn get_etag(&self, user_id: &UserId, version: &str) -> Result<String> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
Ok(utils::u64_from_bytes(
@ -200,40 +217,56 @@ impl service::key_backups::Data for KeyValueDatabase {
version: &str,
) -> Result<BTreeMap<OwnedRoomId, RoomKeyBackup>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(version.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
let mut rooms = BTreeMap::<OwnedRoomId, RoomKeyBackup>::new();
for result in self
.backupkeyid_backup
.scan_prefix(prefix)
.map(|(key, value)| {
let mut parts = key.rsplit(|&b| b == 0xff);
for result in
self.backupkeyid_backup.scan_prefix(prefix).map(|(key, value)| {
let mut parts = key.rsplit(|&b| b == 0xFF);
let session_id =
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| {
Error::bad_database("backupkeyid_backup session_id is invalid.")
})?;
let room_id = RoomId::parse(
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
let session_id = utils::string_from_bytes(
parts.next().ok_or_else(|| {
Error::bad_database(
"backupkeyid_backup key is invalid.",
)
})?,
)
.map_err(|_| {
Error::bad_database("backupkeyid_backup room_id is invalid room id.")
Error::bad_database(
"backupkeyid_backup session_id is invalid.",
)
})?;
let key_data = serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
let room_id = RoomId::parse(
utils::string_from_bytes(parts.next().ok_or_else(
|| {
Error::bad_database(
"backupkeyid_backup key is invalid.",
)
},
)?)
.map_err(|_| {
Error::bad_database(
"backupkeyid_backup room_id is invalid.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"backupkeyid_backup room_id is invalid room id.",
)
})?;
let key_data =
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database(
"KeyBackupData in backupkeyid_backup is invalid.",
)
})?;
Ok::<_, Error>((room_id, session_id, key_data))
})
{
@ -257,30 +290,38 @@ impl service::key_backups::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Result<BTreeMap<String, Raw<KeyBackupData>>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(version.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(room_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
Ok(self
.backupkeyid_backup
.scan_prefix(prefix)
.map(|(key, value)| {
let mut parts = key.rsplit(|&b| b == 0xff);
let mut parts = key.rsplit(|&b| b == 0xFF);
let session_id =
utils::string_from_bytes(parts.next().ok_or_else(|| {
Error::bad_database("backupkeyid_backup key is invalid.")
})?)
.map_err(|_| {
Error::bad_database("backupkeyid_backup session_id is invalid.")
})?;
let key_data = serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
let session_id = utils::string_from_bytes(
parts.next().ok_or_else(|| {
Error::bad_database(
"backupkeyid_backup key is invalid.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"backupkeyid_backup session_id is invalid.",
)
})?;
let key_data =
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database(
"KeyBackupData in backupkeyid_backup is invalid.",
)
})?;
Ok::<_, Error>((session_id, key_data))
})
.filter_map(Result::ok)
@ -295,18 +336,20 @@ impl service::key_backups::Data for KeyValueDatabase {
session_id: &str,
) -> Result<Option<Raw<KeyBackupData>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(session_id.as_bytes());
self.backupkeyid_backup
.get(&key)?
.map(|value| {
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
Error::bad_database(
"KeyBackupData in backupkeyid_backup is invalid.",
)
})
})
.transpose()
@ -314,9 +357,9 @@ impl service::key_backups::Data for KeyValueDatabase {
fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.push(0xFF);
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
self.backupkeyid_backup.remove(&outdated_key)?;
@ -325,13 +368,18 @@ impl service::key_backups::Data for KeyValueDatabase {
Ok(())
}
fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()> {
fn delete_room_keys(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.push(0xFF);
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
self.backupkeyid_backup.remove(&outdated_key)?;
@ -348,11 +396,11 @@ impl service::key_backups::Data for KeyValueDatabase {
session_id: &str,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(version.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(session_id.as_bytes());
for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {

43
src/database/key_value/media.rs
View file

@ -12,22 +12,19 @@ impl service::media::Data for KeyValueDatabase {
content_type: Option<&str>,
) -> Result<Vec<u8>> {
let mut key = mxc.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(&width.to_be_bytes());
key.extend_from_slice(&height.to_be_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(
content_disposition
.as_ref()
.map(|f| f.as_bytes())
.unwrap_or_default(),
);
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(
content_type
.as_ref()
.map(|c| c.as_bytes())
.unwrap_or_default(),
content_type.as_ref().map(|c| c.as_bytes()).unwrap_or_default(),
);
self.mediaid_file.insert(&key, &[])?;
@ -42,24 +39,25 @@ impl service::media::Data for KeyValueDatabase {
height: u32,
) -> Result<(Option<String>, Option<String>, Vec<u8>)> {
let mut prefix = mxc.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(&width.to_be_bytes());
prefix.extend_from_slice(&height.to_be_bytes());
prefix.push(0xff);
prefix.push(0xFF);
let (key, _) = self
.mediaid_file
.scan_prefix(prefix)
.next()
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
let (key, _) =
self.mediaid_file.scan_prefix(prefix).next().ok_or(
Error::BadRequest(ErrorKind::NotFound, "Media not found"),
)?;
let mut parts = key.rsplit(|&b| b == 0xff);
let mut parts = key.rsplit(|&b| b == 0xFF);
let content_type = parts
.next()
.map(|bytes| {
utils::string_from_bytes(bytes).map_err(|_| {
Error::bad_database("Content type in mediaid_file is invalid unicode.")
Error::bad_database(
"Content type in mediaid_file is invalid unicode.",
)
})
})
.transpose()?;
@ -71,11 +69,14 @@ impl service::media::Data for KeyValueDatabase {
let content_disposition = if content_disposition_bytes.is_empty() {
None
} else {
Some(
utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
Error::bad_database("Content Disposition in mediaid_file is invalid unicode.")
})?,
)
Some(utils::string_from_bytes(content_disposition_bytes).map_err(
|_| {
Error::bad_database(
"Content Disposition in mediaid_file is invalid \
unicode.",
)
},
)?)
};
Ok((content_disposition, content_type, key))
}

41
src/database/key_value/pusher.rs
View file

@ -6,30 +6,39 @@ use ruma::{
use crate::{database::KeyValueDatabase, service, utils, Error, Result};
impl service::pusher::Data for KeyValueDatabase {
fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> {
fn set_pusher(
&self,
sender: &UserId,
pusher: set_pusher::v3::PusherAction,
) -> Result<()> {
match &pusher {
set_pusher::v3::PusherAction::Post(data) => {
let mut key = sender.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(data.pusher.ids.pushkey.as_bytes());
self.senderkey_pusher.insert(
&key,
&serde_json::to_vec(&pusher).expect("Pusher is valid JSON value"),
&serde_json::to_vec(&pusher)
.expect("Pusher is valid JSON value"),
)?;
Ok(())
}
set_pusher::v3::PusherAction::Delete(ids) => {
let mut key = sender.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(ids.pushkey.as_bytes());
self.senderkey_pusher.remove(&key).map_err(Into::into)
}
}
}
fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result<Option<Pusher>> {
fn get_pusher(
&self,
sender: &UserId,
pushkey: &str,
) -> Result<Option<Pusher>> {
let mut senderkey = sender.as_bytes().to_vec();
senderkey.push(0xff);
senderkey.push(0xFF);
senderkey.extend_from_slice(pushkey.as_bytes());
self.senderkey_pusher
@ -43,7 +52,7 @@ impl service::pusher::Data for KeyValueDatabase {
fn get_pushers(&self, sender: &UserId) -> Result<Vec<Pusher>> {
let mut prefix = sender.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
self.senderkey_pusher
.scan_prefix(prefix)
@ -59,16 +68,20 @@ impl service::pusher::Data for KeyValueDatabase {
sender: &UserId,
) -> Box<dyn Iterator<Item = Result<String>> + 'a> {
let mut prefix = sender.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(self.senderkey_pusher.scan_prefix(prefix).map(|(k, _)| {
let mut parts = k.splitn(2, |&b| b == 0xff);
let mut parts = k.splitn(2, |&b| b == 0xFF);
let _senderkey = parts.next();
let push_key = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid senderkey_pusher in db"))?;
let push_key_string = utils::string_from_bytes(push_key)
.map_err(|_| Error::bad_database("Invalid pusher bytes in senderkey_pusher"))?;
let push_key = parts.next().ok_or_else(|| {
Error::bad_database("Invalid senderkey_pusher in db")
})?;
let push_key_string =
utils::string_from_bytes(push_key).map_err(|_| {
Error::bad_database(
"Invalid pusher bytes in senderkey_pusher",
)
})?;
Ok(push_key_string)
}))

49
src/database/key_value/rooms/alias.rs
View file

@ -1,6 +1,11 @@
use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId};
use ruma::{
api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, RoomAliasId,
RoomId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::rooms::alias::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
@ -8,17 +13,20 @@ impl service::rooms::alias::Data for KeyValueDatabase {
self.alias_roomid
.insert(alias.alias().as_bytes(), room_id.as_bytes())?;
let mut aliasid = room_id.as_bytes().to_vec();
aliasid.push(0xff);
aliasid.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
aliasid.push(0xFF);
aliasid
.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
self.aliasid_alias.insert(&aliasid, alias.as_bytes())?;
Ok(())
}
#[tracing::instrument(skip(self))]
fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> {
if let Some(room_id) = self.alias_roomid.get(alias.alias().as_bytes())? {
if let Some(room_id) =
self.alias_roomid.get(alias.alias().as_bytes())?
{
let mut prefix = room_id.clone();
prefix.push(0xff);
prefix.push(0xFF);
for (key, _) in self.aliasid_alias.scan_prefix(prefix) {
self.aliasid_alias.remove(&key)?;
@ -34,14 +42,23 @@ impl service::rooms::alias::Data for KeyValueDatabase {
}
#[tracing::instrument(skip(self))]
fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result<Option<OwnedRoomId>> {
fn resolve_local_alias(
&self,
alias: &RoomAliasId,
) -> Result<Option<OwnedRoomId>> {
self.alias_roomid
.get(alias.alias().as_bytes())?
.map(|bytes| {
RoomId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Room ID in alias_roomid is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid."))
RoomId::parse(utils::string_from_bytes(&bytes).map_err(
|_| {
Error::bad_database(
"Room ID in alias_roomid is invalid unicode.",
)
},
)?)
.map_err(|_| {
Error::bad_database("Room ID in alias_roomid is invalid.")
})
})
.transpose()
}
@ -52,13 +69,17 @@ impl service::rooms::alias::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Box<dyn Iterator<Item = Result<OwnedRoomAliasId>> + 'a> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(self.aliasid_alias.scan_prefix(prefix).map(|(_, bytes)| {
utils::string_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid alias bytes in aliasid_alias."))?
.map_err(|_| {
Error::bad_database("Invalid alias bytes in aliasid_alias.")
})?
.try_into()
.map_err(|_| Error::bad_database("Invalid alias in aliasid_alias."))
.map_err(|_| {
Error::bad_database("Invalid alias in aliasid_alias.")
})
}))
}
}

24
src/database/key_value/rooms/auth_chain.rs
View file

@ -3,9 +3,13 @@ use std::{collections::HashSet, mem::size_of, sync::Arc};
use crate::{database::KeyValueDatabase, service, utils, Result};
impl service::rooms::auth_chain::Data for KeyValueDatabase {
fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result<Option<Arc<HashSet<u64>>>> {
fn get_cached_eventid_authchain(
&self,
key: &[u64],
) -> Result<Option<Arc<HashSet<u64>>>> {
// Check RAM cache
if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) {
if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key)
{
return Ok(Some(Arc::clone(result)));
}
@ -18,7 +22,10 @@ impl service::rooms::auth_chain::Data for KeyValueDatabase {
.map(|chain| {
chain
.chunks_exact(size_of::<u64>())
.map(|chunk| utils::u64_from_bytes(chunk).expect("byte length is correct"))
.map(|chunk| {
utils::u64_from_bytes(chunk)
.expect("byte length is correct")
})
.collect()
});
@ -38,7 +45,11 @@ impl service::rooms::auth_chain::Data for KeyValueDatabase {
Ok(None)
}
fn cache_auth_chain(&self, key: Vec<u64>, auth_chain: Arc<HashSet<u64>>) -> Result<()> {
fn cache_auth_chain(
&self,
key: Vec<u64>,
auth_chain: Arc<HashSet<u64>>,
) -> Result<()> {
// Only persist single events in db
if key.len() == 1 {
self.shorteventid_authchain.insert(
@ -51,10 +62,7 @@ impl service::rooms::auth_chain::Data for KeyValueDatabase {
}
// Cache in RAM
self.auth_chain_cache
.lock()
.unwrap()
.insert(key, auth_chain);
self.auth_chain_cache.lock().unwrap().insert(key, auth_chain);
Ok(())
}

18
src/database/key_value/rooms/directory.rs
View file

@ -19,14 +19,18 @@ impl service::rooms::directory::Data for KeyValueDatabase {
}
#[tracing::instrument(skip(self))]
fn public_rooms<'a>(&'a self) -> Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a> {
fn public_rooms<'a>(
&'a self,
) -> Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a> {
Box::new(self.publicroomids.iter().map(|(bytes, _)| {
RoomId::parse(
utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Room ID in publicroomids is invalid unicode.")
})?,
)
.map_err(|_| Error::bad_database("Room ID in publicroomids is invalid."))
RoomId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database(
"Room ID in publicroomids is invalid unicode.",
)
})?)
.map_err(|_| {
Error::bad_database("Room ID in publicroomids is invalid.")
})
}))
}
}

101
src/database/key_value/rooms/edus/read_receipt.rs
View file

@ -1,10 +1,13 @@
use std::mem;
use ruma::{
events::receipt::ReceiptEvent, serde::Raw, CanonicalJsonObject, OwnedUserId, RoomId, UserId,
events::receipt::ReceiptEvent, serde::Raw, CanonicalJsonObject,
OwnedUserId, RoomId, UserId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
fn readreceipt_update(
@ -14,7 +17,7 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
event: ReceiptEvent,
) -> Result<()> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let mut last_possible_key = prefix.clone();
last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
@ -25,7 +28,7 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
.iter_from(&last_possible_key, true)
.take_while(|(key, _)| key.starts_with(&prefix))
.find(|(key, _)| {
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element")
== user_id.as_bytes()
@ -36,13 +39,15 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
}
let mut room_latest_id = prefix;
room_latest_id.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
room_latest_id.push(0xff);
room_latest_id
.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
room_latest_id.push(0xFF);
room_latest_id.extend_from_slice(user_id.as_bytes());
self.readreceiptid_readreceipt.insert(
&room_latest_id,
&serde_json::to_vec(&event).expect("EduEvent::to_string always works"),
&serde_json::to_vec(&event)
.expect("EduEvent::to_string always works"),
)?;
Ok(())
@ -64,7 +69,7 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
> + 'a,
> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let prefix2 = prefix.clone();
let mut first_possible_edu = prefix.clone();
@ -79,21 +84,35 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
let count = utils::u64_from_bytes(
&k[prefix.len()..prefix.len() + mem::size_of::<u64>()],
)
.map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?;
.map_err(|_| {
Error::bad_database(
"Invalid readreceiptid count in db.",
)
})?;
let user_id = UserId::parse(
utils::string_from_bytes(&k[prefix.len() + mem::size_of::<u64>() + 1..])
.map_err(|_| {
Error::bad_database("Invalid readreceiptid userid bytes in db.")
})?,
utils::string_from_bytes(
&k[prefix.len() + mem::size_of::<u64>() + 1..],
)
.map_err(|_| {
Error::bad_database(
"Invalid readreceiptid userid bytes in db.",
)
})?,
)
.map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?;
.map_err(|_| {
Error::bad_database(
"Invalid readreceiptid userid in db.",
)
})?;
let mut json =
serde_json::from_slice::<CanonicalJsonObject>(&v).map_err(|_| {
Error::bad_database(
"Read receipt in roomlatestid_roomlatest is invalid json.",
)
})?;
serde_json::from_slice::<CanonicalJsonObject>(&v)
.map_err(|_| {
Error::bad_database(
"Read receipt in roomlatestid_roomlatest \
is invalid json.",
)
})?;
json.remove("room_id");
Ok((
@ -109,36 +128,46 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
}
#[tracing::instrument(skip(self))]
fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> {
fn private_read_set(
&self,
room_id: &RoomId,
user_id: &UserId,
count: u64,
) -> Result<()> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_privateread
.insert(&key, &count.to_be_bytes())?;
self.roomuserid_privateread.insert(&key, &count.to_be_bytes())?;
self.roomuserid_lastprivatereadupdate
.insert(&key, &services().globals.next_count()?.to_be_bytes())
}
#[tracing::instrument(skip(self))]
fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
fn private_read_get(
&self,
room_id: &RoomId,
user_id: &UserId,
) -> Result<Option<u64>> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_privateread
.get(&key)?
.map_or(Ok(None), |v| {
Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
Error::bad_database("Invalid private read marker bytes")
})?))
})
self.roomuserid_privateread.get(&key)?.map_or(Ok(None), |v| {
Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
Error::bad_database("Invalid private read marker bytes")
})?))
})
}
fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
fn last_privateread_update(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<u64> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
Ok(self
@ -146,7 +175,9 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
.get(&key)?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.")
Error::bad_database(
"Count in roomuserid_lastprivatereadupdate is invalid.",
)
})
})
.transpose()?

18
src/database/key_value/rooms/lazy_load.rs
View file

@ -11,11 +11,11 @@ impl service::rooms::lazy_loading::Data for KeyValueDatabase {
ll_user: &UserId,
) -> Result<bool> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(device_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(ll_user.as_bytes());
Ok(self.lazyloadedids.get(&key)?.is_some())
}
@ -28,11 +28,11 @@ impl service::rooms::lazy_loading::Data for KeyValueDatabase {
confirmed_user_ids: &mut dyn Iterator<Item = &UserId>,
) -> Result<()> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(device_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(room_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
for ll_id in confirmed_user_ids {
let mut key = prefix.clone();
@ -50,11 +50,11 @@ impl service::rooms::lazy_loading::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Result<()> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(device_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(room_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
for (key, _) in self.lazyloadedids.scan_prefix(prefix) {
self.lazyloadedids.remove(&key)?;

22
src/database/key_value/rooms/metadata.rs
View file

@ -1,6 +1,8 @@
use ruma::{OwnedRoomId, RoomId};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::rooms::metadata::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
@ -19,14 +21,18 @@ impl service::rooms::metadata::Data for KeyValueDatabase {
.is_some())
}
fn iter_ids<'a>(&'a self) -> Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a> {
fn iter_ids<'a>(
&'a self,
) -> Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a> {
Box::new(self.roomid_shortroomid.iter().map(|(bytes, _)| {
RoomId::parse(
utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Room ID in publicroomids is invalid unicode.")
})?,
)
.map_err(|_| Error::bad_database("Room ID in roomid_shortroomid is invalid."))
RoomId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database(
"Room ID in publicroomids is invalid unicode.",
)
})?)
.map_err(|_| {
Error::bad_database("Room ID in roomid_shortroomid is invalid.")
})
}))
}

35
src/database/key_value/rooms/outlier.rs
View file

@ -3,24 +3,35 @@ use ruma::{CanonicalJsonObject, EventId};
use crate::{database::KeyValueDatabase, service, Error, PduEvent, Result};
impl service::rooms::outlier::Data for KeyValueDatabase {
fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result<Option<CanonicalJsonObject>> {
self.eventid_outlierpdu
.get(event_id.as_bytes())?
.map_or(Ok(None), |pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
})
fn get_outlier_pdu_json(
&self,
event_id: &EventId,
) -> Result<Option<CanonicalJsonObject>> {
self.eventid_outlierpdu.get(event_id.as_bytes())?.map_or(
Ok(None),
|pdu| {
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
},
)
}
fn get_outlier_pdu(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
self.eventid_outlierpdu
.get(event_id.as_bytes())?
.map_or(Ok(None), |pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
})
self.eventid_outlierpdu.get(event_id.as_bytes())?.map_or(
Ok(None),
|pdu| {
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
},
)
}
#[tracing::instrument(skip(self, pdu))]
fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> {
fn add_pdu_outlier(
&self,
event_id: &EventId,
pdu: &CanonicalJsonObject,
) -> Result<()> {
self.eventid_outlierpdu.insert(
event_id.as_bytes(),
&serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"),

33
src/database/key_value/rooms/pdu_metadata.rs
View file

@ -22,7 +22,8 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
shortroomid: u64,
target: u64,
until: PduCount,
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>> {
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>>
{
let prefix = target.to_be_bytes().to_vec();
let mut current = prefix.clone();
@ -40,8 +41,12 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
.iter_from(&current, true)
.take_while(move |(k, _)| k.starts_with(&prefix))
.map(move |(tofrom, _data)| {
let from = utils::u64_from_bytes(&tofrom[(mem::size_of::<u64>())..])
.map_err(|_| Error::bad_database("Invalid count in tofrom_relation."))?;
let from = utils::u64_from_bytes(
&tofrom[(mem::size_of::<u64>())..],
)
.map_err(|_| {
Error::bad_database("Invalid count in tofrom_relation.")
})?;
let mut pduid = shortroomid.to_be_bytes().to_vec();
pduid.extend_from_slice(&from.to_be_bytes());
@ -50,7 +55,11 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
.rooms
.timeline
.get_pdu_from_id(&pduid)?
.ok_or_else(|| Error::bad_database("Pdu in tofrom_relation is invalid."))?;
.ok_or_else(|| {
Error::bad_database(
"Pdu in tofrom_relation is invalid.",
)
})?;
if pdu.sender != user_id {
pdu.remove_transaction_id()?;
}
@ -59,7 +68,11 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
))
}
fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc<EventId>]) -> Result<()> {
fn mark_as_referenced(
&self,
room_id: &RoomId,
event_ids: &[Arc<EventId>],
) -> Result<()> {
for prev in event_ids {
let mut key = room_id.as_bytes().to_vec();
key.extend_from_slice(prev.as_bytes());
@ -69,7 +82,11 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
Ok(())
}
fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result<bool> {
fn is_event_referenced(
&self,
room_id: &RoomId,
event_id: &EventId,
) -> Result<bool> {
let mut key = room_id.as_bytes().to_vec();
key.extend_from_slice(event_id.as_bytes());
Ok(self.referencedevents.get(&key)?.is_some())
@ -80,8 +97,6 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
}
fn is_event_soft_failed(&self, event_id: &EventId) -> Result<bool> {
self.softfailedeventids
.get(event_id.as_bytes())
.map(|o| o.is_some())
self.softfailedeventids.get(event_id.as_bytes()).map(|o| o.is_some())
}
}

18
src/database/key_value/rooms/search.rs
View file

@ -4,7 +4,12 @@ use crate::{database::KeyValueDatabase, service, services, utils, Result};
impl service::rooms::search::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> {
fn index_pdu(
&self,
shortroomid: u64,
pdu_id: &[u8],
message_body: &str,
) -> Result<()> {
let mut batch = message_body
.split_terminator(|c: char| !c.is_alphanumeric())
.filter(|s| !s.is_empty())
@ -13,7 +18,7 @@ impl service::rooms::search::Data for KeyValueDatabase {
.map(|word| {
let mut key = shortroomid.to_be_bytes().to_vec();
key.extend_from_slice(word.as_bytes());
key.push(0xff);
key.push(0xFF);
// TODO: currently we save the room id a second time here
key.extend_from_slice(pdu_id);
(key, Vec::new())
@ -28,7 +33,8 @@ impl service::rooms::search::Data for KeyValueDatabase {
&'a self,
room_id: &RoomId,
search_string: &str,
) -> Result<Option<(Box<dyn Iterator<Item = Vec<u8>> + 'a>, Vec<String>)>> {
) -> Result<Option<(Box<dyn Iterator<Item = Vec<u8>> + 'a>, Vec<String>)>>
{
let prefix = services()
.rooms
.short
@ -46,7 +52,7 @@ impl service::rooms::search::Data for KeyValueDatabase {
let iterators = words.clone().into_iter().map(move |word| {
let mut prefix2 = prefix.clone();
prefix2.extend_from_slice(word.as_bytes());
prefix2.push(0xff);
prefix2.push(0xFF);
let prefix3 = prefix2.clone();
let mut last_possible_id = prefix2.clone();
@ -60,7 +66,9 @@ impl service::rooms::search::Data for KeyValueDatabase {
});
// We compare b with a because we reversed the iterator earlier
let Some(common_elements) = utils::common_elements(iterators, |a, b| b.cmp(a)) else {
let Some(common_elements) =
utils::common_elements(iterators, |a, b| b.cmp(a))
else {
return Ok(None);
};

159
src/database/key_value/rooms/short.rs
View file

@ -2,26 +2,32 @@ use std::sync::Arc;
use ruma::{events::StateEventType, EventId, RoomId};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::rooms::short::Data for KeyValueDatabase {
fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result<u64> {
if let Some(short) = self.eventidshort_cache.lock().unwrap().get_mut(event_id) {
if let Some(short) =
self.eventidshort_cache.lock().unwrap().get_mut(event_id)
{
return Ok(*short);
}
let short =
if let Some(shorteventid) = self.eventid_shorteventid.get(event_id.as_bytes())? {
utils::u64_from_bytes(&shorteventid)
.map_err(|_| Error::bad_database("Invalid shorteventid in db."))?
} else {
let shorteventid = services().globals.next_count()?;
self.eventid_shorteventid
.insert(event_id.as_bytes(), &shorteventid.to_be_bytes())?;
self.shorteventid_eventid
.insert(&shorteventid.to_be_bytes(), event_id.as_bytes())?;
shorteventid
};
let short = if let Some(shorteventid) =
self.eventid_shorteventid.get(event_id.as_bytes())?
{
utils::u64_from_bytes(&shorteventid).map_err(|_| {
Error::bad_database("Invalid shorteventid in db.")
})?
} else {
let shorteventid = services().globals.next_count()?;
self.eventid_shorteventid
.insert(event_id.as_bytes(), &shorteventid.to_be_bytes())?;
self.shorteventid_eventid
.insert(&shorteventid.to_be_bytes(), event_id.as_bytes())?;
shorteventid
};
self.eventidshort_cache
.lock()
@ -46,15 +52,16 @@ impl service::rooms::short::Data for KeyValueDatabase {
}
let mut db_key = event_type.to_string().as_bytes().to_vec();
db_key.push(0xff);
db_key.push(0xFF);
db_key.extend_from_slice(state_key.as_bytes());
let short = self
.statekey_shortstatekey
.get(&db_key)?
.map(|shortstatekey| {
utils::u64_from_bytes(&shortstatekey)
.map_err(|_| Error::bad_database("Invalid shortstatekey in db."))
utils::u64_from_bytes(&shortstatekey).map_err(|_| {
Error::bad_database("Invalid shortstatekey in db.")
})
})
.transpose()?;
@ -83,12 +90,15 @@ impl service::rooms::short::Data for KeyValueDatabase {
}
let mut db_key = event_type.to_string().as_bytes().to_vec();
db_key.push(0xff);
db_key.push(0xFF);
db_key.extend_from_slice(state_key.as_bytes());
let short = if let Some(shortstatekey) = self.statekey_shortstatekey.get(&db_key)? {
utils::u64_from_bytes(&shortstatekey)
.map_err(|_| Error::bad_database("Invalid shortstatekey in db."))?
let short = if let Some(shortstatekey) =
self.statekey_shortstatekey.get(&db_key)?
{
utils::u64_from_bytes(&shortstatekey).map_err(|_| {
Error::bad_database("Invalid shortstatekey in db.")
})?
} else {
let shortstatekey = services().globals.next_count()?;
self.statekey_shortstatekey
@ -106,12 +116,12 @@ impl service::rooms::short::Data for KeyValueDatabase {
Ok(short)
}
fn get_eventid_from_short(&self, shorteventid: u64) -> Result<Arc<EventId>> {
if let Some(id) = self
.shorteventid_cache
.lock()
.unwrap()
.get_mut(&shorteventid)
fn get_eventid_from_short(
&self,
shorteventid: u64,
) -> Result<Arc<EventId>> {
if let Some(id) =
self.shorteventid_cache.lock().unwrap().get_mut(&shorteventid)
{
return Ok(Arc::clone(id));
}
@ -119,12 +129,20 @@ impl service::rooms::short::Data for KeyValueDatabase {
let bytes = self
.shorteventid_eventid
.get(&shorteventid.to_be_bytes())?
.ok_or_else(|| Error::bad_database("Shorteventid does not exist"))?;
.ok_or_else(|| {
Error::bad_database("Shorteventid does not exist")
})?;
let event_id = EventId::parse_arc(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("EventID in shorteventid_eventid is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?;
let event_id = EventId::parse_arc(
utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database(
"EventID in shorteventid_eventid is invalid unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database("EventId in shorteventid_eventid is invalid.")
})?;
self.shorteventid_cache
.lock()
@ -134,12 +152,12 @@ impl service::rooms::short::Data for KeyValueDatabase {
Ok(event_id)
}
fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> {
if let Some(id) = self
.shortstatekey_cache
.lock()
.unwrap()
.get_mut(&shortstatekey)
fn get_statekey_from_short(
&self,
shortstatekey: u64,
) -> Result<(StateEventType, String)> {
if let Some(id) =
self.shortstatekey_cache.lock().unwrap().get_mut(&shortstatekey)
{
return Ok(id.clone());
}
@ -147,23 +165,32 @@ impl service::rooms::short::Data for KeyValueDatabase {
let bytes = self
.shortstatekey_statekey
.get(&shortstatekey.to_be_bytes())?
.ok_or_else(|| Error::bad_database("Shortstatekey does not exist"))?;
.ok_or_else(|| {
Error::bad_database("Shortstatekey does not exist")
})?;
let mut parts = bytes.splitn(2, |&b| b == 0xff);
let eventtype_bytes = parts.next().expect("split always returns one entry");
let statekey_bytes = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid statekey in shortstatekey_statekey."))?;
let event_type =
StateEventType::from(utils::string_from_bytes(eventtype_bytes).map_err(|_| {
Error::bad_database("Event type in shortstatekey_statekey is invalid unicode.")
})?);
let state_key = utils::string_from_bytes(statekey_bytes).map_err(|_| {
Error::bad_database("Statekey in shortstatekey_statekey is invalid unicode.")
let mut parts = bytes.splitn(2, |&b| b == 0xFF);
let eventtype_bytes =
parts.next().expect("split always returns one entry");
let statekey_bytes = parts.next().ok_or_else(|| {
Error::bad_database("Invalid statekey in shortstatekey_statekey.")
})?;
let event_type = StateEventType::from(
utils::string_from_bytes(eventtype_bytes).map_err(|_| {
Error::bad_database(
"Event type in shortstatekey_statekey is invalid unicode.",
)
})?,
);
let state_key =
utils::string_from_bytes(statekey_bytes).map_err(|_| {
Error::bad_database(
"Statekey in shortstatekey_statekey is invalid unicode.",
)
})?;
let result = (event_type, state_key);
self.shortstatekey_cache
@ -175,12 +202,18 @@ impl service::rooms::short::Data for KeyValueDatabase {
}
/// Returns `(shortstatehash, already_existed)`
fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)> {
fn get_or_create_shortstatehash(
&self,
state_hash: &[u8],
) -> Result<(u64, bool)> {
Ok(
if let Some(shortstatehash) = self.statehash_shortstatehash.get(state_hash)? {
if let Some(shortstatehash) =
self.statehash_shortstatehash.get(state_hash)?
{
(
utils::u64_from_bytes(&shortstatehash)
.map_err(|_| Error::bad_database("Invalid shortstatehash in db."))?,
utils::u64_from_bytes(&shortstatehash).map_err(|_| {
Error::bad_database("Invalid shortstatehash in db.")
})?,
true,
)
} else {
@ -196,17 +229,21 @@ impl service::rooms::short::Data for KeyValueDatabase {
self.roomid_shortroomid
.get(room_id.as_bytes())?
.map(|bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid shortroomid in db."))
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid shortroomid in db.")
})
})
.transpose()
}
fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result<u64> {
Ok(
if let Some(short) = self.roomid_shortroomid.get(room_id.as_bytes())? {
utils::u64_from_bytes(&short)
.map_err(|_| Error::bad_database("Invalid shortroomid in db."))?
if let Some(short) =
self.roomid_shortroomid.get(room_id.as_bytes())?
{
utils::u64_from_bytes(&short).map_err(|_| {
Error::bad_database("Invalid shortroomid in db.")
})?
} else {
let short = services().globals.next_count()?;
self.roomid_shortroomid

55
src/database/key_value/rooms/state.rs
View file

@ -1,20 +1,22 @@
use ruma::{EventId, OwnedEventId, RoomId};
use std::collections::HashSet;
use std::{collections::HashSet, sync::Arc};
use std::sync::Arc;
use ruma::{EventId, OwnedEventId, RoomId};
use tokio::sync::MutexGuard;
use crate::{database::KeyValueDatabase, service, utils, Error, Result};
impl service::rooms::state::Data for KeyValueDatabase {
fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result<Option<u64>> {
self.roomid_shortstatehash
.get(room_id.as_bytes())?
.map_or(Ok(None), |bytes| {
self.roomid_shortstatehash.get(room_id.as_bytes())?.map_or(
Ok(None),
|bytes| {
Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid shortstatehash in roomid_shortstatehash")
Error::bad_database(
"Invalid shortstatehash in roomid_shortstatehash",
)
})?))
})
},
)
}
fn set_room_state(
@ -29,23 +31,40 @@ impl service::rooms::state::Data for KeyValueDatabase {
Ok(())
}
fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) -> Result<()> {
self.shorteventid_shortstatehash
.insert(&shorteventid.to_be_bytes(), &shortstatehash.to_be_bytes())?;
fn set_event_state(
&self,
shorteventid: u64,
shortstatehash: u64,
) -> Result<()> {
self.shorteventid_shortstatehash.insert(
&shorteventid.to_be_bytes(),
&shortstatehash.to_be_bytes(),
)?;
Ok(())
}
fn get_forward_extremities(&self, room_id: &RoomId) -> Result<HashSet<Arc<EventId>>> {
fn get_forward_extremities(
&self,
room_id: &RoomId,
) -> Result<HashSet<Arc<EventId>>> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
self.roomid_pduleaves
.scan_prefix(prefix)
.map(|(_, bytes)| {
EventId::parse_arc(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("EventID in roomid_pduleaves is invalid unicode.")
})?)
.map_err(|_| Error::bad_database("EventId in roomid_pduleaves is invalid."))
EventId::parse_arc(utils::string_from_bytes(&bytes).map_err(
|_| {
Error::bad_database(
"EventID in roomid_pduleaves is invalid unicode.",
)
},
)?)
.map_err(|_| {
Error::bad_database(
"EventId in roomid_pduleaves is invalid.",
)
})
})
.collect()
}
@ -58,7 +77,7 @@ impl service::rooms::state::Data for KeyValueDatabase {
_mutex_lock: &MutexGuard<'_, ()>,
) -> Result<()> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
for (key, _) in self.roomid_pduleaves.scan_prefix(prefix.clone()) {
self.roomid_pduleaves.remove(&key)?;

47
src/database/key_value/rooms/state_accessor.rs
View file

@ -1,12 +1,19 @@
use std::{collections::HashMap, sync::Arc};
use crate::{database::KeyValueDatabase, service, services, utils, Error, PduEvent, Result};
use async_trait::async_trait;
use ruma::{events::StateEventType, EventId, RoomId};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, PduEvent,
Result,
};
#[async_trait]
impl service::rooms::state_accessor::Data for KeyValueDatabase {
async fn state_full_ids(&self, shortstatehash: u64) -> Result<HashMap<u64, Arc<EventId>>> {
async fn state_full_ids(
&self,
shortstatehash: u64,
) -> Result<HashMap<u64, Arc<EventId>>> {
let full_state = services()
.rooms
.state_compressor
@ -56,7 +63,11 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
pdu.kind.to_string().into(),
pdu.state_key
.as_ref()
.ok_or_else(|| Error::bad_database("State event has no state key."))?
.ok_or_else(|| {
Error::bad_database(
"State event has no state key.",
)
})?
.clone(),
),
pdu,
@ -72,17 +83,16 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
Ok(result)
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
/// Returns a single PDU from `room_id` with key (`event_type`,
/// `state_key`).
fn state_get_id(
&self,
shortstatehash: u64,
event_type: &StateEventType,
state_key: &str,
) -> Result<Option<Arc<EventId>>> {
let Some(shortstatekey) = services()
.rooms
.short
.get_shortstatekey(event_type, state_key)?
let Some(shortstatekey) =
services().rooms.short.get_shortstatekey(event_type, state_key)?
else {
return Ok(None);
};
@ -106,7 +116,8 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
}))
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
/// Returns a single PDU from `room_id` with key (`event_type`,
/// `state_key`).
fn state_get(
&self,
shortstatehash: u64,
@ -121,20 +132,22 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
/// Returns the state hash for this pdu.
fn pdu_shortstatehash(&self, event_id: &EventId) -> Result<Option<u64>> {
self.eventid_shorteventid
.get(event_id.as_bytes())?
.map_or(Ok(None), |shorteventid| {
self.eventid_shorteventid.get(event_id.as_bytes())?.map_or(
Ok(None),
|shorteventid| {
self.shorteventid_shortstatehash
.get(&shorteventid)?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database(
"Invalid shortstatehash bytes in shorteventid_shortstatehash",
"Invalid shortstatehash bytes in \
shorteventid_shortstatehash",
)
})
})
.transpose()
})
},
)
}
/// Returns the full room state.
@ -151,7 +164,8 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
}
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
/// Returns a single PDU from `room_id` with key (`event_type`,
/// `state_key`).
fn room_state_get_id(
&self,
room_id: &RoomId,
@ -167,7 +181,8 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
}
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
/// Returns a single PDU from `room_id` with key (`event_type`,
/// `state_key`).
fn room_state_get(
&self,
room_id: &RoomId,

399
src/database/key_value/rooms/state_cache.rs
View file

@ -13,20 +13,24 @@ use crate::{
};
impl service::rooms::state_cache::Data for KeyValueDatabase {
fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> {
fn mark_as_once_joined(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<()> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
self.roomuseroncejoinedids.insert(&userroom_id, &[])
}
fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> {
let mut roomuser_id = room_id.as_bytes().to_vec();
roomuser_id.push(0xff);
roomuser_id.push(0xFF);
roomuser_id.extend_from_slice(user_id.as_bytes());
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
self.userroomid_joined.insert(&userroom_id, &[])?;
@ -46,11 +50,11 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
last_state: Option<Vec<Raw<AnyStrippedStateEvent>>>,
) -> Result<()> {
let mut roomuser_id = room_id.as_bytes().to_vec();
roomuser_id.push(0xff);
roomuser_id.push(0xFF);
roomuser_id.extend_from_slice(user_id.as_bytes());
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
self.userroomid_invitestate.insert(
@ -72,11 +76,11 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> {
let mut roomuser_id = room_id.as_bytes().to_vec();
roomuser_id.push(0xff);
roomuser_id.push(0xFF);
roomuser_id.extend_from_slice(user_id.as_bytes());
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
// TODO
@ -112,7 +116,9 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
joinedcount += 1;
}
for _invited in self.room_members_invited(room_id).filter_map(Result::ok) {
for _invited in
self.room_members_invited(room_id).filter_map(Result::ok)
{
invitedcount += 1;
}
@ -127,15 +133,17 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
.unwrap()
.insert(room_id.to_owned(), Arc::new(real_users));
for old_joined_server in self.room_servers(room_id).filter_map(Result::ok) {
for old_joined_server in
self.room_servers(room_id).filter_map(Result::ok)
{
if !joined_servers.remove(&old_joined_server) {
// Server not in room anymore
let mut roomserver_id = room_id.as_bytes().to_vec();
roomserver_id.push(0xff);
roomserver_id.push(0xFF);
roomserver_id.extend_from_slice(old_joined_server.as_bytes());
let mut serverroom_id = old_joined_server.as_bytes().to_vec();
serverroom_id.push(0xff);
serverroom_id.push(0xFF);
serverroom_id.extend_from_slice(room_id.as_bytes());
self.roomserverids.remove(&roomserver_id)?;
@ -146,49 +154,45 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
// Now only new servers are in joined_servers anymore
for server in joined_servers {
let mut roomserver_id = room_id.as_bytes().to_vec();
roomserver_id.push(0xff);
roomserver_id.push(0xFF);
roomserver_id.extend_from_slice(server.as_bytes());
let mut serverroom_id = server.as_bytes().to_vec();
serverroom_id.push(0xff);
serverroom_id.push(0xFF);
serverroom_id.extend_from_slice(room_id.as_bytes());
self.roomserverids.insert(&roomserver_id, &[])?;
self.serverroomids.insert(&serverroom_id, &[])?;
}
self.appservice_in_room_cache
.write()
.unwrap()
.remove(room_id);
self.appservice_in_room_cache.write().unwrap().remove(room_id);
Ok(())
}
#[tracing::instrument(skip(self, room_id))]
fn get_our_real_users(&self, room_id: &RoomId) -> Result<Arc<HashSet<OwnedUserId>>> {
let maybe = self
.our_real_users_cache
.read()
.unwrap()
.get(room_id)
.cloned();
fn get_our_real_users(
&self,
room_id: &RoomId,
) -> Result<Arc<HashSet<OwnedUserId>>> {
let maybe =
self.our_real_users_cache.read().unwrap().get(room_id).cloned();
if let Some(users) = maybe {
Ok(users)
} else {
self.update_joined_count(room_id)?;
Ok(Arc::clone(
self.our_real_users_cache
.read()
.unwrap()
.get(room_id)
.unwrap(),
self.our_real_users_cache.read().unwrap().get(room_id).unwrap(),
))
}
}
#[tracing::instrument(skip(self, room_id, appservice))]
fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result<bool> {
fn appservice_in_room(
&self,
room_id: &RoomId,
appservice: &RegistrationInfo,
) -> Result<bool> {
let maybe = self
.appservice_in_room_cache
.read()
@ -206,11 +210,13 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
)
.ok();
let in_room = bridge_user_id
.map_or(false, |id| self.is_joined(&id, room_id).unwrap_or(false))
|| self.room_members(room_id).any(|userid| {
userid.map_or(false, |userid| appservice.users.is_match(userid.as_str()))
});
let in_room = bridge_user_id.map_or(false, |id| {
self.is_joined(&id, room_id).unwrap_or(false)
}) || self.room_members(room_id).any(|userid| {
userid.map_or(false, |userid| {
appservice.users.is_match(userid.as_str())
})
});
self.appservice_in_room_cache
.write()
@ -227,11 +233,11 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
let mut roomuser_id = room_id.as_bytes().to_vec();
roomuser_id.push(0xff);
roomuser_id.push(0xFF);
roomuser_id.extend_from_slice(user_id.as_bytes());
self.userroomid_leftstate.remove(&userroom_id)?;
@ -247,51 +253,66 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Box<dyn Iterator<Item = Result<OwnedServerName>> + 'a> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(self.roomserverids.scan_prefix(prefix).map(|(key, _)| {
ServerName::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Server name in roomserverids is invalid unicode.")
Error::bad_database(
"Server name in roomserverids is invalid unicode.",
)
})?,
)
.map_err(|_| Error::bad_database("Server name in roomserverids is invalid."))
.map_err(|_| {
Error::bad_database("Server name in roomserverids is invalid.")
})
}))
}
#[tracing::instrument(skip(self))]
fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result<bool> {
fn server_in_room(
&self,
server: &ServerName,
room_id: &RoomId,
) -> Result<bool> {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
self.serverroomids.get(&key).map(|o| o.is_some())
}
/// Returns an iterator of all rooms a server participates in (as far as we know).
/// Returns an iterator of all rooms a server participates in (as far as we
/// know).
#[tracing::instrument(skip(self))]
fn server_rooms<'a>(
&'a self,
server: &ServerName,
) -> Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a> {
let mut prefix = server.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(self.serverroomids.scan_prefix(prefix).map(|(key, _)| {
RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("RoomId in serverroomids is invalid unicode."))?,
.map_err(|_| {
Error::bad_database(
"RoomId in serverroomids is invalid unicode.",
)
})?,
)
.map_err(|_| Error::bad_database("RoomId in serverroomids is invalid."))
.map_err(|_| {
Error::bad_database("RoomId in serverroomids is invalid.")
})
}))
}
@ -302,20 +323,24 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(self.roomuserid_joined.scan_prefix(prefix).map(|(key, _)| {
UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("User ID in roomuserid_joined is invalid unicode.")
Error::bad_database(
"User ID in roomuserid_joined is invalid unicode.",
)
})?,
)
.map_err(|_| Error::bad_database("User ID in roomuserid_joined is invalid."))
.map_err(|_| {
Error::bad_database("User ID in roomuserid_joined is invalid.")
})
}))
}
@ -324,8 +349,9 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
self.roomid_joinedcount
.get(room_id.as_bytes())?
.map(|b| {
utils::u64_from_bytes(&b)
.map_err(|_| Error::bad_database("Invalid joinedcount in db."))
utils::u64_from_bytes(&b).map_err(|_| {
Error::bad_database("Invalid joinedcount in db.")
})
})
.transpose()
}
@ -335,8 +361,9 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
self.roomid_invitedcount
.get(room_id.as_bytes())?
.map(|b| {
utils::u64_from_bytes(&b)
.map_err(|_| Error::bad_database("Invalid joinedcount in db."))
utils::u64_from_bytes(&b).map_err(|_| {
Error::bad_database("Invalid joinedcount in db.")
})
})
.transpose()
}
@ -348,27 +375,30 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(
self.roomuseroncejoinedids
.scan_prefix(prefix)
.map(|(key, _)| {
UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database(
"User ID in room_useroncejoined is invalid unicode.",
)
})?,
Box::new(self.roomuseroncejoinedids.scan_prefix(prefix).map(
|(key, _)| {
UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("User ID in room_useroncejoined is invalid."))
}),
)
.map_err(|_| {
Error::bad_database(
"User ID in room_useroncejoined is invalid \
unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"User ID in room_useroncejoined is invalid.",
)
})
},
))
}
/// Returns an iterator over all invited members of a room.
@ -378,53 +408,64 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(
self.roomuserid_invitecount
.scan_prefix(prefix)
.map(|(key, _)| {
UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("User ID in roomuserid_invited is invalid unicode.")
})?,
Box::new(self.roomuserid_invitecount.scan_prefix(prefix).map(
|(key, _)| {
UserId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| Error::bad_database("User ID in roomuserid_invited is invalid."))
}),
)
.map_err(|_| {
Error::bad_database(
"User ID in roomuserid_invited is invalid unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"User ID in roomuserid_invited is invalid.",
)
})
},
))
}
#[tracing::instrument(skip(self))]
fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
fn get_invite_count(
&self,
room_id: &RoomId,
user_id: &UserId,
) -> Result<Option<u64>> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_invitecount
.get(&key)?
.map_or(Ok(None), |bytes| {
Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid invitecount in db.")
})?))
})
self.roomuserid_invitecount.get(&key)?.map_or(Ok(None), |bytes| {
Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid invitecount in db.")
})?))
})
}
#[tracing::instrument(skip(self))]
fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
fn get_left_count(
&self,
room_id: &RoomId,
user_id: &UserId,
) -> Result<Option<u64>> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
self.roomuserid_leftcount
.get(&key)?
.map(|bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid leftcount in db."))
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid leftcount in db.")
})
})
.transpose()
}
@ -441,15 +482,22 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
.map(|(key, _)| {
RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Room ID in userroomid_joined is invalid unicode.")
Error::bad_database(
"Room ID in userroomid_joined is invalid \
unicode.",
)
})?,
)
.map_err(|_| Error::bad_database("Room ID in userroomid_joined is invalid."))
.map_err(|_| {
Error::bad_database(
"Room ID in userroomid_joined is invalid.",
)
})
}),
)
}
@ -460,35 +508,43 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
fn rooms_invited<'a>(
&'a self,
user_id: &UserId,
) -> Box<dyn Iterator<Item = Result<(OwnedRoomId, Vec<Raw<AnyStrippedStateEvent>>)>> + 'a> {
) -> Box<
dyn Iterator<
Item = Result<(OwnedRoomId, Vec<Raw<AnyStrippedStateEvent>>)>,
> + 'a,
> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(
self.userroomid_invitestate
.scan_prefix(prefix)
.map(|(key, state)| {
let room_id = RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Room ID in userroomid_invited is invalid unicode.")
})?,
Box::new(self.userroomid_invitestate.scan_prefix(prefix).map(
|(key, state)| {
let room_id = RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Room ID in userroomid_invited is invalid.")
})?;
Error::bad_database(
"Room ID in userroomid_invited is invalid unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"Room ID in userroomid_invited is invalid.",
)
})?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database("Invalid state in userroomid_invitestate.")
})?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database(
"Invalid state in userroomid_invitestate.",
)
})?;
Ok((room_id, state))
}),
)
Ok((room_id, state))
},
))
}
#[tracing::instrument(skip(self))]
@ -498,14 +554,17 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Result<Option<Vec<Raw<AnyStrippedStateEvent>>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
self.userroomid_invitestate
.get(&key)?
.map(|state| {
let state = serde_json::from_slice(&state)
.map_err(|_| Error::bad_database("Invalid state in userroomid_invitestate."))?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database(
"Invalid state in userroomid_invitestate.",
)
})?;
Ok(state)
})
@ -519,14 +578,17 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
room_id: &RoomId,
) -> Result<Option<Vec<Raw<AnyStrippedStateEvent>>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(room_id.as_bytes());
self.userroomid_leftstate
.get(&key)?
.map(|state| {
let state = serde_json::from_slice(&state)
.map_err(|_| Error::bad_database("Invalid state in userroomid_leftstate."))?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database(
"Invalid state in userroomid_leftstate.",
)
})?;
Ok(state)
})
@ -539,41 +601,48 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
fn rooms_left<'a>(
&'a self,
user_id: &UserId,
) -> Box<dyn Iterator<Item = Result<(OwnedRoomId, Vec<Raw<AnySyncStateEvent>>)>> + 'a> {
) -> Box<
dyn Iterator<Item = Result<(OwnedRoomId, Vec<Raw<AnySyncStateEvent>>)>>
+ 'a,
> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
Box::new(
self.userroomid_leftstate
.scan_prefix(prefix)
.map(|(key, state)| {
let room_id = RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xff)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Room ID in userroomid_invited is invalid unicode.")
})?,
Box::new(self.userroomid_leftstate.scan_prefix(prefix).map(
|(key, state)| {
let room_id = RoomId::parse(
utils::string_from_bytes(
key.rsplit(|&b| b == 0xFF)
.next()
.expect("rsplit always returns an element"),
)
.map_err(|_| {
Error::bad_database("Room ID in userroomid_invited is invalid.")
})?;
Error::bad_database(
"Room ID in userroomid_invited is invalid unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database(
"Room ID in userroomid_invited is invalid.",
)
})?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database("Invalid state in userroomid_leftstate.")
})?;
let state = serde_json::from_slice(&state).map_err(|_| {
Error::bad_database(
"Invalid state in userroomid_leftstate.",
)
})?;
Ok((room_id, state))
}),
)
Ok((room_id, state))
},
))
}
#[tracing::instrument(skip(self))]
fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
Ok(self.roomuseroncejoinedids.get(&userroom_id)?.is_some())
@ -582,7 +651,7 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
Ok(self.userroomid_joined.get(&userroom_id)?.is_some())
@ -591,7 +660,7 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
Ok(self.userroomid_invitestate.get(&userroom_id)?.is_some())
@ -600,7 +669,7 @@ impl service::rooms::state_cache::Data for KeyValueDatabase {
#[tracing::instrument(skip(self))]
fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
Ok(self.userroomid_leftstate.get(&userroom_id)?.is_some())

13
src/database/key_value/rooms/state_compressor.rs
View file

@ -12,8 +12,8 @@ impl service::rooms::state_compressor::Data for KeyValueDatabase {
.shortstatehash_statediff
.get(&shortstatehash.to_be_bytes())?
.ok_or_else(|| Error::bad_database("State hash does not exist"))?;
let parent =
utils::u64_from_bytes(&value[0..size_of::<u64>()]).expect("bytes have right length");
let parent = utils::u64_from_bytes(&value[0..size_of::<u64>()])
.expect("bytes have right length");
let parent = (parent != 0).then_some(parent);
let mut add_mode = true;
@ -30,7 +30,8 @@ impl service::rooms::state_compressor::Data for KeyValueDatabase {
if add_mode {
added.insert(v.try_into().expect("we checked the size above"));
} else {
removed.insert(v.try_into().expect("we checked the size above"));
removed
.insert(v.try_into().expect("we checked the size above"));
}
i += 2 * size_of::<u64>();
}
@ -42,7 +43,11 @@ impl service::rooms::state_compressor::Data for KeyValueDatabase {
})
}
fn save_statediff(&self, shortstatehash: u64, diff: StateDiff) -> Result<()> {
fn save_statediff(
&self,
shortstatehash: u64,
diff: StateDiff,
) -> Result<()> {
let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec();
for new in diff.added.iter() {
value.extend_from_slice(&new[..]);

55
src/database/key_value/rooms/threads.rs
View file

@ -1,8 +1,14 @@
use std::mem;
use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId};
use ruma::{
api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId,
UserId,
};
use crate::{database::KeyValueDatabase, service, services, utils, Error, PduEvent, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, PduEvent,
Result,
};
impl service::rooms::threads::Data for KeyValueDatabase {
fn threads_until<'a>(
@ -28,14 +34,22 @@ impl service::rooms::threads::Data for KeyValueDatabase {
.iter_from(&current, true)
.take_while(move |(k, _)| k.starts_with(&prefix))
.map(move |(pduid, _users)| {
let count = utils::u64_from_bytes(&pduid[(mem::size_of::<u64>())..])
.map_err(|_| Error::bad_database("Invalid pduid in threadid_userids."))?;
let count = utils::u64_from_bytes(
&pduid[(mem::size_of::<u64>())..],
)
.map_err(|_| {
Error::bad_database(
"Invalid pduid in threadid_userids.",
)
})?;
let mut pdu = services()
.rooms
.timeline
.get_pdu_from_id(&pduid)?
.ok_or_else(|| {
Error::bad_database("Invalid pduid reference in threadid_userids")
Error::bad_database(
"Invalid pduid reference in threadid_userids",
)
})?;
if pdu.sender != user_id {
pdu.remove_transaction_id()?;
@ -45,28 +59,43 @@ impl service::rooms::threads::Data for KeyValueDatabase {
))
}
fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()> {
fn update_participants(
&self,
root_id: &[u8],
participants: &[OwnedUserId],
) -> Result<()> {
let users = participants
.iter()
.map(|user| user.as_bytes())
.collect::<Vec<_>>()
.join(&[0xff][..]);
.join(&[0xFF][..]);
self.threadid_userids.insert(root_id, &users)?;
Ok(())
}
fn get_participants(&self, root_id: &[u8]) -> Result<Option<Vec<OwnedUserId>>> {
fn get_participants(
&self,
root_id: &[u8],
) -> Result<Option<Vec<OwnedUserId>>> {
if let Some(users) = self.threadid_userids.get(root_id)? {
Ok(Some(
users
.split(|b| *b == 0xff)
.split(|b| *b == 0xFF)
.map(|bytes| {
UserId::parse(utils::string_from_bytes(bytes).map_err(|_| {
Error::bad_database("Invalid UserId bytes in threadid_userids.")
})?)
.map_err(|_| Error::bad_database("Invalid UserId in threadid_userids."))
UserId::parse(utils::string_from_bytes(bytes).map_err(
|_| {
Error::bad_database(
"Invalid UserId bytes in threadid_userids.",
)
},
)?)
.map_err(|_| {
Error::bad_database(
"Invalid UserId in threadid_userids.",
)
})
})
.filter_map(Result::ok)
.collect(),

114
src/database/key_value/rooms/timeline.rs
View file

@ -1,16 +1,23 @@
use std::{collections::hash_map, mem::size_of, sync::Arc};
use ruma::{
api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId,
api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedUserId,
RoomId, UserId,
};
use service::rooms::timeline::PduCount;
use tracing::error;
use crate::{database::KeyValueDatabase, service, services, utils, Error, PduEvent, Result};
use service::rooms::timeline::PduCount;
use crate::{
database::KeyValueDatabase, service, services, utils, Error, PduEvent,
Result,
};
impl service::rooms::timeline::Data for KeyValueDatabase {
fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result<PduCount> {
fn last_timeline_count(
&self,
sender_user: &UserId,
room_id: &RoomId,
) -> Result<PduCount> {
match self
.lasttimelinecount_cache
.lock()
@ -45,14 +52,18 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
}
/// Returns the json of a pdu.
fn get_pdu_json(&self, event_id: &EventId) -> Result<Option<CanonicalJsonObject>> {
fn get_pdu_json(
&self,
event_id: &EventId,
) -> Result<Option<CanonicalJsonObject>> {
self.get_non_outlier_pdu_json(event_id)?.map_or_else(
|| {
self.eventid_outlierpdu
.get(event_id.as_bytes())?
.map(|pdu| {
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
serde_json::from_slice(&pdu).map_err(|_| {
Error::bad_database("Invalid PDU in db.")
})
})
.transpose()
},
@ -61,17 +72,21 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
}
/// Returns the json of a pdu.
fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result<Option<CanonicalJsonObject>> {
fn get_non_outlier_pdu_json(
&self,
event_id: &EventId,
) -> Result<Option<CanonicalJsonObject>> {
self.eventid_pduid
.get(event_id.as_bytes())?
.map(|pduid| {
self.pduid_pdu
.get(&pduid)?
.ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid."))
self.pduid_pdu.get(&pduid)?.ok_or_else(|| {
Error::bad_database("Invalid pduid in eventid_pduid.")
})
})
.transpose()?
.map(|pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
})
.transpose()
}
@ -82,17 +97,21 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
}
/// Returns the pdu.
fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
fn get_non_outlier_pdu(
&self,
event_id: &EventId,
) -> Result<Option<PduEvent>> {
self.eventid_pduid
.get(event_id.as_bytes())?
.map(|pduid| {
self.pduid_pdu
.get(&pduid)?
.ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid."))
self.pduid_pdu.get(&pduid)?.ok_or_else(|| {
Error::bad_database("Invalid pduid in eventid_pduid.")
})
})
.transpose()?
.map(|pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
})
.transpose()
}
@ -112,8 +131,9 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
self.eventid_outlierpdu
.get(event_id.as_bytes())?
.map(|pdu| {
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
serde_json::from_slice(&pdu).map_err(|_| {
Error::bad_database("Invalid PDU in db.")
})
})
.transpose()
},
@ -144,7 +164,10 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
}
/// Returns the pdu as a `BTreeMap<String, CanonicalJsonValue>`.
fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result<Option<CanonicalJsonObject>> {
fn get_pdu_json_from_id(
&self,
pdu_id: &[u8],
) -> Result<Option<CanonicalJsonObject>> {
self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| {
Ok(Some(
serde_json::from_slice(&pdu)
@ -162,7 +185,8 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
) -> Result<()> {
self.pduid_pdu.insert(
pdu_id,
&serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"),
&serde_json::to_vec(json)
.expect("CanonicalJsonObject is always a valid"),
)?;
self.lasttimelinecount_cache
@ -184,7 +208,8 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
) -> Result<()> {
self.pduid_pdu.insert(
pdu_id,
&serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"),
&serde_json::to_vec(json)
.expect("CanonicalJsonObject is always a valid"),
)?;
self.eventid_pduid.insert(event_id.as_bytes(), pdu_id)?;
@ -203,7 +228,8 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
if self.pduid_pdu.get(pdu_id)?.is_some() {
self.pduid_pdu.insert(
pdu_id,
&serde_json::to_vec(pdu_json).expect("CanonicalJsonObject is always a valid"),
&serde_json::to_vec(pdu_json)
.expect("CanonicalJsonObject is always a valid"),
)?;
} else {
return Err(Error::BadRequest(
@ -212,22 +238,21 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
));
}
self.pdu_cache
.lock()
.unwrap()
.remove(&(*pdu.event_id).to_owned());
self.pdu_cache.lock().unwrap().remove(&(*pdu.event_id).to_owned());
Ok(())
}
/// Returns an iterator over all events and their tokens in a room that happened before the
/// event with id `until` in reverse-chronological order.
/// Returns an iterator over all events and their tokens in a room that
/// happened before the event with id `until` in reverse-chronological
/// order.
fn pdus_until<'a>(
&'a self,
user_id: &UserId,
room_id: &RoomId,
until: PduCount,
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>> {
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>>
{
let (prefix, current) = count_to_id(room_id, until, 1, true)?;
let user_id = user_id.to_owned();
@ -238,7 +263,9 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
.take_while(move |(k, _)| k.starts_with(&prefix))
.map(move |(pdu_id, v)| {
let mut pdu = serde_json::from_slice::<PduEvent>(&v)
.map_err(|_| Error::bad_database("PDU in db is invalid."))?;
.map_err(|_| {
Error::bad_database("PDU in db is invalid.")
})?;
if pdu.sender != user_id {
pdu.remove_transaction_id()?;
}
@ -254,7 +281,8 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
user_id: &UserId,
room_id: &RoomId,
from: PduCount,
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>> {
) -> Result<Box<dyn Iterator<Item = Result<(PduCount, PduEvent)>> + 'a>>
{
let (prefix, current) = count_to_id(room_id, from, 1, false)?;
let user_id = user_id.to_owned();
@ -265,7 +293,9 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
.take_while(move |(k, _)| k.starts_with(&prefix))
.map(move |(pdu_id, v)| {
let mut pdu = serde_json::from_slice::<PduEvent>(&v)
.map_err(|_| Error::bad_database("PDU in db is invalid."))?;
.map_err(|_| {
Error::bad_database("PDU in db is invalid.")
})?;
if pdu.sender != user_id {
pdu.remove_transaction_id()?;
}
@ -286,13 +316,13 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
let mut highlights_batch = Vec::new();
for user in notifies {
let mut userroom_id = user.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
notifies_batch.push(userroom_id);
}
for user in highlights {
let mut userroom_id = user.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
highlights_batch.push(userroom_id);
}
@ -307,10 +337,12 @@ impl service::rooms::timeline::Data for KeyValueDatabase {
/// Returns the `count` of this pdu's id.
fn pdu_count(pdu_id: &[u8]) -> Result<PduCount> {
let last_u64 = utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::<u64>()..])
.map_err(|_| Error::bad_database("PDU has invalid count bytes."))?;
let last_u64 =
utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::<u64>()..])
.map_err(|_| Error::bad_database("PDU has invalid count bytes."))?;
let second_last_u64 = utils::u64_from_bytes(
&pdu_id[pdu_id.len() - 2 * size_of::<u64>()..pdu_id.len() - size_of::<u64>()],
&pdu_id[pdu_id.len() - 2 * size_of::<u64>()
..pdu_id.len() - size_of::<u64>()],
);
if matches!(second_last_u64, Ok(0)) {
@ -330,7 +362,9 @@ fn count_to_id(
.rooms
.short
.get_shortroomid(room_id)?
.ok_or_else(|| Error::bad_database("Looked for bad shortroomid in timeline"))?
.ok_or_else(|| {
Error::bad_database("Looked for bad shortroomid in timeline")
})?
.to_be_bytes()
.to_vec();
let mut pdu_id = prefix.clone();

110
src/database/key_value/rooms/user.rs
View file

@ -1,14 +1,20 @@
use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId};
use crate::{database::KeyValueDatabase, service, services, utils, Error, Result};
use crate::{
database::KeyValueDatabase, service, services, utils, Error, Result,
};
impl service::rooms::user::Data for KeyValueDatabase {
fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> {
fn reset_notification_counts(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<()> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
let mut roomuser_id = room_id.as_bytes().to_vec();
roomuser_id.push(0xff);
roomuser_id.push(0xFF);
roomuser_id.extend_from_slice(user_id.as_bytes());
self.userroomid_notificationcount
@ -24,35 +30,51 @@ impl service::rooms::user::Data for KeyValueDatabase {
Ok(())
}
fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
fn notification_count(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<u64> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
self.userroomid_notificationcount
.get(&userroom_id)?
.map_or(Ok(0), |bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid notification count in db."))
})
self.userroomid_notificationcount.get(&userroom_id)?.map_or(
Ok(0),
|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid notification count in db.")
})
},
)
}
fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
fn highlight_count(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<u64> {
let mut userroom_id = user_id.as_bytes().to_vec();
userroom_id.push(0xff);
userroom_id.push(0xFF);
userroom_id.extend_from_slice(room_id.as_bytes());
self.userroomid_highlightcount
.get(&userroom_id)?
.map_or(Ok(0), |bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid highlight count in db."))
})
self.userroomid_highlightcount.get(&userroom_id)?.map_or(
Ok(0),
|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid highlight count in db.")
})
},
)
}
fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
fn last_notification_read(
&self,
user_id: &UserId,
room_id: &RoomId,
) -> Result<u64> {
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(user_id.as_bytes());
Ok(self
@ -60,7 +82,9 @@ impl service::rooms::user::Data for KeyValueDatabase {
.get(&key)?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.")
Error::bad_database(
"Count in roomuserid_lastprivatereadupdate is invalid.",
)
})
})
.transpose()?
@ -86,7 +110,11 @@ impl service::rooms::user::Data for KeyValueDatabase {
.insert(&key, &shortstatehash.to_be_bytes())
}
fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result<Option<u64>> {
fn get_token_shortstatehash(
&self,
room_id: &RoomId,
token: u64,
) -> Result<Option<u64>> {
let shortroomid = services()
.rooms
.short
@ -100,7 +128,10 @@ impl service::rooms::user::Data for KeyValueDatabase {
.get(&key)?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid shortstatehash in roomsynctoken_shortstatehash")
Error::bad_database(
"Invalid shortstatehash in \
roomsynctoken_shortstatehash",
)
})
})
.transpose()
@ -112,7 +143,7 @@ impl service::rooms::user::Data for KeyValueDatabase {
) -> Result<Box<dyn Iterator<Item = Result<OwnedRoomId>> + 'a>> {
let iterators = users.into_iter().map(move |user_id| {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
self.userroomid_joined
.scan_prefix(prefix)
@ -121,8 +152,12 @@ impl service::rooms::user::Data for KeyValueDatabase {
let roomid_index = key
.iter()
.enumerate()
.find(|(_, &b)| b == 0xff)
.ok_or_else(|| Error::bad_database("Invalid userroomid_joined in db."))?
.find(|(_, &b)| b == 0xFF)
.ok_or_else(|| {
Error::bad_database(
"Invalid userroomid_joined in db.",
)
})?
.0
+ 1;
@ -133,15 +168,24 @@ impl service::rooms::user::Data for KeyValueDatabase {
.filter_map(Result::ok)
});
// We use the default compare function because keys are sorted correctly (not reversed)
// We use the default compare function because keys are sorted correctly
// (not reversed)
Ok(Box::new(
utils::common_elements(iterators, Ord::cmp)
.expect("users is not empty")
.map(|bytes| {
RoomId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid RoomId bytes in userroomid_joined")
})?)
.map_err(|_| Error::bad_database("Invalid RoomId in userroomid_joined."))
RoomId::parse(utils::string_from_bytes(&bytes).map_err(
|_| {
Error::bad_database(
"Invalid RoomId bytes in userroomid_joined",
)
},
)?)
.map_err(|_| {
Error::bad_database(
"Invalid RoomId in userroomid_joined.",
)
})
}),
))
}

138
src/database/key_value/sending.rs
View file

@ -12,31 +12,34 @@ use crate::{
impl service::sending::Data for KeyValueDatabase {
fn active_requests<'a>(
&'a self,
) -> Box<dyn Iterator<Item = Result<(Vec<u8>, OutgoingKind, SendingEventType)>> + 'a> {
Box::new(
self.servercurrentevent_data
.iter()
.map(|(key, v)| parse_servercurrentevent(&key, v).map(|(k, e)| (key, k, e))),
)
) -> Box<
dyn Iterator<Item = Result<(Vec<u8>, OutgoingKind, SendingEventType)>>
+ 'a,
> {
Box::new(self.servercurrentevent_data.iter().map(|(key, v)| {
parse_servercurrentevent(&key, v).map(|(k, e)| (key, k, e))
}))
}
fn active_requests_for<'a>(
&'a self,
outgoing_kind: &OutgoingKind,
) -> Box<dyn Iterator<Item = Result<(Vec<u8>, SendingEventType)>> + 'a> {
) -> Box<dyn Iterator<Item = Result<(Vec<u8>, SendingEventType)>> + 'a>
{
let prefix = outgoing_kind.get_prefix();
Box::new(
self.servercurrentevent_data
.scan_prefix(prefix)
.map(|(key, v)| parse_servercurrentevent(&key, v).map(|(_, e)| (key, e))),
)
Box::new(self.servercurrentevent_data.scan_prefix(prefix).map(
|(key, v)| parse_servercurrentevent(&key, v).map(|(_, e)| (key, e)),
))
}
fn delete_active_request(&self, key: Vec<u8>) -> Result<()> {
self.servercurrentevent_data.remove(&key)
}
fn delete_all_active_requests_for(&self, outgoing_kind: &OutgoingKind) -> Result<()> {
fn delete_all_active_requests_for(
&self,
outgoing_kind: &OutgoingKind,
) -> Result<()> {
let prefix = outgoing_kind.get_prefix();
for (key, _) in self.servercurrentevent_data.scan_prefix(prefix) {
self.servercurrentevent_data.remove(&key)?;
@ -45,9 +48,13 @@ impl service::sending::Data for KeyValueDatabase {
Ok(())
}
fn delete_all_requests_for(&self, outgoing_kind: &OutgoingKind) -> Result<()> {
fn delete_all_requests_for(
&self,
outgoing_kind: &OutgoingKind,
) -> Result<()> {
let prefix = outgoing_kind.get_prefix();
for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) {
for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone())
{
self.servercurrentevent_data.remove(&key).unwrap();
}
@ -69,7 +76,9 @@ impl service::sending::Data for KeyValueDatabase {
if let SendingEventType::Pdu(value) = &event {
key.extend_from_slice(value);
} else {
key.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
key.extend_from_slice(
&services().globals.next_count()?.to_be_bytes(),
);
}
let value = if let SendingEventType::Edu(value) = &event {
&**value
@ -79,24 +88,25 @@ impl service::sending::Data for KeyValueDatabase {
batch.push((key.clone(), value.to_owned()));
keys.push(key);
}
self.servernameevent_data
.insert_batch(&mut batch.into_iter())?;
self.servernameevent_data.insert_batch(&mut batch.into_iter())?;
Ok(keys)
}
fn queued_requests<'a>(
&'a self,
outgoing_kind: &OutgoingKind,
) -> Box<dyn Iterator<Item = Result<(SendingEventType, Vec<u8>)>> + 'a> {
) -> Box<dyn Iterator<Item = Result<(SendingEventType, Vec<u8>)>> + 'a>
{
let prefix = outgoing_kind.get_prefix();
return Box::new(
self.servernameevent_data
.scan_prefix(prefix)
.map(|(k, v)| parse_servercurrentevent(&k, v).map(|(_, ev)| (ev, k))),
);
return Box::new(self.servernameevent_data.scan_prefix(prefix).map(
|(k, v)| parse_servercurrentevent(&k, v).map(|(_, ev)| (ev, k)),
));
}
fn mark_as_active(&self, events: &[(SendingEventType, Vec<u8>)]) -> Result<()> {
fn mark_as_active(
&self,
events: &[(SendingEventType, Vec<u8>)],
) -> Result<()> {
for (e, key) in events {
let value = if let SendingEventType::Edu(value) = &e {
&**value
@ -110,18 +120,24 @@ impl service::sending::Data for KeyValueDatabase {
Ok(())
}
fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) -> Result<()> {
fn set_latest_educount(
&self,
server_name: &ServerName,
last_count: u64,
) -> Result<()> {
self.servername_educount
.insert(server_name.as_bytes(), &last_count.to_be_bytes())
}
fn get_latest_educount(&self, server_name: &ServerName) -> Result<u64> {
self.servername_educount
.get(server_name.as_bytes())?
.map_or(Ok(0), |bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid u64 in servername_educount."))
})
self.servername_educount.get(server_name.as_bytes())?.map_or(
Ok(0),
|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid u64 in servername_educount.")
})
},
)
}
}
@ -132,15 +148,17 @@ fn parse_servercurrentevent(
) -> Result<(OutgoingKind, SendingEventType)> {
// Appservices start with a plus
Ok::<_, Error>(if key.starts_with(b"+") {
let mut parts = key[1..].splitn(2, |&b| b == 0xff);
let mut parts = key[1..].splitn(2, |&b| b == 0xFF);
let server = parts.next().expect("splitn always returns one element");
let event = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let event = parts.next().ok_or_else(|| {
Error::bad_database("Invalid bytes in servercurrentpdus.")
})?;
let server = utils::string_from_bytes(server).map_err(|_| {
Error::bad_database("Invalid server bytes in server_currenttransaction")
Error::bad_database(
"Invalid server bytes in server_currenttransaction",
)
})?;
(
@ -152,23 +170,27 @@ fn parse_servercurrentevent(
},
)
} else if key.starts_with(b"$") {
let mut parts = key[1..].splitn(3, |&b| b == 0xff);
let mut parts = key[1..].splitn(3, |&b| b == 0xFF);
let user = parts.next().expect("splitn always returns one element");
let user_string = utils::string_from_bytes(user)
.map_err(|_| Error::bad_database("Invalid user string in servercurrentevent"))?;
let user_id = UserId::parse(user_string)
.map_err(|_| Error::bad_database("Invalid user id in servercurrentevent"))?;
let user_string = utils::string_from_bytes(user).map_err(|_| {
Error::bad_database("Invalid user string in servercurrentevent")
})?;
let user_id = UserId::parse(user_string).map_err(|_| {
Error::bad_database("Invalid user id in servercurrentevent")
})?;
let pushkey = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let pushkey_string = utils::string_from_bytes(pushkey)
.map_err(|_| Error::bad_database("Invalid pushkey in servercurrentevent"))?;
let pushkey = parts.next().ok_or_else(|| {
Error::bad_database("Invalid bytes in servercurrentpdus.")
})?;
let pushkey_string =
utils::string_from_bytes(pushkey).map_err(|_| {
Error::bad_database("Invalid pushkey in servercurrentevent")
})?;
let event = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let event = parts.next().ok_or_else(|| {
Error::bad_database("Invalid bytes in servercurrentpdus.")
})?;
(
OutgoingKind::Push(user_id, pushkey_string),
@ -180,20 +202,24 @@ fn parse_servercurrentevent(
},
)
} else {
let mut parts = key.splitn(2, |&b| b == 0xff);
let mut parts = key.splitn(2, |&b| b == 0xFF);
let server = parts.next().expect("splitn always returns one element");
let event = parts
.next()
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
let event = parts.next().ok_or_else(|| {
Error::bad_database("Invalid bytes in servercurrentpdus.")
})?;
let server = utils::string_from_bytes(server).map_err(|_| {
Error::bad_database("Invalid server bytes in server_currenttransaction")
Error::bad_database(
"Invalid server bytes in server_currenttransaction",
)
})?;
(
OutgoingKind::Normal(ServerName::parse(server).map_err(|_| {
Error::bad_database("Invalid server string in server_currenttransaction")
Error::bad_database(
"Invalid server string in server_currenttransaction",
)
})?),
if value.is_empty() {
SendingEventType::Pdu(event.to_vec())

16
src/database/key_value/transaction_ids.rs
View file

@ -11,9 +11,11 @@ impl service::transaction_ids::Data for KeyValueDatabase {
data: &[u8],
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(
device_id.map(DeviceId::as_bytes).unwrap_or_default(),
);
key.push(0xFF);
key.extend_from_slice(txn_id.as_bytes());
self.userdevicetxnid_response.insert(&key, data)?;
@ -28,9 +30,11 @@ impl service::transaction_ids::Data for KeyValueDatabase {
txn_id: &TransactionId,
) -> Result<Option<Vec<u8>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(
device_id.map(DeviceId::as_bytes).unwrap_or_default(),
);
key.push(0xFF);
key.extend_from_slice(txn_id.as_bytes());
// If there's no entry, this is a new transaction

35
src/database/key_value/uiaa.rs
View file

@ -13,13 +13,10 @@ impl service::uiaa::Data for KeyValueDatabase {
session: &str,
request: &CanonicalJsonValue,
) -> Result<()> {
self.userdevicesessionid_uiaarequest
.write()
.unwrap()
.insert(
(user_id.to_owned(), device_id.to_owned(), session.to_owned()),
request.to_owned(),
);
self.userdevicesessionid_uiaarequest.write().unwrap().insert(
(user_id.to_owned(), device_id.to_owned(), session.to_owned()),
request.to_owned(),
);
Ok(())
}
@ -33,7 +30,11 @@ impl service::uiaa::Data for KeyValueDatabase {
self.userdevicesessionid_uiaarequest
.read()
.unwrap()
.get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned()))
.get(&(
user_id.to_owned(),
device_id.to_owned(),
session.to_owned(),
))
.map(ToOwned::to_owned)
}
@ -45,19 +46,19 @@ impl service::uiaa::Data for KeyValueDatabase {
uiaainfo: Option<&UiaaInfo>,
) -> Result<()> {
let mut userdevicesessionid = user_id.as_bytes().to_vec();
userdevicesessionid.push(0xff);
userdevicesessionid.push(0xFF);
userdevicesessionid.extend_from_slice(device_id.as_bytes());
userdevicesessionid.push(0xff);
userdevicesessionid.push(0xFF);
userdevicesessionid.extend_from_slice(session.as_bytes());
if let Some(uiaainfo) = uiaainfo {
self.userdevicesessionid_uiaainfo.insert(
&userdevicesessionid,
&serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"),
&serde_json::to_vec(&uiaainfo)
.expect("UiaaInfo::to_vec always works"),
)?;
} else {
self.userdevicesessionid_uiaainfo
.remove(&userdevicesessionid)?;
self.userdevicesessionid_uiaainfo.remove(&userdevicesessionid)?;
}
Ok(())
@ -70,9 +71,9 @@ impl service::uiaa::Data for KeyValueDatabase {
session: &str,
) -> Result<UiaaInfo> {
let mut userdevicesessionid = user_id.as_bytes().to_vec();
userdevicesessionid.push(0xff);
userdevicesessionid.push(0xFF);
userdevicesessionid.extend_from_slice(device_id.as_bytes());
userdevicesessionid.push(0xff);
userdevicesessionid.push(0xFF);
userdevicesessionid.extend_from_slice(session.as_bytes());
serde_json::from_slice(
@ -84,6 +85,8 @@ impl service::uiaa::Data for KeyValueDatabase {
"UIAA session does not exist.",
))?,
)
.map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))
.map_err(|_| {
Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid.")
})
}
}

535
src/database/key_value/users.rs
View file

@ -5,8 +5,8 @@ use ruma::{
encryption::{CrossSigningKey, DeviceKeys, OneTimeKey},
events::{AnyToDeviceEvent, StateEventType},
serde::Raw,
DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId,
OwnedDeviceKeyId, OwnedMxcUri, OwnedUserId, UInt, UserId,
DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch,
OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedUserId, UInt, UserId,
};
use tracing::warn;
@ -40,67 +40,100 @@ impl service::users::Data for KeyValueDatabase {
}
/// Find out which user an access token belongs to.
fn find_from_token(&self, token: &str) -> Result<Option<(OwnedUserId, String)>> {
self.token_userdeviceid
.get(token.as_bytes())?
.map_or(Ok(None), |bytes| {
let mut parts = bytes.split(|&b| b == 0xff);
fn find_from_token(
&self,
token: &str,
) -> Result<Option<(OwnedUserId, String)>> {
self.token_userdeviceid.get(token.as_bytes())?.map_or(
Ok(None),
|bytes| {
let mut parts = bytes.split(|&b| b == 0xFF);
let user_bytes = parts.next().ok_or_else(|| {
Error::bad_database("User ID in token_userdeviceid is invalid.")
Error::bad_database(
"User ID in token_userdeviceid is invalid.",
)
})?;
let device_bytes = parts.next().ok_or_else(|| {
Error::bad_database("Device ID in token_userdeviceid is invalid.")
Error::bad_database(
"Device ID in token_userdeviceid is invalid.",
)
})?;
Ok(Some((
UserId::parse(utils::string_from_bytes(user_bytes).map_err(|_| {
Error::bad_database("User ID in token_userdeviceid is invalid unicode.")
})?)
UserId::parse(
utils::string_from_bytes(user_bytes).map_err(|_| {
Error::bad_database(
"User ID in token_userdeviceid is invalid \
unicode.",
)
})?,
)
.map_err(|_| {
Error::bad_database("User ID in token_userdeviceid is invalid.")
Error::bad_database(
"User ID in token_userdeviceid is invalid.",
)
})?,
utils::string_from_bytes(device_bytes).map_err(|_| {
Error::bad_database("Device ID in token_userdeviceid is invalid.")
Error::bad_database(
"Device ID in token_userdeviceid is invalid.",
)
})?,
)))
})
},
)
}
/// Returns an iterator over all users on this homeserver.
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
fn iter<'a>(
&'a self,
) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
Box::new(self.userid_password.iter().map(|(bytes, _)| {
UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("User ID in userid_password is invalid unicode.")
Error::bad_database(
"User ID in userid_password is invalid unicode.",
)
})?)
.map_err(|_| Error::bad_database("User ID in userid_password is invalid."))
.map_err(|_| {
Error::bad_database("User ID in userid_password is invalid.")
})
}))
}
/// Returns a list of local users as list of usernames.
///
/// A user account is considered `local` if the length of it's password is greater then zero.
/// A user account is considered `local` if the length of it's password is
/// greater then zero.
fn list_local_users(&self) -> Result<Vec<String>> {
let users: Vec<String> = self
.userid_password
.iter()
.filter_map(|(username, pw)| get_username_with_valid_password(&username, &pw))
.filter_map(|(username, pw)| {
get_username_with_valid_password(&username, &pw)
})
.collect();
Ok(users)
}
/// Returns the password hash for the given user.
fn password_hash(&self, user_id: &UserId) -> Result<Option<String>> {
self.userid_password
.get(user_id.as_bytes())?
.map_or(Ok(None), |bytes| {
self.userid_password.get(user_id.as_bytes())?.map_or(
Ok(None),
|bytes| {
Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Password hash in db is not valid string.")
Error::bad_database(
"Password hash in db is not valid string.",
)
})?))
})
},
)
}
/// Hash and set the user's password to the Argon2 hash
fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> {
fn set_password(
&self,
user_id: &UserId,
password: Option<&str>,
) -> Result<()> {
if let Some(password) = password {
if let Ok(hash) = utils::calculate_password_hash(password) {
self.userid_password
@ -120,17 +153,23 @@ impl service::users::Data for KeyValueDatabase {
/// Returns the `displayname` of a user on this homeserver.
fn displayname(&self, user_id: &UserId) -> Result<Option<String>> {
self.userid_displayname
.get(user_id.as_bytes())?
.map_or(Ok(None), |bytes| {
self.userid_displayname.get(user_id.as_bytes())?.map_or(
Ok(None),
|bytes| {
Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Displayname in db is invalid.")
})?))
})
},
)
}
/// Sets a new `displayname` or removes it if `displayname` is `None`. You still need to nofify all rooms of this change.
fn set_displayname(&self, user_id: &UserId, displayname: Option<String>) -> Result<()> {
/// Sets a new `displayname` or removes it if `displayname` is `None`. You
/// still need to nofify all rooms of this change.
fn set_displayname(
&self,
user_id: &UserId,
displayname: Option<String>,
) -> Result<()> {
if let Some(displayname) = displayname {
self.userid_displayname
.insert(user_id.as_bytes(), displayname.as_bytes())?;
@ -147,17 +186,25 @@ impl service::users::Data for KeyValueDatabase {
.get(user_id.as_bytes())?
.map(|bytes| {
utils::string_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Avatar URL in db is invalid."))
.map_err(|_| {
Error::bad_database("Avatar URL in db is invalid.")
})
.map(Into::into)
})
.transpose()
}
/// Sets a new `avatar_url` or removes it if `avatar_url` is `None`.
fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<OwnedMxcUri>) -> Result<()> {
fn set_avatar_url(
&self,
user_id: &UserId,
avatar_url: Option<OwnedMxcUri>,
) -> Result<()> {
if let Some(avatar_url) = avatar_url {
self.userid_avatarurl
.insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?;
self.userid_avatarurl.insert(
user_id.as_bytes(),
avatar_url.to_string().as_bytes(),
)?;
} else {
self.userid_avatarurl.remove(user_id.as_bytes())?;
}
@ -170,8 +217,9 @@ impl service::users::Data for KeyValueDatabase {
self.userid_blurhash
.get(user_id.as_bytes())?
.map(|bytes| {
let s = utils::string_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?;
let s = utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Avatar URL in db is invalid.")
})?;
Ok(s)
})
@ -179,7 +227,11 @@ impl service::users::Data for KeyValueDatabase {
}
/// Sets a new `avatar_url` or removes it if `avatar_url` is `None`.
fn set_blurhash(&self, user_id: &UserId, blurhash: Option<String>) -> Result<()> {
fn set_blurhash(
&self,
user_id: &UserId,
blurhash: Option<String>,
) -> Result<()> {
if let Some(blurhash) = blurhash {
self.userid_blurhash
.insert(user_id.as_bytes(), blurhash.as_bytes())?;
@ -204,11 +256,10 @@ impl service::users::Data for KeyValueDatabase {
);
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
self.userid_devicelistversion
.increment(user_id.as_bytes())?;
self.userid_devicelistversion.increment(user_id.as_bytes())?;
self.userdeviceid_metadata.insert(
&userdeviceid,
@ -228,9 +279,13 @@ impl service::users::Data for KeyValueDatabase {
}
/// Removes a device from a user.
fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> {
fn remove_device(
&self,
user_id: &UserId,
device_id: &DeviceId,
) -> Result<()> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
// Remove tokens
@ -241,7 +296,7 @@ impl service::users::Data for KeyValueDatabase {
// Remove todevice events
let mut prefix = userdeviceid.clone();
prefix.push(0xff);
prefix.push(0xFF);
for (key, _) in self.todeviceid_events.scan_prefix(prefix) {
self.todeviceid_events.remove(&key)?;
@ -249,8 +304,7 @@ impl service::users::Data for KeyValueDatabase {
// TODO: Remove onetimekeys
self.userid_devicelistversion
.increment(user_id.as_bytes())?;
self.userid_devicelistversion.increment(user_id.as_bytes())?;
self.userdeviceid_metadata.remove(&userdeviceid)?;
@ -263,29 +317,34 @@ impl service::users::Data for KeyValueDatabase {
user_id: &UserId,
) -> Box<dyn Iterator<Item = Result<OwnedDeviceId>> + 'a> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
// All devices have metadata
Box::new(
self.userdeviceid_metadata
.scan_prefix(prefix)
.map(|(bytes, _)| {
Ok(utils::string_from_bytes(
bytes.rsplit(|&b| b == 0xff).next().ok_or_else(|| {
Error::bad_database("UserDevice ID in db is invalid.")
})?,
Box::new(self.userdeviceid_metadata.scan_prefix(prefix).map(
|(bytes, _)| {
Ok(utils::string_from_bytes(
bytes.rsplit(|&b| b == 0xFF).next().ok_or_else(|| {
Error::bad_database("UserDevice ID in db is invalid.")
})?,
)
.map_err(|_| {
Error::bad_database(
"Device ID in userdeviceid_metadata is invalid.",
)
.map_err(|_| {
Error::bad_database("Device ID in userdeviceid_metadata is invalid.")
})?
.into())
}),
)
})?
.into())
},
))
}
/// Replaces the access token of one device.
fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
fn set_token(
&self,
user_id: &UserId,
device_id: &DeviceId,
token: &str,
) -> Result<()> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
assert!(
@ -300,10 +359,8 @@ impl service::users::Data for KeyValueDatabase {
}
// Assign token to user device combination
self.userdeviceid_token
.insert(&userdeviceid, token.as_bytes())?;
self.token_userdeviceid
.insert(token.as_bytes(), &userdeviceid)?;
self.userdeviceid_token.insert(&userdeviceid, token.as_bytes())?;
self.token_userdeviceid.insert(token.as_bytes(), &userdeviceid)?;
Ok(())
}
@ -316,17 +373,19 @@ impl service::users::Data for KeyValueDatabase {
one_time_key_value: &Raw<OneTimeKey>,
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(device_id.as_bytes());
assert!(
self.userdeviceid_metadata.get(&key)?.is_some(),
"devices should have metadata and this method should only be called with existing devices"
"devices should have metadata and this method should only be \
called with existing devices"
);
key.push(0xff);
// TODO: Use DeviceKeyId::to_string when it's available (and update everything,
// because there are no wrapping quotation marks anymore)
key.push(0xFF);
// TODO: Use DeviceKeyId::to_string when it's available (and update
// everything, because there are no wrapping quotation marks
// anymore)
key.extend_from_slice(
serde_json::to_string(one_time_key_key)
.expect("DeviceKeyId::to_string always works")
@ -335,7 +394,8 @@ impl service::users::Data for KeyValueDatabase {
self.onetimekeyid_onetimekeys.insert(
&key,
&serde_json::to_vec(&one_time_key_value).expect("OneTimeKey::to_vec always works"),
&serde_json::to_vec(&one_time_key_value)
.expect("OneTimeKey::to_vec always works"),
)?;
self.userid_lastonetimekeyupdate.insert(
@ -347,13 +407,16 @@ impl service::users::Data for KeyValueDatabase {
}
fn last_one_time_keys_update(&self, user_id: &UserId) -> Result<u64> {
self.userid_lastonetimekeyupdate
.get(user_id.as_bytes())?
.map_or(Ok(0), |bytes| {
self.userid_lastonetimekeyupdate.get(user_id.as_bytes())?.map_or(
Ok(0),
|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
Error::bad_database(
"Count in roomid_lastroomactiveupdate is invalid.",
)
})
})
},
)
}
fn take_one_time_key(
@ -363,9 +426,9 @@ impl service::users::Data for KeyValueDatabase {
key_algorithm: &DeviceKeyAlgorithm,
) -> Result<Option<(OwnedDeviceKeyId, Raw<OneTimeKey>)>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(device_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
// Annoying quotation mark
prefix.push(b'"');
prefix.extend_from_slice(key_algorithm.as_ref().as_bytes());
@ -384,13 +447,18 @@ impl service::users::Data for KeyValueDatabase {
Ok((
serde_json::from_slice(
key.rsplit(|&b| b == 0xff)
.next()
.ok_or_else(|| Error::bad_database("OneTimeKeyId in db is invalid."))?,
key.rsplit(|&b| b == 0xFF).next().ok_or_else(|| {
Error::bad_database(
"OneTimeKeyId in db is invalid.",
)
})?,
)
.map_err(|_| Error::bad_database("OneTimeKeyId in db is invalid."))?,
serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("OneTimeKeys in db are invalid."))?,
.map_err(|_| {
Error::bad_database("OneTimeKeyId in db is invalid.")
})?,
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("OneTimeKeys in db are invalid.")
})?,
))
})
.transpose()
@ -402,25 +470,31 @@ impl service::users::Data for KeyValueDatabase {
device_id: &DeviceId,
) -> Result<BTreeMap<DeviceKeyAlgorithm, UInt>> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
let mut counts = BTreeMap::new();
for algorithm in
self.onetimekeyid_onetimekeys
.scan_prefix(userdeviceid)
.map(|(bytes, _)| {
Ok::<_, Error>(
serde_json::from_slice::<OwnedDeviceKeyId>(
bytes.rsplit(|&b| b == 0xff).next().ok_or_else(|| {
Error::bad_database("OneTimeKey ID in db is invalid.")
})?,
)
.map_err(|_| Error::bad_database("DeviceKeyId in db is invalid."))?
.algorithm(),
for algorithm in self
.onetimekeyid_onetimekeys
.scan_prefix(userdeviceid)
.map(|(bytes, _)| {
Ok::<_, Error>(
serde_json::from_slice::<OwnedDeviceKeyId>(
bytes.rsplit(|&b| b == 0xFF).next().ok_or_else(
|| {
Error::bad_database(
"OneTimeKey ID in db is invalid.",
)
},
)?,
)
})
.map_err(|_| {
Error::bad_database("DeviceKeyId in db is invalid.")
})?
.algorithm(),
)
})
{
*counts.entry(algorithm?).or_default() += UInt::from(1_u32);
}
@ -435,12 +509,13 @@ impl service::users::Data for KeyValueDatabase {
device_keys: &Raw<DeviceKeys>,
) -> Result<()> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
self.keyid_key.insert(
&userdeviceid,
&serde_json::to_vec(&device_keys).expect("DeviceKeys::to_vec always works"),
&serde_json::to_vec(&device_keys)
.expect("DeviceKeys::to_vec always works"),
)?;
self.mark_device_key_update(user_id)?;
@ -458,30 +533,33 @@ impl service::users::Data for KeyValueDatabase {
) -> Result<()> {
// TODO: Check signatures
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let (master_key_key, _) = self.parse_master_key(user_id, master_key)?;
self.keyid_key
.insert(&master_key_key, master_key.json().get().as_bytes())?;
self.userid_masterkeyid
.insert(user_id.as_bytes(), &master_key_key)?;
self.userid_masterkeyid.insert(user_id.as_bytes(), &master_key_key)?;
// Self-signing key
if let Some(self_signing_key) = self_signing_key {
let mut self_signing_key_ids = self_signing_key
.deserialize()
.map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key")
Error::BadRequest(
ErrorKind::InvalidParam,
"Invalid self signing key",
)
})?
.keys
.into_values();
let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Self signing key contained no key.",
))?;
let self_signing_key_id =
self_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Self signing key contained no key.",
))?;
if self_signing_key_ids.next().is_some() {
return Err(Error::BadRequest(
@ -491,7 +569,8 @@ impl service::users::Data for KeyValueDatabase {
}
let mut self_signing_key_key = prefix.clone();
self_signing_key_key.extend_from_slice(self_signing_key_id.as_bytes());
self_signing_key_key
.extend_from_slice(self_signing_key_id.as_bytes());
self.keyid_key.insert(
&self_signing_key_key,
@ -507,15 +586,19 @@ impl service::users::Data for KeyValueDatabase {
let mut user_signing_key_ids = user_signing_key
.deserialize()
.map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key")
Error::BadRequest(
ErrorKind::InvalidParam,
"Invalid user signing key",
)
})?
.keys
.into_values();
let user_signing_key_id = user_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"User signing key contained no key.",
))?;
let user_signing_key_id =
user_signing_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"User signing key contained no key.",
))?;
if user_signing_key_ids.next().is_some() {
return Err(Error::BadRequest(
@ -525,7 +608,8 @@ impl service::users::Data for KeyValueDatabase {
}
let mut user_signing_key_key = prefix;
user_signing_key_key.extend_from_slice(user_signing_key_id.as_bytes());
user_signing_key_key
.extend_from_slice(user_signing_key_id.as_bytes());
self.keyid_key.insert(
&user_signing_key_key,
@ -551,32 +635,44 @@ impl service::users::Data for KeyValueDatabase {
sender_id: &UserId,
) -> Result<()> {
let mut key = target_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(key_id.as_bytes());
let mut cross_signing_key: serde_json::Value =
serde_json::from_slice(&self.keyid_key.get(&key)?.ok_or(Error::BadRequest(
let mut cross_signing_key: serde_json::Value = serde_json::from_slice(
&self.keyid_key.get(&key)?.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Tried to sign nonexistent key.",
))?)
.map_err(|_| Error::bad_database("key in keyid_key is invalid."))?;
))?,
)
.map_err(|_| Error::bad_database("key in keyid_key is invalid."))?;
let signatures = cross_signing_key
.get_mut("signatures")
.ok_or_else(|| Error::bad_database("key in keyid_key has no signatures field."))?
.ok_or_else(|| {
Error::bad_database("key in keyid_key has no signatures field.")
})?
.as_object_mut()
.ok_or_else(|| Error::bad_database("key in keyid_key has invalid signatures field."))?
.ok_or_else(|| {
Error::bad_database(
"key in keyid_key has invalid signatures field.",
)
})?
.entry(sender_id.to_string())
.or_insert_with(|| serde_json::Map::new().into());
signatures
.as_object_mut()
.ok_or_else(|| Error::bad_database("signatures in keyid_key for a user is invalid."))?
.ok_or_else(|| {
Error::bad_database(
"signatures in keyid_key for a user is invalid.",
)
})?
.insert(signature.0, signature.1.into());
self.keyid_key.insert(
&key,
&serde_json::to_vec(&cross_signing_key).expect("CrossSigningKey::to_vec always works"),
&serde_json::to_vec(&cross_signing_key)
.expect("CrossSigningKey::to_vec always works"),
)?;
self.mark_device_key_update(target_id)?;
@ -591,7 +687,7 @@ impl service::users::Data for KeyValueDatabase {
to: Option<u64>,
) -> Box<dyn Iterator<Item = Result<OwnedUserId>> + 'a> {
let mut prefix = user_or_room_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let mut start = prefix.clone();
start.extend_from_slice(&(from + 1).to_be_bytes());
@ -603,26 +699,39 @@ impl service::users::Data for KeyValueDatabase {
.iter_from(&start, false)
.take_while(move |(k, _)| {
k.starts_with(&prefix)
&& if let Some(current) = k.splitn(2, |&b| b == 0xff).nth(1) {
&& if let Some(current) =
k.splitn(2, |&b| b == 0xFF).nth(1)
{
if let Ok(c) = utils::u64_from_bytes(current) {
c <= to
} else {
warn!("BadDatabase: Could not parse keychangeid_userid bytes");
warn!(
"BadDatabase: Could not parse \
keychangeid_userid bytes"
);
false
}
} else {
warn!("BadDatabase: Could not parse keychangeid_userid");
warn!(
"BadDatabase: Could not parse \
keychangeid_userid"
);
false
}
})
.map(|(_, bytes)| {
UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| {
Error::bad_database(
"User ID in devicekeychangeid_userid is invalid unicode.",
)
})?)
UserId::parse(utils::string_from_bytes(&bytes).map_err(
|_| {
Error::bad_database(
"User ID in devicekeychangeid_userid is \
invalid unicode.",
)
},
)?)
.map_err(|_| {
Error::bad_database("User ID in devicekeychangeid_userid is invalid.")
Error::bad_database(
"User ID in devicekeychangeid_userid is invalid.",
)
})
}),
)
@ -647,14 +756,14 @@ impl service::users::Data for KeyValueDatabase {
}
let mut key = room_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(&count);
self.keychangeid_userid.insert(&key, user_id.as_bytes())?;
}
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(&count);
self.keychangeid_userid.insert(&key, user_id.as_bytes())?;
@ -667,7 +776,7 @@ impl service::users::Data for KeyValueDatabase {
device_id: &DeviceId,
) -> Result<Option<Raw<DeviceKeys>>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(device_id.as_bytes());
self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
@ -683,11 +792,11 @@ impl service::users::Data for KeyValueDatabase {
master_key: &Raw<CrossSigningKey>,
) -> Result<(Vec<u8>, CrossSigningKey)> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
let master_key = master_key
.deserialize()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))?;
let master_key = master_key.deserialize().map_err(|_| {
Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key")
})?;
let mut master_key_ids = master_key.keys.values();
let master_key_id = master_key_ids.next().ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
@ -712,8 +821,12 @@ impl service::users::Data for KeyValueDatabase {
allowed_signatures: &dyn Fn(&UserId) -> bool,
) -> Result<Option<Raw<CrossSigningKey>>> {
self.keyid_key.get(key)?.map_or(Ok(None), |bytes| {
let mut cross_signing_key = serde_json::from_slice::<serde_json::Value>(&bytes)
.map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?;
let mut cross_signing_key = serde_json::from_slice::<
serde_json::Value,
>(&bytes)
.map_err(|_| {
Error::bad_database("CrossSigningKey in db is invalid.")
})?;
clean_signatures(
&mut cross_signing_key,
sender_user,
@ -754,16 +867,20 @@ impl service::users::Data for KeyValueDatabase {
})
}
fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<Raw<CrossSigningKey>>> {
self.userid_usersigningkeyid
.get(user_id.as_bytes())?
.map_or(Ok(None), |key| {
fn get_user_signing_key(
&self,
user_id: &UserId,
) -> Result<Option<Raw<CrossSigningKey>>> {
self.userid_usersigningkeyid.get(user_id.as_bytes())?.map_or(
Ok(None),
|key| {
self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
Ok(Some(serde_json::from_slice(&bytes).map_err(|_| {
Error::bad_database("CrossSigningKey in db is invalid.")
})?))
})
})
},
)
}
fn add_to_device_event(
@ -775,9 +892,9 @@ impl service::users::Data for KeyValueDatabase {
content: serde_json::Value,
) -> Result<()> {
let mut key = target_user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(target_device_id.as_bytes());
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(&services().globals.next_count()?.to_be_bytes());
let mut json = serde_json::Map::new();
@ -785,7 +902,8 @@ impl service::users::Data for KeyValueDatabase {
json.insert("sender".to_owned(), sender.to_string().into());
json.insert("content".to_owned(), content);
let value = serde_json::to_vec(&json).expect("Map::to_vec always works");
let value =
serde_json::to_vec(&json).expect("Map::to_vec always works");
self.todeviceid_events.insert(&key, &value)?;
@ -800,15 +918,14 @@ impl service::users::Data for KeyValueDatabase {
let mut events = Vec::new();
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(device_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
for (_, value) in self.todeviceid_events.scan_prefix(prefix) {
events.push(
serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("Event in todeviceid_events is invalid."))?,
);
events.push(serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("Event in todeviceid_events is invalid.")
})?);
}
Ok(events)
@ -821,9 +938,9 @@ impl service::users::Data for KeyValueDatabase {
until: u64,
) -> Result<()> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
prefix.push(0xFF);
prefix.extend_from_slice(device_id.as_bytes());
prefix.push(0xff);
prefix.push(0xFF);
let mut last = prefix.clone();
last.extend_from_slice(&until.to_be_bytes());
@ -836,8 +953,14 @@ impl service::users::Data for KeyValueDatabase {
.map(|(key, _)| {
Ok::<_, Error>((
key.clone(),
utils::u64_from_bytes(&key[key.len() - size_of::<u64>()..key.len()])
.map_err(|_| Error::bad_database("ToDeviceId has invalid count bytes."))?,
utils::u64_from_bytes(
&key[key.len() - size_of::<u64>()..key.len()],
)
.map_err(|_| {
Error::bad_database(
"ToDeviceId has invalid count bytes.",
)
})?,
))
})
.filter_map(Result::ok)
@ -856,7 +979,7 @@ impl service::users::Data for KeyValueDatabase {
device: &Device,
) -> Result<()> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
assert!(
@ -864,12 +987,12 @@ impl service::users::Data for KeyValueDatabase {
"this method should only be called with existing devices"
);
self.userid_devicelistversion
.increment(user_id.as_bytes())?;
self.userid_devicelistversion.increment(user_id.as_bytes())?;
self.userdeviceid_metadata.insert(
&userdeviceid,
&serde_json::to_vec(device).expect("Device::to_string always works"),
&serde_json::to_vec(device)
.expect("Device::to_string always works"),
)?;
Ok(())
@ -882,26 +1005,32 @@ impl service::users::Data for KeyValueDatabase {
device_id: &DeviceId,
) -> Result<Option<Device>> {
let mut userdeviceid = user_id.as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.push(0xFF);
userdeviceid.extend_from_slice(device_id.as_bytes());
self.userdeviceid_metadata
.get(&userdeviceid)?
.map_or(Ok(None), |bytes| {
self.userdeviceid_metadata.get(&userdeviceid)?.map_or(
Ok(None),
|bytes| {
Ok(Some(serde_json::from_slice(&bytes).map_err(|_| {
Error::bad_database("Metadata in userdeviceid_metadata is invalid.")
Error::bad_database(
"Metadata in userdeviceid_metadata is invalid.",
)
})?))
})
},
)
}
fn get_devicelist_version(&self, user_id: &UserId) -> Result<Option<u64>> {
self.userid_devicelistversion
.get(user_id.as_bytes())?
.map_or(Ok(None), |bytes| {
self.userid_devicelistversion.get(user_id.as_bytes())?.map_or(
Ok(None),
|bytes| {
utils::u64_from_bytes(&bytes)
.map_err(|_| Error::bad_database("Invalid devicelistversion in db."))
.map_err(|_| {
Error::bad_database("Invalid devicelistversion in db.")
})
.map(Some)
})
},
)
}
fn all_devices_metadata<'a>(
@ -909,25 +1038,29 @@ impl service::users::Data for KeyValueDatabase {
user_id: &UserId,
) -> Box<dyn Iterator<Item = Result<Device>> + 'a> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
Box::new(
self.userdeviceid_metadata
.scan_prefix(key)
.map(|(_, bytes)| {
serde_json::from_slice::<Device>(&bytes).map_err(|_| {
Error::bad_database("Device in userdeviceid_metadata is invalid.")
})
}),
)
Box::new(self.userdeviceid_metadata.scan_prefix(key).map(
|(_, bytes)| {
serde_json::from_slice::<Device>(&bytes).map_err(|_| {
Error::bad_database(
"Device in userdeviceid_metadata is invalid.",
)
})
},
))
}
/// Creates a new sync filter. Returns the filter id.
fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result<String> {
fn create_filter(
&self,
user_id: &UserId,
filter: &FilterDefinition,
) -> Result<String> {
let filter_id = utils::random_string(4);
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(filter_id.as_bytes());
self.userfilterid_filter.insert(
@ -938,9 +1071,13 @@ impl service::users::Data for KeyValueDatabase {
Ok(filter_id)
}
fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result<Option<FilterDefinition>> {
fn get_filter(
&self,
user_id: &UserId,
filter_id: &str,
) -> Result<Option<FilterDefinition>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.push(0xFF);
key.extend_from_slice(filter_id.as_bytes());
let raw = self.userfilterid_filter.get(&key)?;
@ -956,9 +1093,12 @@ impl service::users::Data for KeyValueDatabase {
/// Will only return with Some(username) if the password was not empty and the
/// username could be successfully parsed.
/// If [`utils::string_from_bytes`] returns an error that username will be skipped
/// and the error will be logged.
fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option<String> {
/// If [`utils::string_from_bytes`] returns an error that username will be
/// skipped and the error will be logged.
fn get_username_with_valid_password(
username: &[u8],
password: &[u8],
) -> Option<String> {
// A valid password is not empty
if password.is_empty() {
None
@ -967,7 +1107,8 @@ fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option<
Ok(u) => Some(u),
Err(e) => {
warn!(
"Failed to parse username while calling get_local_users(): {}",
"Failed to parse username while calling \
get_local_users(): {}",
e.to_string()
);
None