backups/grapevine
1
0
Fork 0
mirror of https://gitlab.computer.surgery/matrix/grapevine.git synced 2025-12-17 15:51:23 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Convert giant tuple in state_compressor to struct

Browse source
This commit is contained in:
Lambda 2024-05-20 10:13:01 +00:00 • committed by Charles Hall
parent 8d09a7e490
commit edfccea30a
4 changed files with 39 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/database.rs
View file

@ -722,12 +722,12 @@ impl KeyValueDatabase {
states_parents.last() states_parents.last()
{ {
let statediffnew = current_state let statediffnew = current_state
.difference(&parent_stateinfo.1) .difference(&parent_stateinfo.full_state)
.copied() .copied()
.collect::<HashSet<_>>(); .collect::<HashSet<_>>();
let statediffremoved = parent_stateinfo let statediffremoved = parent_stateinfo
.1 .full_state
.difference(&current_state) .difference(&current_state)
.copied() .copied()
.collect::<HashSet<_>>(); .collect::<HashSet<_>>();

6
src/database/key_value/rooms/state_accessor.rs
View file

@ -20,7 +20,7 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
.load_shortstatehash_info(shortstatehash)? .load_shortstatehash_info(shortstatehash)?
.pop() .pop()
.expect("there is always one layer") .expect("there is always one layer")
.1; .full_state;
let mut result = HashMap::new(); let mut result = HashMap::new();
let mut i = 0; let mut i = 0;
for compressed in full_state.iter() { for compressed in full_state.iter() {
@ -48,7 +48,7 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
.load_shortstatehash_info(shortstatehash)? .load_shortstatehash_info(shortstatehash)?
.pop() .pop()
.expect("there is always one layer") .expect("there is always one layer")
.1; .full_state;
let mut result = HashMap::new(); let mut result = HashMap::new();
let mut i = 0; let mut i = 0;
@ -102,7 +102,7 @@ impl service::rooms::state_accessor::Data for KeyValueDatabase {
.load_shortstatehash_info(shortstatehash)? .load_shortstatehash_info(shortstatehash)?
.pop() .pop()
.expect("there is always one layer") .expect("there is always one layer")
.1; .full_state;
Ok(full_state Ok(full_state
.iter() .iter()
.find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes()))

8
src/service/rooms/state.rs
View file

@ -159,12 +159,12 @@ impl Service {
let (statediffnew, statediffremoved) = let (statediffnew, statediffremoved) =
if let Some(parent_stateinfo) = states_parents.last() { if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: HashSet<_> = state_ids_compressed let statediffnew: HashSet<_> = state_ids_compressed
.difference(&parent_stateinfo.1) .difference(&parent_stateinfo.full_state)
.copied() .copied()
.collect(); .collect();
let statediffremoved: HashSet<_> = parent_stateinfo let statediffremoved: HashSet<_> = parent_stateinfo
.1 .full_state
.difference(&state_ids_compressed) .difference(&state_ids_compressed)
.copied() .copied()
.collect(); .collect();
@ -231,7 +231,7 @@ impl Service {
let replaces = states_parents let replaces = states_parents
.last() .last()
.map(|info| { .map(|info| {
info.1.iter().find(|bytes| { info.full_state.iter().find(|bytes| {
bytes.starts_with(&shortstatekey.to_be_bytes()) bytes.starts_with(&shortstatekey.to_be_bytes())
}) })
}) })
@ -436,7 +436,7 @@ impl Service {
.load_shortstatehash_info(shortstatehash)? .load_shortstatehash_info(shortstatehash)?
.pop() .pop()
.expect("there is always one layer") .expect("there is always one layer")
.1; .full_state;
Ok(full_state Ok(full_state
.iter() .iter()

82
src/service/rooms/state_compressor.rs
View file

@ -12,25 +12,19 @@ use ruma::{EventId, RoomId};
use self::data::StateDiff; use self::data::StateDiff;
use crate::{services, utils, Result}; use crate::{services, utils, Result};
#[derive(Clone)]
pub(crate) struct CompressedStateLayer {
pub(crate) shortstatehash: u64,
pub(crate) full_state: Arc<HashSet<CompressedStateEvent>>,
pub(crate) added: Arc<HashSet<CompressedStateEvent>>,
pub(crate) removed: Arc<HashSet<CompressedStateEvent>>,
}
pub(crate) struct Service { pub(crate) struct Service {
pub(crate) db: &'static dyn Data, pub(crate) db: &'static dyn Data,
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
pub(crate) stateinfo_cache: Mutex< pub(crate) stateinfo_cache: Mutex<LruCache<u64, Vec<CompressedStateLayer>>>,
LruCache<
u64,
Vec<(
// shortstatehash
u64,
// full state
Arc<HashSet<CompressedStateEvent>>,
// added
Arc<HashSet<CompressedStateEvent>>,
// removed
Arc<HashSet<CompressedStateEvent>>,
)>,
>,
>,
} }
pub(crate) type CompressedStateEvent = [u8; 2 * size_of::<u64>()]; pub(crate) type CompressedStateEvent = [u8; 2 * size_of::<u64>()];
@ -43,18 +37,7 @@ impl Service {
pub(crate) fn load_shortstatehash_info( pub(crate) fn load_shortstatehash_info(
&self, &self,
shortstatehash: u64, shortstatehash: u64,
) -> Result< ) -> Result<Vec<CompressedStateLayer>> {
Vec<(
// shortstatehash
u64,
// full state
Arc<HashSet<CompressedStateEvent>>,
// added
Arc<HashSet<CompressedStateEvent>>,
// removed
Arc<HashSet<CompressedStateEvent>>,
)>,
> {
if let Some(r) = if let Some(r) =
self.stateinfo_cache.lock().unwrap().get_mut(&shortstatehash) self.stateinfo_cache.lock().unwrap().get_mut(&shortstatehash)
{ {
@ -69,19 +52,19 @@ impl Service {
if let Some(parent) = parent { if let Some(parent) = parent {
let mut response = self.load_shortstatehash_info(parent)?; let mut response = self.load_shortstatehash_info(parent)?;
let mut state = (*response.last().unwrap().1).clone(); let mut state = (*response.last().unwrap().full_state).clone();
state.extend(added.iter().copied()); state.extend(added.iter().copied());
let removed = (*removed).clone(); let removed = (*removed).clone();
for r in &removed { for r in &removed {
state.remove(r); state.remove(r);
} }
response.push(( response.push(CompressedStateLayer {
shortstatehash, shortstatehash,
Arc::new(state), full_state: Arc::new(state),
added, added,
Arc::new(removed), removed: Arc::new(removed),
)); });
self.stateinfo_cache self.stateinfo_cache
.lock() .lock()
@ -90,8 +73,12 @@ impl Service {
Ok(response) Ok(response)
} else { } else {
let response = let response = vec![CompressedStateLayer {
vec![(shortstatehash, added.clone(), added, removed)]; shortstatehash,
full_state: added.clone(),
added,
removed,
}];
self.stateinfo_cache self.stateinfo_cache
.lock() .lock()
.unwrap() .unwrap()
@ -167,16 +154,7 @@ impl Service {
statediffnew: Arc<HashSet<CompressedStateEvent>>, statediffnew: Arc<HashSet<CompressedStateEvent>>,
statediffremoved: Arc<HashSet<CompressedStateEvent>>, statediffremoved: Arc<HashSet<CompressedStateEvent>>,
diff_to_sibling: usize, diff_to_sibling: usize,
mut parent_states: Vec<( mut parent_states: Vec<CompressedStateLayer>,
// shortstatehash
u64,
// full state
Arc<HashSet<CompressedStateEvent>>,
// added
Arc<HashSet<CompressedStateEvent>>,
// removed
Arc<HashSet<CompressedStateEvent>>,
)>,
) -> Result<()> { ) -> Result<()> {
let diffsum = statediffnew.len() + statediffremoved.len(); let diffsum = statediffnew.len() + statediffremoved.len();
@ -185,8 +163,8 @@ impl Service {
// To many layers, we have to go deeper // To many layers, we have to go deeper
let parent = parent_states.pop().unwrap(); let parent = parent_states.pop().unwrap();
let mut parent_new = (*parent.2).clone(); let mut parent_new = (*parent.added).clone();
let mut parent_removed = (*parent.3).clone(); let mut parent_removed = (*parent.removed).clone();
for removed in statediffremoved.iter() { for removed in statediffremoved.iter() {
if !parent_new.remove(removed) { if !parent_new.remove(removed) {
@ -236,12 +214,12 @@ impl Service {
// 2. We replace a layer above // 2. We replace a layer above
let parent = parent_states.pop().unwrap(); let parent = parent_states.pop().unwrap();
let parent_diff = parent.2.len() + parent.3.len(); let parent_diff = parent.added.len() + parent.removed.len();
if diffsum * diffsum >= 2 * diff_to_sibling * parent_diff { if diffsum * diffsum >= 2 * diff_to_sibling * parent_diff {
// Diff too big, we replace above layer(s) // Diff too big, we replace above layer(s)
let mut parent_new = (*parent.2).clone(); let mut parent_new = (*parent.added).clone();
let mut parent_removed = (*parent.3).clone(); let mut parent_removed = (*parent.removed).clone();
for removed in statediffremoved.iter() { for removed in statediffremoved.iter() {
if !parent_new.remove(removed) { if !parent_new.remove(removed) {
@ -273,7 +251,7 @@ impl Service {
self.db.save_statediff( self.db.save_statediff(
shortstatehash, shortstatehash,
StateDiff { StateDiff {
parent: Some(parent.0), parent: Some(parent.shortstatehash),
added: statediffnew, added: statediffnew,
removed: statediffremoved, removed: statediffremoved,
}, },
@ -325,12 +303,12 @@ impl Service {
let (statediffnew, statediffremoved) = let (statediffnew, statediffremoved) =
if let Some(parent_stateinfo) = states_parents.last() { if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: HashSet<_> = new_state_ids_compressed let statediffnew: HashSet<_> = new_state_ids_compressed
.difference(&parent_stateinfo.1) .difference(&parent_stateinfo.full_state)
.copied() .copied()
.collect(); .collect();
let statediffremoved: HashSet<_> = parent_stateinfo let statediffremoved: HashSet<_> = parent_stateinfo
.1 .full_state
.difference(&new_state_ids_compressed) .difference(&new_state_ids_compressed)
.copied() .copied()
.collect(); .collect();