include mxcs from dangling thumbnails in service::media::iter_all

When requesting remote thumbnails over federation, we can end up with a
thumbnail in the media db without an associated original file. Because
of this, skipping thumbnails is insufficient to get a list of all MXCs.
This commit is contained in:
Olivia Lee 2024-11-29 11:33:13 -08:00
parent 230192be1b
commit 916088a22f
No known key found for this signature in database
GPG key ID: 54D568A15B9CD1F9
3 changed files with 19 additions and 10 deletions

View file

@ -208,11 +208,6 @@ impl service::media::Data for KeyValueDatabase {
let key = MediaFileKey::new(key);
let parts = MediaFileKeyParts::try_from(&key)?;
if parts.width != 0 || parts.height != 0 {
// Skip thumbnails
return Ok(None);
};
Ok(Some((parts.mxc, parts.meta, key)))
})
.filter_map(Result::transpose),

View file

@ -169,11 +169,27 @@ impl Service {
/// List all media stored in the database.
///
/// Each MXC is list once. Thumbnails are not included separately from the
/// Each MXC is listed once. Thumbnails are not included separately from the
/// original media.
#[tracing::instrument(skip(self))]
pub(crate) fn iter_all(&self) -> impl Iterator<Item = Result<OwnedMxcUri>> {
self.db.all_file_metadata().map(|media| media.map(|(mxc, ..)| mxc))
let mut prev_mxc = None;
self.db
.all_file_metadata()
.map(|media| media.map(|(mxc, ..)| mxc))
.filter(move |mxc| {
if let Ok(mxc) = mxc {
// Skip mxcs that we have already seen. All files associated
// with a given mxc should appear consecutively in the db
// iterator, so we only need to check against the previous
// value.
if prev_mxc.as_ref() == Some(mxc) {
return false;
}
prev_mxc = Some(mxc.clone());
}
true
})
}
/// Returns width, height of the thumbnail and whether it should be cropped.

View file

@ -30,9 +30,7 @@ pub(crate) trait Data: Send + Sync {
mxc: OwnedMxcUri,
) -> Result<Vec<(FileMeta, MediaFileKey)>>;
/// Returns an iterator over metadata for all media.
///
/// Thumbnails are not included.
/// Returns an iterator over metadata for all media, including thumbnails.
fn all_file_metadata(
&self,
) -> Box<