From 5a5608e0884656c50c170bf8dce01c8d253d055b Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Fri, 28 Feb 2025 10:56:08 -0800 Subject: [PATCH] separate media and database paths The primary motivation for this change is to support databases that don't take a path, e.g. out of process databases. This configuration structure leaves the door open for other media storage mechanisms in the future, such as S3. It's also structured to avoid `#[serde(flatten)]` so that we can use `#[serde(deny_unknown_fields)]`. --- book/changelog.md | 3 ++ book/installing/migrating-conduit.md | 48 +++++++++++++++++++ src/config.rs | 17 ++++++- src/service/globals.rs | 10 ++-- .../fixtures/check_config/minimal-valid.toml | 6 ++- .../fixtures/check_config/valid.toml | 6 ++- 6 files changed, 82 insertions(+), 8 deletions(-) diff --git a/book/changelog.md b/book/changelog.md index 43928b1b..4ca6dcf4 100644 --- a/book/changelog.md +++ b/book/changelog.md @@ -146,6 +146,9 @@ This will be the first release of Grapevine since it was forked from Conduit the server is now behind the `serve` command, so `grapevine --config ...` becomes `grapevine serve --config ...`. ([!108](https://gitlab.computer.surgery/matrix/grapevine/-/merge_requests/108)) +13. **BREAKING:** The path to media files is now specified separately from the + database path. + ([!40](https://gitlab.computer.surgery/matrix/grapevine/-/merge_requests/140)) ### Fixed diff --git a/book/installing/migrating-conduit.md b/book/installing/migrating-conduit.md index f847c7af..42cde3aa 100644 --- a/book/installing/migrating-conduit.md +++ b/book/installing/migrating-conduit.md @@ -23,6 +23,54 @@ automatically migrating existing configs to the new schema. [room]: https://matrix.to/#/#grapevine:computer.surgery [config-migration-issue]: https://gitlab.computer.surgery/matrix/grapevine/-/issues/38 +## Filesystem + +Grapevine requires database data and media data to live in **separate**, +**non-nested** directories, which are configurable. Here is a typical example, +starting with the filesystem structure: + +```text +/var/lib/grapevine ++ database/ +| + database-file-1 +| + ... +| + database-file-n ++ media/ + + media-file-1 + + ... + + media-file-n +``` + +And here is the matching configuration: + +```toml +[database] +path = "/var/lib/grapevine/database" + +[media.backend] +type = "filesystem" +path = "/var/lib/grapevine/media" +``` + +On the other hand, Conduit's filesystem layout looks like this: + +```text +/var/lib/conduit ++ media/ +| + media-file-1 +| + ... +| + media-file-n ++ database-file-1 ++ ... ++ database-file-n +``` + +Which **nests** the media directory inside the database directory. Grapevine +will reject this layout, so the filesystem layout must be changed before +starting Grapevine. It is important to migrate the filesystem layout before +starting Grapevine, because otherwise it will create a fresh database instead of +using the existing one. + ## Database Grapevine is currently compatible with the Conduit 0.7.0 database format. It is diff --git a/src/config.rs b/src/config.rs index b904a903..aed1244b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,7 +44,6 @@ pub(crate) struct Config { pub(crate) server_discovery: ServerDiscovery, pub(crate) database: DatabaseConfig, - #[serde(default)] pub(crate) media: MediaConfig, #[serde(default)] pub(crate) federation: FederationConfig, @@ -74,13 +73,27 @@ pub(crate) struct Config { pub(crate) emergency_password: Option, } -#[derive(Debug, Deserialize, Default)] +#[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct MediaConfig { + pub(crate) backend: MediaBackendConfig, + #[serde(default)] pub(crate) allow_unauthenticated_access: bool, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] +pub(crate) enum MediaBackendConfig { + Filesystem(MediaFilesystemConfig), +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MediaFilesystemConfig { + pub(crate) path: PathBuf, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields, default)] pub(crate) struct CacheConfig { diff --git a/src/service/globals.rs b/src/service/globals.rs index c572fb1f..ea1dce4f 100644 --- a/src/service/globals.rs +++ b/src/service/globals.rs @@ -39,6 +39,7 @@ use trust_dns_resolver::TokioAsyncResolver; use crate::{ api::server_server::FedDest, + config::{MediaBackendConfig, MediaFilesystemConfig}, observability::FilterReloadHandles, service::media::MediaFileKey, services, @@ -602,10 +603,11 @@ impl Service { } pub(crate) fn get_media_folder(&self) -> PathBuf { - let mut r = PathBuf::new(); - r.push(self.config.database.path.clone()); - r.push("media"); - r + let MediaBackendConfig::Filesystem(MediaFilesystemConfig { + path, + }) = &self.config.media.backend; + + path.clone() } pub(crate) fn get_media_file(&self, key: &MediaFileKey) -> PathBuf { diff --git a/tests/integrations/fixtures/check_config/minimal-valid.toml b/tests/integrations/fixtures/check_config/minimal-valid.toml index e8a93d0e..3c114f2e 100644 --- a/tests/integrations/fixtures/check_config/minimal-valid.toml +++ b/tests/integrations/fixtures/check_config/minimal-valid.toml @@ -5,4 +5,8 @@ client.base_url = "https://matrix.example.com" [database] backend = "rocksdb" -path = "/var/lib/grapevine" +path = "/var/lib/grapevine/database" + +[media.backend] +type = "filesystem" +path = "/var/lib/grapevine/media" diff --git a/tests/integrations/fixtures/check_config/valid.toml b/tests/integrations/fixtures/check_config/valid.toml index 8d8a0bae..6b6e1a3d 100644 --- a/tests/integrations/fixtures/check_config/valid.toml +++ b/tests/integrations/fixtures/check_config/valid.toml @@ -10,7 +10,11 @@ client.base_url = "https://matrix.example.com" [database] backend = "rocksdb" -path = "/var/lib/grapevine" +path = "/var/lib/grapevine/database" + +[media.backend] +type = "filesystem" +path = "/var/lib/grapevine/media" [federation] enable = true