separate media and database paths

The primary motivation for this change is to support databases that
don't take a path, e.g. out of process databases.

This configuration structure leaves the door open for other media
storage mechanisms in the future, such as S3.

It's also structured to avoid `#[serde(flatten)]` so that we can use
`#[serde(deny_unknown_fields)]`.

book/changelog.md

@@ -146,6 +146,9 @@ This will be the first release of Grapevine since it was forked from Conduit
     the server is now behind the `serve` command, so `grapevine --config ...`
     becomes `grapevine serve --config ...`.
     ([!108](https://gitlab.computer.surgery/matrix/grapevine/-/merge_requests/108))
+13. **BREAKING:** The path to media files is now specified separately from the
+    database path.
+    ([!40](https://gitlab.computer.surgery/matrix/grapevine/-/merge_requests/140))
 
 ### Fixed
 

book/installing/migrating-conduit.md

@@ -23,6 +23,54 @@ automatically migrating existing configs to the new schema.
 [room]: https://matrix.to/#/#grapevine:computer.surgery
 [config-migration-issue]: https://gitlab.computer.surgery/matrix/grapevine/-/issues/38
 
+## Filesystem
+
+Grapevine requires database data and media data to live in **separate**,
+**non-nested** directories, which are configurable. Here is a typical example,
+starting with the filesystem structure:
+
+```text
+/var/lib/grapevine
++ database/
+| + database-file-1
+| + ...
+| + database-file-n
++ media/
+  + media-file-1
+  + ...
+  + media-file-n
+```
+
+And here is the matching configuration:
+
+```toml
+[database]
+path = "/var/lib/grapevine/database"
+
+[media.backend]
+type = "filesystem"
+path = "/var/lib/grapevine/media"
+```
+
+On the other hand, Conduit's filesystem layout looks like this:
+
+```text
+/var/lib/conduit
++ media/
+| + media-file-1
+| + ...
+| + media-file-n
++ database-file-1
++ ...
++ database-file-n
+```
+
+Which **nests** the media directory inside the database directory.  Grapevine
+will reject this layout, so the filesystem layout must be changed before
+starting Grapevine. It is important to migrate the filesystem layout before
+starting Grapevine, because otherwise it will create a fresh database instead of
+using the existing one.
+
 ## Database
 
 Grapevine is currently compatible with the Conduit 0.7.0 database format. It is

src/config.rs

@@ -44,7 +44,6 @@ pub(crate) struct Config {
 
     pub(crate) server_discovery: ServerDiscovery,
     pub(crate) database: DatabaseConfig,
-    #[serde(default)]
     pub(crate) media: MediaConfig,
     #[serde(default)]
     pub(crate) federation: FederationConfig,
@@ -74,13 +73,27 @@ pub(crate) struct Config {
     pub(crate) emergency_password: Option<String>,
 }
 
-#[derive(Debug, Deserialize, Default)]
+#[derive(Debug, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct MediaConfig {
+    pub(crate) backend: MediaBackendConfig,
+
     #[serde(default)]
     pub(crate) allow_unauthenticated_access: bool,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")]
+pub(crate) enum MediaBackendConfig {
+    Filesystem(MediaFilesystemConfig),
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct MediaFilesystemConfig {
+    pub(crate) path: PathBuf,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(deny_unknown_fields, default)]
 pub(crate) struct CacheConfig {

src/service/globals.rs

@@ -39,6 +39,7 @@ use trust_dns_resolver::TokioAsyncResolver;
 
 use crate::{
     api::server_server::FedDest,
+    config::{MediaBackendConfig, MediaFilesystemConfig},
     observability::FilterReloadHandles,
     service::media::MediaFileKey,
     services,
@@ -602,10 +603,11 @@ impl Service {
     }
 
     pub(crate) fn get_media_folder(&self) -> PathBuf {
-        let mut r = PathBuf::new();
-        r.push(self.config.database.path.clone());
-        r.push("media");
-        r
+        let MediaBackendConfig::Filesystem(MediaFilesystemConfig {
+            path,
+        }) = &self.config.media.backend;
+
+        path.clone()
     }
 
     pub(crate) fn get_media_file(&self, key: &MediaFileKey) -> PathBuf {

tests/integrations/fixtures/check_config/minimal-valid.toml

@@ -5,4 +5,8 @@ client.base_url = "https://matrix.example.com"
 
 [database]
 backend = "rocksdb"
-path = "/var/lib/grapevine"
+path = "/var/lib/grapevine/database"
+
+[media.backend]
+type = "filesystem"
+path = "/var/lib/grapevine/media"

tests/integrations/fixtures/check_config/valid.toml

@@ -10,7 +10,11 @@ client.base_url = "https://matrix.example.com"
 
 [database]
 backend = "rocksdb"
-path = "/var/lib/grapevine"
+path = "/var/lib/grapevine/database"
+
+[media.backend]
+type = "filesystem"
+path = "/var/lib/grapevine/media"
 
 [federation]
 enable = true