Make JSON exports faster

CHANGELOG.md

@@ -14,6 +14,9 @@ Procedure when bumping the version number:
 
 ## Unreleased
 
+### Changed
+- Improved JSON export performance
+
 ## v0.6.0 - 2023-04-04
 
 ### Added

src/export/json.rs

@@ -8,14 +8,13 @@ pub async fn export<W: Write>(vault: &EuphRoomVault, file: &mut W) -> anyhow::Re
     write!(file, "[")?;
 
     let mut total = 0;
-    let mut offset = 0;
+    let mut last_msg_id = None;
     loop {
-        let messages = vault.chunk_at_offset(CHUNK_SIZE, offset).await?;
+        let messages = vault.chunk_after(last_msg_id, CHUNK_SIZE).await?;
-        offset += messages.len();
+        last_msg_id = Some(match messages.last() {
-
+            Some(last_msg) => last_msg.id,
-        if messages.is_empty() {
+            None => break, // No more messages, export finished
-            break;
+        });
-        }
 
         for message in messages {
             if total == 0 {
@@ -40,14 +39,13 @@ pub async fn export<W: Write>(vault: &EuphRoomVault, file: &mut W) -> anyhow::Re
 
 pub async fn export_stream<W: Write>(vault: &EuphRoomVault, file: &mut W) -> anyhow::Result<()> {
     let mut total = 0;
-    let mut offset = 0;
+    let mut last_msg_id = None;
     loop {
-        let messages = vault.chunk_at_offset(CHUNK_SIZE, offset).await?;
+        let messages = vault.chunk_after(last_msg_id, CHUNK_SIZE).await?;
-        offset += messages.len();
+        last_msg_id = Some(match messages.last() {
-
+            Some(last_msg) => last_msg.id,
-        if messages.is_empty() {
+            None => break, // No more messages, export finished
-            break;
+        });
-        }
 
         for message in messages {
             serde_json::to_writer(&mut *file, &message)?; // Fancy reborrow! :D

src/vault/euph.rs

@@ -5,7 +5,7 @@ use async_trait::async_trait;
 use cookie::{Cookie, CookieJar};
 use euphoxide::api::{Message, MessageId, SessionId, SessionView, Snowflake, Time, UserId};
 use rusqlite::types::{FromSql, FromSqlError, ToSqlOutput, Value, ValueRef};
-use rusqlite::{named_params, params, Connection, OptionalExtension, ToSql, Transaction};
+use rusqlite::{named_params, params, Connection, OptionalExtension, Row, ToSql, Transaction};
 use time::OffsetDateTime;
 use vault::Action;
 
@@ -240,7 +240,7 @@ euph_room_vault_actions! {
     GetUnseenMsgsCount : unseen_msgs_count() -> usize;
     SetSeen : set_seen(id: MessageId, seen: bool) -> ();
     SetOlderSeen : set_older_seen(id: MessageId, seen: bool) -> ();
-    GetChunkAtOffset : chunk_at_offset(amount: usize, offset: usize) -> Vec<Message>;
+    GetChunkAfter : chunk_after(id: Option<MessageId>, amount: usize) -> Vec<Message>;
 }
 
 impl Action for Join {
@@ -961,25 +961,11 @@ impl Action for SetOlderSeen {
     }
 }
 
-impl Action for GetChunkAtOffset {
+impl Action for GetChunkAfter {
     type Result = Vec<Message>;
 
     fn run(self, conn: &mut Connection) -> rusqlite::Result<Self::Result> {
-        let mut query = conn.prepare(
+        fn row2msg(row: &Row<'_>) -> rusqlite::Result<Message> {
-            "
-            SELECT
-                id, parent, previous_edit_id, time, content, encryption_key_id, edited, deleted, truncated,
-                user_id, name, server_id, server_era, session_id, is_staff, is_manager, client_address, real_client_address
-            FROM euph_msgs
-            WHERE room = ?
-            ORDER BY id ASC
-            LIMIT ?
-            OFFSET ?
-            ",
-        )?;
-
-        let messages = query
-            .query_map(params![self.room, self.amount, self.offset], |row| {
             Ok(Message {
                 id: MessageId(row.get::<_, WSnowflake>(0)?.0),
                 parent: row.get::<_, Option<WSnowflake>>(1)?.map(|s| MessageId(s.0)),
@@ -1002,8 +988,35 @@ impl Action for GetChunkAtOffset {
                     real_client_address: row.get(17)?,
                 },
             })
-            })?
+        }
-            .collect::<rusqlite::Result<_>>()?;
+
+        let messages = if let Some(id) = self.id {
+            conn.prepare("
+                SELECT
+                    id, parent, previous_edit_id, time, content, encryption_key_id, edited, deleted, truncated,
+                    user_id, name, server_id, server_era, session_id, is_staff, is_manager, client_address, real_client_address
+                FROM euph_msgs
+                WHERE room = ?
+                AND id > ?
+                ORDER BY id ASC
+                LIMIT ?
+            ")?
+            .query_map(params![self.room, WSnowflake(id.0), self.amount], row2msg)?
+            .collect::<rusqlite::Result<_>>()?
+        } else {
+            conn.prepare("
+                SELECT
+                    id, parent, previous_edit_id, time, content, encryption_key_id, edited, deleted, truncated,
+                    user_id, name, server_id, server_era, session_id, is_staff, is_manager, client_address, real_client_address
+                FROM euph_msgs
+                WHERE room = ?
+                ORDER BY id ASC
+                LIMIT ?
+            ")?
+            .query_map(params![self.room, self.amount], row2msg)?
+            .collect::<rusqlite::Result<_>>()?
+        };
+
         Ok(messages)
     }
 }