Add json export

src/euph/api/types.rs

@@ -39,9 +39,11 @@ pub struct Message {
     /// The id of the message (unique within a room).
     pub id: Snowflake,
     /// The id of the message's parent, or null if top-level.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub parent: Option<Snowflake>,
     /// The edit id of the most recent edit of this message, or null if it's
     /// never been edited.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub previous_edit_id: Option<Snowflake>,
     /// The unix timestamp of when the message was posted.
     pub time: Time,
@@ -50,15 +52,18 @@ pub struct Message {
     /// The content of the message (client-defined).
     pub content: String,
     /// The id of the key that encrypts the message in storage.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub encryption_key_id: Option<String>,
     /// The unix timestamp of when the message was last edited.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub edited: Option<Time>,
     /// The unix timestamp of when the message was deleted.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub deleted: Option<Time>,
     /// If true, then the full content of this message is not included (see
     /// [`GetMessage`](super::GetMessage) to obtain the message with full
     /// content).
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub truncated: bool,
 }
 
@@ -267,14 +272,16 @@ pub struct SessionView {
     /// Id of the session, unique across all sessions globally.
     pub session_id: String,
     /// If true, this session belongs to a member of staff.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub is_staff: bool,
     /// If true, this session belongs to a manager of the room.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub is_manager: bool,
     /// For hosts and staff, the virtual address of the client.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub client_address: Option<String>,
     /// For staff, the real address of the client.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub real_client_address: Option<String>,
 }
 

src/export.rs

@@ -1,5 +1,6 @@
 //! Export logs from the vault to plain text files.
 
+mod json;
 mod text;
 
 use std::fs::File;
@@ -11,18 +12,22 @@ use crate::vault::Vault;
 pub enum Format {
     /// Human-readable tree-structured messages.
     Text,
+    /// List of message objects in the same format as the euphoria API uses.
+    Json,
 }
 
 impl Format {
     fn name(&self) -> &'static str {
         match self {
             Self::Text => "text",
+            Self::Json => "json",
         }
     }
 
     fn extension(&self) -> &'static str {
         match self {
             Self::Text => "txt",
+            Self::Json => "json",
         }
     }
 }
@@ -81,6 +86,7 @@ pub async fn export(vault: &Vault, mut args: Args) -> anyhow::Result<()> {
         let mut file = BufWriter::new(File::create(out)?);
         match args.format {
             Format::Text => text::export_to_file(vault, room, &mut file).await?,
+            Format::Json => json::export_to_file(vault, room, &mut file).await?,
         }
         file.flush()?;
     }

src/export/json.rs

@@ -0,0 +1,47 @@
+use std::fs::File;
+use std::io::{BufWriter, Write};
+
+use crate::vault::Vault;
+
+const CHUNK_SIZE: usize = 10000;
+
+pub async fn export_to_file(
+    vault: &Vault,
+    room: String,
+    file: &mut BufWriter<File>,
+) -> anyhow::Result<()> {
+    let vault = vault.euph(room);
+
+    write!(file, "[")?;
+
+    let mut total = 0;
+    let mut offset = 0;
+    loop {
+        let messages = vault.chunk_at_offset(CHUNK_SIZE, offset).await;
+        offset += messages.len();
+
+        if messages.is_empty() {
+            break;
+        }
+
+        for message in messages {
+            if total == 0 {
+                writeln!(file)?;
+            } else {
+                writeln!(file, ",")?;
+            }
+            serde_json::to_writer(&mut *file, &message)?; // Fancy reborrow! :D
+            total += 1;
+        }
+
+        if total % 100000 == 0 {
+            println!("  {total} messages");
+        }
+    }
+
+    write!(file, "\n]")?;
+
+    println!("  {total} messages in total");
+
+    Ok(())
+}

src/vault/euph.rs

@@ -8,7 +8,7 @@ use rusqlite::{named_params, params, Connection, OptionalExtension, ToSql, Trans
 use time::OffsetDateTime;
 use tokio::sync::oneshot;
 
-use crate::euph::api::{Message, Snowflake, Time, UserId};
+use crate::euph::api::{Message, SessionView, Snowflake, Time, UserId};
 use crate::euph::SmallMessage;
 use crate::store::{MsgStore, Path, Tree};
 
@@ -139,6 +139,19 @@ impl EuphVault {
         let _ = self.vault.tx.send(request.into());
         rx.await.unwrap()
     }
+
+    pub async fn chunk_at_offset(&self, amount: usize, offset: usize) -> Vec<Message> {
+        // TODO vault::Error
+        let (tx, rx) = oneshot::channel();
+        let request = EuphRequest::GetChunkAtOffset {
+            room: self.room.clone(),
+            amount,
+            offset,
+            result: tx,
+        };
+        let _ = self.vault.tx.send(request.into());
+        rx.await.unwrap()
+    }
 }
 
 #[async_trait]
@@ -457,6 +470,12 @@ pub(super) enum EuphRequest {
         id: Snowflake,
         seen: bool,
     },
+    GetChunkAtOffset {
+        room: String,
+        amount: usize,
+        offset: usize,
+        result: oneshot::Sender<Vec<Message>>,
+    },
 }
 
 impl EuphRequest {
@@ -525,6 +544,12 @@ impl EuphRequest {
             EuphRequest::SetOlderSeen { room, id, seen } => {
                 Self::set_older_seen(conn, room, id, seen)
             }
+            EuphRequest::GetChunkAtOffset {
+                room,
+                amount,
+                offset,
+                result,
+            } => Self::get_chunk_at_offset(conn, room, amount, offset, result),
         };
         if let Err(e) = result {
             // If an error occurs here, the rest of the UI will likely panic and
@@ -1235,4 +1260,54 @@ impl EuphRequest {
         )?;
         Ok(())
     }
+
+    fn get_chunk_at_offset(
+        conn: &Connection,
+        room: String,
+        amount: usize,
+        offset: usize,
+        result: oneshot::Sender<Vec<Message>>,
+    ) -> rusqlite::Result<()> {
+        let mut query = conn.prepare(
+            "
+            SELECT
+                id, parent, previous_edit_id, time, content, encryption_key_id, edited, deleted, truncated,
+                user_id, name, server_id, server_era, session_id, is_staff, is_manager, client_address, real_client_address
+            FROM euph_msgs
+            WHERE room = ?
+            ORDER BY id ASC
+            LIMIT ?
+            OFFSET ?
+            ",
+        )?;
+
+        let messages = query
+            .query_map(params![room, amount, offset], |row| {
+                Ok(Message {
+                    id: row.get(0)?,
+                    parent: row.get(1)?,
+                    previous_edit_id: row.get(2)?,
+                    time: row.get(3)?,
+                    content: row.get(4)?,
+                    encryption_key_id: row.get(5)?,
+                    edited: row.get(6)?,
+                    deleted: row.get(7)?,
+                    truncated: row.get(8)?,
+                    sender: SessionView {
+                        id: UserId(row.get(9)?),
+                        name: row.get(10)?,
+                        server_id: row.get(11)?,
+                        server_era: row.get(12)?,
+                        session_id: row.get(13)?,
+                        is_staff: row.get(14)?,
+                        is_manager: row.get(15)?,
+                        client_address: row.get(16)?,
+                        real_client_address: row.get(17)?,
+                    },
+                })
+            })?
+            .collect::<rusqlite::Result<_>>()?;
+        let _ = result.send(messages);
+        Ok(())
+    }
 }