From 0ceaffc608b2401f16e6422a05871a74bd3ff145 Mon Sep 17 00:00:00 2001
From: Joscha <joscha@plugh.de>
Date: Sun, 12 Feb 2023 00:53:50 +0100
Subject: [PATCH] Add json-stream export format

---
 CHANGELOG.md       |  1 +
 src/export.rs      |  8 ++++++--
 src/export/json.rs | 30 ++++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 660d36c..e4e4dcb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ Procedure when bumping the version number:
 
 ### Added
 - `--verbose` flag
+- `json-stream` room export format
 
 ### Changed
 - Respect colon-delimited emoji when calculating nick hue
diff --git a/src/export.rs b/src/export.rs
index c76009c..a2831a4 100644
--- a/src/export.rs
+++ b/src/export.rs
@@ -12,8 +12,10 @@ use crate::vault::EuphVault;
 pub enum Format {
     /// Human-readable tree-structured messages.
     Text,
-    /// List of message objects in the same format as the euphoria API uses.
+    /// Array of message objects in the same format as the euphoria API uses.
     Json,
+    /// Message objects in the same format as the euphoria API uses, one per line.
+    JsonStream,
 }
 
 impl Format {
@@ -21,13 +23,14 @@ impl Format {
         match self {
             Self::Text => "text",
             Self::Json => "json",
+            Self::JsonStream => "json stream",
         }
     }
 
     fn extension(&self) -> &'static str {
         match self {
             Self::Text => "txt",
-            Self::Json => "json",
+            Self::Json | Self::JsonStream => "json",
         }
     }
 }
@@ -88,6 +91,7 @@ pub async fn export(vault: &EuphVault, mut args: Args) -> anyhow::Result<()> {
         match args.format {
             Format::Text => text::export_to_file(&vault, &mut file).await?,
             Format::Json => json::export_to_file(&vault, &mut file).await?,
+            Format::JsonStream => json::export_stream_to_file(&vault, &mut file).await?,
         }
         file.flush()?;
     }
diff --git a/src/export/json.rs b/src/export/json.rs
index 5b9222c..998087e 100644
--- a/src/export/json.rs
+++ b/src/export/json.rs
@@ -42,3 +42,33 @@ pub async fn export_to_file(
 
     Ok(())
 }
+
+pub async fn export_stream_to_file(
+    vault: &EuphRoomVault,
+    file: &mut BufWriter<File>,
+) -> anyhow::Result<()> {
+    let mut total = 0;
+    let mut offset = 0;
+    loop {
+        let messages = vault.chunk_at_offset(CHUNK_SIZE, offset).await;
+        offset += messages.len();
+
+        if messages.is_empty() {
+            break;
+        }
+
+        for message in messages {
+            serde_json::to_writer(&mut *file, &message)?; // Fancy reborrow! :D
+            writeln!(file)?;
+            total += 1;
+        }
+
+        if total % 100000 == 0 {
+            println!("  {total} messages");
+        }
+    }
+
+    println!("  {total} messages in total");
+
+    Ok(())
+}