Store more commit info in db

.sqlx/query-35e5c16c2952f550783b234f27839cf5110d11798d749f523d0e0c98a77194f5.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\nINSERT OR IGNORE INTO commits (hash, author, author_date, committer, committer_date, message)\nVALUES (?, ?, ?, ?, ?, ?)\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 6
+    },
+    "nullable": []
+  },
+  "hash": "35e5c16c2952f550783b234f27839cf5110d11798d749f523d0e0c98a77194f5"
+}

.sqlx/query-afe943820305632281456f0a629065b3221314c59b1865c50b9bff51529c93d7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "SQLite",
-  "query": "\nWITH RECURSIVE reachable(hash) AS (\n    SELECT hash FROM tracked_refs\n    UNION\n    SELECT parent FROM commit_links\n    JOIN reachable ON hash = child\n)\n\nUPDATE commits\nSET tracked = (hash IN reachable)\n",
+  "query": "\nWITH RECURSIVE reachable(hash) AS (\n    SELECT hash FROM tracked_refs\n    UNION\n    SELECT parent FROM commit_links\n    JOIN reachable ON hash = child\n)\n\nUPDATE commits\nSET reachable = (hash IN reachable)\n",
   "describe": {
     "columns": [],
     "parameters": {
@@ -8,5 +8,5 @@
     },
     "nullable": []
   },
-  "hash": "6398e5b1dce1142d3460f9d067588bcb42b0be5278ce7524280a4786c8b41786"
+  "hash": "afe943820305632281456f0a629065b3221314c59b1865c50b9bff51529c93d7"
 }

.sqlx/query-d06de282a27e84bae58b7ea73fce0f0e205e82b8c17406b4f613066a7f3979da.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "INSERT OR IGNORE INTO commits (hash) VALUES (?)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": []
-  },
-  "hash": "d06de282a27e84bae58b7ea73fce0f0e205e82b8c17406b4f613066a7f3979da"
-}

.sqlx/query-e58a4211444bfe1c965c021085f0e204ca93fd93778b360730976a76e299ffef.json

@@ -1,6 +1,6 @@
 {
   "db_name": "SQLite",
-  "query": "\nSELECT child, tracked FROM commit_links\nJOIN commits ON hash = child\nWHERE parent = ?\n    ",
+  "query": "\nSELECT child, reachable FROM commit_links\nJOIN commits ON hash = child\nWHERE parent = ?\n    ",
   "describe": {
     "columns": [
       {
@@ -9,7 +9,7 @@
         "type_info": "Text"
       },
       {
-        "name": "tracked",
+        "name": "reachable",
         "ordinal": 1,
         "type_info": "Int64"
       }
@@ -22,5 +22,5 @@
       false
     ]
   },
-  "hash": "2846970f979d84f4ba4ba2cd659fc37347ea280ebdf584bc535f135f63610a66"
+  "hash": "e58a4211444bfe1c965c021085f0e204ca93fd93778b360730976a76e299ffef"
 }

migrations/20230805101911_commits.sql

@@ -1,7 +1,12 @@
 CREATE TABLE commits (
-    hash    TEXT NOT NULL PRIMARY KEY,
+    hash           TEXT NOT NULL PRIMARY KEY,
-    new     INT  NOT NULL DEFAULT 1,
+    author         TEXT NOT NULL,
-    tracked INT  NOT NULL DEFAULT 0
+    author_date    TEXT NOT NULL,
+    committer      TEXT NOT NULL,
+    committer_date TEXT NOT NULL,
+    message        TEXT NOT NULL,
+    reachable      INT  NOT NULL DEFAULT 0,
+    new            INT  NOT NULL DEFAULT 1
 ) STRICT;
 
 CREATE TABLE commit_links (

src/recurring/repo.rs

@@ -6,13 +6,13 @@
 use std::collections::HashSet;
 
 use futures::TryStreamExt;
-use gix::{objs::Kind, traverse::commit::Info, ObjectId, Repository};
+use gix::{date::time::format::ISO8601_STRICT, objs::Kind, Commit, ObjectId, Repository};
 use sqlx::{Acquire, SqliteConnection, SqlitePool};
 use tracing::{debug, info};
 
-use crate::somehow;
+use crate::{repo, somehow};
 
-async fn get_all_commits_from_db(
+async fn get_all_commit_hashes_from_db(
     conn: &mut SqliteConnection,
 ) -> somehow::Result<HashSet<ObjectId>> {
     let hashes = sqlx::query!("SELECT hash FROM commits")
@@ -25,14 +25,14 @@ async fn get_all_commits_from_db(
     Ok(hashes)
 }
 
-fn get_new_commits_from_repo(
+fn get_new_commits_from_repo<'a, 'b: 'a>(
-    repo: &Repository,
+    repo: &'a Repository,
-    old: &HashSet<ObjectId>,
+    old: &'b HashSet<ObjectId>,
-) -> somehow::Result<Vec<Info>> {
+) -> somehow::Result<Vec<Commit<'a>>> {
     // Collect all references starting with "refs"
     let mut all_references: Vec<ObjectId> = vec![];
     for reference in repo.references()?.prefixed("refs")? {
-        let reference = reference.map_err(|e| somehow::Error(anyhow::anyhow!(e)))?;
+        let reference = reference.map_err(somehow::Error::from_box)?;
         let id = reference.into_fully_peeled_id()?;
 
         // Some repos *cough*linuxkernel*cough* have refs that don't point to
@@ -45,29 +45,57 @@ fn get_new_commits_from_repo(
     }
 
     // Walk from those until hitting old references
-    let new_commits = repo
+    let mut new = vec![];
+    for commit in repo
         .rev_walk(all_references)
         .selected(|c| !old.contains(c))?
-        .map(|r| r.map(|i| i.detach()))
+    {
-        .collect::<Result<Vec<_>, _>>()?;
+        let commit = commit?.id().object()?.try_into_commit()?;
+        new.push(commit);
+    }
 
-    Ok(new_commits)
+    Ok(new)
 }
 
-async fn insert_new_commits(conn: &mut SqliteConnection, new: &[Info]) -> somehow::Result<()> {
+async fn insert_new_commits(
+    conn: &mut SqliteConnection,
+    new: &[Commit<'_>],
+) -> somehow::Result<()> {
     for commit in new {
         let hash = commit.id.to_string();
-        sqlx::query!("INSERT OR IGNORE INTO commits (hash) VALUES (?)", hash)
+        let author_info = commit.author()?;
-            .execute(&mut *conn)
+        let author = repo::format_actor(author_info.actor())?;
-            .await?;
+        let author_date = author_info.time.format(ISO8601_STRICT);
+        let committer_info = commit.committer()?;
+        let committer = repo::format_actor(committer_info.actor())?;
+        let committer_date = committer_info.time.format(ISO8601_STRICT);
+        let message = commit.message_raw()?.to_string();
+
+        sqlx::query!(
+            "
+INSERT OR IGNORE INTO commits (hash, author, author_date, committer, committer_date, message)
+VALUES (?, ?, ?, ?, ?, ?)
+",
+            hash,
+            author,
+            author_date,
+            committer,
+            committer_date,
+            message
+        )
+        .execute(&mut *conn)
+        .await?;
     }
     Ok(())
 }
 
-async fn insert_new_commit_links(conn: &mut SqliteConnection, new: &[Info]) -> somehow::Result<()> {
+async fn insert_new_commit_links(
+    conn: &mut SqliteConnection,
+    new: &[Commit<'_>],
+) -> somehow::Result<()> {
     for commit in new {
         let child = commit.id.to_string();
-        for parent in &commit.parent_ids {
+        for parent in commit.parent_ids() {
             let parent = parent.to_string();
             // Commits *cough*linuxkernel*cough* may list the same parent
             // multiple times, so we just ignore duplicates during insert.
@@ -148,7 +176,7 @@ WITH RECURSIVE reachable(hash) AS (
 )
 
 UPDATE commits
-SET tracked = (hash IN reachable)
+SET reachable = (hash IN reachable)
 "
     )
     .execute(conn)
@@ -161,7 +189,7 @@ pub async fn update(db: &SqlitePool, repo: &Repository) -> somehow::Result<()> {
     let mut tx = db.begin().await?;
     let conn = tx.acquire().await?;
 
-    let old = get_all_commits_from_db(&mut *conn).await?;
+    let old = get_all_commit_hashes_from_db(&mut *conn).await?;
     debug!("Loaded {} commits from the db", old.len());
 
     let repo_is_new = old.is_empty();

src/repo.rs

@@ -7,7 +7,7 @@ use crate::somehow;
 
 pub fn format_actor(author: IdentityRef<'_>) -> somehow::Result<String> {
     let mut buffer = vec![];
-    author.write_to(&mut buffer)?;
+    author.trim().write_to(&mut buffer)?;
     Ok(String::from_utf8_lossy(&buffer).to_string())
 }
 

src/somehow.rs

@@ -8,6 +8,12 @@ use axum::{
 /// Wrapper around [`anyhow::Error`] that implements additional type classes.
 pub struct Error(pub anyhow::Error);
 
+impl Error {
+    pub fn from_box(err: Box<dyn error::Error + Send + Sync + 'static>) -> Self {
+        Self(anyhow::anyhow!(err))
+    }
+}
+
 impl<E> From<E> for Error
 where
     E: error::Error + Send + Sync + 'static,

src/web/commit_hash.rs

@@ -53,7 +53,7 @@ pub async fn get(
     // Do this first because a &Repository can't be kept across awaits.
     let child_rows = sqlx::query!(
         "
-SELECT child, tracked FROM commit_links
+SELECT child, reachable FROM commit_links
 JOIN commits ON hash = child
 WHERE parent = ?
     ",
@@ -79,7 +79,7 @@ WHERE parent = ?
     let mut children = vec![];
     for row in child_rows {
         let id = row.child.parse::<ObjectId>()?.attach(&repo);
-        children.push(Commit::new(id, row.tracked != 0)?);
+        children.push(Commit::new(id, row.reachable != 0)?);
     }
 
     Ok(CommitIdTemplate {