Switch from tracing to log crate

This commit is contained in:
Joscha 2023-08-17 03:09:24 +02:00
parent 1faf42bd82
commit f12da915a9
16 changed files with 149 additions and 202 deletions

97
Cargo.lock generated
View file

@ -678,6 +678,19 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "env_logger"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
dependencies = [
"humantime",
"is-terminal",
"log",
"regex",
"termcolor",
]
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.1" version = "1.0.1"
@ -1970,9 +1983,9 @@ dependencies = [
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.19" version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]] [[package]]
name = "matchit" name = "matchit"
@ -2065,16 +2078,6 @@ dependencies = [
"minimal-lexical", "minimal-lexical",
] ]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]] [[package]]
name = "num-bigint-dig" name = "num-bigint-dig"
version = "0.8.4" version = "0.8.4"
@ -2180,12 +2183,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"
@ -2803,15 +2800,6 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "sharded-slab"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "shellexpand" name = "shellexpand"
version = "2.1.2" version = "2.1.2"
@ -3178,11 +3166,13 @@ dependencies = [
"bytes", "bytes",
"clap", "clap",
"directories", "directories",
"env_logger",
"flate2", "flate2",
"futures", "futures",
"gethostname", "gethostname",
"gix", "gix",
"humantime", "humantime",
"log",
"mime_guess", "mime_guess",
"open", "open",
"rand", "rand",
@ -3199,8 +3189,6 @@ dependencies = [
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"toml", "toml",
"tracing",
"tracing-subscriber",
"vergen", "vergen",
"walkdir", "walkdir",
] ]
@ -3228,6 +3216,15 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "termcolor"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
dependencies = [
"winapi-util",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.44" version = "1.0.44"
@ -3248,16 +3245,6 @@ dependencies = [
"syn 2.0.28", "syn 2.0.28",
] ]
[[package]]
name = "thread_local"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.25" version = "0.3.25"
@ -3462,32 +3449,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922"
dependencies = [
"lazy_static",
"log",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
dependencies = [
"nu-ansi-term",
"sharded-slab",
"smallvec",
"thread_local",
"tracing-core",
"tracing-log",
] ]
[[package]] [[package]]
@ -3582,12 +3543,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.15" version = "0.2.15"

View file

@ -12,10 +12,12 @@ axum-extra = { version = "0.7.7", features = ["typed-routing", "query"] }
bytes = "1.4.0" bytes = "1.4.0"
clap = { version = "4.3.19", features = ["derive", "deprecated"] } clap = { version = "4.3.19", features = ["derive", "deprecated"] }
directories = "5.0.1" directories = "5.0.1"
env_logger = "0.10.0"
flate2 = "1.0.26" flate2 = "1.0.26"
futures = "0.3.28" futures = "0.3.28"
gethostname = "0.4.3" gethostname = "0.4.3"
humantime = "2.1.0" humantime = "2.1.0"
log = "0.4.20"
mime_guess = "2.0.4" mime_guess = "2.0.4"
open = "5.0.0" open = "5.0.0"
rand = "0.8.5" rand = "0.8.5"
@ -30,8 +32,6 @@ tempfile = "3.7.1"
tokio = { version = "1.29.1", features = ["full"] } tokio = { version = "1.29.1", features = ["full"] }
tokio-stream = "0.1.14" tokio-stream = "0.1.14"
toml = "0.7.6" toml = "0.7.6"
tracing = "0.1.37"
tracing-subscriber = "0.3.17"
walkdir = "2.3.3" walkdir = "2.3.3"
[dependencies.gix] [dependencies.gix]

View file

@ -3,8 +3,8 @@
use std::{collections::HashMap, fs, net::SocketAddr, path::PathBuf, time::Duration}; use std::{collections::HashMap, fs, net::SocketAddr, path::PathBuf, time::Duration};
use directories::ProjectDirs; use directories::ProjectDirs;
use log::{info, trace};
use serde::Deserialize; use serde::Deserialize;
use tracing::{debug, info};
use crate::{ use crate::{
args::{Args, Command}, args::{Args, Command},
@ -254,13 +254,13 @@ impl Config {
pub fn load(args: &Args) -> somehow::Result<Self> { pub fn load(args: &Args) -> somehow::Result<Self> {
let path = Self::path(args); let path = Self::path(args);
info!(path = %path.display(), "Loading config"); info!("Loading config from {}", path.display());
let raw = fs::read_to_string(path)?; let raw = fs::read_to_string(path)?;
let raw = toml::from_str::<RawConfig>(&raw)?; let raw = toml::from_str::<RawConfig>(&raw)?;
debug!("Loaded raw config: {raw:#?}"); trace!("Raw config: {raw:#?}");
let config = Self::from_raw_config(raw, args); let config = Self::from_raw_config(raw, args);
debug!("Loaded config: {config:#?}"); trace!("Config: {config:#?}");
Ok(config) Ok(config)
} }
} }

View file

@ -10,15 +10,18 @@ mod shared;
mod somehow; mod somehow;
mod worker; mod worker;
use std::{collections::HashMap, io, net::IpAddr, process, time::Duration}; use std::{
collections::HashMap,
io::{self, Write},
net::IpAddr,
process,
time::Duration,
};
use clap::Parser; use clap::Parser;
use config::ServerConfig; use config::ServerConfig;
use log::{debug, error, info, LevelFilter};
use tokio::{select, signal::unix::SignalKind}; use tokio::{select, signal::unix::SignalKind};
use tracing::{debug, error, info, Level};
use tracing_subscriber::{
filter::Targets, prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt,
};
use crate::{ use crate::{
args::{Args, Command, NAME, VERSION}, args::{Args, Command, NAME, VERSION},
@ -28,33 +31,34 @@ use crate::{
}; };
fn set_up_logging(verbose: u8) { fn set_up_logging(verbose: u8) {
let filter = Targets::new() let level = match verbose {
.with_default(Level::TRACE) 0 => LevelFilter::Info,
.with_target("hyper", Level::INFO) 1 => LevelFilter::Debug,
.with_target("sqlx", Level::INFO); 2.. => LevelFilter::Trace,
match verbose { };
0 => tracing_subscriber::fmt()
.with_max_level(Level::INFO) env_logger::builder()
.without_time() .filter_level(level)
.with_target(false) .filter_module("hyper", LevelFilter::Warn)
.init(), .filter_module("sqlx", LevelFilter::Warn)
1 => tracing_subscriber::fmt() .filter_module("tracing", LevelFilter::Warn)
.with_max_level(Level::TRACE) .format(|f, record| {
.with_target(false) // By prefixing <syslog_level> to the logged messages, they will
.finish() // show up in journalctl with their appropriate level.
.with(filter) // https://unix.stackexchange.com/a/349148
.init(), // https://0pointer.de/blog/projects/journal-submit.html
2 => tracing_subscriber::fmt() // https://en.wikipedia.org/wiki/Syslog#Severity_level
.with_max_level(Level::TRACE) let syslog_level = match record.level() {
.pretty() log::Level::Error => 3,
.finish() log::Level::Warn => 4,
.with(filter) log::Level::Info => 6,
.init(), log::Level::Debug | log::Level::Trace => 7,
_ => tracing_subscriber::fmt() };
.with_max_level(Level::TRACE) let level = f.default_styled_level(record.level());
.pretty() let args = record.args();
.init(), writeln!(f, "<{syslog_level}>[{level:>5}] {args}")
} })
.init();
} }
async fn wait_for_signal() -> io::Result<()> { async fn wait_for_signal() -> io::Result<()> {
@ -131,7 +135,7 @@ async fn launch_local_workers(config: &'static Config, amount: u8) {
); );
let worker_config = Box::leak(Box::new(worker_config)); let worker_config = Box::leak(Box::new(worker_config));
info!("Launching local worker {}", worker_config.name); info!("Starting local worker {}", worker_config.name);
let worker = Worker::new(worker_config); let worker = Worker::new(worker_config);
tokio::spawn(async move { worker.run().await }); tokio::spawn(async move { worker.run().await });
} }
@ -147,6 +151,8 @@ async fn run() -> somehow::Result<()> {
match args.command { match args.command {
Command::Server(command) => { Command::Server(command) => {
info!("Starting server");
if command.open { if command.open {
tokio::task::spawn(open_in_browser(&config.server)); tokio::task::spawn(open_in_browser(&config.server));
} }
@ -176,6 +182,8 @@ async fn run() -> somehow::Result<()> {
} }
} }
Command::Worker => { Command::Worker => {
info!("Starting worker");
let worker = Worker::new(&config.worker); let worker = Worker::new(&config.worker);
select! { select! {

View file

@ -11,18 +11,18 @@ use std::{
use axum::extract::FromRef; use axum::extract::FromRef;
use gix::ThreadSafeRepository; use gix::ThreadSafeRepository;
use log::{debug, info};
use sqlx::{ use sqlx::{
sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}, sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous},
SqlitePool, SqlitePool,
}; };
use tokio::select; use tokio::select;
use tracing::{debug, info};
use crate::{args::ServerCommand, config::ServerConfig, somehow}; use crate::{args::ServerCommand, config::ServerConfig, somehow};
use self::workers::Workers; use self::workers::Workers;
async fn open_db(db_path: &Path) -> sqlx::Result<SqlitePool> { async fn open_db(path: &Path) -> sqlx::Result<SqlitePool> {
let options = SqliteConnectOptions::new() let options = SqliteConnectOptions::new()
// https://www.sqlite.org/pragma.html#pragma_journal_mode // https://www.sqlite.org/pragma.html#pragma_journal_mode
.journal_mode(SqliteJournalMode::Wal) .journal_mode(SqliteJournalMode::Wal)
@ -34,14 +34,14 @@ async fn open_db(db_path: &Path) -> sqlx::Result<SqlitePool> {
// https://www.sqlite.org/pragma.html#pragma_trusted_schema // https://www.sqlite.org/pragma.html#pragma_trusted_schema
// The docs recommend always turning this off // The docs recommend always turning this off
.pragma("trusted_schema", "false") .pragma("trusted_schema", "false")
.filename(db_path) .filename(path)
.create_if_missing(true) .create_if_missing(true)
// https://www.sqlite.org/lang_analyze.html#recommended_usage_pattern // https://www.sqlite.org/lang_analyze.html#recommended_usage_pattern
// https://www.sqlite.org/pragma.html#pragma_analysis_limit // https://www.sqlite.org/pragma.html#pragma_analysis_limit
// https://www.sqlite.org/pragma.html#pragma_optimize // https://www.sqlite.org/pragma.html#pragma_optimize
.optimize_on_close(true, Some(1000)); .optimize_on_close(true, Some(1000));
info!(path = %db_path.display(), "Opening db"); info!("Opening db at {}", path.display());
let pool = SqlitePoolOptions::new() let pool = SqlitePoolOptions::new()
// Regularly optimize the db as recommended by the sqlite docs // Regularly optimize the db as recommended by the sqlite docs
// https://www.sqlite.org/lang_analyze.html#recommended_usage_pattern // https://www.sqlite.org/lang_analyze.html#recommended_usage_pattern
@ -77,7 +77,7 @@ impl Server {
command: ServerCommand, command: ServerCommand,
) -> somehow::Result<Self> { ) -> somehow::Result<Self> {
let repo = if let Some(path) = command.repo.as_ref() { let repo = if let Some(path) = command.repo.as_ref() {
info!(path = %path.display(), "Opening repo"); info!("Opening repo at {}", path.display());
let repo = ThreadSafeRepository::open(path)?; let repo = ThreadSafeRepository::open(path)?;
Some(Repo(Arc::new(repo))) Some(Repo(Arc::new(repo)))
} else { } else {
@ -85,7 +85,7 @@ impl Server {
}; };
let bench_repo = if let Some(path) = command.bench_repo.as_ref() { let bench_repo = if let Some(path) = command.bench_repo.as_ref() {
info!(path = %path.display(), "Opening repo"); info!("Opening bench repo at {}", path.display());
let repo = ThreadSafeRepository::open(path)?; let repo = ThreadSafeRepository::open(path)?;
Some(BenchRepo(Arc::new(repo))) Some(BenchRepo(Arc::new(repo)))
} else { } else {

View file

@ -4,37 +4,13 @@ mod fetch;
mod queue; mod queue;
mod repo; mod repo;
use tracing::{debug_span, error, warn_span, Instrument};
use super::{Repo, Server}; use super::{Repo, Server};
async fn recurring_task(state: &Server, repo: Repo) {
fetch::update(state.config, repo.clone())
.instrument(debug_span!("fetch refs"))
.await;
async {
if let Err(e) = repo::update(&state.db, repo).await {
error!("Error updating repo:\n{e:?}");
};
}
.instrument(debug_span!("update repo"))
.await;
async {
if let Err(e) = queue::update(&state.db).await {
error!("Error updating queue:\n{e:?}");
};
}
.instrument(debug_span!("update queue"))
.await;
}
pub(super) async fn run(server: Server, repo: Repo) { pub(super) async fn run(server: Server, repo: Repo) {
loop { loop {
recurring_task(&server, repo.clone()) fetch::update(server.config, repo.clone()).await;
.instrument(warn_span!("update")) repo::update(&server.db, repo.clone()).await;
.await; queue::update(&server.db).await;
tokio::time::sleep(server.config.repo_update).await; tokio::time::sleep(server.config.repo_update).await;
} }

View file

@ -2,8 +2,8 @@
use std::process::Command; use std::process::Command;
use gix::bstr::ByteVec; use gix::bstr::ByteSlice;
use tracing::{info, warn}; use log::{info, warn};
use crate::{config::ServerConfig, server::Repo, somehow}; use crate::{config::ServerConfig, server::Repo, somehow};
@ -25,9 +25,15 @@ fn fetch(repo: Repo, url: &str, refspecs: &[String]) -> somehow::Result<()> {
let output = command.output()?; let output = command.output()?;
if output.status.success() { if output.status.success() {
} else { } else {
warn!(exitcode = %output.status, "'git fetch' failed"); warn!(
warn!(output = "stdout", "{}", output.stdout.into_string_lossy()); "Error fetching refs:\n\
warn!(output = "stderr", "{}", output.stderr.into_string_lossy()); {command:?} exited with code {}\n\
STDOUT:\n{}\n\
STDERR:\n{}",
output.status,
output.stdout.to_str_lossy(),
output.stderr.to_str_lossy()
);
} }
Ok(()) Ok(())
@ -38,7 +44,7 @@ async fn inner(repo: Repo, url: &'static str, refspecs: &'static [String]) -> so
Ok(()) Ok(())
} }
pub async fn update(config: &'static ServerConfig, repo: Repo) { pub(super) async fn update(config: &'static ServerConfig, repo: Repo) {
if let Some(url) = &config.repo_fetch_url { if let Some(url) = &config.repo_fetch_url {
if let Err(e) = inner(repo, url, &config.repo_fetch_refspecs).await { if let Err(e) = inner(repo, url, &config.repo_fetch_refspecs).await {
warn!("Error fetching refs:\n{e:?}"); warn!("Error fetching refs:\n{e:?}");

View file

@ -1,10 +1,10 @@
use log::{debug, info, warn};
use sqlx::{Acquire, SqlitePool}; use sqlx::{Acquire, SqlitePool};
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::debug;
use crate::somehow; use crate::somehow;
pub async fn update(db: &SqlitePool) -> somehow::Result<()> { async fn inner(db: &SqlitePool) -> somehow::Result<()> {
debug!("Updating queue"); debug!("Updating queue");
let mut tx = db.begin().await?; let mut tx = db.begin().await?;
let conn = tx.acquire().await?; let conn = tx.acquire().await?;
@ -13,20 +13,22 @@ pub async fn update(db: &SqlitePool) -> somehow::Result<()> {
let new = sqlx::query!("SELECT hash FROM commits WHERE new AND reachable = 2") let new = sqlx::query!("SELECT hash FROM commits WHERE new AND reachable = 2")
.fetch_all(&mut *conn) .fetch_all(&mut *conn)
.await?; .await?;
let new_len = new.len();
// Insert them into the queue // Insert them into the queue
for row in new { for row in new {
let date = OffsetDateTime::now_utc(); let date = OffsetDateTime::now_utc();
sqlx::query!( let result = sqlx::query!(
"INSERT OR IGNORE INTO queue (hash, date) VALUES (?, ?)", "INSERT OR IGNORE INTO queue (hash, date) VALUES (?, ?)",
row.hash, row.hash,
date, date,
) )
.execute(&mut *conn) .execute(&mut *conn)
.await?; .await?;
if result.rows_affected() > 0 {
info!("Added new commit {} to the queue", row.hash);
}
} }
debug!("Added {new_len} commits to the queue");
// Mark all commits as old // Mark all commits as old
sqlx::query!("UPDATE commits SET new = false") sqlx::query!("UPDATE commits SET new = false")
@ -34,6 +36,11 @@ pub async fn update(db: &SqlitePool) -> somehow::Result<()> {
.await?; .await?;
tx.commit().await?; tx.commit().await?;
debug!("Updated queue");
Ok(()) Ok(())
} }
pub(super) async fn update(db: &SqlitePool) {
if let Err(e) = inner(db).await {
warn!("Error updating queue:\n{e:?}");
}
}

View file

@ -4,9 +4,9 @@ use std::collections::HashSet;
use futures::TryStreamExt; use futures::TryStreamExt;
use gix::{date::Time, objs::Kind, prelude::ObjectIdExt, refs::Reference, ObjectId, Repository}; use gix::{date::Time, objs::Kind, prelude::ObjectIdExt, refs::Reference, ObjectId, Repository};
use log::{debug, info, warn};
use sqlx::{Acquire, SqliteConnection, SqlitePool}; use sqlx::{Acquire, SqliteConnection, SqlitePool};
use time::{OffsetDateTime, UtcOffset}; use time::{OffsetDateTime, UtcOffset};
use tracing::{debug, info};
use crate::{ use crate::{
server::{util, Repo}, server::{util, Repo},
@ -111,11 +111,11 @@ async fn insert_new_commits(
.execute(&mut *conn) .execute(&mut *conn)
.await?; .await?;
// So the user has something to look at while importing big repos
if (i + 1) % 100000 == 0 { if (i + 1) % 100000 == 0 {
debug!("Inserted {} commits so far", i + 1); info!("(1/2) Inserting commits: {}/{}", i + 1, new.len());
} }
} }
debug!("Inserted {} commits in total", new.len());
Ok(()) Ok(())
} }
@ -140,11 +140,11 @@ async fn insert_new_commit_links(
.await?; .await?;
} }
// So the user has something to look at while importing big repos
if (i + 1) % 100000 == 0 { if (i + 1) % 100000 == 0 {
debug!("Inserted {} commits' links so far", i + 1); info!("(2/2) Inserting links: {}/{}", i + 1, new.len());
} }
} }
debug!("Inserted {} commits' links in total", new.len());
Ok(()) Ok(())
} }
@ -234,8 +234,8 @@ async fn update_commit_tracked_status(conn: &mut SqliteConnection) -> somehow::R
Ok(()) Ok(())
} }
pub(super) async fn update(db: &SqlitePool, repo: Repo) -> somehow::Result<()> { pub async fn inner(db: &SqlitePool, repo: Repo) -> somehow::Result<()> {
debug!("Updating repo"); debug!("Updating repo data");
let thread_local_repo = repo.0.to_thread_local(); let thread_local_repo = repo.0.to_thread_local();
let mut tx = db.begin().await?; let mut tx = db.begin().await?;
let conn = tx.acquire().await?; let conn = tx.acquire().await?;
@ -254,7 +254,11 @@ pub(super) async fn update(db: &SqlitePool, repo: Repo) -> somehow::Result<()> {
get_all_refs_and_new_commits_from_repo(&repo.0.to_thread_local(), &old) get_all_refs_and_new_commits_from_repo(&repo.0.to_thread_local(), &old)
}) })
.await??; .await??;
debug!("Found {} new commits in repo", new.len()); if new.is_empty() {
debug!("Found no new commits in repo");
} else {
info!("Found {} new commits in repo", new.len());
}
// Defer foreign key checks until the end of the transaction to improve // Defer foreign key checks until the end of the transaction to improve
// insert performance. // insert performance.
@ -282,6 +286,11 @@ pub(super) async fn update(db: &SqlitePool, repo: Repo) -> somehow::Result<()> {
if repo_is_new { if repo_is_new {
info!("Initialized new repo"); info!("Initialized new repo");
} }
debug!("Updated repo");
Ok(()) Ok(())
} }
pub(super) async fn update(db: &SqlitePool, repo: Repo) {
if let Err(e) = inner(db, repo).await {
warn!("Error updating repo data:\n{e:?}");
}
}

View file

@ -3,6 +3,7 @@ use axum::{
response::{IntoResponse, Redirect}, response::{IntoResponse, Redirect},
Form, Form,
}; };
use log::info;
use serde::Deserialize; use serde::Deserialize;
use sqlx::SqlitePool; use sqlx::SqlitePool;
use time::OffsetDateTime; use time::OffsetDateTime;
@ -46,6 +47,11 @@ pub async fn post_admin_queue_add(
.execute(&db) .execute(&db)
.await?; .await?;
info!(
"Admin added {} to queue with priority {}",
form.hash, form.priority,
);
let link = Base::link_with_config(config, PathQueue {}); let link = Base::link_with_config(config, PathQueue {});
Ok(Redirect::to(&format!("{link}"))) Ok(Redirect::to(&format!("{link}")))
} }
@ -65,6 +71,8 @@ pub async fn post_admin_queue_delete(
.execute(&db) .execute(&db)
.await?; .await?;
info!("Admin deleted {} from queue", form.hash);
let link = Base::link_with_config(config, PathQueue {}); let link = Base::link_with_config(config, PathQueue {});
Ok(Redirect::to(&format!("{link}"))) Ok(Redirect::to(&format!("{link}")))
} }
@ -87,6 +95,8 @@ pub async fn post_admin_queue_increase(
.execute(&db) .execute(&db)
.await?; .await?;
info!("Admin increased queue priority of {} by one", form.hash);
let link = Base::link_with_config(config, PathQueue {}); let link = Base::link_with_config(config, PathQueue {});
Ok(Redirect::to(&format!("{link}"))) Ok(Redirect::to(&format!("{link}")))
} }
@ -109,6 +119,8 @@ pub async fn post_admin_queue_decrease(
.execute(&db) .execute(&db)
.await?; .await?;
info!("Admin decreased queue priority of {} by one", form.hash);
let link = Base::link_with_config(config, PathQueue {}); let link = Base::link_with_config(config, PathQueue {});
Ok(Redirect::to(&format!("{link}"))) Ok(Redirect::to(&format!("{link}")))
} }

View file

@ -13,9 +13,9 @@ use axum::{
Json, TypedHeader, Json, TypedHeader,
}; };
use gix::{ObjectId, ThreadSafeRepository}; use gix::{ObjectId, ThreadSafeRepository};
use log::{debug, info};
use sqlx::{Acquire, SqlitePool}; use sqlx::{Acquire, SqlitePool};
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::debug;
use crate::{ use crate::{
config::ServerConfig, config::ServerConfig,
@ -136,8 +136,10 @@ pub async fn post_api_worker_status(
Ok(name) => name, Ok(name) => name,
Err(response) => return Ok(response), Err(response) => return Ok(response),
}; };
debug!("Received status update from {name}");
if let Some(run) = request.submit_run { if let Some(run) = request.submit_run {
info!("Received run {} for {} from {name}", run.id, run.hash);
save_work(run, &name, &request.info, &db).await?; save_work(run, &name, &request.info, &db).await?;
} }
@ -178,7 +180,6 @@ pub async fn post_api_worker_status(
(work, abort_work) (work, abort_work)
}; };
debug!("Received status update from {name}");
Ok(Json(ServerResponse { Ok(Json(ServerResponse {
run: work, run: work,
abort_run: abort_work, abort_run: abort_work,
@ -208,10 +209,11 @@ pub async fn get_api_worker_repo_by_hash_tree_tar_gz(
State(repo): State<Option<Repo>>, State(repo): State<Option<Repo>>,
auth: Option<TypedHeader<Authorization<Basic>>>, auth: Option<TypedHeader<Authorization<Basic>>>,
) -> somehow::Result<Response> { ) -> somehow::Result<Response> {
let _name = match auth::authenticate(config, auth) { let name = match auth::authenticate(config, auth) {
Ok(name) => name, Ok(name) => name,
Err(response) => return Ok(response), Err(response) => return Ok(response),
}; };
debug!("Worker {name} is downloading repo hash {}", path.hash);
let Some(repo) = repo else { let Some(repo) = repo else {
return Ok(StatusCode::NOT_FOUND.into_response()); return Ok(StatusCode::NOT_FOUND.into_response());
@ -227,10 +229,11 @@ pub async fn get_api_worker_bench_repo_by_hash_tree_tar_gz(
State(bench_repo): State<Option<BenchRepo>>, State(bench_repo): State<Option<BenchRepo>>,
auth: Option<TypedHeader<Authorization<Basic>>>, auth: Option<TypedHeader<Authorization<Basic>>>,
) -> somehow::Result<Response> { ) -> somehow::Result<Response> {
let _name = match auth::authenticate(config, auth) { let name = match auth::authenticate(config, auth) {
Ok(name) => name, Ok(name) => name,
Err(response) => return Ok(response), Err(response) => return Ok(response),
}; };
debug!("Worker {name} is downloading bench repo hash {}", path.hash);
let Some(bench_repo) = bench_repo else { let Some(bench_repo) = bench_repo else {
return Ok(StatusCode::NOT_FOUND.into_response()); return Ok(StatusCode::NOT_FOUND.into_response());

View file

@ -12,9 +12,9 @@ use gix::{
bstr::ByteSlice, objs::tree::EntryMode, prelude::ObjectIdExt, worktree::stream::Entry, bstr::ByteSlice, objs::tree::EntryMode, prelude::ObjectIdExt, worktree::stream::Entry,
ObjectId, ThreadSafeRepository, ObjectId, ThreadSafeRepository,
}; };
use log::warn;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream; use tokio_stream::wrappers::ReceiverStream;
use tracing::{debug, warn};
const BLOCK_SIZE: usize = 1024 * 1024; const BLOCK_SIZE: usize = 1024 * 1024;
const COMPRESSION_LEVEL: Compression = Compression::fast(); const COMPRESSION_LEVEL: Compression = Compression::fast();
@ -114,8 +114,6 @@ pub fn tar_and_gzip(
if let Err(e) = write_worktree(repo, id, tx.clone()) { if let Err(e) = write_worktree(repo, id, tx.clone()) {
warn!("Error while streaming tar:\n{e:?}"); warn!("Error while streaming tar:\n{e:?}");
let _ = tx.blocking_send(Err(e)); let _ = tx.blocking_send(Err(e));
} else {
debug!("Tar streamed successfully");
} }
}); });
ReceiverStream::new(rx) ReceiverStream::new(rx)

View file

@ -1,6 +1,6 @@
mod util; mod util;
use std::{collections::HashMap, time::Instant}; use std::collections::HashMap;
use askama::Template; use askama::Template;
use axum::{extract::State, response::IntoResponse, Json}; use axum::{extract::State, response::IntoResponse, Json};
@ -9,7 +9,6 @@ use futures::{StreamExt, TryStreamExt};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sqlx::{Acquire, SqlitePool}; use sqlx::{Acquire, SqlitePool};
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::debug;
use crate::{ use crate::{
config::ServerConfig, config::ServerConfig,
@ -150,12 +149,6 @@ struct GraphData {
measurements: HashMap<String, Vec<Option<f64>>>, measurements: HashMap<String, Vec<Option<f64>>>,
} }
fn f(t0: &mut Instant, name: &str) {
let now = Instant::now();
debug!("{name:>20} {}", (now - *t0).as_millis());
*t0 = Instant::now();
}
pub async fn get_graph_data( pub async fn get_graph_data(
_path: PathGraphData, _path: PathGraphData,
State(db): State<SqlitePool>, State(db): State<SqlitePool>,
@ -170,9 +163,6 @@ pub async fn get_graph_data(
// TODO Limit by date or amount // TODO Limit by date or amount
// TODO Limit to tracked commits // TODO Limit to tracked commits
let mut t0 = Instant::now();
debug!("");
let mut unsorted_hashes = Vec::<String>::new(); let mut unsorted_hashes = Vec::<String>::new();
let mut times_by_hash = HashMap::<String, i64>::new(); let mut times_by_hash = HashMap::<String, i64>::new();
let mut rows = sqlx::query!( let mut rows = sqlx::query!(
@ -193,8 +183,6 @@ pub async fn get_graph_data(
} }
drop(rows); drop(rows);
f(&mut t0, "hashes, times");
let parent_child_pairs = sqlx::query!( let parent_child_pairs = sqlx::query!(
"\ "\
SELECT parent, child \ SELECT parent, child \
@ -209,13 +197,9 @@ pub async fn get_graph_data(
.try_collect::<Vec<_>>() .try_collect::<Vec<_>>()
.await?; .await?;
f(&mut t0, "parent-child");
let mut hashes = util::sort_topologically(&unsorted_hashes, &parent_child_pairs); let mut hashes = util::sort_topologically(&unsorted_hashes, &parent_child_pairs);
hashes.sort_by_key(|hash| times_by_hash[hash]); hashes.sort_by_key(|hash| times_by_hash[hash]);
f(&mut t0, "sort");
let sorted_hash_indices = hashes let sorted_hash_indices = hashes
.iter() .iter()
.cloned() .cloned()
@ -223,8 +207,6 @@ pub async fn get_graph_data(
.map(|(i, hash)| (hash, i)) .map(|(i, hash)| (hash, i))
.collect::<HashMap<_, _>>(); .collect::<HashMap<_, _>>();
f(&mut t0, "hash indices");
let mut parents = HashMap::<usize, Vec<usize>>::new(); let mut parents = HashMap::<usize, Vec<usize>>::new();
for (parent, child) in &parent_child_pairs { for (parent, child) in &parent_child_pairs {
if let Some(parent_idx) = sorted_hash_indices.get(parent) { if let Some(parent_idx) = sorted_hash_indices.get(parent) {
@ -234,24 +216,18 @@ pub async fn get_graph_data(
} }
} }
f(&mut t0, "parents");
// Collect times // Collect times
let times = hashes let times = hashes
.iter() .iter()
.map(|hash| times_by_hash[hash]) .map(|hash| times_by_hash[hash])
.collect::<Vec<_>>(); .collect::<Vec<_>>();
f(&mut t0, "times");
// permutation[unsorted_index] = sorted_index // permutation[unsorted_index] = sorted_index
let permutation = unsorted_hashes let permutation = unsorted_hashes
.iter() .iter()
.map(|hash| sorted_hash_indices[hash]) .map(|hash| sorted_hash_indices[hash])
.collect::<Vec<_>>(); .collect::<Vec<_>>();
f(&mut t0, "permutation");
// Collect and permutate measurements // Collect and permutate measurements
let mut measurements = HashMap::new(); let mut measurements = HashMap::new();
for metric in form.metric { for metric in form.metric {
@ -284,9 +260,6 @@ pub async fn get_graph_data(
measurements.insert(metric, values); measurements.insert(metric, values);
} }
f(&mut t0, "measurements");
debug!("");
Ok(Json(GraphData { Ok(Json(GraphData {
hashes, hashes,
parents, parents,

View file

@ -4,10 +4,10 @@ mod tree;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use log::{error, info, warn};
use reqwest::Client; use reqwest::Client;
use time::OffsetDateTime; use time::OffsetDateTime;
use tokio::sync::Mutex as AsyncMutex; use tokio::sync::Mutex as AsyncMutex;
use tracing::{error, info, warn};
use crate::{ use crate::{
config::WorkerConfig, config::WorkerConfig,

View file

@ -5,8 +5,8 @@ use std::{
sync::{Arc, Mutex}, sync::{Arc, Mutex},
}; };
use log::error;
use tokio::sync::Notify; use tokio::sync::Notify;
use tracing::error;
use crate::{ use crate::{
config::WorkerServerConfig, config::WorkerServerConfig,

View file

@ -1,9 +1,9 @@
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use log::{debug, warn};
use reqwest::Client; use reqwest::Client;
use tempfile::TempDir; use tempfile::TempDir;
use tokio::sync::Mutex as AsyncMutex; use tokio::sync::Mutex as AsyncMutex;
use tracing::{debug, warn};
use crate::{ use crate::{
config::{WorkerConfig, WorkerServerConfig}, config::{WorkerConfig, WorkerServerConfig},