perf(metadata): parallelize metadata fetch, use gzip on request

Author: QaidVoid

Diff for: Cargo.lock

@@ -25,8 +25,9 @@ panic = "abort"
 futures = "0.3.31"
 rayon = "1.10.0"
 regex = { version = "1.11.1", default-features = false, features = ["unicode-case", "unicode-perl", "std"] }
-reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls", "blocking", "http2", "json", "stream"] }
+reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls", "blocking", "http2", "json", "stream", "gzip"] }
 rusqlite = { version = "0.33.0", features = ["bundled", "rusqlite-macros"] }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = { version = "1.0.135", features = ["indexmap"] }
 soar-dl = "0.3.3"
+tracing = { version = "0.1.41", default-features = false }

Diff for: Cargo.toml

@@ -32,5 +32,5 @@ soar-core = { version = "0.1.2", path = "../soar-core" }
 soar-dl = { workspace = true }
 tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread"] }
 toml = "0.8.19"
-tracing = { version = "0.1.41", default-features = false }
+tracing = { workspace = true }
 tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt", "json", "nu-ansi-term"] }

Diff for: soar-cli/Cargo.toml

@@ -10,6 +10,7 @@ use soar_core::{
     config::{get_config, Config},
     constants::CORE_MIGRATIONS,
     database::{connection::Database, migration::MigrationManager},
+    error::SoarError,
     metadata::fetch_metadata,
     SoarResult,
 };
@@ -39,13 +40,22 @@ impl AppState {
     }
 
     async fn init_repo_dbs(&self) -> SoarResult<()> {
+        let mut tasks = Vec::new();
+
         for repo in &self.inner.config.repositories {
             let db_file = repo.get_path()?.join("metadata.db");
             if !db_file.exists() {
                 fs::create_dir_all(repo.get_path()?)?;
                 File::create(&db_file)?;
             }
-            fetch_metadata(repo.clone()).await?;
+            let repo_clone = repo.clone();
+            let task = tokio::task::spawn(async move { fetch_metadata(repo_clone).await });
+            tasks.push(task);
+        }
+
+        for task in tasks {
+            task.await
+                .map_err(|err| SoarError::Custom(format!("Join handle error: {}", err)))??;
         }
         Ok(())
     }

Diff for: soar-cli/src/state.rs

@@ -28,3 +28,4 @@ soar-dl = { workspace = true }
 squishy = { version = "0.3.1", features = ["appimage"] }
 thiserror = "2.0.6"
 toml = "0.8.19"
+tracing = { workspace = true }

Diff for: soar-core/Cargo.toml

@@ -61,5 +61,5 @@ CREATE TABLE packages (
   download_count INTEGER,
   download_count_week INTEGER,
   download_count_month INTEGER,
-  UNIQUE (pkg_id, pkg_name)
+  UNIQUE (pkg_id, pkg_name, version)
 );

Diff for: soar-core/migrations/metadata/V1_initial.sql

@@ -42,7 +42,7 @@ impl<'a> DbStatements<'a> {
                     ?26, jsonb(?27), jsonb(?28), jsonb(?29), jsonb(?30), jsonb(?31), ?32, ?33, ?34, ?35, ?36,
                     jsonb(?37), jsonb(?38), jsonb(?39), ?40, ?41, ?42
                 )
-                ON CONFLICT (pkg_id, pkg_name) DO NOTHING",
+                ON CONFLICT (pkg_id, pkg_name, version) DO NOTHING",
             )?,
         })
     }

Diff for: soar-core/src/database/statements.rs

@@ -1,7 +1,9 @@
 use std::fs::{self, File};
 
+use futures::TryStreamExt;
 use reqwest::header::{self, HeaderMap};
 use rusqlite::Connection;
+use tracing::info;
 
 use crate::{
     config::Repository,
@@ -30,32 +32,50 @@ pub async fn fetch_metadata(repo: Repository) -> SoarResult<()> {
 
     let metadata_db = repo_path.join("metadata.db");
 
-    let conn = Connection::open(&metadata_db)?;
-    let etag: String = conn
-        .query_row("SELECT etag FROM repository", [], |row| row.get(0))
-        .unwrap_or_default();
-
-    let etag = if let Some(remote_etag) = resp.headers().get(header::ETAG) {
-        let remote_etag = remote_etag.to_str().unwrap();
-        if etag == remote_etag {
-            return Ok(());
+    let etag = {
+        let conn = Connection::open(&metadata_db)?;
+        let etag: String = conn
+            .query_row("SELECT etag FROM repository", [], |row| row.get(0))
+            .unwrap_or_default();
+
+        match resp.headers().get(header::ETAG) {
+            Some(remote_etag) => {
+                let remote_etag = remote_etag.to_str().unwrap();
+                if etag == remote_etag {
+                    return Ok(());
+                }
+                remote_etag.to_string()
+            }
+            None => {
+                return Err(SoarError::Custom(
+                    "etag is required in metadata response header.".to_string(),
+                ))
+            }
         }
-        remote_etag.to_string()
-    } else {
-        return Err(SoarError::Custom(
-            "etag is required in metadata response header.".to_string(),
-        ));
     };
-    let _ = conn.close();
 
     let _ = fs::remove_file(&metadata_db);
     File::create(&metadata_db)?;
 
+    info!("Fetching metadata from {}", repo.url);
+
     let conn = Connection::open(&metadata_db)?;
     let mut manager = MigrationManager::new(conn)?;
     manager.migrate_from_dir(METADATA_MIGRATIONS)?;
 
-    let remote_metadata: Vec<RemotePackage> = resp.json().await?;
+    let mut content = Vec::new();
+    let mut stream = resp.bytes_stream();
+
+    while let Ok(Some(chunk)) = stream.try_next().await {
+        content.extend_from_slice(&chunk);
+    }
+
+    let remote_metadata: Vec<RemotePackage> = serde_json::from_slice(&content).map_err(|err| {
+        SoarError::Custom(format!(
+            "Failed to parse metadata response from {}: {:#?}",
+            repo.url, err
+        ))
+    })?;
 
     let db = Database::new(metadata_db)?;
     db.from_remote_metadata(remote_metadata.as_ref(), &repo.name, &etag)?;