diff --git a/Cargo.lock b/Cargo.lock index ebe6c2d873..c7c515b5f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,56 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anstream" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.98" @@ -55,6 +105,17 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "castaway" version = "0.2.3" @@ -64,12 +125,67 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cc" +version = "1.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +[[package]] +name = "clap" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.103", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "compact_str" version = "0.9.0" @@ -166,6 +282,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "edit" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f364860e764787163c8c8f58231003839be31276e821e2ad2092ddf496b1aa09" +dependencies = [ + "tempfile", + "which", +] + [[package]] name = "either" version = "1.15.0" @@ -178,6 +304,34 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -219,6 +373,30 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.4", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys", +] + [[package]] name = "indexmap" version = "2.9.0" @@ -229,6 +407,12 @@ dependencies = [ "hashbrown 0.15.4", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.11.0" @@ -259,6 +443,17 @@ version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +[[package]] +name = "libsqlite3-sys" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91632f3b4fb6bd1d72aa3d78f41ffecfcf2b1a6648d8c241dbe7dbfaf4875e15" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libyml" version = "0.0.5" @@ -269,6 +464,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "lock_api" version = "0.4.13" @@ -380,6 +587,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "parking_lot_core" version = "0.9.11" @@ -406,6 +619,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "pori" version = "0.0.0" @@ -544,12 +763,52 @@ dependencies = [ "serde", ] +[[package]] +name = "rusqlite" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de23c3319433716cf134eed225fe9986bc24f63bed9be9f20c329029e672dc7" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.9.4", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.21" @@ -624,6 +883,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "shell-escape" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "smallvec" version = "1.15.1" @@ -636,6 +907,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "1.0.109" @@ -658,6 +935,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.0.7", + "windows-sys", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -705,6 +995,18 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -723,15 +1025,20 @@ version = "0.0.0" dependencies = [ "anyhow", "bincode", + "bstr", + "clap", "compact_str", "dashmap", "diff-struct", + "edit", "itertools 0.14.0", "petgraph", "rayon", "relative-path", + "rusqlite", "serde", "serde_json", + "shell-escape", "twox-hash", "vite_workspace", "wax", @@ -785,6 +1092,18 @@ dependencies = [ "walkdir", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi-util" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 03ea7b6e30..fe646fa493 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,11 @@ serde_yml = "0.0.12" twox-hash = "2.1.1" vite_workspace = { path = "crates/vite_workspace" } wax = "0.6.0" +bstr = "1.12.0" +clap = "4.5.40" +edit = "0.1.5" +rusqlite = "0.36.0" +shell-escape = "0.1.5" [profile.dev] # Disabling debug info speeds up local and CI builds, diff --git a/crates/vite_task/Cargo.toml b/crates/vite_task/Cargo.toml index 9056adc85d..f36f23c1b5 100644 --- a/crates/vite_task/Cargo.toml +++ b/crates/vite_task/Cargo.toml @@ -14,15 +14,20 @@ workspace = true [dependencies] anyhow = { workspace = true } bincode = { workspace = true, features = ["derive"] } +bstr = { workspace = true } +clap = { workspace = true, features = ["derive"] } compact_str = { workspace = true, features = ["serde"] } dashmap = { workspace = true } diff-struct = { workspace = true } +edit = { workspace = true } itertools = { workspace = true } petgraph = { workspace = true } rayon = { workspace = true } relative-path = { workspace = true } +rusqlite = { workspace = true, features = ["bundled"] } serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } +shell-escape = { workspace = true } twox-hash = { workspace = true } vite_workspace = { workspace = true } wax = { workspace = true } diff --git a/crates/vite_task/src/cache.rs b/crates/vite_task/src/cache.rs index 10ea4ea4bf..c27ca07d90 100644 --- a/crates/vite_task/src/cache.rs +++ b/crates/vite_task/src/cache.rs @@ -1,12 +1,10 @@ -use crate::collections::HashMap; -use std::fs::{self, File}; -use std::io::{BufReader, BufWriter}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::path::Path; +use std::sync::{Arc, Mutex}; // use bincode::config::{Configuration, standard}; -use bincode::{Decode, Encode}; -use serde::{Deserialize, Serialize}; +use bincode::{Decode, Encode, decode_from_slice, encode_to_vec}; +use rusqlite::{Connection, OptionalExtension as _}; +use serde::Serialize; use crate::config::ResolvedTask; use crate::execute::{ExecutedTask, StdOutput}; @@ -14,7 +12,7 @@ use crate::fingerprint::{FingerprintMismatch, TaskFingerprint}; use crate::fs::FileSystem; use crate::str::Str; -#[derive(Debug, Encode, Decode, Serialize, Deserialize)] +#[derive(Debug, Encode, Decode, Serialize)] pub struct CachedTask { pub fingerprint: TaskFingerprint, pub std_outputs: Arc<[StdOutput]>, @@ -33,11 +31,16 @@ impl CachedTask { } pub struct TaskCache { - cached_tasks_by_name: HashMap, - path: PathBuf, + conn: Mutex, } -// const BINCODE_CONFIG: Configuration = standard(); +#[derive(Debug, Hash, Encode, Decode, Serialize)] +pub struct TaskCacheKey { + pub task_name: Str, + pub args: Arc<[Str]>, +} + +const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); #[derive(Debug)] pub enum CacheMiss { @@ -48,55 +51,95 @@ pub enum CacheMiss { impl TaskCache { pub fn load_from_file(path: impl AsRef) -> anyhow::Result { let path = path.as_ref(); - let cached_tasks_by_name: HashMap = match File::open(path) { - Ok(file) => { - let reader = BufReader::new(file); - // Using json for easy debugging - // Will switch to bincode for better performance - serde_json::from_reader(reader)? - // bincode::decode_from_std_read(&mut reader, BINCODE_CONFIG)? - } - Err(err) => { - if err.kind() == std::io::ErrorKind::NotFound { - HashMap::new() - } else { - return Err(err.into()); + let conn = Connection::open(path)?; + conn.execute_batch("PRAGMA journal_mode=WAL")?; + conn.execute("BEGIN EXCLUSIVE", ())?; + loop { + let user_version: u32 = conn.query_one("PRAGMA user_version", (), |row| row.get(0))?; + match user_version { + 0 => { + // fresh new db + conn.execute("CREATE TABLE tasks (key BLOB PRIMARY KEY, value BLOB);", ())?; + conn.execute("PRAGMA user_version = 1", ())?; } + // Migration done here + 1 => break, + 2.. => anyhow::bail!("Unrecognized cache db version: {user_version}"), } - }; - Ok(Self { cached_tasks_by_name, path: path.to_path_buf() }) - } - pub fn save(&self) -> anyhow::Result<()> { - let path = self.path.as_path(); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; } - let file = File::create(path)?; - let mut writer = BufWriter::new(file); - serde_json::to_writer_pretty(&mut writer, &self.cached_tasks_by_name)?; - // bincode::encode_into_std_write(&self.cached_tasks_by_name, &mut writer, BINCODE_CONFIG)?; - writer.into_inner()?.sync_data()?; + Ok(Self { conn: Mutex::new(conn) }) + } + pub fn save(self) -> anyhow::Result<()> { + let conn = self.conn.into_inner().unwrap(); + conn.execute("COMMIT", ())?; Ok(()) } - pub fn update(&mut self, task_name: Str, cached_task: CachedTask) -> anyhow::Result<()> { - self.cached_tasks_by_name.insert(task_name, cached_task); + pub fn update( + &mut self, + task_name: Str, + args: Arc<[Str]>, + cached_task: CachedTask, + ) -> anyhow::Result<()> { + let key = TaskCacheKey { task_name, args }; + let conn = self.conn.lock().unwrap(); + let key_blob = encode_to_vec(&key, BINCODE_CONFIG)?; + let value_blob = encode_to_vec(&cached_task, BINCODE_CONFIG)?; + let mut update_stmt = conn.prepare_cached( + "INSERT INTO tasks (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value=?2" + )?; + update_stmt.execute([key_blob, value_blob])?; + Ok(()) + } + + pub fn get_cache( + &self, + task_name: Str, + args: Arc<[Str]>, + ) -> anyhow::Result> { + let conn = self.conn.lock().unwrap(); + let mut select_stmt = conn.prepare_cached("SELECT value FROM tasks WHERE key=?")?; + let key_blob = encode_to_vec(&TaskCacheKey { task_name, args }, BINCODE_CONFIG)?; + let Some(value_blob) = + select_stmt.query_row::, _, _>([key_blob], |row| row.get(0)).optional()? + else { + return Ok(None); + }; + let (cached_task, _) = decode_from_slice::(&value_blob, BINCODE_CONFIG)?; + Ok(Some(cached_task)) + } + + pub fn list_cache( + &self, + mut f: impl FnMut(TaskCacheKey, CachedTask) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { + let conn = self.conn.lock().unwrap(); + let mut select_stmt = conn.prepare_cached("SELECT key, value FROM tasks")?; + let cache_list = select_stmt.query_and_then((), |row| { + let key_blob: Vec = row.get(0)?; + let value_blob: Vec = row.get(1)?; + let (key, _) = decode_from_slice::(&key_blob, BINCODE_CONFIG)?; + let (cached_task, _) = decode_from_slice::(&value_blob, BINCODE_CONFIG)?; + anyhow::Ok((key, cached_task)) + })?; + for cache in cache_list { + let (key, cached_task) = cache?; + f(key, cached_task)?; + } Ok(()) } /// Tries to get the task cache if the fingerprint matches, otherwise returns why the cache misses - pub fn try_hit<'me>( - &'me self, + pub fn try_hit( + &self, task: &ResolvedTask, fs: &impl FileSystem, base_dir: &Path, - ) -> anyhow::Result> { - let Some(cached_task) = self.cached_tasks_by_name.get(&task.name) else { + ) -> anyhow::Result> { + let Some(cached_task) = self.get_cache(task.name.clone(), task.args.clone())? else { return Ok(Err(CacheMiss::NotFound)); }; - if let Some(fingerprint_mismatch) = - cached_task.fingerprint.validate(&task.config, fs, base_dir)? - { + if let Some(fingerprint_mismatch) = cached_task.fingerprint.validate(task, fs, base_dir)? { return Ok(Err(CacheMiss::FingerprintMismatch(fingerprint_mismatch))); } Ok(Ok(cached_task)) diff --git a/crates/vite_task/src/config.rs b/crates/vite_task/src/config.rs index 53b48bbb0f..2c59b0a0d5 100644 --- a/crates/vite_task/src/config.rs +++ b/crates/vite_task/src/config.rs @@ -1,7 +1,9 @@ use std::{ + collections::BTreeSet, + ffi::OsStr, fs::File, io::BufReader, - iter::once, + iter::{self}, path::{Path, PathBuf}, sync::Arc, }; @@ -16,9 +18,10 @@ use crate::{ use anyhow::Context; use bincode::{Decode, Encode}; -use compact_str::CompactString; use diff::Diff; +use itertools::Itertools; use petgraph::{graph::NodeIndex, stable_graph::StableDiGraph}; +use relative_path::RelativePath; use serde::{Deserialize, Serialize}; use vite_workspace::PackageInfo; @@ -66,9 +69,52 @@ pub struct Workspace { #[derive(Debug)] pub struct ResolvedTask { pub name: Str, + pub args: Arc<[Str]>, + pub resolved_config: ResolvedTaskConfig, + pub resolved_command: ResolvedTaskCommand, +} + +#[derive(Encode, Decode, Debug, Serialize, PartialEq, Eq, Diff)] +#[diff(attr(#[derive(Debug)]))] +pub struct ResolvedTaskConfig { pub config_dir: Str, pub config: TaskConfig, - pub envs: TaskEnvs, +} + +impl ResolvedTaskConfig { + fn resolve_command(&self, task_args: &[Str]) -> anyhow::Result { + let cwd = RelativePath::new(&self.config_dir).join(self.config.cwd.as_str()); + let command_line = iter::once(self.config.command.clone()) + .chain( + task_args + .iter() + .map(|arg| shell_escape::escape(arg.as_str().into()).as_ref().into()), + ) + .join(" "); + let task_envs = TaskEnvs::resolve(&self.config)?; + Ok(ResolvedTaskCommand { + fingerprint: CommandFingerprint { + cwd: cwd.as_str().into(), + command_line: command_line.as_str().into(), + envs_without_pass_through: task_envs.envs_without_pass_through, + }, + all_envs: task_envs.all_envs, + }) + } +} + +#[derive(Debug)] +pub struct ResolvedTaskCommand { + pub fingerprint: CommandFingerprint, + pub all_envs: HashMap>, +} + +#[derive(Encode, Decode, Debug, Serialize, PartialEq, Eq, Diff)] +#[diff(attr(#[derive(Debug)]))] +pub struct CommandFingerprint { + pub cwd: Str, + pub command_line: Str, + pub envs_without_pass_through: HashMap, } impl Workspace { @@ -91,30 +137,33 @@ impl Workspace { Err(err) => { if err.kind() == std::io::ErrorKind::NotFound { continue; - } else { - return Err(err.into()); } + return Err(err.into()); } }))?; vite_task_jsons.push((vite_task_json, pkg)); } - let cache_path = dir.join("node_modules/.vite/task-cache.json"); + let cache_path = dir.join("node_modules/.vite/task-cache.db"); let task_cache = TaskCache::load_from_file(&cache_path)?; Ok(Self { vite_task_jsons, dir, task_cache, fs: CachedFileSystem::default() }) } + pub const fn cache(&self) -> &TaskCache { + &self.task_cache + } pub fn unload(self) -> anyhow::Result<()> { self.task_cache.save()?; Ok(()) } - pub fn to_task_graph( + pub fn resolve_tasks( &self, - mut task_names: Vec, + task_names: &[Str], + task_args: Arc<[Str]>, ) -> anyhow::Result> { - let mut tasks_by_full_name: HashMap = + let mut task_configs_by_full_name: HashMap = HashMap::new(); for (task_json, package_info) in &self.vite_task_jsons { for (task_name, task_config_json) in &task_json.tasks { @@ -123,7 +172,7 @@ impl Workspace { } else { format!("{}#{}", &package_info.name, task_name).as_str().into() }; - if tasks_by_full_name + if task_configs_by_full_name .insert(full_name.clone(), (task_config_json.clone(), package_info.clone())) .is_some() { @@ -131,30 +180,38 @@ impl Workspace { } } } - // let mut vite_task_json = self.vite_task_json.clone(); - // let capacity = vite_task_json.tasks.len(); + + let mut task_names: BTreeSet = task_names.iter().cloned().collect(); + let mut task_graph = StableDiGraph::::new(); let mut ids_by_task_name = HashMap::::new(); let mut edges = Vec::<(Str, Str)>::new(); - while let Some(task_name) = task_names.pop() { - let (task_config, package_info) = tasks_by_full_name + while let Some(task_name) = task_names.pop_first() { + let (task_config_with_deps, package_info) = task_configs_by_full_name .remove(&task_name) .with_context(|| format!("Task '{}' not found", &task_name))?; + let resolved_config = ResolvedTaskConfig { + config_dir: package_info.path.as_str().into(), + config: task_config_with_deps.config, + }; + + let resolved_command = resolved_config.resolve_command(&task_args)?; + let id = task_graph.add_node(ResolvedTask { name: task_name.clone(), - config_dir: package_info.path.as_str().into(), - envs: TaskEnvs::resolve(&task_config.config)?, - config: task_config.config, + args: task_args.clone(), + resolved_command, + resolved_config, }); if ids_by_task_name.insert(task_name.clone(), id).is_some() { anyhow::bail!("Duplicated task name '{}'", &task_name) } - for dep in task_config.depends_on { + for dep in task_config_with_deps.depends_on { edges.push((dep.clone(), task_name.clone())); - task_names.push(dep); + task_names.insert(dep); } } diff --git a/crates/vite_task/src/execute.rs b/crates/vite_task/src/execute.rs index 75688ea157..0ae34ab74b 100644 --- a/crates/vite_task/src/execute.rs +++ b/crates/vite_task/src/execute.rs @@ -16,6 +16,7 @@ use wax::Glob; use crate::{ collections::{HashMap, HashSet}, config::{ResolvedTask, TaskConfig}, + maybe_str::MaybeString, str::Str, }; @@ -25,10 +26,10 @@ pub enum OutputKind { StdErr, } -#[derive(Debug, Encode, Decode, Serialize, Deserialize)] +#[derive(Debug, Encode, Decode, Serialize)] pub struct StdOutput { pub kind: OutputKind, - pub content: Vec, + pub content: MaybeString, } /// Contains info that is available after executing the task @@ -62,7 +63,7 @@ fn collect_std_outputs( { last.content.extend_from_slice(content); } else { - outputs.push(StdOutput { kind, content: content.to_vec() }); + outputs.push(StdOutput { kind, content: content.to_vec().into() }); } } } @@ -70,7 +71,7 @@ fn collect_std_outputs( #[derive(Debug)] pub struct TaskEnvs { pub all_envs: HashMap>, - pub env_fingerprint: HashMap, + pub envs_without_pass_through: HashMap, } impl TaskEnvs { @@ -94,13 +95,7 @@ impl TaskEnvs { }) .collect(); - let env_path = - all_envs.entry("PATH".into()).or_insert_with(|| Arc::::from(OsStr::new(""))); - let paths = split_paths(env_path); - let node_modules_bin = Path::new(&task.cwd).join("node_modules/.bin"); - *env_path = join_paths(iter::once(node_modules_bin).chain(paths))?.into(); - - let mut env_fingerprint = HashMap::::new(); + let mut envs_without_pass_through = HashMap::::new(); for name in &task.envs { let Some(value) = all_envs.get(name) else { continue; @@ -112,13 +107,21 @@ impl TaskEnvs { value ); }; - env_fingerprint.insert(name.clone(), value.into()); + envs_without_pass_through.insert(name.clone(), value.into()); } - Ok(Self { all_envs, env_fingerprint }) + + let env_path = + all_envs.entry("PATH".into()).or_insert_with(|| Arc::::from(OsStr::new(""))); + let paths = split_paths(env_path); + let node_modules_bin = Path::new(&task.cwd).join("node_modules/.bin"); + *env_path = join_paths(iter::once(node_modules_bin).chain(paths))?.into(); + + Ok(Self { all_envs, envs_without_pass_through }) } } pub fn execute_task(task: &ResolvedTask, base_dir: &Path) -> anyhow::Result { + let command = &task.resolved_command; let mut child = if cfg!(windows) { let mut cmd = Command::new("cmd.exe"); // https://github.com/nodejs/node/blob/dbd24b165128affb7468ca42f69edaf7e0d85a9a/lib/child_process.js#L633 @@ -129,12 +132,12 @@ pub fn execute_task(task: &ResolvedTask, base_dir: &Path) -> anyhow::Result anyhow::Result anyhow::Result>> { // Task inferring to be implemented here - if task.config.inputs.is_empty() { + let inputs = &task.resolved_config.config.inputs; + if inputs.is_empty() { return Ok(HashSet::new()); } - let glob = format!("{{{}}}", itertools::Itertools::join(&mut task.config.inputs.iter(), ",")); // TODO: handle "," inside globs + let glob = format!("{{{}}}", itertools::Itertools::join(&mut inputs.iter(), ",")); // TODO: handle "," inside globs let glob = Glob::new(&glob)?; let mut paths: HashSet> = HashSet::new(); - for entry in glob.walk(base_dir.join(&task.config_dir)) { + for entry in glob.walk(base_dir.join(&task.resolved_config.config_dir)) { let entry = entry?; paths.insert(entry.into_path().into_os_string().into()); } diff --git a/crates/vite_task/src/fingerprint.rs b/crates/vite_task/src/fingerprint.rs index c72ac39c0f..d9f3df82f7 100644 --- a/crates/vite_task/src/fingerprint.rs +++ b/crates/vite_task/src/fingerprint.rs @@ -2,24 +2,27 @@ use std::{ffi::OsStr, fmt::Display, path::Path, sync::Arc}; use crate::{ collections::HashMap, - config::{ResolvedTask, TaskConfig, TaskConfigDiff}, - execute::{ExecutedTask, TaskEnvs}, + config::{ + CommandFingerprint, CommandFingerprintDiff, ResolvedTask, ResolvedTaskConfig, + ResolvedTaskConfigDiff, + }, + execute::ExecutedTask, fs::FileSystem, str::Str, }; use bincode::{Decode, Encode}; -use diff::{Diff as _, HashMapDiff}; +use diff::Diff as _; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use relative_path::RelativePath; use serde::{Deserialize, Serialize}; /// The fingerprint of a task. Determines if the task needs to be re-executed -#[derive(Encode, Decode, Debug, Serialize, Deserialize)] +#[derive(Encode, Decode, Debug, Serialize)] pub struct TaskFingerprint { - pub config: TaskConfig, + pub resolved_config: ResolvedTaskConfig, + pub command_fingerprint: CommandFingerprint, pub inputs: HashMap, - pub envs: HashMap, } #[derive(Encode, Decode, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] @@ -30,9 +33,9 @@ pub enum PathFingerprint { #[derive(Debug)] pub enum FingerprintMismatch { - ConfigChanged(TaskConfigDiff), + ConfigChanged(ResolvedTaskConfigDiff), InputContentChanged { path: Str }, - EnvChanged(HashMapDiff), + ResolvedCommandChanged(CommandFingerprintDiff), } impl Display for FingerprintMismatch { @@ -44,8 +47,8 @@ impl Display for FingerprintMismatch { Self::InputContentChanged { path } => { write!(f, "File content changed: {path:?}") } - Self::EnvChanged(env_diff) => { - write!(f, "Environment variables changed: {env_diff:?}") + Self::ResolvedCommandChanged(env_diff) => { + write!(f, "Resolved command changed: {env_diff:?}") } } } @@ -55,17 +58,19 @@ impl TaskFingerprint { /// Checks if the cached fingerprint is still valid. Returns why if not. pub fn validate( &self, - current_config: &TaskConfig, + resolved_task: &ResolvedTask, fs: &impl FileSystem, base_dir: &Path, ) -> anyhow::Result> { - let task_envs = TaskEnvs::resolve(current_config)?; - // TODO: use diff result instead of eq - Ok(if &self.config != current_config { - Some(FingerprintMismatch::ConfigChanged(self.config.diff(current_config))) - } else if self.envs != task_envs.env_fingerprint { - Some(FingerprintMismatch::EnvChanged(self.envs.diff(&task_envs.env_fingerprint))) + Ok(if self.resolved_config != resolved_task.resolved_config { + Some(FingerprintMismatch::ConfigChanged( + self.resolved_config.diff(&resolved_task.resolved_config), + )) + } else if self.command_fingerprint != resolved_task.resolved_command.fingerprint { + Some(FingerprintMismatch::ResolvedCommandChanged( + self.command_fingerprint.diff(&resolved_task.resolved_command.fingerprint), + )) } else { let input_mismatch = self.inputs.par_iter().find_map_any(|(input_relative_path, path_fingerprint)| { @@ -108,6 +113,10 @@ impl TaskFingerprint { })()) }) .collect::>>()?; - Ok(Self { config: task.config.clone(), inputs, envs: task.envs.env_fingerprint }) + Ok(Self { + resolved_config: task.resolved_config, + command_fingerprint: task.resolved_command.fingerprint, + inputs, + }) } } diff --git a/crates/vite_task/src/lib.rs b/crates/vite_task/src/lib.rs index 8ffd21e23d..fcc998f1a9 100644 --- a/crates/vite_task/src/lib.rs +++ b/crates/vite_task/src/lib.rs @@ -4,26 +4,70 @@ mod config; mod execute; mod fingerprint; mod fs; +mod maybe_str; mod schedule; mod str; +use std::iter; use std::path::PathBuf; +use std::sync::Arc; +use clap::Parser; +use itertools::Itertools; + +use crate::cache::CachedTask; +use crate::collections::HashMap; use crate::str::Str; use crate::{config::Workspace, schedule::ExecutionPlan}; -#[derive(Debug)] +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] pub struct Args { + /// A list of tasks to run. + #[clap(num_args = 0..)] pub tasks: Vec, + + /// Optional arguments for the tasks, captured after '--'. + #[clap(last = true)] + pub task_args: Vec, + + /// Display cache for debugging. + #[clap(short, long)] + pub debug: bool, } pub fn main(cwd: PathBuf, args: Args) -> anyhow::Result<()> { let mut workspace = Workspace::load(cwd)?; - let task_graph = workspace.to_task_graph(args.tasks)?; - let plan = ExecutionPlan::plan(task_graph)?; - plan.execute(&mut workspace)?; + let task_args = Arc::<[Str]>::from(args.task_args); + let task_graph = workspace.resolve_tasks(&args.tasks, task_args.clone())?; + if args.debug { + let cache = workspace.cache(); + let mut task_cache_map = HashMap::>::new(); + if args.tasks.is_empty() { + cache.list_cache(|cache_key, cached_task| { + let key = iter::once(cache_key.task_name.clone()) + .chain(cache_key.args.iter().cloned()) + .join(" "); + task_cache_map.insert(key, Some(cached_task)); + Ok(()) + })?; + } else { + for resolved_task in task_graph.node_weights() { + let key = iter::once(resolved_task.name.clone()) + .chain(task_args.iter().cloned()) + .join(" "); + let cached_task = cache.get_cache(resolved_task.name.clone(), task_args.clone())?; + task_cache_map.insert(key, cached_task); + } + } + let cache_debug_json = serde_json::to_string_pretty(&task_cache_map)?; + let _ = edit::edit(&cache_debug_json)?; + } else { + let plan = ExecutionPlan::plan(task_graph)?; + plan.execute(&mut workspace)?; - workspace.unload()?; + workspace.unload()?; + } Ok(()) } diff --git a/crates/vite_task/src/main.rs b/crates/vite_task/src/main.rs index 149a51e1cb..ea08beec8f 100644 --- a/crates/vite_task/src/main.rs +++ b/crates/vite_task/src/main.rs @@ -1,10 +1,9 @@ -use std::env::{args, current_dir}; +use std::env::current_dir; +use clap::Parser as _; use vite_task::Args; fn main() -> anyhow::Result<()> { - vite_task::main( - current_dir()?, - Args { tasks: args().skip(1).map(|arg| arg.as_str().into()).collect() }, - ) + let args = Args::parse(); + vite_task::main(current_dir()?, args) } diff --git a/crates/vite_task/src/maybe_str.rs b/crates/vite_task/src/maybe_str.rs new file mode 100644 index 0000000000..99677698b2 --- /dev/null +++ b/crates/vite_task/src/maybe_str.rs @@ -0,0 +1,48 @@ +use std::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +use bincode::{Decode, Encode}; +use bstr::BStr; +use serde::Serialize; + +/// Similar to `bstr::BString`, but also implements `bincode::{Encode`, Decode}, +/// and serializes losslessly to utf8 for outputing debug json + +#[derive(Encode, Decode)] +pub struct MaybeString(Vec); + +impl From> for MaybeString { + fn from(value: Vec) -> Self { + Self(value) + } +} + +impl Deref for MaybeString { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl DerefMut for MaybeString { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Debug for MaybeString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(BStr::new(&self.0), f) + } +} + +impl Serialize for MaybeString { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(&bstr::ByteSlice::escape_bytes(self.0.as_slice())) + } +} diff --git a/crates/vite_task/src/schedule.rs b/crates/vite_task/src/schedule.rs index dce1ae4d4f..d23dbb6434 100644 --- a/crates/vite_task/src/schedule.rs +++ b/crates/vite_task/src/schedule.rs @@ -30,7 +30,9 @@ impl ExecutionPlan { pub fn execute(self, workspace: &mut Workspace) -> anyhow::Result<()> { for step in self.steps { println!("------- {} -------", &step.name); - let command = step.config.command.clone(); + + let command_line = step.resolved_command.fingerprint.command_line.clone(); + let (cache_miss, execute_or_replay) = get_cached_or_execute( step, &mut workspace.task_cache, @@ -40,11 +42,11 @@ impl ExecutionPlan { match cache_miss { Some(CacheMiss::NotFound) => { println!("Cache Not Found, executing task"); - println!("> {command}"); + println!("> {command_line}"); } Some(CacheMiss::FingerprintMismatch(mismatch)) => { println!("{mismatch}, executing task"); - println!("> {command}"); + println!("> {command_line}"); } None => { println!("Cache hit, replaying previously executed task"); @@ -89,8 +91,9 @@ fn get_cached_or_execute<'a>( Box::new(move || { let executed_task = execute_task(&task, base_dir)?; let task_name = task.name.clone(); + let task_args = task.args.clone(); let cached_task = CachedTask::create(task, executed_task, fs, base_dir)?; - cache.update(task_name, cached_task)?; + cache.update(task_name, task_args, cached_task)?; anyhow::Ok(()) }), ),