From 792b97d8c71dba9e9b16fe0785b040f07faeda0c Mon Sep 17 00:00:00 2001 From: DmitryAstafyev Date: Fri, 24 Nov 2023 12:09:03 +0100 Subject: [PATCH 001/174] Draft of cli based build system --- application/apps/indexer/Cargo.lock | 3493 +++++++++++++++++++++++++++ cli/.gitignore | 6 + cli/Cargo.lock | 1128 +++++++++ cli/Cargo.toml | 20 + cli/src/fstools.rs | 70 + cli/src/location.rs | 41 + cli/src/main.rs | 125 + cli/src/modules/app.rs | 58 + cli/src/modules/binding.rs | 38 + cli/src/modules/cli.rs | 31 + cli/src/modules/client.rs | 31 + cli/src/modules/core.rs | 31 + cli/src/modules/mod.rs | 161 ++ cli/src/modules/shared.rs | 31 + cli/src/modules/wrapper.rs | 50 + cli/src/spawner.rs | 140 ++ cli/src/target.rs | 56 + cli/src/tools.rs | 17 + cli/src/tracker.rs | 223 ++ 19 files changed, 5750 insertions(+) create mode 100644 application/apps/indexer/Cargo.lock create mode 100644 cli/.gitignore create mode 100644 cli/Cargo.lock create mode 100644 cli/Cargo.toml create mode 100644 cli/src/fstools.rs create mode 100644 cli/src/location.rs create mode 100644 cli/src/main.rs create mode 100644 cli/src/modules/app.rs create mode 100644 cli/src/modules/binding.rs create mode 100644 cli/src/modules/cli.rs create mode 100644 cli/src/modules/client.rs create mode 100644 cli/src/modules/core.rs create mode 100644 cli/src/modules/mod.rs create mode 100644 cli/src/modules/shared.rs create mode 100644 cli/src/modules/wrapper.rs create mode 100644 cli/src/spawner.rs create mode 100644 cli/src/target.rs create mode 100644 cli/src/tools.rs create mode 100644 cli/src/tracker.rs diff --git a/application/apps/indexer/Cargo.lock b/application/apps/indexer/Cargo.lock new file mode 100644 index 000000000..15ea7d272 --- /dev/null +++ b/application/apps/indexer/Cargo.lock @@ -0,0 +1,3493 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "CoreFoundation-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e9889e6db118d49d88d84728d0e964d973a5680befb5f85f55141beea5c20b" +dependencies = [ + "libc", + "mach 0.1.2", +] + +[[package]] +name = "IOKit-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99696c398cbaf669d2368076bdb3d627fb0ce51a26899d7c61228c5c0af3bf4a" +dependencies = [ + "CoreFoundation-sys", + "libc", + "mach 0.1.2", +] + +[[package]] +name = "addon" +version = "0.1.0" +dependencies = [ + "chrono", + "dlt-core", + "env_logger", + "futures 0.3.29", + "indexer_base", + "log", + "parsers", + "processor", + "rand 0.8.5", + "serde", + "serde_json", + "sources", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + +[[package]] +name = "blake3" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", +] + +[[package]] +name = "bstr" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", + "slice-deque", +] + +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "git+https://github.com/DmitryAstafyev/buf_redux.git#595d13446d3d90eb4834a3cee67c0f79e28f01d8" +dependencies = [ + "memchr", + "safemem", + "slice-deque", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytecount" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "chrono-tz" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "circular" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags 1.3.2", + "clap_lex", + "indexmap", + "textwrap 0.16.0", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clipboard-win" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +dependencies = [ + "error-code", + "str-buf", + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.25", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", + "memoffset 0.9.0", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2 1.0.69", + "quote 1.0.33", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2 1.0.69", + "quote 1.0.33", + "rustc_version 0.4.0", + "syn 1.0.109", +] + +[[package]] +name = "destructure_traitobject" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c877555693c14d2f84191cfd3ad8582790fc52b5e2274b40b59cf5f5cea25c7" + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "dlt-core" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6fd69a328826e883613fbbcf468a33a53df1198c3e39bae7ad079c449431bfd" +dependencies = [ + "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "bytes 1.5.0", + "derive_more", + "lazy_static", + "log", + "memchr", + "nom", + "quick-xml 0.29.0", + "rustc-hash", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "env_logger" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "envvars" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce438a0fc581e838d1e1d1cd14abf08256dc2eb0f29b49b1c6df50436159d51a" +dependencies = [ + "blake3", + "fs_extra", + "home", + "lazy_static", + "log", + "serde", + "serde_json", + "thiserror", + "uuid", +] + +[[package]] +name = "errno" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "etherparse" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "827292ea592108849932ad8e30218f8b1f21c0dfd0696698a18b5d0aed62d990" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fd-lock" +version = "3.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ae6b3d9530211fb3b12a95374b8b0823be812f53d09e18c5675c0146b09642" +dependencies = [ + "cfg-if 1.0.0", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "grep-matcher" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3902ca28f26945fe35cad349d776f163981d777fee382ccd6ef451126f51b319" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-printer" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14551578f49da1f774b70da5bd1b8c20bbbead01620c426cb0a217536d95a6a" +dependencies = [ + "base64", + "bstr", + "grep-matcher", + "grep-searcher", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997598b41d53a37a2e3fc5300d5c11d825368c054420a9c65125b8fe1078463f" +dependencies = [ + "aho-corasick 0.7.20", + "bstr", + "grep-matcher", + "log", + "regex", + "regex-syntax 0.6.29", + "thread_local", +] + +[[package]] +name = "grep-searcher" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5601c4b9f480f0c9ebb40b1f6cbf447b8a50c5369223937a6c5214368c58779f" +dependencies = [ + "bstr", + "bytecount", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memmap2", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexer_base" +version = "0.1.0" +dependencies = [ + "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel", + "log", + "pretty_assertions", + "regex", + "rustc-hash", + "serde", + "serde_json", + "tempdir", + "test-generator", + "thiserror", + "tokio", +] + +[[package]] +name = "indexer_cli" +version = "0.35.2" +dependencies = [ + "addon", + "anyhow", + "async-stream", + "bytes 1.5.0", + "chrono", + "crossbeam-channel", + "dirs", + "dlt-core", + "env_logger", + "futures 0.3.29", + "indexer_base", + "indicatif", + "lazy_static", + "log", + "parsers", + "processor", + "rustyline", + "serde", + "serde_json", + "session", + "someip-messages", + "someip-payload", + "sources", + "structopt", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.3", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.3", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.0.2", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "libudev" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b324152da65df7bb95acfcaab55e3097ceaab02fb19b228a9eb74d55f135e0" +dependencies = [ + "libc", + "libudev-sys", +] + +[[package]] +name = "libudev-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8469b4a23b962c1396b9b451dda50ef5b283e8dd309d69033475fa9b334324" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "serde", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" + +[[package]] +name = "log4rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36ca1786d9e79b8193a68d480a0907b612f109537115c6ff655a3a1967533fd" +dependencies = [ + "anyhow", + "arc-swap", + "chrono", + "derivative", + "fnv", + "humantime", + "libc", + "log", + "log-mdc", + "parking_lot 0.12.1", + "serde", + "serde-value", + "serde_json", + "serde_yaml", + "thiserror", + "thread-id", + "typemap-ors", + "winapi 0.3.9", +] + +[[package]] +name = "mach" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd13ee2dd61cc82833ba05ade5a30bb3d63f7ced605ef827063c63078302de9" +dependencies = [ + "libc", +] + +[[package]] +name = "mach" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1" +dependencies = [ + "libc", +] + +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "merging" +version = "0.1.0" +dependencies = [ + "crossbeam-channel", + "indexer_base", + "log", + "log4rs", + "pretty_assertions", + "processor", + "regex", + "serde", + "serde_json", + "tempdir", + "test-generator", + "thiserror", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio 0.6.23", + "miow 0.3.7", + "winapi 0.3.9", +] + +[[package]] +name = "mio-serial" +version = "5.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20a4c60ca5c9c0e114b3bd66ff4aa5f9b2b175442be51ca6c4365d687a97a2ac" +dependencies = [ + "log", + "mio 0.8.9", + "nix", + "serialport", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec 1.11.2", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "libc", + "memoffset 0.7.1", + "pin-utils", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version 0.2.3", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api 0.4.11", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version 0.2.3", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.4.1", + "smallvec 1.11.2", + "windows-targets 0.48.5", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + +[[package]] +name = "parsers" +version = "0.1.0" +dependencies = [ + "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "chrono", + "chrono-tz", + "crossbeam-channel", + "dlt-core", + "env_logger", + "humantime", + "lazy_static", + "log", + "memchr", + "rand 0.8.5", + "serde", + "someip-messages", + "someip-payload", + "stringreader", + "thiserror", + "tokio-util", +] + +[[package]] +name = "pcap-parser" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79dfb40aef938ed2082c9ae9443f4eba21b79c1a9d6cfa071f5c2bd8d829491" +dependencies = [ + "circular", + "nom", + "rusticata-macros", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "processor" +version = "0.2.0" +dependencies = [ + "bincode", + "buf_redux 0.8.4 (git+https://github.com/DmitryAstafyev/buf_redux.git)", + "bytecount", + "chrono", + "criterion", + "crossbeam-channel", + "dirs", + "encoding_rs_io", + "env_logger", + "futures 0.3.29", + "grep-matcher", + "grep-printer", + "grep-regex", + "grep-searcher", + "indexer_base", + "itertools", + "lazy_static", + "log", + "nom", + "parsers", + "pretty_assertions", + "proptest", + "proptest-derive", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "sources", + "tempfile", + "termcolor", + "test-generator", + "thiserror", + "tokio", + "tokio-util", + "uuid", +] + +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.0.2", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b46295382dc76166cb7cf2bb4a97952464e4b7ed5a43e6cd34e1fec3349ddc" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2 1.0.69", +] + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick 1.1.2", + "memchr", + "regex-automata", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick 1.1.2", + "memchr", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.20", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "rustyline" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "clipboard-win", + "dirs-next", + "fd-lock", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "scopeguard", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi 0.3.9", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "serialport" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c32634e2bd4311420caa504404a55fad2131292c485c97014cbed89a5899885f" +dependencies = [ + "CoreFoundation-sys", + "IOKit-sys", + "bitflags 2.0.2", + "cfg-if 1.0.0", + "libudev", + "mach2", + "nix", + "regex", + "scopeguard", + "winapi 0.3.9", +] + +[[package]] +name = "session" +version = "0.1.0" +dependencies = [ + "blake3", + "crossbeam-channel", + "dirs", + "dlt-core", + "envvars", + "futures 0.3.29", + "indexer_base", + "lazy_static", + "log", + "merging", + "mime_guess", + "parsers", + "processor", + "rustc-hash", + "serde", + "serde_json", + "serialport", + "sources", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "uuid", + "walkdir", +] + +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slice-deque" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7" +dependencies = [ + "libc", + "mach 0.2.3", + "winapi 0.3.9", +] + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "someip-messages" +version = "0.3.1" +source = "git+https://github.com/esrlabs/someip#de0b37b2cd023a36410a89d4b65770679310b791" +dependencies = [ + "byteorder", + "derive_builder", + "thiserror", +] + +[[package]] +name = "someip-payload" +version = "0.1.1" +source = "git+https://github.com/esrlabs/someip-payload#9a58a561a3d284e37a25ea2dc52188c70694682d" +dependencies = [ + "byteorder", + "log", + "quick-xml 0.22.0", + "regex", + "thiserror", + "ux", + "voca_rs", +] + +[[package]] +name = "sources" +version = "0.1.0" +dependencies = [ + "async-stream", + "async-trait", + "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 1.5.0", + "crossbeam-channel", + "env_logger", + "etherparse", + "futures 0.3.29", + "indexer_base", + "lazy_static", + "log", + "parsers", + "pcap-parser", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "shellexpand", + "thiserror", + "tokio", + "tokio-process", + "tokio-serial", + "tokio-stream", + "tokio-util", + "uuid", +] + +[[package]] +name = "stfu8" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1310970b29733b601839578f8ba24991a97057dbedc4ac0decea835474054ee7" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "str-buf" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" + +[[package]] +name = "stringreader" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913e7b03d63752f6cdd2df77da36749d82669904798fe8944b9ec3d23f159905" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "unicode-ident", +] + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "test-generator" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b23be2add79223226e1cb6446cb3e37506a5927089870687a0f1149bb7a073a" +dependencies = [ + "glob", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "thread-id" +version = "4.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0ec81c46e9eb50deaa257be2f148adf052d1fb7701cfd55ccfab2525280b70b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +dependencies = [ + "backtrace", + "bytes 1.5.0", + "libc", + "mio 0.8.9", + "num_cpus", + "parking_lot 0.12.1", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + +[[package]] +name = "tokio-process" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382d90f43fa31caebe5d3bc6cfd854963394fff3b8cb59d5146607aaae7e7e43" +dependencies = [ + "crossbeam-queue", + "futures 0.1.31", + "lazy_static", + "libc", + "log", + "mio 0.6.23", + "mio-named-pipes", + "tokio-io", + "tokio-reactor", + "tokio-signal", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-serial" +version = "5.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa6e2e4cf0520a99c5f87d5abb24172b5bd220de57c3181baaaa5440540c64aa" +dependencies = [ + "cfg-if 1.0.0", + "futures 0.3.29", + "log", + "mio-serial", + "tokio", +] + +[[package]] +name = "tokio-signal" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" +dependencies = [ + "futures 0.1.31", + "libc", + "mio 0.6.23", + "mio-uds", + "signal-hook-registry", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes 1.5.0", + "futures-core", + "futures-io", + "futures-sink", + "futures-util", + "hashbrown 0.14.2", + "pin-project-lite", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typemap-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68c24b707f02dd18f1e4ccceb9d49f2058c2fb86384ef9972592904d7a28867" +dependencies = [ + "unsafe-any-ors", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unsafe-any-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a303d30665362d9680d7d91d78b23f5f899504d4f08b3c4cf08d055d87c0ad" +dependencies = [ + "destructure_traitobject", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "ux" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb3ff47e36907a6267572c1e398ff32ef78ac5131de8aa272e53846592c207e" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "voca_rs" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e44efbf25e32768d5ecd22244feacc3d3b3eca72d318f5ef0a4764c2c158e18" +dependencies = [ + "regex", + "stfu8", + "unicode-segmentation", +] + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote 1.0.33", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] diff --git a/cli/.gitignore b/cli/.gitignore new file mode 100644 index 000000000..e9a83dd15 --- /dev/null +++ b/cli/.gitignore @@ -0,0 +1,6 @@ +/target +**/*.rs.bk +*.out +lineMetadata.json +vim-markdown-preview.html +.DS_Store diff --git a/cli/Cargo.lock b/cli/Cargo.lock new file mode 100644 index 000000000..25b3c917f --- /dev/null +++ b/cli/Cargo.lock @@ -0,0 +1,1128 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" +dependencies = [ + "concurrent-queue", + "event-listener 3.1.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" +dependencies = [ + "async-lock 3.1.1", + "async-task", + "concurrent-queue", + "fastrand 2.0.1", + "futures-lite 2.0.1", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" +dependencies = [ + "async-channel 2.1.0", + "async-executor", + "async-io 2.2.0", + "async-lock 3.1.1", + "blocking", + "futures-lite 2.0.1", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9d5715c2d329bf1b4da8d60455b99b187f27ba726df2883799af9af60997" +dependencies = [ + "async-lock 3.1.1", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.0.1", + "parking", + "polling 3.3.0", + "rustix 0.38.25", + "slab", + "tracing", + "waker-fn", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" +dependencies = [ + "event-listener 3.1.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" +dependencies = [ + "async-io 1.13.0", + "async-lock 2.8.0", + "async-signal", + "blocking", + "cfg-if", + "event-listener 3.1.0", + "futures-lite 1.13.0", + "rustix 0.38.25", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-signal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +dependencies = [ + "async-io 2.2.0", + "async-lock 2.8.0", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 0.38.25", + "signal-hook-registry", + "slab", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel 2.1.0", + "async-lock 3.1.1", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.0.1", + "piper", + "tracing", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "cli" +version = "0.1.0" +dependencies = [ + "async-channel 1.9.0", + "async-io 1.13.0", + "async-process", + "async-std", + "async-trait", + "clap", + "console", + "fs_extra", + "futures", + "futures-lite 1.13.0", + "indicatif", + "lazy_static", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "concurrent-queue" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "errno" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" +dependencies = [ + "event-listener 3.1.0", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53b6af1f60f36f8c2ac2aad5459d75a5a9b4be1e8cdd40264f315d78193e531" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.25", + "tracing", + "windows-sys 0.48.0", +] + +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys 0.4.11", + "windows-sys 0.48.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "value-bag" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" + +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 000000000..eb3e7c3b3 --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-channel = "1.9.0" +async-io = "1.13.0" +async-process = "1.7.0" +async-std = "1.12.0" +async-trait = "0.1.73" +clap = { version = "4.4.4", features = ["derive"] } +console = "0.15.7" +fs_extra = "1.3.0" +futures = "0.3.28" +futures-lite = "1.13.0" +indicatif = "0.17.7" +lazy_static = "1.4.0" diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs new file mode 100644 index 000000000..45940a723 --- /dev/null +++ b/cli/src/fstools.rs @@ -0,0 +1,70 @@ +extern crate fs_extra; +use crate::TRACKER; +use async_std::task::spawn; +use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; +use std::sync::mpsc; +use std::{fs, io::Error, path::PathBuf}; + +pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { + let sequence = TRACKER.start("copy file", None).await?; + fs::copy(&src, &dest)?; + TRACKER + .success( + sequence, + &format!( + "copied: {} to {}", + src.to_string_lossy(), + dest.to_string_lossy() + ), + ) + .await; + Ok(()) +} + +pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { + let sequence = TRACKER.start("copy folder", None).await?; + let options = CopyOptions::new(); + let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); + let msg = format!( + "copied: {} to {}", + src.to_string_lossy(), + dest.to_string_lossy() + ); + spawn(async move { + if let Err(e) = copy_with_progress(src, dest, &options, |info| { + if tx.send(info).is_err() { + eprintln!("Fail to send copying progress"); + } + TransitProcessResult::ContinueOrAbort + }) { + panic!("Fail to copy: {e}") + } + }) + .await; + while let Ok(info) = rx.recv() { + TRACKER + .msg( + sequence, + &format!( + "copied: {} bytes; current: {}", + info.copied_bytes, info.file_name + ), + ) + .await; + TRACKER.progress(sequence, None).await; + } + TRACKER.success(sequence, &msg).await; + Ok(()) +} + +pub async fn rm_folder(path: PathBuf) -> Result<(), Error> { + if !path.exists() { + return Ok(()); + } + let sequence = TRACKER.start("remove folder", None).await?; + fs::remove_dir_all(&path)?; + TRACKER + .success(sequence, &format!("removed: {}", path.to_string_lossy(),)) + .await; + Ok(()) +} diff --git a/cli/src/location.rs b/cli/src/location.rs new file mode 100644 index 000000000..ba30d84b4 --- /dev/null +++ b/cli/src/location.rs @@ -0,0 +1,41 @@ +use crate::LOCATION; +use std::ffi::OsStr; +use std::{ + env::current_dir, + io::{Error, ErrorKind}, + path::PathBuf, +}; + +#[derive(Clone, Debug)] +pub struct Location { + pub root: PathBuf, +} + +impl Location { + pub fn new() -> Result { + let mut root = current_dir()?; + let mut len = root.iter().collect::>().len(); + loop { + if len == 0 { + return Err(Error::new( + ErrorKind::NotFound, + "Fail to find ICSMW location", + )); + } + // TODO: better compare folders stucts or some file, like some git config file + if root.ends_with("chipmunk") || root.ends_with("logviewer") { + break; + } + if len > 0 { + len = len.saturating_sub(1); + } + root.pop(); + } + Ok(Self { root }) + } +} + +pub fn to_relative_path(path: PathBuf) -> String { + let path_str = path.to_string_lossy().to_string(); + path_str.replace(&LOCATION.root.to_string_lossy().to_string(), "") +} diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 000000000..d252f35c3 --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,125 @@ +mod fstools; +mod location; +mod modules; +mod spawner; +mod target; +mod tools; +mod tracker; + +use clap::{Parser, Subcommand}; +use futures::future::join_all; +use location::Location; +use modules::Manager; +use std::io::{Error, ErrorKind}; +use target::Target; +use tools::RemoveDuplicates; +use tracker::Tracker; + +#[macro_use] +extern crate lazy_static; + +lazy_static! { + static ref LOCATION: Location = Location::new().expect("Fail to setup location of ICSMW"); + static ref TRACKER: Tracker = Tracker::new(); +} + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// Build release version + #[arg(short, long, action = clap::ArgAction::Count)] + release: u8, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand, Debug, Clone)] +enum Command { + /// runs linting & clippy + Lint { + /// target to build, by default whole application will be built + #[arg(short, long, num_args(0..))] + target: Option>, + }, + /// build + Build { + /// target to build, by default whole application will be built + #[arg(short, long, num_args(0..))] + target: Option>, + }, + /// Clean + Clean { + /// target to build, by default whole application will be built + #[arg(short, long, num_args(0..))] + target: Option>, + }, +} + +fn main() -> Result<(), Error> { + async_io::block_on(async { + let cli = Cli::parse(); + let production = cli.release > 0; + if let Some(ref command) = cli.command { + let targets: Vec> = if let Some(mut list) = + match command.clone() { + Command::Lint { target } => target, + Command::Build { target } => target, + Command::Clean { target } => target, + } { + list.remove_duplicates(); + list.iter().map(|target| target.get()).collect() + } else { + Target::all() + }; + let results = match command { + Command::Lint { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.check()) + .collect::>(), + ) + .await + } + Command::Build { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.build(production)) + .collect::>(), + ) + .await + } + Command::Clean { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.reset()) + .collect::>(), + ) + .await + } + }; + TRACKER.shutdown().await?; + let mut success: bool = true; + results.iter().for_each(|res| match res { + Ok(status) => { + if !status.status.success() { + eprintln!("Failed with errors"); + println!("{}:\n{}", status.job, status.stderr.join("")); + success = false; + } + } + Err(err) => { + eprintln!("Builder error: {err}"); + success = false; + } + }); + if !success { + return Err(Error::new(ErrorKind::Other, "Some task were failed")); + } + } + Ok(()) + }) +} diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs new file mode 100644 index 000000000..219546f7d --- /dev/null +++ b/cli/src/modules/app.rs @@ -0,0 +1,58 @@ +use super::{Kind, Manager}; +use crate::{fstools, spawner::SpawnResult, Target, LOCATION}; +use async_trait::async_trait; +use std::{ + fs, + io::{Error, ErrorKind}, + path::PathBuf, +}; + +const PATH: &str = "application/holder"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::App + } + fn kind(&self) -> Kind { + Kind::Ts + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![Target::Shared, Target::Wrapper, Target::Client] + } + fn install_cmd(&self, _prod: bool) -> Option { + // For app we don't need --production + Some(String::from("yarn install")) + } + async fn after(&self) -> Result, Error> { + let src = Target::Client.get().cwd().join("dist/client"); + let dest = self.cwd().join("dist"); + if !src.exists() { + return Err(Error::new( + ErrorKind::NotFound, + format!("Not found: {}", src.to_string_lossy()), + )); + } + if !dest.exists() { + fs::create_dir(&dest)?; + } + let prev = dest.join("client"); + if prev.exists() { + fstools::rm_folder(prev).await?; + } + fstools::cp_folder(src, dest).await?; + Ok(None) + } +} diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs new file mode 100644 index 000000000..7eec01a2e --- /dev/null +++ b/cli/src/modules/binding.rs @@ -0,0 +1,38 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "application/apps/rustcore/rs-bindings"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Binding + } + fn kind(&self) -> Kind { + Kind::Rs + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![] + } + fn build_cmd(&self, prod: bool) -> Option { + Some(format!( + "/{}/node_modules/.bin/electron-build-env nj-cli build{}", + Target::Wrapper.get().cwd().to_string_lossy(), + if prod { " --release" } else { "" } + )) + } +} diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs new file mode 100644 index 000000000..f0e16fbb1 --- /dev/null +++ b/cli/src/modules/cli.rs @@ -0,0 +1,31 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "cli"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Cli + } + fn kind(&self) -> Kind { + Kind::Rs + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![] + } +} diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs new file mode 100644 index 000000000..8ae990429 --- /dev/null +++ b/cli/src/modules/client.rs @@ -0,0 +1,31 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "application/client"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Client + } + fn kind(&self) -> Kind { + Kind::Ts + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![Target::Shared] + } +} diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs new file mode 100644 index 000000000..83b4ce6f2 --- /dev/null +++ b/cli/src/modules/core.rs @@ -0,0 +1,31 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "application/apps/indexer"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Core + } + fn kind(&self) -> Kind { + Kind::Rs + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![] + } +} diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs new file mode 100644 index 000000000..ce20afba1 --- /dev/null +++ b/cli/src/modules/mod.rs @@ -0,0 +1,161 @@ +pub mod app; +pub mod binding; +pub mod cli; +pub mod client; +pub mod core; +pub mod shared; +pub mod wrapper; + +use crate::{ + fstools, + spawner::{spawn, SpawnResult}, + Target, LOCATION, +}; +use async_trait::async_trait; +use std::{io::Error, path::PathBuf}; + +#[derive(Debug, Clone)] +pub enum Kind { + Ts, + Rs, +} + +impl Kind { + pub fn build_cmd(&self, prod: bool) -> String { + match self { + Kind::Ts => format!("yarn run {}", if prod { "prod" } else { "build" }), + Kind::Rs => format!( + "cargo build --color always{}", + if prod { " --release" } else { "" } + ), + } + } + pub fn install_cmd(&self, prod: bool) -> Option { + match self { + Kind::Ts => Some(format!( + "yarn install{}", + if prod { " --production" } else { "" } + )), + Kind::Rs => None, + } + } +} + +#[async_trait] +pub trait Manager { + fn kind(&self) -> Kind; + fn owner(&self) -> Target; + fn cwd(&self) -> PathBuf; + fn deps(&self) -> Vec; + fn build_cmd(&self, _prod: bool) -> Option { + None + } + fn install_cmd(&self, _prod: bool) -> Option { + None + } + async fn reset(&self) -> Result { + self.clean().await?; + fstools::rm_folder(self.cwd().join("dist")).await?; + Ok(SpawnResult::empty()) + } + async fn clean(&self) -> Result<(), Error> { + match self.kind() { + Kind::Ts => { + fstools::rm_folder(self.cwd().join("node_modules")).await?; + } + Kind::Rs => { + fstools::rm_folder(self.cwd().join("target")).await?; + } + } + Ok(()) + } + async fn install(&self, prod: bool) -> Result { + let cmd = if self.install_cmd(prod).is_some() { + self.install_cmd(prod) + } else { + self.kind().install_cmd(prod) + }; + if let Some(cmd) = cmd { + spawn(&cmd, Some(self.cwd()), Some(&cmd)).await + } else { + Ok(SpawnResult::empty()) + } + } + async fn install_if_need(&self, prod: bool) -> Result { + match self.kind() { + Kind::Ts => { + if self.cwd().join("node_modules").exists() { + Ok(SpawnResult::empty()) + } else { + self.install(prod).await + } + } + Kind::Rs => Ok(SpawnResult::empty()), + } + } + async fn after(&self) -> Result, Error> { + Ok(None) + } + async fn build(&self, prod: bool) -> Result { + self.install(false).await?; + let deps: Vec> = + self.deps().iter().map(|target| target.get()).collect(); + for module in deps { + let status = module.build(prod).await?; + if !status.status.success() { + return Ok(status); + } + } + let path = LOCATION.root.clone().join(self.cwd()); + let cmd = if let Some(cmd) = self.build_cmd(prod) { + cmd + } else { + self.kind().build_cmd(prod) + }; + match spawn(&cmd, Some(path), Some(&cmd)).await { + Ok(status) => { + if !status.status.success() { + Ok(status) + } else { + let res = self.after().await?; + if matches!(self.kind(), Kind::Ts) && prod { + self.clean().await?; + self.install(prod).await?; + } + if let Some(res) = res { + Ok(res) + } else { + Ok(SpawnResult::empty()) + } + } + } + Err(err) => Err(err), + } + } + async fn check(&self) -> Result { + match self.kind() { + Kind::Ts => { + self.install(false).await?; + self.lint().await + } + Kind::Rs => self.clippy().await, + } + } + async fn lint(&self) -> Result { + let path = LOCATION.root.clone().join(self.cwd()); + let status = spawn("yarn run lint", Some(path.clone()), Some("linting")).await?; + if !status.status.success() { + return Ok(status); + } + spawn("yarn run build", Some(path), Some("TS compilation")).await + } + async fn clippy(&self) -> Result { + let path = LOCATION.root.clone().join(self.cwd()); + spawn( + "cargo clippy --color always --all --all-features -- -D warnings", + Some(path), + Some("clippy"), + ) + .await + } +} diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs new file mode 100644 index 000000000..686c4f1d0 --- /dev/null +++ b/cli/src/modules/shared.rs @@ -0,0 +1,31 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "application/platform"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Shared + } + fn kind(&self) -> Kind { + Kind::Ts + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![] + } +} diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs new file mode 100644 index 000000000..caca879f2 --- /dev/null +++ b/cli/src/modules/wrapper.rs @@ -0,0 +1,50 @@ +use super::{Kind, Manager}; +use crate::{fstools, spawner::SpawnResult, Target, LOCATION}; +use async_trait::async_trait; +use std::{ + fs, + io::{Error, ErrorKind}, + path::PathBuf, +}; + +const PATH: &str = "application/apps/rustcore/ts-bindings"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Wrapper + } + fn kind(&self) -> Kind { + Kind::Ts + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + fn deps(&self) -> Vec { + vec![Target::Binding, Target::Shared] + } + async fn after(&self) -> Result, Error> { + let src = Target::Binding.get().cwd().join("dist/index.node"); + let dest = self.cwd().join("dist/native"); + if !src.exists() { + return Err(Error::new( + ErrorKind::NotFound, + format!("Not found: {}", src.to_string_lossy()), + )); + } + if !dest.exists() { + fs::create_dir(&dest)?; + } + fstools::cp_file(src, dest.join("index.node")).await?; + Ok(None) + } +} diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs new file mode 100644 index 000000000..25f4d0150 --- /dev/null +++ b/cli/src/spawner.rs @@ -0,0 +1,140 @@ +// cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); +use crate::{ + location::to_relative_path, + {LOCATION, TRACKER}, +}; +use async_process::{Command, ExitStatus, Stdio}; +use async_std::{io::BufReader, prelude::*}; +use futures_lite::{future, FutureExt}; +use std::{env::vars, io, path::PathBuf}; + +#[derive(Clone, Debug)] +pub struct SpawnResult { + pub stdout: Vec, + pub stderr: Vec, + pub status: ExitStatus, + pub job: String, +} + +impl SpawnResult { + pub fn empty() -> Self { + SpawnResult { + stdout: vec![], + stderr: vec![], + status: ExitStatus::default(), + job: String::new(), + } + } +} + +pub async fn spawn( + command: &str, + cwd: Option, + caption: Option<&str>, +) -> Result { + let cwd = if let Some(cwd) = cwd { + cwd + } else { + LOCATION.root.clone() + }; + let mut parts = command.split(' ').collect::>(); + let cmd = parts.remove(0); + #[allow(clippy::useless_vec)] + let mut child = Command::new(cmd) + .current_dir(&cwd) + .args(parts) + .envs( + vec![ + vars().collect::>(), + vec![(String::from("TERM"), String::from("xterm-256color"))], + ] + .concat(), + ) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let job_title = if let Some(caption) = caption { + caption + } else { + cmd + }; + let sequence = TRACKER + .start(&format!("{}: {}", to_relative_path(cwd), job_title), None) + .await?; + let mut stdout_lines: Vec = vec![]; + let drain_stdout = { + let storage = &mut stdout_lines; + let stdout = child.stdout.take().unwrap(); + async move { + let mut buf = BufReader::new(stdout); + loop { + let mut line = String::new(); + let read_lines = buf.read_line(&mut line).await?; + if read_lines == 0 { + break; + } else { + TRACKER.msg(sequence, &line).await; + TRACKER.progress(sequence, None).await; + storage.push(line); + } + } + future::pending::<()>().await; + Ok::, io::Error>(None) + } + }; + + let mut stderr_lines: Vec = vec![]; + let drain_stderr = { + let storage = &mut stderr_lines; + let stderr = child.stderr.take().unwrap(); + async move { + let mut buf = BufReader::new(stderr); + loop { + let mut line = String::new(); + let read_lines = buf.read_line(&mut line).await?; + if read_lines == 0 { + break; + } else { + TRACKER.progress(sequence, None).await; + if !line.trim().is_empty() { + storage.push(line); + } + } + } + future::pending::<()>().await; + Ok::, io::Error>(None) + } + }; + let status = match drain_stdout + .or(drain_stderr) + .or(async move { Ok(Some(child.status().await?)) }) + .await + { + Ok(status) => status, + Err(err) => { + TRACKER.fail(sequence, &err.to_string()).await; + return Err(err); + } + }; + if let Some(status) = status { + if status.success() { + TRACKER.success(sequence, "").await; + } else { + TRACKER.fail(sequence, "finished with errors").await; + } + Ok(SpawnResult { + stdout: stdout_lines, + stderr: stderr_lines, + status, + job: job_title.to_string(), + }) + } else { + TRACKER + .fail(sequence, "Fail to get exist status of spawned command") + .await; + Err(io::Error::new( + io::ErrorKind::Other, + "Fail to get exist status of spawned command", + )) + } +} diff --git a/cli/src/target.rs b/cli/src/target.rs new file mode 100644 index 000000000..c8108ba04 --- /dev/null +++ b/cli/src/target.rs @@ -0,0 +1,56 @@ +use crate::{modules, modules::Manager}; +use clap::ValueEnum; + +#[derive(ValueEnum, Clone, Debug, Hash, PartialEq, Eq)] +pub enum Target { + Core, + Binding, + Wrapper, + Client, + Shared, + App, + Cli, +} + +impl std::fmt::Display for Target { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Target::Core => "Core", + Target::Wrapper => "Wrapper", + Target::Binding => "Binding", + Target::Cli => "Cli", + Target::Client => "Client", + Target::Shared => "Shared", + Target::App => "App", + } + ) + } +} + +impl Target { + pub fn all() -> Vec> { + vec![ + Box::new(modules::binding::Module::new()), + Box::new(modules::cli::Module::new()), + Box::new(modules::app::Module::new()), + Box::new(modules::core::Module::new()), + Box::new(modules::wrapper::Module::new()), + Box::new(modules::shared::Module::new()), + Box::new(modules::client::Module::new()), + ] + } + pub fn get(&self) -> Box { + match self { + Target::Binding => Box::new(modules::binding::Module::new()), + Target::Cli => Box::new(modules::cli::Module::new()), + Target::Client => Box::new(modules::client::Module::new()), + Target::Core => Box::new(modules::core::Module::new()), + Target::Wrapper => Box::new(modules::wrapper::Module::new()), + Target::Shared => Box::new(modules::shared::Module::new()), + Target::App => Box::new(modules::app::Module::new()), + } + } +} diff --git a/cli/src/tools.rs b/cli/src/tools.rs new file mode 100644 index 000000000..049ff219c --- /dev/null +++ b/cli/src/tools.rs @@ -0,0 +1,17 @@ +use crate::target::Target; +use std::collections::HashSet; + +pub trait RemoveDuplicates { + fn remove_duplicates(&mut self); +} + +impl RemoveDuplicates for Vec { + fn remove_duplicates(&mut self) { + let mut seen = HashSet::new(); + self.retain(|c| { + let is_first = !seen.contains(c); + seen.insert(c.clone()); + is_first + }); + } +} diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs new file mode 100644 index 000000000..8615679b9 --- /dev/null +++ b/cli/src/tracker.rs @@ -0,0 +1,223 @@ +use async_channel::{bounded, unbounded, Receiver, Sender}; +use async_std::task; +use console::style; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use std::{ + collections::HashMap, + io::{Error, ErrorKind}, + time::Instant, +}; + +#[derive(Clone, Debug)] +pub enum OperationResult { + Success, + Failed, +} + +impl std::fmt::Display for OperationResult { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + match self { + OperationResult::Success => style("done").bold().green(), + OperationResult::Failed => style("fail").bold().red(), + } + ) + } +} + +#[derive(Clone, Debug)] +pub enum Tick { + Started(String, Option, Sender), + Progress(usize, Option), + Message(usize, String), + Finished(usize, OperationResult, String), + #[allow(dead_code)] + Print(String), + Shutdown(Sender<()>), +} + +#[derive(Clone, Debug)] +pub struct Tracker { + tx: Sender, +} + +fn order_offset(num: usize, total: usize) -> String { + " ".repeat(format!("[{total}/{total}]").len() - format!("[{num}/{total}]").len()) + .to_string() +} + +impl Tracker { + pub fn new() -> Self { + let (tx, rx): (Sender, Receiver) = unbounded(); + task::spawn(Tracker::run(rx)); + Self { tx } + } + + pub async fn run(rx: Receiver) -> Result<(), Error> { + let spinner_style = ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}") + .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))? + .tick_chars("▂▃▅▆▇▆▅▃▂ "); + async move { + let mut sequence: usize = 0; + let max = u64::MAX; + let mut bars: HashMap)> = + HashMap::new(); + let mp = MultiProgress::new(); + while let Ok(tick) = rx.recv().await { + match tick { + Tick::Started(job, len, tx_response) => { + sequence += 1; + let bar = mp.add(ProgressBar::new(if let Some(len) = len { + len + } else { + max + })); + bar.set_style(spinner_style.clone()); + bars.insert(sequence, (bar, Instant::now(), job, None)); + bars.iter_mut().for_each(|(k, (bar, _, job, result))| { + if let Some(result) = result { + bar.set_prefix(format!( + "[{k}/{sequence}]{}[{result}][{job}]", + order_offset(*k, sequence) + )); + } else { + bar.set_prefix(format!( + "[{k}/{sequence}]{}[....][{job}]", + order_offset(*k, sequence) + )); + } + }); + if let Err(e) = tx_response.send(sequence).await { + let _ = mp.println(format!("Fail to send response: {e}")); + } + } + Tick::Message(sequence, log) => { + if let Some((bar, _, _, _)) = bars.get(&sequence) { + bar.set_message(log); + } + } + Tick::Progress(sequence, pos) => { + if let Some((bar, _, _, _)) = bars.get(&sequence) { + if let Some(pos) = pos { + bar.set_position(pos); + } else { + bar.inc(1); + } + } + } + Tick::Finished(seq, result, msg) => { + if let Some((bar, instant, job, res)) = bars.get_mut(&seq) { + bar.set_prefix(format!( + "[{seq}/{sequence}]{}[{result}][{job}]", + order_offset(seq, sequence) + )); + bar.finish_with_message(format!( + "Done in {}s. {msg}", + instant.elapsed().as_secs() + )); + res.replace(result); + } + } + Tick::Print(msg) => { + let _ = mp.println(msg); + } + Tick::Shutdown(tx_response) => { + bars.iter_mut().for_each(|(_, (bar, instant, _, _))| { + if !bar.is_finished() { + bar.finish_with_message(format!( + "Done in {}s.", + instant.elapsed().as_secs() + )); + } + }); + bars.clear(); + // let _ = mp.clear(); + if let Err(e) = tx_response.send(()).await { + let _ = mp.println(format!("Fail to send response: {e}")); + } + break; + } + } + } + } + .await; + Ok(()) + } + + pub async fn start(&self, job: &str, max: Option) -> Result { + let (tx_response, rx_response) = bounded(1); + self.tx + .send(Tick::Started(job.to_string(), max, tx_response)) + .await + .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; + rx_response + .recv() + .await + .map_err(|e| Error::new(ErrorKind::NotConnected, e.to_string())) + } + + pub async fn progress(&self, sequence: usize, pos: Option) { + if let Err(e) = self.tx.send(Tick::Progress(sequence, pos)).await { + eprintln!("Fail to communicate with tracker: {e}"); + } + } + + pub async fn msg(&self, sequence: usize, log: &str) { + if let Err(e) = self.tx.send(Tick::Message(sequence, log.to_string())).await { + eprintln!("Fail to communicate with tracker: {e}"); + } + } + + pub async fn success(&self, sequence: usize, msg: &str) { + if let Err(e) = self + .tx + .send(Tick::Finished( + sequence, + OperationResult::Success, + msg.to_string(), + )) + .await + { + eprintln!("Fail to communicate with tracker: {e}"); + } + } + + pub async fn fail(&self, sequence: usize, msg: &str) { + if let Err(e) = self + .tx + .send(Tick::Finished( + sequence, + OperationResult::Failed, + msg.to_string(), + )) + .await + { + eprintln!("Fail to communicate with tracker: {e}"); + } + } + + pub async fn shutdown(&self) -> Result<(), Error> { + let (tx_response, rx_response) = bounded(1); + self.tx + .send(Tick::Shutdown(tx_response)) + .await + .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; + rx_response + .recv() + .await + .map_err(|e| Error::new(ErrorKind::NotConnected, e.to_string())) + } + + pub async fn _print(&self, msg: String) { + if let Err(e) = self + .tx + .send(Tick::Print(msg)) + .await + .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}"))) + { + eprintln!("Fail to communicate with tracker: {e}"); + } + } +} From 962d812c61c1a440f4b510d11d110aa428684fca Mon Sep 17 00:00:00 2001 From: DmitryAstafyev Date: Thu, 30 Nov 2023 09:19:52 +0100 Subject: [PATCH 002/174] Correct error messages --- cli/src/location.rs | 2 +- cli/src/main.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/src/location.rs b/cli/src/location.rs index ba30d84b4..e151f33ed 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -19,7 +19,7 @@ impl Location { if len == 0 { return Err(Error::new( ErrorKind::NotFound, - "Fail to find ICSMW location", + "Fail to find project's root location", )); } // TODO: better compare folders stucts or some file, like some git config file diff --git a/cli/src/main.rs b/cli/src/main.rs index d252f35c3..c3ec44266 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -19,7 +19,8 @@ use tracker::Tracker; extern crate lazy_static; lazy_static! { - static ref LOCATION: Location = Location::new().expect("Fail to setup location of ICSMW"); + static ref LOCATION: Location = + Location::new().expect("Fail to setup location of root of project"); static ref TRACKER: Tracker = Tracker::new(); } From dd4af6c65ee50a25a1d32fec5882556e95d54bd9 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 2 Dec 2023 21:40:36 +0100 Subject: [PATCH 003/174] Build-CLI: Replace to_string_lossy with display - Display is more suitable when printing infos for the users --- cli/src/fstools.rs | 14 +++----------- cli/src/modules/app.rs | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 45940a723..02fa67d2f 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -11,11 +11,7 @@ pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { TRACKER .success( sequence, - &format!( - "copied: {} to {}", - src.to_string_lossy(), - dest.to_string_lossy() - ), + &format!("copied: {} to {}", src.display(), dest.display()), ) .await; Ok(()) @@ -25,11 +21,7 @@ pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { let sequence = TRACKER.start("copy folder", None).await?; let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); - let msg = format!( - "copied: {} to {}", - src.to_string_lossy(), - dest.to_string_lossy() - ); + let msg = format!("copied: {} to {}", src.display(), dest.display()); spawn(async move { if let Err(e) = copy_with_progress(src, dest, &options, |info| { if tx.send(info).is_err() { @@ -64,7 +56,7 @@ pub async fn rm_folder(path: PathBuf) -> Result<(), Error> { let sequence = TRACKER.start("remove folder", None).await?; fs::remove_dir_all(&path)?; TRACKER - .success(sequence, &format!("removed: {}", path.to_string_lossy(),)) + .success(sequence, &format!("removed: {}", path.display(),)) .await; Ok(()) } diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 219546f7d..143c42055 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -42,7 +42,7 @@ impl Manager for Module { if !src.exists() { return Err(Error::new( ErrorKind::NotFound, - format!("Not found: {}", src.to_string_lossy()), + format!("Not found: {}", src.display()), )); } if !dest.exists() { From f272d411ce779be6099bf9380fc3b45ba0e9295a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 2 Dec 2023 22:10:09 +0100 Subject: [PATCH 004/174] Build-CLI: Refactor to_relative_path - Use rust built-in path methods - Change method signature to return Path instead of String --- cli/src/location.rs | 6 +++--- cli/src/spawner.rs | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cli/src/location.rs b/cli/src/location.rs index e151f33ed..d2723efc4 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,5 +1,6 @@ use crate::LOCATION; use std::ffi::OsStr; +use std::path::Path; use std::{ env::current_dir, io::{Error, ErrorKind}, @@ -35,7 +36,6 @@ impl Location { } } -pub fn to_relative_path(path: PathBuf) -> String { - let path_str = path.to_string_lossy().to_string(); - path_str.replace(&LOCATION.root.to_string_lossy().to_string(), "") +pub fn to_relative_path(path: &PathBuf) -> &Path { + path.strip_prefix(&LOCATION.root).unwrap_or(path) } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 25f4d0150..ac07d735e 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -59,7 +59,10 @@ pub async fn spawn( cmd }; let sequence = TRACKER - .start(&format!("{}: {}", to_relative_path(cwd), job_title), None) + .start( + &format!("{}: {}", to_relative_path(&cwd).display(), job_title), + None, + ) .await?; let mut stdout_lines: Vec = vec![]; let drain_stdout = { From e4b777b4826c1bc33cbc6e3eebef1e66bf6a2aa0 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 2 Dec 2023 22:18:06 +0100 Subject: [PATCH 005/174] Build-CLI: Refactor finding root - Function is rewritten in a more idiomatic way without having to allocate memory and keeping track of the length manually --- cli/src/location.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cli/src/location.rs b/cli/src/location.rs index d2723efc4..da44b1214 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,5 +1,4 @@ use crate::LOCATION; -use std::ffi::OsStr; use std::path::Path; use std::{ env::current_dir, @@ -15,23 +14,16 @@ pub struct Location { impl Location { pub fn new() -> Result { let mut root = current_dir()?; - let mut len = root.iter().collect::>().len(); - loop { - if len == 0 { + // TODO: better compare folders stucts or some file, like some git config file + while !root.ends_with("chipmunk") && !root.ends_with("logviewer") { + if !root.pop() { return Err(Error::new( ErrorKind::NotFound, "Fail to find project's root location", )); } - // TODO: better compare folders stucts or some file, like some git config file - if root.ends_with("chipmunk") || root.ends_with("logviewer") { - break; - } - if len > 0 { - len = len.saturating_sub(1); - } - root.pop(); } + Ok(Self { root }) } } From 6071ee3b3a427464eb9b29142105517d51bcd4e9 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 2 Dec 2023 22:25:13 +0100 Subject: [PATCH 006/174] Build-CLI: Refactor Build Command - Use Path built-in methods to set the path of the build command --- cli/src/modules/binding.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 7eec01a2e..803366359 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -29,9 +29,13 @@ impl Manager for Module { vec![] } fn build_cmd(&self, prod: bool) -> Option { + let path = Target::Wrapper + .get() + .cwd() + .join("node_modules/.bin/electron-build-env"); Some(format!( - "/{}/node_modules/.bin/electron-build-env nj-cli build{}", - Target::Wrapper.get().cwd().to_string_lossy(), + "{} nj-cli build{}", + path.to_string_lossy(), if prod { " --release" } else { "" } )) } From bd07982184f8aa2f5e901e61c0b9de7f7409518b Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 30 Dec 2023 20:57:43 +0100 Subject: [PATCH 007/174] CLI Wasm: Create wasm module + build command - Module for wasm management have been created with the needed implementation to integrate in the system. - Some Todos are left to be clarified later. --- cli/src/modules/mod.rs | 1 + cli/src/modules/wasm.rs | 40 ++++++++++++++++++++++++++++++++++++++++ cli/src/target.rs | 3 +++ 3 files changed, 44 insertions(+) create mode 100644 cli/src/modules/wasm.rs diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index ce20afba1..032d0a457 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -4,6 +4,7 @@ pub mod cli; pub mod client; pub mod core; pub mod shared; +pub mod wasm; pub mod wrapper; use crate::{ diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs new file mode 100644 index 000000000..64f408758 --- /dev/null +++ b/cli/src/modules/wasm.rs @@ -0,0 +1,40 @@ +use super::{Kind, Manager}; +use crate::{Target, LOCATION}; +use async_trait::async_trait; +use std::path::PathBuf; + +const PATH: &str = "application/apps/rustcore/wasm-bindings"; + +#[derive(Clone, Debug)] +pub struct Module {} + +impl Module { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl Manager for Module { + fn owner(&self) -> Target { + Target::Wasm + } + fn kind(&self) -> Kind { + Kind::Rs + } + fn cwd(&self) -> PathBuf { + LOCATION.root.clone().join(PATH) + } + + fn deps(&self) -> Vec { + //TODO: Do we have dependencies here? + vec![] + } + fn build_cmd(&self, prod: bool) -> Option { + //TODO: It's possible to set the environment with wasm-pack, but it wasn't set in the ruby + //version. Check if we can set it here. + let env = if prod { "--release" } else { "--dev" }; + + Some(format!("wasm-pack build {env} --target bundler")) + } +} diff --git a/cli/src/target.rs b/cli/src/target.rs index c8108ba04..a71a4906b 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -10,6 +10,7 @@ pub enum Target { Shared, App, Cli, + Wasm, } impl std::fmt::Display for Target { @@ -25,6 +26,7 @@ impl std::fmt::Display for Target { Target::Client => "Client", Target::Shared => "Shared", Target::App => "App", + Target::Wasm => "Wasm", } ) } @@ -51,6 +53,7 @@ impl Target { Target::Wrapper => Box::new(modules::wrapper::Module::new()), Target::Shared => Box::new(modules::shared::Module::new()), Target::App => Box::new(modules::app::Module::new()), + Target::Wasm => Box::new(modules::wasm::Module::new()), } } } From 6dc897d59bb1218323a7c1a5a655da06d0390e41 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 30 Dec 2023 22:30:27 +0100 Subject: [PATCH 008/174] CLI Wasm: Prepare the implementation to run tests - Method to run the tests has been implemented to use when running the tests is built in the cli app --- cli/src/modules/wasm.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 64f408758..4954a8123 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,7 +1,11 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{ + spawner::{spawn, SpawnResult}, + Target, LOCATION, +}; use async_trait::async_trait; -use std::path::PathBuf; +use futures::future::join_all; +use std::{io, path::PathBuf}; const PATH: &str = "application/apps/rustcore/wasm-bindings"; @@ -38,3 +42,21 @@ impl Manager for Module { Some(format!("wasm-pack build {env} --target bundler")) } } + +impl Module { + // TODO: Use this implementation when testing is implemented in the system + #[allow(dead_code)] + pub async fn run_tests(&self) -> Vec> { + let path_karma = self.cwd().join("spec"); + + join_all([ + spawn( + "wasm-pack test --node", + Some(self.cwd()), + Some("wasm-pack test wasm-bindings"), + ), + spawn("npm run test", Some(path_karma), Some("npm test wasm")), + ]) + .await + } +} From 6787d3fd30973439ffe1fc1bce888c7d08e59da8 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 30 Dec 2023 22:41:04 +0100 Subject: [PATCH 009/174] CLI Wasm: Add Wasm module to all targets --- cli/src/target.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/target.rs b/cli/src/target.rs index a71a4906b..18449cc2c 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -42,6 +42,7 @@ impl Target { Box::new(modules::wrapper::Module::new()), Box::new(modules::shared::Module::new()), Box::new(modules::client::Module::new()), + Box::new(modules::wasm::Module::new()), ] } pub fn get(&self) -> Box { From 974f4fdcd5c5bc563455aff4c1eb8f5f9ea76aa6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 3 Jan 2024 13:40:30 +0100 Subject: [PATCH 010/174] CLI Wasm: Remove TODOs & Add color always - TODOs has been removed after clarification - Set Color flag to always in build command --- cli/src/modules/wasm.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 4954a8123..d20b67f19 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -31,15 +31,15 @@ impl Manager for Module { } fn deps(&self) -> Vec { - //TODO: Do we have dependencies here? vec![] } + fn build_cmd(&self, prod: bool) -> Option { - //TODO: It's possible to set the environment with wasm-pack, but it wasn't set in the ruby - //version. Check if we can set it here. let env = if prod { "--release" } else { "--dev" }; - Some(format!("wasm-pack build {env} --target bundler")) + Some(format!( + "wasm-pack build {env} --target bundler --color always" + )) } } From ca4dc04bc431965a6d9f4c1804f4af9c8a3e2c04 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 3 Jan 2024 13:44:56 +0100 Subject: [PATCH 011/174] CLI Wasm: Add Wasm as dependency to Client Manager --- cli/src/modules/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 8ae990429..3791c0594 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -26,6 +26,6 @@ impl Manager for Module { LOCATION.root.clone().join(PATH) } fn deps(&self) -> Vec { - vec![Target::Shared] + vec![Target::Shared, Target::Wasm] } } From f43f5a1d909a2cf1d2309932f5212d4c302f1737 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 3 Jan 2024 15:17:02 +0100 Subject: [PATCH 012/174] CLI Wasm: Implementing running tests - Test Cli Command defined and implemented - Methods for tests in Manager trait has been defined and implemented - Test commands in wasm module added --- cli/src/main.rs | 16 ++++++++++++++ cli/src/modules/mod.rs | 48 +++++++++++++++++++++++++++++++++++++++++ cli/src/modules/wasm.rs | 31 +++++++------------------- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index c3ec44266..aea63691a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -55,6 +55,12 @@ enum Command { #[arg(short, long, num_args(0..))] target: Option>, }, + /// Run tests + Test { + /// target to test, by default whole application will be tested + #[arg(short, long, num_args(0..))] + target: Option>, + }, } fn main() -> Result<(), Error> { @@ -67,6 +73,7 @@ fn main() -> Result<(), Error> { Command::Lint { target } => target, Command::Build { target } => target, Command::Clean { target } => target, + Command::Test { target } => target, } { list.remove_duplicates(); list.iter().map(|target| target.get()).collect() @@ -101,6 +108,15 @@ fn main() -> Result<(), Error> { ) .await } + Command::Test { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.test()) + .collect::>(), + ) + .await + } }; TRACKER.shutdown().await?; let mut success: bool = true; diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 032d0a457..b2f1c1d20 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -13,6 +13,7 @@ use crate::{ Target, LOCATION, }; use async_trait::async_trait; +use futures::future::join_all; use std::{io::Error, path::PathBuf}; #[derive(Debug, Clone)] @@ -42,6 +43,17 @@ impl Kind { } } +pub(crate) struct TestCommand { + command: String, + cwd: PathBuf, +} + +impl TestCommand { + pub(crate) fn new(command: String, cwd: PathBuf) -> Self { + Self { command, cwd } + } +} + #[async_trait] pub trait Manager { fn kind(&self) -> Kind; @@ -54,6 +66,9 @@ pub trait Manager { fn install_cmd(&self, _prod: bool) -> Option { None } + fn test_cmds(&self) -> Vec { + Vec::new() + } async fn reset(&self) -> Result { self.clean().await?; fstools::rm_folder(self.cwd().join("dist")).await?; @@ -159,4 +174,37 @@ pub trait Manager { ) .await } + + // TODO: After using native rust asnyc traits, we can return a vector of results here and use + // flat_map in main() to get rid of the double join calls + async fn test(&self) -> Result { + self.install(false).await?; + // TODO: Check if we need to run the dependencies tests too + + let test_cmds = self.test_cmds(); + if test_cmds.is_empty() { + return Ok(SpawnResult::empty()); + } + + let results = join_all( + test_cmds + .iter() + .map(|cmd| spawn(&cmd.command, Some(cmd.cwd.to_owned()), Some(&cmd.command))), + ) + .await; + + // return the first failed result, or the first one if all was successful + let return_pos = results + .iter() + .position(|res| match res { + Ok(result) => !result.status.success(), + Err(_) => true, + }) + .unwrap_or(0); + + results + .into_iter() + .nth(return_pos) + .expect("Commands has been checked if they are empty before spawning tasks") + } } diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index d20b67f19..3d67ce952 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,11 +1,7 @@ -use super::{Kind, Manager}; -use crate::{ - spawner::{spawn, SpawnResult}, - Target, LOCATION, -}; +use super::{Kind, Manager, TestCommand}; +use crate::{Target, LOCATION}; use async_trait::async_trait; -use futures::future::join_all; -use std::{io, path::PathBuf}; +use std::path::PathBuf; const PATH: &str = "application/apps/rustcore/wasm-bindings"; @@ -41,22 +37,11 @@ impl Manager for Module { "wasm-pack build {env} --target bundler --color always" )) } -} -impl Module { - // TODO: Use this implementation when testing is implemented in the system - #[allow(dead_code)] - pub async fn run_tests(&self) -> Vec> { - let path_karma = self.cwd().join("spec"); - - join_all([ - spawn( - "wasm-pack test --node", - Some(self.cwd()), - Some("wasm-pack test wasm-bindings"), - ), - spawn("npm run test", Some(path_karma), Some("npm test wasm")), - ]) - .await + fn test_cmds(&self) -> Vec { + vec![ + TestCommand::new("wasm-pack test --node --color always".into(), self.cwd()), + TestCommand::new("npm run test".into(), self.cwd().join("spec")), + ] } } From 220132da04d07526eca89ec68aec783faafa5471 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 3 Jan 2024 21:11:50 +0100 Subject: [PATCH 013/174] CLI Tests: Spawn Options & Suppress msg for wasm - Spawn Options has been added to provide more control over how spawn call should behave. - It's used to suppress sending messages in wasm npm test command because it caused weird behavior in the progress bars because when too many lines are sent at the same time --- cli/src/modules/mod.rs | 33 +++++++++++++++++++++------------ cli/src/modules/wasm.rs | 14 +++++++++++--- cli/src/spawner.rs | 11 ++++++++++- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index b2f1c1d20..25239df93 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -9,7 +9,7 @@ pub mod wrapper; use crate::{ fstools, - spawner::{spawn, SpawnResult}, + spawner::{spawn, SpawnOptions, SpawnResult}, Target, LOCATION, }; use async_trait::async_trait; @@ -46,11 +46,16 @@ impl Kind { pub(crate) struct TestCommand { command: String, cwd: PathBuf, + spawn_opts: Option, } impl TestCommand { - pub(crate) fn new(command: String, cwd: PathBuf) -> Self { - Self { command, cwd } + pub(crate) fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { + Self { + command, + cwd, + spawn_opts, + } } } @@ -92,7 +97,7 @@ pub trait Manager { self.kind().install_cmd(prod) }; if let Some(cmd) = cmd { - spawn(&cmd, Some(self.cwd()), Some(&cmd)).await + spawn(&cmd, Some(self.cwd()), Some(&cmd), None).await } else { Ok(SpawnResult::empty()) } @@ -128,7 +133,7 @@ pub trait Manager { } else { self.kind().build_cmd(prod) }; - match spawn(&cmd, Some(path), Some(&cmd)).await { + match spawn(&cmd, Some(path), Some(&cmd), None).await { Ok(status) => { if !status.status.success() { Ok(status) @@ -159,11 +164,11 @@ pub trait Manager { } async fn lint(&self) -> Result { let path = LOCATION.root.clone().join(self.cwd()); - let status = spawn("yarn run lint", Some(path.clone()), Some("linting")).await?; + let status = spawn("yarn run lint", Some(path.clone()), Some("linting"), None).await?; if !status.status.success() { return Ok(status); } - spawn("yarn run build", Some(path), Some("TS compilation")).await + spawn("yarn run build", Some(path), Some("TS compilation"), None).await } async fn clippy(&self) -> Result { let path = LOCATION.root.clone().join(self.cwd()); @@ -171,6 +176,7 @@ pub trait Manager { "cargo clippy --color always --all --all-features -- -D warnings", Some(path), Some("clippy"), + None, ) .await } @@ -186,11 +192,14 @@ pub trait Manager { return Ok(SpawnResult::empty()); } - let results = join_all( - test_cmds - .iter() - .map(|cmd| spawn(&cmd.command, Some(cmd.cwd.to_owned()), Some(&cmd.command))), - ) + let results = join_all(test_cmds.iter().map(|cmd| { + spawn( + &cmd.command, + Some(cmd.cwd.to_owned()), + Some(&cmd.command), + cmd.spawn_opts.clone(), + ) + })) .await; // return the first failed result, or the first one if all was successful diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 3d67ce952..8676a707f 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager, TestCommand}; -use crate::{Target, LOCATION}; +use crate::{spawner::SpawnOptions, Target, LOCATION}; use async_trait::async_trait; use std::path::PathBuf; @@ -40,8 +40,16 @@ impl Manager for Module { fn test_cmds(&self) -> Vec { vec![ - TestCommand::new("wasm-pack test --node --color always".into(), self.cwd()), - TestCommand::new("npm run test".into(), self.cwd().join("spec")), + TestCommand::new( + "wasm-pack test --node --color always".into(), + self.cwd(), + None, + ), + TestCommand::new( + "npm run test".into(), + self.cwd().join("spec"), + Some(SpawnOptions { suppress_msg: true }), + ), ] } } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index ac07d735e..a78c57ef3 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -27,11 +27,18 @@ impl SpawnResult { } } +#[derive(Debug, Clone, Default)] +pub(crate) struct SpawnOptions { + pub suppress_msg: bool, +} + pub async fn spawn( command: &str, cwd: Option, caption: Option<&str>, + opts: Option, ) -> Result { + let opts = opts.unwrap_or_default(); let cwd = if let Some(cwd) = cwd { cwd } else { @@ -76,7 +83,9 @@ pub async fn spawn( if read_lines == 0 { break; } else { - TRACKER.msg(sequence, &line).await; + if !opts.suppress_msg { + TRACKER.msg(sequence, &line).await; + } TRACKER.progress(sequence, None).await; storage.push(line); } From 1cf4132a0c05e1d61a01e9290ff967219789919c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 4 Dec 2023 19:34:24 +0100 Subject: [PATCH 014/174] Build-CLI: Refactoring tracker UI... - Rewrite jobs alignment to keep the braces and the symbols in the same horizontal alignment among all lines - Small refactoring to get the length or max --- cli/src/tracker.rs | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 8615679b9..d7f42eeb4 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -43,11 +43,6 @@ pub struct Tracker { tx: Sender, } -fn order_offset(num: usize, total: usize) -> String { - " ".repeat(format!("[{total}/{total}]").len() - format!("[{num}/{total}]").len()) - .to_string() -} - impl Tracker { pub fn new() -> Self { let (tx, rx): (Sender, Receiver) = unbounded(); @@ -69,25 +64,20 @@ impl Tracker { match tick { Tick::Started(job, len, tx_response) => { sequence += 1; - let bar = mp.add(ProgressBar::new(if let Some(len) = len { - len - } else { - max - })); + let sequence_txt = sequence.to_string(); + let bar = mp.add(ProgressBar::new(len.unwrap_or(max))); bar.set_style(spinner_style.clone()); bars.insert(sequence, (bar, Instant::now(), job, None)); bars.iter_mut().for_each(|(k, (bar, _, job, result))| { - if let Some(result) = result { - bar.set_prefix(format!( - "[{k}/{sequence}]{}[{result}][{job}]", - order_offset(*k, sequence) - )); - } else { - bar.set_prefix(format!( - "[{k}/{sequence}]{}[....][{job}]", - order_offset(*k, sequence) - )); - } + bar.set_prefix(format!( + "[{:0seq_width$}/{:0seq_width$}][{}][{job}]", + k, + sequence_txt, + result + .as_ref() + .map_or_else(|| String::from("...."), |res| res.to_string()), + seq_width = sequence_txt.len() + )); }); if let Err(e) = tx_response.send(sequence).await { let _ = mp.println(format!("Fail to send response: {e}")); @@ -109,9 +99,13 @@ impl Tracker { } Tick::Finished(seq, result, msg) => { if let Some((bar, instant, job, res)) = bars.get_mut(&seq) { + let sequence_txt = sequence.to_string(); bar.set_prefix(format!( - "[{seq}/{sequence}]{}[{result}][{job}]", - order_offset(seq, sequence) + "[{:0seq_width$}/{:0seq_width$}][{}][{job}]", + seq, + sequence_txt, + result, + seq_width = sequence_txt.len() )); bar.finish_with_message(format!( "Done in {}s. {msg}", From b0817fdd0225fe6ba456eb3b7cf9cc7058e8dca0 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 4 Dec 2023 20:42:11 +0100 Subject: [PATCH 015/174] Build-CLI UI: Change jobs placeholder to spaces - Change job numbers placeholder from zero to spaces in tracker --- cli/src/tracker.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index d7f42eeb4..93b015a11 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -70,9 +70,8 @@ impl Tracker { bars.insert(sequence, (bar, Instant::now(), job, None)); bars.iter_mut().for_each(|(k, (bar, _, job, result))| { bar.set_prefix(format!( - "[{:0seq_width$}/{:0seq_width$}][{}][{job}]", + "[{:seq_width$}/{sequence_txt}][{}][{job}]", k, - sequence_txt, result .as_ref() .map_or_else(|| String::from("...."), |res| res.to_string()), @@ -101,9 +100,8 @@ impl Tracker { if let Some((bar, instant, job, res)) = bars.get_mut(&seq) { let sequence_txt = sequence.to_string(); bar.set_prefix(format!( - "[{:0seq_width$}/{:0seq_width$}][{}][{job}]", + "[{:seq_width$}/{sequence_txt}][{}][{job}]", seq, - sequence_txt, result, seq_width = sequence_txt.len() )); From 196367899484b9cdf78c57d13941a4c47f85863b Mon Sep 17 00:00:00 2001 From: DmitryAstafyev Date: Mon, 22 Jan 2024 10:27:37 +0100 Subject: [PATCH 016/174] Add missed script links --- .../apps/rustcore/ts-bindings/package.json | 1 + application/holder/package.json | 1 + application/platform/package.json | 1 + cli/src/modules/app.rs | 16 +++++++++++++--- cli/src/modules/client.rs | 7 +++++++ cli/src/modules/mod.rs | 7 +++++-- cli/src/modules/wrapper.rs | 2 +- 7 files changed, 29 insertions(+), 6 deletions(-) diff --git a/application/apps/rustcore/ts-bindings/package.json b/application/apps/rustcore/ts-bindings/package.json index a8fc62b7b..57d2b7a70 100644 --- a/application/apps/rustcore/ts-bindings/package.json +++ b/application/apps/rustcore/ts-bindings/package.json @@ -6,6 +6,7 @@ "types": "./dist/index.d.ts", "scripts": { "build": "node_modules/.bin/tsc -p tsconfig.json", + "prod": "node_modules/.bin/tsc -p tsconfig.json", "test_cancel": "node_modules/.bin/electron node_modules/jasmine-ts/lib/index.js ./spec/session.cancel.spec.ts", "lint": "node_modules/.bin/eslint . --ext .ts --max-warnings=0", "check": "node_modules/.bin/tsc -p tsconfig.json --noemit" diff --git a/application/holder/package.json b/application/holder/package.json index 658e1ce3a..73561e35a 100644 --- a/application/holder/package.json +++ b/application/holder/package.json @@ -115,6 +115,7 @@ "electron": "npm run build && ./node_modules/.bin/electron --inspect ./dist/app.js", "electron-win": "node_modules/.bin/electron --inspect ./dist/app.js", "build": "node_modules/.bin/tsc -p tsconfig.json", + "prod": "node_modules/.bin/tsc -p tsconfig.json", "start": "npm run build-ts && npm run electron", "postinstall": "electron-builder install-app-deps", "build-darwin": "node_modules/.bin/electron-builder --mac --dir", diff --git a/application/platform/package.json b/application/platform/package.json index ee3393bfc..451833d81 100644 --- a/application/platform/package.json +++ b/application/platform/package.json @@ -34,6 +34,7 @@ }, "scripts": { "build": "node_modules/.bin/tsc -p tsconfig.json", + "prod": "node_modules/.bin/tsc -p tsconfig.json", "lint": "node_modules/.bin/eslint . --ext .ts --max-warnings=0 ", "check": "node_modules/.bin/tsc -p tsconfig.json --noemit" }, diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 143c42055..e1198b658 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -36,8 +36,11 @@ impl Manager for Module { // For app we don't need --production Some(String::from("yarn install")) } - async fn after(&self) -> Result, Error> { - let src = Target::Client.get().cwd().join("dist/client"); + async fn after(&self, prod: bool) -> Result, Error> { + let src = Target::Client.get().dist_path(prod).ok_or(Error::new( + ErrorKind::NotFound, + "Fail to get client artifacts", + ))?; let dest = self.cwd().join("dist"); if !src.exists() { return Err(Error::new( @@ -52,7 +55,14 @@ impl Manager for Module { if prev.exists() { fstools::rm_folder(prev).await?; } - fstools::cp_folder(src, dest).await?; + fstools::cp_folder(src.clone(), dest.clone()).await?; + std::fs::rename( + dest.join(src.file_name().ok_or(Error::new( + ErrorKind::NotFound, + "Fail to parse client artifacts path", + ))?), + dest.join("client"), + )?; Ok(None) } } diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 3791c0594..978791a1c 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -28,4 +28,11 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![Target::Shared, Target::Wasm] } + fn dist_path(&self, prod: bool) -> Option { + Some(LOCATION.root.clone().join(PATH).join(if prod { + "dist/release" + } else { + "dist/debug" + })) + } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 25239df93..e486a1731 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -64,6 +64,9 @@ pub trait Manager { fn kind(&self) -> Kind; fn owner(&self) -> Target; fn cwd(&self) -> PathBuf; + fn dist_path(&self, _prod: bool) -> Option { + None + } fn deps(&self) -> Vec; fn build_cmd(&self, _prod: bool) -> Option { None @@ -114,7 +117,7 @@ pub trait Manager { Kind::Rs => Ok(SpawnResult::empty()), } } - async fn after(&self) -> Result, Error> { + async fn after(&self, _prod: bool) -> Result, Error> { Ok(None) } async fn build(&self, prod: bool) -> Result { @@ -138,7 +141,7 @@ pub trait Manager { if !status.status.success() { Ok(status) } else { - let res = self.after().await?; + let res = self.after(prod).await?; if matches!(self.kind(), Kind::Ts) && prod { self.clean().await?; self.install(prod).await?; diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index caca879f2..024b71d69 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -32,7 +32,7 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] } - async fn after(&self) -> Result, Error> { + async fn after(&self, _prod: bool) -> Result, Error> { let src = Target::Binding.get().cwd().join("dist/index.node"); let dest = self.cwd().join("dist/native"); if !src.exists() { From 0e4865dadf7de0f062bbe3ac001949245fcba6bc Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Jan 2024 19:53:38 +0100 Subject: [PATCH 017/174] CLI: UI Changes in jobs' bars & Add total time - Job duration is moved to the prefix part of the bars. - Total time is appended to the results. --- cli/src/tracker.rs | 96 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 93b015a11..5328adad4 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -43,6 +43,11 @@ pub struct Tracker { tx: Sender, } +enum TimeIndicator { + Running(Instant), + Finished(u64), +} + impl Tracker { pub fn new() -> Self { let (tx, rx): (Sender, Receiver) = unbounded(); @@ -56,10 +61,12 @@ impl Tracker { .tick_chars("▂▃▅▆▇▆▅▃▂ "); async move { let mut sequence: usize = 0; + let mut max_time_len = 0; let max = u64::MAX; - let mut bars: HashMap)> = + let mut bars: HashMap)> = HashMap::new(); let mp = MultiProgress::new(); + let start_time = Instant::now(); while let Ok(tick) = rx.recv().await { match tick { Tick::Started(job, len, tx_response) => { @@ -67,16 +74,29 @@ impl Tracker { let sequence_txt = sequence.to_string(); let bar = mp.add(ProgressBar::new(len.unwrap_or(max))); bar.set_style(spinner_style.clone()); - bars.insert(sequence, (bar, Instant::now(), job, None)); - bars.iter_mut().for_each(|(k, (bar, _, job, result))| { - bar.set_prefix(format!( - "[{:seq_width$}/{sequence_txt}][{}][{job}]", - k, - result - .as_ref() - .map_or_else(|| String::from("...."), |res| res.to_string()), - seq_width = sequence_txt.len() - )); + bars.insert(sequence, (bar, TimeIndicator::Running(Instant::now()), job, None)); + bars.iter_mut().for_each(|(k, (bar, time_indicator, job, result))| { + let seq_width = sequence_txt.len(); + let line_prefix = match time_indicator { + TimeIndicator::Running(_) => { + format!( + "[{:seq_width$}/{sequence_txt}][{}][{job}]", + k, + String::from("....") + ) + }, + TimeIndicator::Finished(time) => { + format!( + "[{:seq_width$}/{sequence_txt}][{}][{time:max_time_len$}s][{job}].", + k, + result + .as_ref() + .expect("Job must be finished here") + ) + }, + }; + + bar.set_prefix(line_prefix); }); if let Err(e) = tx_response.send(sequence).await { let _ = mp.println(format!("Fail to send response: {e}")); @@ -97,18 +117,20 @@ impl Tracker { } } Tick::Finished(seq, result, msg) => { - if let Some((bar, instant, job, res)) = bars.get_mut(&seq) { + if let Some((bar, time_indicator, job, res)) = bars.get_mut(&seq) { let sequence_txt = sequence.to_string(); + let TimeIndicator::Running(instant) = time_indicator else {panic!("{job} can finish only once")}; + // It doesn't make sense to show that a job is done in 0 seconds + let time = instant.elapsed().as_secs().max(1); + *time_indicator = TimeIndicator::Finished(time); + + max_time_len = max_time_len.max(Self::count_digits(time)); + + let seq_width = sequence_txt.len(); bar.set_prefix(format!( - "[{:seq_width$}/{sequence_txt}][{}][{job}]", - seq, - result, - seq_width = sequence_txt.len() - )); - bar.finish_with_message(format!( - "Done in {}s. {msg}", - instant.elapsed().as_secs() + "[{seq:seq_width$}/{sequence_txt}][{result}][{time:max_time_len$}s][{job}].", )); + bar.finish_with_message(msg); res.replace(result); } } @@ -116,14 +138,24 @@ impl Tracker { let _ = mp.println(msg); } Tick::Shutdown(tx_response) => { - bars.iter_mut().for_each(|(_, (bar, instant, _, _))| { + bars.iter_mut().for_each(|(_, (bar, time_indicator, job, _))| { if !bar.is_finished() { - bar.finish_with_message(format!( - "Done in {}s.", - instant.elapsed().as_secs() - )); + let TimeIndicator::Running(instant) = time_indicator else {panic!("{job} can finish only once")}; + let time = instant.elapsed().as_secs().max(1); + *time_indicator = TimeIndicator::Finished(time); + max_time_len = max_time_len.max(Self::count_digits(time)); + + bar.finish(); } }); + + // Insert total time bar + let total_time = start_time.elapsed().as_secs().max(1); + let total_bar = mp.add(ProgressBar::new((bars.len() + 1) as u64)); + total_bar.set_style(spinner_style.clone()); + total_bar.set_prefix(format!("[total] done all in {total_time}s.")); + total_bar.finish(); + bars.clear(); // let _ = mp.clear(); if let Err(e) = tx_response.send(()).await { @@ -138,6 +170,20 @@ impl Tracker { Ok(()) } + /// Counts the digits in a number without allocating new string + fn count_digits(mut num: u64) -> usize { + if num == 0 { + return 1; // Special case for zero + } + + let mut count = 0; + while num > 0 { + num /= 10; + count += 1; + } + count + } + pub async fn start(&self, job: &str, max: Option) -> Result { let (tx_response, rx_response) = bounded(1); self.tx From 1dcf5bbcb7b5c777325593e428e6a72e34c695de Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Jan 2024 23:58:34 +0100 Subject: [PATCH 018/174] Refactoring: Define Struct for each job-bar-state The states of jobs has been growing and getting too complex for a tuple --- cli/src/tracker.rs | 82 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 5328adad4..ea82f62b2 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -43,9 +43,24 @@ pub struct Tracker { tx: Sender, } -enum TimeIndicator { - Running(Instant), - Finished(u64), +struct JobBarState { + name: String, + bar: ProgressBar, + start_time: Instant, + result: Option<(OperationResult, u64)>, +} + +impl JobBarState { + fn start_job(name: String, bar: ProgressBar) -> Self { + let start_time = Instant::now(); + + Self { + name, + bar, + start_time, + result: None, + } + } } impl Tracker { @@ -63,7 +78,7 @@ impl Tracker { let mut sequence: usize = 0; let mut max_time_len = 0; let max = u64::MAX; - let mut bars: HashMap)> = + let mut bars: HashMap = HashMap::new(); let mp = MultiProgress::new(); let start_time = Instant::now(); @@ -74,78 +89,71 @@ impl Tracker { let sequence_txt = sequence.to_string(); let bar = mp.add(ProgressBar::new(len.unwrap_or(max))); bar.set_style(spinner_style.clone()); - bars.insert(sequence, (bar, TimeIndicator::Running(Instant::now()), job, None)); - bars.iter_mut().for_each(|(k, (bar, time_indicator, job, result))| { + let job_bar = JobBarState::start_job(job, bar); + bars.insert(sequence, job_bar); + bars.iter_mut().for_each(|(k, job_bar)| { let seq_width = sequence_txt.len(); - let line_prefix = match time_indicator { - TimeIndicator::Running(_) => { + let job = job_bar.name.as_str(); + let line_prefix = match job_bar.result.as_ref() { + None => { format!( - "[{:seq_width$}/{sequence_txt}][{}][{job}]", - k, + "[{k:seq_width$}/{sequence_txt}][{}][{job}]", String::from("....") ) }, - TimeIndicator::Finished(time) => { - format!( - "[{:seq_width$}/{sequence_txt}][{}][{time:max_time_len$}s][{job}].", - k, - result - .as_ref() - .expect("Job must be finished here") - ) + Some((res, time)) => { + format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") }, }; - bar.set_prefix(line_prefix); + job_bar.bar.set_prefix(line_prefix); }); if let Err(e) = tx_response.send(sequence).await { let _ = mp.println(format!("Fail to send response: {e}")); } } Tick::Message(sequence, log) => { - if let Some((bar, _, _, _)) = bars.get(&sequence) { - bar.set_message(log); + if let Some(job_bar) = bars.get(&sequence) { + job_bar.bar.set_message(log); } } Tick::Progress(sequence, pos) => { - if let Some((bar, _, _, _)) = bars.get(&sequence) { + if let Some(job_bar) = bars.get(&sequence) { if let Some(pos) = pos { - bar.set_position(pos); + job_bar.bar.set_position(pos); } else { - bar.inc(1); + job_bar.bar.inc(1); } } } Tick::Finished(seq, result, msg) => { - if let Some((bar, time_indicator, job, res)) = bars.get_mut(&seq) { + if let Some(job_bar) = bars.get_mut(&seq) { let sequence_txt = sequence.to_string(); - let TimeIndicator::Running(instant) = time_indicator else {panic!("{job} can finish only once")}; // It doesn't make sense to show that a job is done in 0 seconds - let time = instant.elapsed().as_secs().max(1); - *time_indicator = TimeIndicator::Finished(time); + let time = job_bar.start_time.elapsed().as_secs().max(1); max_time_len = max_time_len.max(Self::count_digits(time)); let seq_width = sequence_txt.len(); - bar.set_prefix(format!( + let job = job_bar.name.as_str(); + job_bar.bar.set_prefix(format!( "[{seq:seq_width$}/{sequence_txt}][{result}][{time:max_time_len$}s][{job}].", )); - bar.finish_with_message(msg); - res.replace(result); + job_bar.bar.finish_with_message(msg); + job_bar.result.replace((result, time)); } } Tick::Print(msg) => { let _ = mp.println(msg); } Tick::Shutdown(tx_response) => { - bars.iter_mut().for_each(|(_, (bar, time_indicator, job, _))| { - if !bar.is_finished() { - let TimeIndicator::Running(instant) = time_indicator else {panic!("{job} can finish only once")}; - let time = instant.elapsed().as_secs().max(1); - *time_indicator = TimeIndicator::Finished(time); + bars.iter_mut().for_each(|(_, job_bar)| { + if !job_bar.bar.is_finished() { + let time = job_bar.start_time.elapsed().as_secs().max(1); + job_bar.result.replace((OperationResult::Success, time)); max_time_len = max_time_len.max(Self::count_digits(time)); - bar.finish(); + job_bar.bar.finish(); } }); From b58ace6e4e4cb0bad0cf3c9a79c4cde3bd25d9a5 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 27 Jan 2024 00:10:16 +0100 Subject: [PATCH 019/174] CLI: Refresh all bars on each job's finish. The padding of finishing time can change on job ends which led to miss-alignment in the bars in some cases --- cli/src/tracker.rs | 50 +++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index ea82f62b2..0b0b969bb 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -78,36 +78,18 @@ impl Tracker { let mut sequence: usize = 0; let mut max_time_len = 0; let max = u64::MAX; - let mut bars: HashMap = - HashMap::new(); + let mut bars: HashMap = HashMap::new(); let mp = MultiProgress::new(); let start_time = Instant::now(); while let Ok(tick) = rx.recv().await { match tick { Tick::Started(job, len, tx_response) => { sequence += 1; - let sequence_txt = sequence.to_string(); let bar = mp.add(ProgressBar::new(len.unwrap_or(max))); bar.set_style(spinner_style.clone()); let job_bar = JobBarState::start_job(job, bar); bars.insert(sequence, job_bar); - bars.iter_mut().for_each(|(k, job_bar)| { - let seq_width = sequence_txt.len(); - let job = job_bar.name.as_str(); - let line_prefix = match job_bar.result.as_ref() { - None => { - format!( - "[{k:seq_width$}/{sequence_txt}][{}][{job}]", - String::from("....") - ) - }, - Some((res, time)) => { - format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") - }, - }; - - job_bar.bar.set_prefix(line_prefix); - }); + Self::refresh_all_bars(&mut bars, sequence, max_time_len); if let Err(e) = tx_response.send(sequence).await { let _ = mp.println(format!("Fail to send response: {e}")); } @@ -141,6 +123,8 @@ impl Tracker { )); job_bar.bar.finish_with_message(msg); job_bar.result.replace((result, time)); + + Self::refresh_all_bars(&mut bars, sequence, max_time_len); } } Tick::Print(msg) => { @@ -178,6 +162,32 @@ impl Tracker { Ok(()) } + fn refresh_all_bars( + bars: &mut HashMap, + sequence: usize, + max_time_len: usize, + ) { + let sequence_txt = sequence.to_string(); + + bars.iter_mut().for_each(|(k, job_bar)| { + let seq_width = sequence_txt.len(); + let job = job_bar.name.as_str(); + let line_prefix = match job_bar.result.as_ref() { + None => { + format!( + "[{k:seq_width$}/{sequence_txt}][{}][{job}]", + String::from("....") + ) + } + Some((res, time)) => { + format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") + } + }; + + job_bar.bar.set_prefix(line_prefix); + }); + } + /// Counts the digits in a number without allocating new string fn count_digits(mut num: u64) -> usize { if num == 0 { From 03523fb17712d2ecddb1c76dddc926ec7685d06f Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 27 Jan 2024 11:19:51 +0100 Subject: [PATCH 020/174] CLI: Insert duration bar on each job by shut down --- cli/src/tracker.rs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 0b0b969bb..43770d164 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -8,6 +8,8 @@ use std::{ time::Instant, }; +const TIME_BAR_WIDTH: usize = 5; + #[derive(Clone, Debug)] pub enum OperationResult { Success, @@ -89,7 +91,7 @@ impl Tracker { bar.set_style(spinner_style.clone()); let job_bar = JobBarState::start_job(job, bar); bars.insert(sequence, job_bar); - Self::refresh_all_bars(&mut bars, sequence, max_time_len); + Self::refresh_all_bars(&mut bars, sequence, max_time_len, None); if let Err(e) = tx_response.send(sequence).await { let _ = mp.println(format!("Fail to send response: {e}")); } @@ -124,7 +126,7 @@ impl Tracker { job_bar.bar.finish_with_message(msg); job_bar.result.replace((result, time)); - Self::refresh_all_bars(&mut bars, sequence, max_time_len); + Self::refresh_all_bars(&mut bars, sequence, max_time_len, None); } } Tick::Print(msg) => { @@ -141,8 +143,11 @@ impl Tracker { } }); + // Insert graphic bar for the running duration of each bars + let total_time = start_time.elapsed().as_secs().max(1) as usize; + Self::refresh_all_bars(&mut bars, sequence, max_time_len, Some(total_time)); + // Insert total time bar - let total_time = start_time.elapsed().as_secs().max(1); let total_bar = mp.add(ProgressBar::new((bars.len() + 1) as u64)); total_bar.set_style(spinner_style.clone()); total_bar.set_prefix(format!("[total] done all in {total_time}s.")); @@ -166,6 +171,7 @@ impl Tracker { bars: &mut HashMap, sequence: usize, max_time_len: usize, + total_time: Option, ) { let sequence_txt = sequence.to_string(); @@ -174,13 +180,16 @@ impl Tracker { let job = job_bar.name.as_str(); let line_prefix = match job_bar.result.as_ref() { None => { - format!( - "[{k:seq_width$}/{sequence_txt}][{}][{job}]", - String::from("....") - ) + format!("[{k:seq_width$}/{sequence_txt}][....][{job}]") } Some((res, time)) => { - format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") + if let Some(total_time) = total_time { + let finish_limit = (*time as usize * TIME_BAR_WIDTH) / total_time; + let time_bar: String = (0..TIME_BAR_WIDTH).map(|idx| if idx <= finish_limit {'█'}else {'░'}).collect(); + format!("[{k:seq_width$}/{sequence_txt}][{res}][{time_bar} {time:max_time_len$}s][{job}].") + }else { + format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") + } } }; From 2e4ffb5c9f52224f7783f59ae1172013f67594be Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 4 Feb 2024 16:03:38 +0100 Subject: [PATCH 021/174] CLI: Update bars on both stdout and stderr Some build commands send big part of their infos to stderr than stdout, which gives the app a little more responsive looking if we update the bars on stderr too --- cli/src/spawner.rs | 72 ++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index a78c57ef3..401c30bb1 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -71,54 +71,56 @@ pub async fn spawn( None, ) .await?; + let mut stdout_lines: Vec = vec![]; - let drain_stdout = { - let storage = &mut stdout_lines; + let mut stderr_lines: Vec = vec![]; + let drain_stdout_stderr = { let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + let storage_out = &mut stdout_lines; + let storage_err = &mut stderr_lines; async move { - let mut buf = BufReader::new(stdout); + use futures::{select, FutureExt}; + + let mut stdout_buf = BufReader::new(stdout); + let mut stderr_buf = BufReader::new(stderr); loop { - let mut line = String::new(); - let read_lines = buf.read_line(&mut line).await?; - if read_lines == 0 { - break; - } else { - if !opts.suppress_msg { - TRACKER.msg(sequence, &line).await; + let mut stdout_line = String::new(); + let mut stderr_line = String::new(); + select! { + out_lines = stdout_buf.read_line(&mut stdout_line).fuse() => { + let out_lines = out_lines?; + if out_lines == 0 { + break; + } else { + if !opts.suppress_msg { + TRACKER.msg(sequence, &stdout_line).await; + } + TRACKER.progress(sequence, None).await; + storage_out.push(stdout_line); + } } - TRACKER.progress(sequence, None).await; - storage.push(line); - } - } - future::pending::<()>().await; - Ok::, io::Error>(None) - } - }; + err_lines = stderr_buf.read_line(&mut stderr_line).fuse() => { + let err_lines = err_lines?; + if err_lines == 0 { + break; + } else { + TRACKER.progress(sequence, None).await; + if !stderr_line.trim().is_empty() { + storage_err.push(stderr_line); + } + } - let mut stderr_lines: Vec = vec![]; - let drain_stderr = { - let storage = &mut stderr_lines; - let stderr = child.stderr.take().unwrap(); - async move { - let mut buf = BufReader::new(stderr); - loop { - let mut line = String::new(); - let read_lines = buf.read_line(&mut line).await?; - if read_lines == 0 { - break; - } else { - TRACKER.progress(sequence, None).await; - if !line.trim().is_empty() { - storage.push(line); } } } + future::pending::<()>().await; Ok::, io::Error>(None) } }; - let status = match drain_stdout - .or(drain_stderr) + + let status = match drain_stdout_stderr .or(async move { Ok(Some(child.status().await?)) }) .await { From 58b89072568351b0bebb7a4981285f67f97f489c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 5 Feb 2024 21:14:01 +0100 Subject: [PATCH 022/174] CLI: Update on stdout & stderr: Small refactoring Better names for the variables --- cli/src/spawner.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 401c30bb1..1fc19ad7a 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -77,8 +77,8 @@ pub async fn spawn( let drain_stdout_stderr = { let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); - let storage_out = &mut stdout_lines; - let storage_err = &mut stderr_lines; + let storage_stdout = &mut stdout_lines; + let storage_stderr = &mut stderr_lines; async move { use futures::{select, FutureExt}; @@ -88,26 +88,26 @@ pub async fn spawn( let mut stdout_line = String::new(); let mut stderr_line = String::new(); select! { - out_lines = stdout_buf.read_line(&mut stdout_line).fuse() => { - let out_lines = out_lines?; - if out_lines == 0 { + stdout_read_result = stdout_buf.read_line(&mut stdout_line).fuse() => { + let stdout_read_bytes = stdout_read_result?; + if stdout_read_bytes == 0 { break; } else { if !opts.suppress_msg { TRACKER.msg(sequence, &stdout_line).await; } TRACKER.progress(sequence, None).await; - storage_out.push(stdout_line); + storage_stdout.push(stdout_line); } } - err_lines = stderr_buf.read_line(&mut stderr_line).fuse() => { - let err_lines = err_lines?; - if err_lines == 0 { + stderr_read_result = stderr_buf.read_line(&mut stderr_line).fuse() => { + let stderr_read_bytes = stderr_read_result?; + if stderr_read_bytes == 0 { break; } else { TRACKER.progress(sequence, None).await; if !stderr_line.trim().is_empty() { - storage_err.push(stderr_line); + storage_stderr.push(stderr_line); } } From ed2664f1851269fab9cdf5ba7f4568ff32db44f1 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 11 Feb 2024 10:26:43 +0100 Subject: [PATCH 023/174] CLI: Replace async-std with tokio --- cli/Cargo.lock | 656 ++++++++++----------------------------------- cli/Cargo.toml | 5 +- cli/src/fstools.rs | 3 +- cli/src/main.rs | 143 +++++----- cli/src/spawner.rs | 23 +- cli/src/tracker.rs | 62 ++--- 6 files changed, 259 insertions(+), 633 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 25b3c917f..946505a72 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "anstream" version = "0.6.4" @@ -50,186 +65,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" -dependencies = [ - "concurrent-queue", - "event-listener 3.1.0", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-executor" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5ea910c42e5ab19012bab31f53cb4d63d54c3a27730f9a833a88efcf4bb52d" -dependencies = [ - "async-lock 3.1.1", - "async-task", - "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.0.1", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" -dependencies = [ - "async-channel 2.1.0", - "async-executor", - "async-io 2.2.0", - "async-lock 3.1.1", - "blocking", - "futures-lite 2.0.1", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2", - "waker-fn", -] - -[[package]] -name = "async-io" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9d5715c2d329bf1b4da8d60455b99b187f27ba726df2883799af9af60997" -dependencies = [ - "async-lock 3.1.1", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.0.1", - "parking", - "polling 3.3.0", - "rustix 0.38.25", - "slab", - "tracing", - "waker-fn", - "windows-sys 0.48.0", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655b9c7fe787d3b25cc0f804a1a8401790f0c5bc395beb5a64dc77d8de079105" -dependencies = [ - "event-listener 3.1.0", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-process" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" -dependencies = [ - "async-io 1.13.0", - "async-lock 2.8.0", - "async-signal", - "blocking", - "cfg-if", - "event-listener 3.1.0", - "futures-lite 1.13.0", - "rustix 0.38.25", - "windows-sys 0.48.0", -] - -[[package]] -name = "async-signal" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" -dependencies = [ - "async-io 2.2.0", - "async-lock 2.8.0", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix 0.38.25", - "signal-hook-registry", - "slab", - "windows-sys 0.48.0", -] - -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-channel 1.9.0", - "async-global-executor", - "async-io 1.13.0", - "async-lock 2.8.0", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite 1.13.0", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" - [[package]] name = "async-trait" version = "0.1.74" @@ -241,18 +76,27 @@ dependencies = [ "syn", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -260,33 +104,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "bitflags" -version = "2.4.1" +name = "bytes" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] -name = "blocking" -version = "1.5.1" +name = "cc" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ - "async-channel 2.1.0", - "async-lock 3.1.1", - "async-task", - "fastrand 2.0.1", - "futures-io", - "futures-lite 2.0.1", - "piper", - "tracing", + "libc", ] -[[package]] -name = "bumpalo" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" - [[package]] name = "cfg-if" version = "1.0.0" @@ -337,18 +168,15 @@ checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" name = "cli" version = "0.1.0" dependencies = [ - "async-channel 1.9.0", - "async-io 1.13.0", - "async-process", - "async-std", "async-trait", "clap", "console", "fs_extra", "futures", - "futures-lite 1.13.0", + "futures-lite", "indicatif", "lazy_static", + "tokio", ] [[package]] @@ -357,15 +185,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "concurrent-queue" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.15.7" @@ -379,58 +198,12 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - [[package]] name = "encode_unicode" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "errno" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" -dependencies = [ - "event-listener 3.1.0", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -440,12 +213,6 @@ dependencies = [ "instant", ] -[[package]] -name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" - [[package]] name = "fs_extra" version = "1.3.0" @@ -506,7 +273,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand 1.9.0", + "fastrand", "futures-core", "futures-io", "memchr", @@ -515,20 +282,6 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "futures-lite" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" -dependencies = [ - "fastrand 2.0.1", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.29" @@ -571,16 +324,10 @@ dependencies = [ ] [[package]] -name = "gloo-timers" -version = "0.2.6" +name = "gimli" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "heck" @@ -616,35 +363,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "js-sys" -version = "0.3.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -658,31 +376,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] -name = "linux-raw-sys" -version = "0.3.8" +name = "lock_api" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] [[package]] -name = "linux-raw-sys" -version = "0.4.11" +name = "memchr" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] -name = "log" -version = "0.4.20" +name = "miniz_oxide" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ - "value-bag", + "adler", ] [[package]] -name = "memchr" -version = "2.6.4" +name = "mio" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] [[package]] name = "number_prefix" @@ -691,10 +428,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] -name = "once_cell" -version = "1.18.0" +name = "object" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] [[package]] name = "parking" @@ -703,57 +443,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "piper" -version = "0.2.1" +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "atomic-waker", - "fastrand 2.0.1", - "futures-io", + "lock_api", + "parking_lot_core", ] [[package]] -name = "polling" -version = "2.8.0" +name = "parking_lot_core" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ - "autocfg", - "bitflags 1.3.2", "cfg-if", - "concurrent-queue", "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", ] [[package]] -name = "polling" -version = "3.3.0" +name = "pin-project-lite" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53b6af1f60f36f8c2ac2aad5459d75a5a9b4be1e8cdd40264f315d78193e531" -dependencies = [ - "cfg-if", - "concurrent-queue", - "pin-project-lite", - "rustix 0.38.25", - "tracing", - "windows-sys 0.48.0", -] +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" @@ -780,31 +502,25 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "0.37.27" +name = "redox_syscall" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", + "bitflags", ] [[package]] -name = "rustix" -version = "0.38.25" +name = "rustc-demangle" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" -dependencies = [ - "bitflags 2.4.1", - "errno", - "libc", - "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", -] +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "signal-hook-registry" @@ -824,14 +540,20 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + [[package]] name = "socket2" -version = "0.4.10" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -852,20 +574,34 @@ dependencies = [ ] [[package]] -name = "tracing" -version = "0.1.40" +name = "tokio" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", "pin-project-lite", - "tracing-core", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", ] [[package]] -name = "tracing-core" -version = "0.1.32" +name = "tokio-macros" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "unicode-ident" @@ -885,12 +621,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "value-bag" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" - [[package]] name = "waker-fn" version = "1.1.1" @@ -898,102 +628,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] -name = "wasm-bindgen" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" - -[[package]] -name = "web-sys" -version = "0.3.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "windows-sys" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index eb3e7c3b3..2eaa82c76 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,10 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-channel = "1.9.0" -async-io = "1.13.0" -async-process = "1.7.0" -async-std = "1.12.0" async-trait = "0.1.73" clap = { version = "4.4.4", features = ["derive"] } console = "0.15.7" @@ -18,3 +14,4 @@ futures = "0.3.28" futures-lite = "1.13.0" indicatif = "0.17.7" lazy_static = "1.4.0" +tokio = { version = "1.36.0", features = ["full"] } diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 02fa67d2f..8b0247592 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,6 +1,5 @@ extern crate fs_extra; use crate::TRACKER; -use async_std::task::spawn; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; use std::sync::mpsc; use std::{fs, io::Error, path::PathBuf}; @@ -22,7 +21,7 @@ pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); let msg = format!("copied: {} to {}", src.display(), dest.display()); - spawn(async move { + let _ = tokio::spawn(async move { if let Err(e) = copy_with_progress(src, dest, &options, |info| { if tx.send(info).is_err() { eprintln!("Fail to send copying progress"); diff --git a/cli/src/main.rs b/cli/src/main.rs index aea63691a..e1320203b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -63,80 +63,79 @@ enum Command { }, } -fn main() -> Result<(), Error> { - async_io::block_on(async { - let cli = Cli::parse(); - let production = cli.release > 0; - if let Some(ref command) = cli.command { - let targets: Vec> = if let Some(mut list) = - match command.clone() { - Command::Lint { target } => target, - Command::Build { target } => target, - Command::Clean { target } => target, - Command::Test { target } => target, - } { - list.remove_duplicates(); - list.iter().map(|target| target.get()).collect() - } else { - Target::all() - }; - let results = match command { - Command::Lint { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.check()) - .collect::>(), - ) - .await - } - Command::Build { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.build(production)) - .collect::>(), - ) - .await - } - Command::Clean { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.reset()) - .collect::>(), - ) - .await - } - Command::Test { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.test()) - .collect::>(), - ) - .await - } - }; - TRACKER.shutdown().await?; - let mut success: bool = true; - results.iter().for_each(|res| match res { - Ok(status) => { - if !status.status.success() { - eprintln!("Failed with errors"); - println!("{}:\n{}", status.job, status.stderr.join("")); - success = false; - } - } - Err(err) => { - eprintln!("Builder error: {err}"); +#[tokio::main] +async fn main() -> Result<(), Error> { + let cli = Cli::parse(); + let production = cli.release > 0; + if let Some(ref command) = cli.command { + let targets: Vec> = if let Some(mut list) = + match command.clone() { + Command::Lint { target } => target, + Command::Build { target } => target, + Command::Clean { target } => target, + Command::Test { target } => target, + } { + list.remove_duplicates(); + list.iter().map(|target| target.get()).collect() + } else { + Target::all() + }; + let results = match command { + Command::Lint { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.check()) + .collect::>(), + ) + .await + } + Command::Build { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.build(production)) + .collect::>(), + ) + .await + } + Command::Clean { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.reset()) + .collect::>(), + ) + .await + } + Command::Test { target: _ } => { + join_all( + targets + .iter() + .map(|module| module.test()) + .collect::>(), + ) + .await + } + }; + TRACKER.shutdown().await?; + let mut success: bool = true; + results.iter().for_each(|res| match res { + Ok(status) => { + if !status.status.success() { + eprintln!("Failed with errors"); + println!("{}:\n{}", status.job, status.stderr.join("")); success = false; } - }); - if !success { - return Err(Error::new(ErrorKind::Other, "Some task were failed")); } + Err(err) => { + eprintln!("Builder error: {err}"); + success = false; + } + }); + if !success { + return Err(Error::new(ErrorKind::Other, "Some task were failed")); } - Ok(()) - }) + } + Ok(()) } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 1fc19ad7a..9f50426f4 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -3,10 +3,17 @@ use crate::{ location::to_relative_path, {LOCATION, TRACKER}, }; -use async_process::{Command, ExitStatus, Stdio}; -use async_std::{io::BufReader, prelude::*}; use futures_lite::{future, FutureExt}; -use std::{env::vars, io, path::PathBuf}; +use std::{ + env::vars, + io, + path::PathBuf, + process::{ExitStatus, Stdio}, +}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, +}; #[derive(Clone, Debug)] pub struct SpawnResult { @@ -80,15 +87,13 @@ pub async fn spawn( let storage_stdout = &mut stdout_lines; let storage_stderr = &mut stderr_lines; async move { - use futures::{select, FutureExt}; - let mut stdout_buf = BufReader::new(stdout); let mut stderr_buf = BufReader::new(stderr); loop { let mut stdout_line = String::new(); let mut stderr_line = String::new(); - select! { - stdout_read_result = stdout_buf.read_line(&mut stdout_line).fuse() => { + tokio::select! { + stdout_read_result = stdout_buf.read_line(&mut stdout_line) => { let stdout_read_bytes = stdout_read_result?; if stdout_read_bytes == 0 { break; @@ -100,7 +105,7 @@ pub async fn spawn( storage_stdout.push(stdout_line); } } - stderr_read_result = stderr_buf.read_line(&mut stderr_line).fuse() => { + stderr_read_result = stderr_buf.read_line(&mut stderr_line) => { let stderr_read_bytes = stderr_read_result?; if stderr_read_bytes == 0 { break; @@ -121,7 +126,7 @@ pub async fn spawn( }; let status = match drain_stdout_stderr - .or(async move { Ok(Some(child.status().await?)) }) + .or(async move { Ok(Some(child.wait().await?)) }) .await { Ok(status) => status, diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 43770d164..8e664aeb9 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -1,5 +1,3 @@ -use async_channel::{bounded, unbounded, Receiver, Sender}; -use async_std::task; use console::style; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use std::{ @@ -7,6 +5,7 @@ use std::{ io::{Error, ErrorKind}, time::Instant, }; +use tokio::sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender}; const TIME_BAR_WIDTH: usize = 5; @@ -42,7 +41,7 @@ pub enum Tick { #[derive(Clone, Debug)] pub struct Tracker { - tx: Sender, + tx: UnboundedSender, } struct JobBarState { @@ -67,12 +66,12 @@ impl JobBarState { impl Tracker { pub fn new() -> Self { - let (tx, rx): (Sender, Receiver) = unbounded(); - task::spawn(Tracker::run(rx)); + let (tx, rx): (UnboundedSender, UnboundedReceiver) = unbounded_channel(); + tokio::spawn(Tracker::run(rx)); Self { tx } } - pub async fn run(rx: Receiver) -> Result<(), Error> { + pub async fn run(mut rx: UnboundedReceiver) -> Result<(), Error> { let spinner_style = ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}") .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))? .tick_chars("▂▃▅▆▇▆▅▃▂ "); @@ -83,7 +82,7 @@ impl Tracker { let mut bars: HashMap = HashMap::new(); let mp = MultiProgress::new(); let start_time = Instant::now(); - while let Ok(tick) = rx.recv().await { + while let Some(tick) = rx.recv().await { match tick { Tick::Started(job, len, tx_response) => { sequence += 1; @@ -212,74 +211,63 @@ impl Tracker { } pub async fn start(&self, job: &str, max: Option) -> Result { - let (tx_response, rx_response) = bounded(1); + let (tx_response, mut rx_response) = channel(1); self.tx .send(Tick::Started(job.to_string(), max, tx_response)) - .await .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; - rx_response - .recv() - .await - .map_err(|e| Error::new(ErrorKind::NotConnected, e.to_string())) + rx_response.recv().await.ok_or(Error::new( + ErrorKind::NotConnected, + "Failed to receive start response", + )) } pub async fn progress(&self, sequence: usize, pos: Option) { - if let Err(e) = self.tx.send(Tick::Progress(sequence, pos)).await { + if let Err(e) = self.tx.send(Tick::Progress(sequence, pos)) { eprintln!("Fail to communicate with tracker: {e}"); } } pub async fn msg(&self, sequence: usize, log: &str) { - if let Err(e) = self.tx.send(Tick::Message(sequence, log.to_string())).await { + if let Err(e) = self.tx.send(Tick::Message(sequence, log.to_string())) { eprintln!("Fail to communicate with tracker: {e}"); } } pub async fn success(&self, sequence: usize, msg: &str) { - if let Err(e) = self - .tx - .send(Tick::Finished( - sequence, - OperationResult::Success, - msg.to_string(), - )) - .await - { + if let Err(e) = self.tx.send(Tick::Finished( + sequence, + OperationResult::Success, + msg.to_string(), + )) { eprintln!("Fail to communicate with tracker: {e}"); } } pub async fn fail(&self, sequence: usize, msg: &str) { - if let Err(e) = self - .tx - .send(Tick::Finished( - sequence, - OperationResult::Failed, - msg.to_string(), - )) - .await - { + if let Err(e) = self.tx.send(Tick::Finished( + sequence, + OperationResult::Failed, + msg.to_string(), + )) { eprintln!("Fail to communicate with tracker: {e}"); } } pub async fn shutdown(&self) -> Result<(), Error> { - let (tx_response, rx_response) = bounded(1); + let (tx_response, mut rx_response) = channel(1); self.tx .send(Tick::Shutdown(tx_response)) - .await .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; rx_response .recv() .await - .map_err(|e| Error::new(ErrorKind::NotConnected, e.to_string())) + .ok_or_else(|| Error::new(ErrorKind::NotConnected, "test")) } pub async fn _print(&self, msg: String) { if let Err(e) = self .tx .send(Tick::Print(msg)) - .await .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}"))) { eprintln!("Fail to communicate with tracker: {e}"); From a9305be863bf202fe85093c4254a4050642bcc78 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 11 Feb 2024 19:47:04 +0100 Subject: [PATCH 024/174] CLI: replace bounded with oneshot channels - Bounded channels with 1 capacity is replaced with oneshot channels. - Unused Clone is removed from Tick enum because the oneshot channels aren't clone --- cli/src/tracker.rs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 8e664aeb9..991eb7af2 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -5,7 +5,10 @@ use std::{ io::{Error, ErrorKind}, time::Instant, }; -use tokio::sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender}; +use tokio::sync::{ + mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, + oneshot, +}; const TIME_BAR_WIDTH: usize = 5; @@ -28,15 +31,15 @@ impl std::fmt::Display for OperationResult { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub enum Tick { - Started(String, Option, Sender), + Started(String, Option, oneshot::Sender), Progress(usize, Option), Message(usize, String), Finished(usize, OperationResult, String), #[allow(dead_code)] Print(String), - Shutdown(Sender<()>), + Shutdown(oneshot::Sender<()>), } #[derive(Clone, Debug)] @@ -91,7 +94,7 @@ impl Tracker { let job_bar = JobBarState::start_job(job, bar); bars.insert(sequence, job_bar); Self::refresh_all_bars(&mut bars, sequence, max_time_len, None); - if let Err(e) = tx_response.send(sequence).await { + if let Err(e) = tx_response.send(sequence) { let _ = mp.println(format!("Fail to send response: {e}")); } } @@ -154,8 +157,8 @@ impl Tracker { bars.clear(); // let _ = mp.clear(); - if let Err(e) = tx_response.send(()).await { - let _ = mp.println(format!("Fail to send response: {e}")); + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response"); } break; } @@ -211,14 +214,13 @@ impl Tracker { } pub async fn start(&self, job: &str, max: Option) -> Result { - let (tx_response, mut rx_response) = channel(1); + let (tx_response, rx_response) = oneshot::channel(); self.tx .send(Tick::Started(job.to_string(), max, tx_response)) .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; - rx_response.recv().await.ok_or(Error::new( - ErrorKind::NotConnected, - "Failed to receive start response", - )) + rx_response + .await + .map_err(|err| Error::new(ErrorKind::NotConnected, err.to_string())) } pub async fn progress(&self, sequence: usize, pos: Option) { @@ -254,14 +256,13 @@ impl Tracker { } pub async fn shutdown(&self) -> Result<(), Error> { - let (tx_response, mut rx_response) = channel(1); + let (tx_response, rx_response) = oneshot::channel(); self.tx .send(Tick::Shutdown(tx_response)) .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; rx_response - .recv() .await - .ok_or_else(|| Error::new(ErrorKind::NotConnected, "test")) + .map_err(|err| Error::new(ErrorKind::NotConnected, err.to_string())) } pub async fn _print(&self, msg: String) { From 5a09c5878e2bb9cc8eb70f93458ef14f191bb53c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 17 Feb 2024 20:31:46 +0100 Subject: [PATCH 025/174] Remove TODO after clarification - Native Async trait can't be applied in near feature --- cli/src/modules/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index e486a1731..f596b5569 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -184,7 +184,6 @@ pub trait Manager { .await } - // TODO: After using native rust asnyc traits, we can return a vector of results here and use // flat_map in main() to get rid of the double join calls async fn test(&self) -> Result { self.install(false).await?; From 2d2b1f10282a62f47ee1ffceae08fa7f343a7825 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 17 Feb 2024 21:06:38 +0100 Subject: [PATCH 026/174] CLI: Small refactoring using idiomatic rust & TODO --- cli/src/main.rs | 1 + cli/src/modules/mod.rs | 15 +++++---------- cli/src/spawner.rs | 12 ++---------- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index e1320203b..606c2ac58 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -75,6 +75,7 @@ async fn main() -> Result<(), Error> { Command::Clean { target } => target, Command::Test { target } => target, } { + //TODO: Check if we can use dedup() and delete the remove_duplicates() method and trait list.remove_duplicates(); list.iter().map(|target| target.get()).collect() } else { diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index f596b5569..fb461d361 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -131,11 +131,9 @@ pub trait Manager { } } let path = LOCATION.root.clone().join(self.cwd()); - let cmd = if let Some(cmd) = self.build_cmd(prod) { - cmd - } else { - self.kind().build_cmd(prod) - }; + let cmd = self + .build_cmd(prod) + .unwrap_or_else(|| self.kind().build_cmd(prod)); match spawn(&cmd, Some(path), Some(&cmd), None).await { Ok(status) => { if !status.status.success() { @@ -146,11 +144,8 @@ pub trait Manager { self.clean().await?; self.install(prod).await?; } - if let Some(res) = res { - Ok(res) - } else { - Ok(SpawnResult::empty()) - } + + res.map_or(Ok(SpawnResult::empty()), Ok) } } Err(err) => Err(err), diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 9f50426f4..17644f414 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -46,11 +46,7 @@ pub async fn spawn( opts: Option, ) -> Result { let opts = opts.unwrap_or_default(); - let cwd = if let Some(cwd) = cwd { - cwd - } else { - LOCATION.root.clone() - }; + let cwd = cwd.unwrap_or_else(|| LOCATION.root.clone()); let mut parts = command.split(' ').collect::>(); let cmd = parts.remove(0); #[allow(clippy::useless_vec)] @@ -67,11 +63,7 @@ pub async fn spawn( .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; - let job_title = if let Some(caption) = caption { - caption - } else { - cmd - }; + let job_title = caption.unwrap_or(cmd); let sequence = TRACKER .start( &format!("{}: {}", to_relative_path(&cwd).display(), job_title), From 9f0cd6a77c48acf79408d491732e7e454e32215d Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 17 Feb 2024 22:36:59 +0100 Subject: [PATCH 027/174] CLI: Replace remove_duplicates method with dedup - The trait `RemoveDuplicates` is removed in favor of using the dedup method from the standard library --- cli/src/main.rs | 5 +---- cli/src/tools.rs | 17 ----------------- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 cli/src/tools.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index 606c2ac58..463ebb0cc 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,7 +3,6 @@ mod location; mod modules; mod spawner; mod target; -mod tools; mod tracker; use clap::{Parser, Subcommand}; @@ -12,7 +11,6 @@ use location::Location; use modules::Manager; use std::io::{Error, ErrorKind}; use target::Target; -use tools::RemoveDuplicates; use tracker::Tracker; #[macro_use] @@ -75,8 +73,7 @@ async fn main() -> Result<(), Error> { Command::Clean { target } => target, Command::Test { target } => target, } { - //TODO: Check if we can use dedup() and delete the remove_duplicates() method and trait - list.remove_duplicates(); + list.dedup(); list.iter().map(|target| target.get()).collect() } else { Target::all() diff --git a/cli/src/tools.rs b/cli/src/tools.rs deleted file mode 100644 index 049ff219c..000000000 --- a/cli/src/tools.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::target::Target; -use std::collections::HashSet; - -pub trait RemoveDuplicates { - fn remove_duplicates(&mut self); -} - -impl RemoveDuplicates for Vec { - fn remove_duplicates(&mut self) { - let mut seen = HashSet::new(); - self.retain(|c| { - let is_first = !seen.contains(c); - seen.insert(c.clone()); - is_first - }); - } -} From e51c5625b49d6081a9dca3e25f0e88cd4094933e Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 18 Feb 2024 14:39:19 +0100 Subject: [PATCH 028/174] CLI: Change release flag from u8 to bool - Boolean is more representative of the release flag since it has two states only --- cli/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 463ebb0cc..b316edc89 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -26,8 +26,8 @@ lazy_static! { #[command(author, version, about, long_about = None)] struct Cli { /// Build release version - #[arg(short, long, action = clap::ArgAction::Count)] - release: u8, + #[arg(short, long, default_value_t = false)] + release: bool, #[command(subcommand)] command: Option, @@ -64,7 +64,7 @@ enum Command { #[tokio::main] async fn main() -> Result<(), Error> { let cli = Cli::parse(); - let production = cli.release > 0; + let production = cli.release; if let Some(ref command) = cli.command { let targets: Vec> = if let Some(mut list) = match command.clone() { From bc3f4c94176442bd7908a36a0011f7f3e9e56f11 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 18 Feb 2024 15:29:03 +0100 Subject: [PATCH 029/174] CLI: Small Fixes to commands descriptions --- cli/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index b316edc89..a5bad3d6f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -35,27 +35,27 @@ struct Cli { #[derive(Subcommand, Debug, Clone)] enum Command { - /// runs linting & clippy + /// Runs linting & clippy Lint { - /// target to build, by default whole application will be built + /// Target to lint, by default whole application will be linted #[arg(short, long, num_args(0..))] target: Option>, }, - /// build + /// Build Build { - /// target to build, by default whole application will be built + /// Target to build, by default whole application will be built #[arg(short, long, num_args(0..))] target: Option>, }, /// Clean Clean { - /// target to build, by default whole application will be built + /// Target to clean, by default whole application will be cleaned #[arg(short, long, num_args(0..))] target: Option>, }, /// Run tests Test { - /// target to test, by default whole application will be tested + /// Target to test, by default whole application will be tested #[arg(short, long, num_args(0..))] target: Option>, }, From 65b43c70c6f2dfd91e51e9c0c9707a1b6e634ff7 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 18 Feb 2024 21:53:12 +0100 Subject: [PATCH 030/174] Changes on the CLI arguments & Commands... - Targets changed to positional arguments with the index 1, so the users enter their targets directly after the command - Unnecessary clone for the targets removed by creating a methods to extract the targets from the optional arguments - Release flag moved to build command since it's used their only & It's name changed to production to free up the short argument -r for the report - report argument added to CLI --- cli/src/main.rs | 180 +++++++++++++++++++++++++++--------------------- 1 file changed, 103 insertions(+), 77 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index a5bad3d6f..9a622955b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -9,7 +9,10 @@ use clap::{Parser, Subcommand}; use futures::future::join_all; use location::Location; use modules::Manager; -use std::io::{Error, ErrorKind}; +use std::{ + io::{Error, ErrorKind}, + path::PathBuf, +}; use target::Target; use tracker::Tracker; @@ -22,15 +25,15 @@ lazy_static! { static ref TRACKER: Tracker = Tracker::new(); } +static REPORT_HELP_TEXT: &str = + "Write report from command logs to the given file or to stdout if no file is defined"; +static REPORT_VALUE_NAME: &str = "FILE-PATH"; + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Cli { - /// Build release version - #[arg(short, long, default_value_t = false)] - release: bool, - #[command(subcommand)] - command: Option, + command: Command, } #[derive(Subcommand, Debug, Clone)] @@ -38,102 +41,125 @@ enum Command { /// Runs linting & clippy Lint { /// Target to lint, by default whole application will be linted - #[arg(short, long, num_args(0..))] + #[arg(index = 1)] target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, }, /// Build Build { /// Target to build, by default whole application will be built - #[arg(short, long, num_args(0..))] + #[arg(index = 1)] target: Option>, + + /// Build release version + #[arg(short, long, default_value_t = false)] + production: bool, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, }, /// Clean Clean { /// Target to clean, by default whole application will be cleaned - #[arg(short, long, num_args(0..))] + #[arg(index = 1)] target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, }, /// Run tests Test { /// Target to test, by default whole application will be tested - #[arg(short, long, num_args(0..))] + #[arg(index = 1)] target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, }, } #[tokio::main] async fn main() -> Result<(), Error> { let cli = Cli::parse(); - let production = cli.release; - if let Some(ref command) = cli.command { - let targets: Vec> = if let Some(mut list) = - match command.clone() { - Command::Lint { target } => target, - Command::Build { target } => target, - Command::Clean { target } => target, - Command::Test { target } => target, - } { - list.dedup(); - list.iter().map(|target| target.get()).collect() - } else { - Target::all() - }; - let results = match command { - Command::Lint { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.check()) - .collect::>(), - ) - .await - } - Command::Build { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.build(production)) - .collect::>(), - ) - .await - } - Command::Clean { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.reset()) - .collect::>(), - ) - .await - } - Command::Test { target: _ } => { - join_all( - targets - .iter() - .map(|module| module.test()) - .collect::>(), - ) - .await - } - }; - TRACKER.shutdown().await?; - let mut success: bool = true; - results.iter().for_each(|res| match res { - Ok(status) => { - if !status.status.success() { - eprintln!("Failed with errors"); - println!("{}:\n{}", status.job, status.stderr.join("")); - success = false; - } - } - Err(err) => { - eprintln!("Builder error: {err}"); + let command = cli.command; + let results = match command { + Command::Lint { target, report } => { + dbg!(report); + let targets = get_targets_or_default(target); + join_all( + targets + .iter() + .map(|module| module.check()) + .collect::>(), + ) + .await + } + Command::Build { + target, + production, + report, + } => { + dbg!(report); + let targets = get_targets_or_default(target); + join_all( + targets + .iter() + .map(|module| module.build(production)) + .collect::>(), + ) + .await + } + Command::Clean { target, report } => { + dbg!(report); + let targets = get_targets_or_default(target); + join_all( + targets + .iter() + .map(|module| module.reset()) + .collect::>(), + ) + .await + } + Command::Test { target, report } => { + dbg!(report); + let targets = get_targets_or_default(target); + join_all( + targets + .iter() + .map(|module| module.test()) + .collect::>(), + ) + .await + } + }; + TRACKER.shutdown().await?; + let mut success: bool = true; + results.iter().for_each(|res| match res { + Ok(status) => { + if !status.status.success() { + eprintln!("Failed with errors"); + println!("{}:\n{}", status.job, status.stderr.join("")); success = false; } - }); - if !success { - return Err(Error::new(ErrorKind::Other, "Some task were failed")); } + Err(err) => { + eprintln!("Builder error: {err}"); + success = false; + } + }); + if !success { + return Err(Error::new(ErrorKind::Other, "Some task were failed")); } Ok(()) } + +fn get_targets_or_default(targets: Option>) -> Vec> { + if let Some(mut list) = targets { + list.dedup(); + list.iter().map(|target| target.get()).collect() + } else { + Target::all() + } +} From 01385333deaf55eabafc21179c714e40f394fc91 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 19 Feb 2024 08:02:30 +0100 Subject: [PATCH 031/174] CLI: Printing command report (in progress)... - Combine stdout and stderr logs to keep the order of the logs in the report. - Handle report argument and integrate it in the system. - Writing the report isn't implemented yet --- cli/src/main.rs | 40 +++++++++++++++++++++++++++++++++++----- cli/src/spawner.rs | 19 +++++++------------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 9a622955b..487cafc25 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -36,6 +36,13 @@ struct Cli { command: Command, } +#[derive(Debug, Clone)] +enum ReportOptions { + None, + Stdout, + File(PathBuf), +} + #[derive(Subcommand, Debug, Clone)] enum Command { /// Runs linting & clippy @@ -84,9 +91,10 @@ enum Command { async fn main() -> Result<(), Error> { let cli = Cli::parse(); let command = cli.command; + let mut report_opt = ReportOptions::None; let results = match command { Command::Lint { target, report } => { - dbg!(report); + report_opt = get_report_option(report); let targets = get_targets_or_default(target); join_all( targets @@ -112,7 +120,7 @@ async fn main() -> Result<(), Error> { .await } Command::Clean { target, report } => { - dbg!(report); + report_opt = get_report_option(report); let targets = get_targets_or_default(target); join_all( targets @@ -123,7 +131,7 @@ async fn main() -> Result<(), Error> { .await } Command::Test { target, report } => { - dbg!(report); + report_opt = get_report_option(report); let targets = get_targets_or_default(target); join_all( targets @@ -138,9 +146,23 @@ async fn main() -> Result<(), Error> { let mut success: bool = true; results.iter().for_each(|res| match res { Ok(status) => { + let print_err = match &report_opt { + ReportOptions::None => true, + ReportOptions::Stdout => { + //TODO: write to stdout + false + } + ReportOptions::File(_file_path) => { + // TODO: write to file. + false + } + }; + if !status.status.success() { - eprintln!("Failed with errors"); - println!("{}:\n{}", status.job, status.stderr.join("")); + if print_err { + eprintln!("Failed with errors"); + eprintln!("{}:\n{}", status.job, status.report.join("")); + } success = false; } } @@ -163,3 +185,11 @@ fn get_targets_or_default(targets: Option>) -> Vec>) -> ReportOptions { + match report_argument { + None => ReportOptions::None, + Some(None) => ReportOptions::Stdout, + Some(Some(path)) => ReportOptions::File(path), + } +} diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 17644f414..2b454e046 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -17,8 +17,7 @@ use tokio::{ #[derive(Clone, Debug)] pub struct SpawnResult { - pub stdout: Vec, - pub stderr: Vec, + pub report: Vec, pub status: ExitStatus, pub job: String, } @@ -26,8 +25,7 @@ pub struct SpawnResult { impl SpawnResult { pub fn empty() -> Self { SpawnResult { - stdout: vec![], - stderr: vec![], + report: Vec::default(), status: ExitStatus::default(), job: String::new(), } @@ -71,13 +69,11 @@ pub async fn spawn( ) .await?; - let mut stdout_lines: Vec = vec![]; - let mut stderr_lines: Vec = vec![]; + let mut report_lines: Vec = vec![]; let drain_stdout_stderr = { let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); - let storage_stdout = &mut stdout_lines; - let storage_stderr = &mut stderr_lines; + let storage_report = &mut report_lines; async move { let mut stdout_buf = BufReader::new(stdout); let mut stderr_buf = BufReader::new(stderr); @@ -94,7 +90,7 @@ pub async fn spawn( TRACKER.msg(sequence, &stdout_line).await; } TRACKER.progress(sequence, None).await; - storage_stdout.push(stdout_line); + storage_report.push(stdout_line); } } stderr_read_result = stderr_buf.read_line(&mut stderr_line) => { @@ -104,7 +100,7 @@ pub async fn spawn( } else { TRACKER.progress(sequence, None).await; if !stderr_line.trim().is_empty() { - storage_stderr.push(stderr_line); + storage_report.push(stderr_line); } } @@ -134,8 +130,7 @@ pub async fn spawn( TRACKER.fail(sequence, "finished with errors").await; } Ok(SpawnResult { - stdout: stdout_lines, - stderr: stderr_lines, + report: report_lines, status, job: job_title.to_string(), }) From 2be551bdee1e08235dab66062475e2dc01a46b4c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 19 Feb 2024 22:55:06 +0100 Subject: [PATCH 032/174] CLI: Writing logs report (in progress) - Report is written to stdout or a file - Message will be printed with the file full path in case of writing the report to a file. - Some infos are still missing from the report --- cli/src/main.rs | 100 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 487cafc25..09da22646 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -9,8 +9,10 @@ use clap::{Parser, Subcommand}; use futures::future::join_all; use location::Location; use modules::Manager; +use spawner::SpawnResult; use std::{ - io::{Error, ErrorKind}, + fs::File, + io::{self, stdout, Error, ErrorKind, Stdout}, path::PathBuf, }; use target::Target; @@ -36,11 +38,11 @@ struct Cli { command: Command, } -#[derive(Debug, Clone)] +#[derive(Debug)] enum ReportOptions { None, - Stdout, - File(PathBuf), + Stdout(Stdout), + File(PathBuf, File), } #[derive(Subcommand, Debug, Clone)] @@ -94,7 +96,7 @@ async fn main() -> Result<(), Error> { let mut report_opt = ReportOptions::None; let results = match command { Command::Lint { target, report } => { - report_opt = get_report_option(report); + report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets @@ -120,7 +122,7 @@ async fn main() -> Result<(), Error> { .await } Command::Clean { target, report } => { - report_opt = get_report_option(report); + report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets @@ -131,7 +133,7 @@ async fn main() -> Result<(), Error> { .await } Command::Test { target, report } => { - report_opt = get_report_option(report); + report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets @@ -144,33 +146,39 @@ async fn main() -> Result<(), Error> { }; TRACKER.shutdown().await?; let mut success: bool = true; - results.iter().for_each(|res| match res { - Ok(status) => { - let print_err = match &report_opt { - ReportOptions::None => true, - ReportOptions::Stdout => { - //TODO: write to stdout - false - } - ReportOptions::File(_file_path) => { - // TODO: write to file. - false - } - }; - - if !status.status.success() { - if print_err { - eprintln!("Failed with errors"); - eprintln!("{}:\n{}", status.job, status.report.join("")); + for (idx, res) in results.iter().enumerate() { + match res { + Ok(status) => { + let print_err = match &report_opt { + ReportOptions::None => true, + ReportOptions::Stdout(stdout) => { + write_report(status, stdout)?; + false + } + ReportOptions::File(path, file) => { + write_report(status, file)?; + if idx == results.len() - 1 { + let full_path = path.canonicalize().unwrap_or_else(|_| path.to_owned()); + println!("Report is written to '{}'", full_path.display()); + } + false + } + }; + + if !status.status.success() { + if print_err { + eprintln!("Failed with errors"); + eprintln!("{}:\n{}", status.job, status.report.join("")); + } + success = false; } + } + Err(err) => { + eprintln!("Builder error: {err}"); success = false; } } - Err(err) => { - eprintln!("Builder error: {err}"); - success = false; - } - }); + } if !success { return Err(Error::new(ErrorKind::Other, "Some task were failed")); } @@ -186,10 +194,34 @@ fn get_targets_or_default(targets: Option>) -> Vec>) -> ReportOptions { +fn get_report_option(report_argument: Option>) -> Result { match report_argument { - None => ReportOptions::None, - Some(None) => ReportOptions::Stdout, - Some(Some(path)) => ReportOptions::File(path), + None => Ok(ReportOptions::None), + Some(None) => Ok(ReportOptions::Stdout(stdout())), + Some(Some(path)) => { + let file = File::create(&path)?; + Ok(ReportOptions::File(path, file)) + } } } + +fn write_report(spawn_result: &SpawnResult, mut sink: impl io::Write) -> Result<(), io::Error> { + let status = if spawn_result.status.success() { + "succeeded" + } else { + "failed" + }; + + writeln!(sink, "Job {} has {status}", spawn_result.job)?; + writeln!(sink, "Logs:")?; + + for line in spawn_result.report.iter() { + sink.write_all(line.as_bytes())?; + } + + writeln!(sink)?; + writeln!(sink, "====================================================")?; + writeln!(sink)?; + + Ok(()) +} From 061e3fe040a2476ebb1c1edf6a5aeea72c01cd97 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 20 Feb 2024 19:54:50 +0100 Subject: [PATCH 033/174] CLI: Skip empty results from the report --- cli/src/main.rs | 10 ++++++++-- cli/src/spawner.rs | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 09da22646..4d2845d17 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -152,11 +152,15 @@ async fn main() -> Result<(), Error> { let print_err = match &report_opt { ReportOptions::None => true, ReportOptions::Stdout(stdout) => { - write_report(status, stdout)?; + if !status.is_empty() { + write_report(status, stdout)?; + } false } ReportOptions::File(path, file) => { - write_report(status, file)?; + if !status.is_empty() { + write_report(status, file)?; + } if idx == results.len() - 1 { let full_path = path.canonicalize().unwrap_or_else(|_| path.to_owned()); println!("Report is written to '{}'", full_path.display()); @@ -206,6 +210,8 @@ fn get_report_option(report_argument: Option>) -> Result Result<(), io::Error> { + assert!(!spawn_result.is_empty()); + let status = if spawn_result.status.success() { "succeeded" } else { diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 2b454e046..7695f6941 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -30,6 +30,13 @@ impl SpawnResult { job: String::new(), } } + + pub fn is_empty(&self) -> bool { + self.report.is_empty() && + self.job.is_empty() && + // Default status indicates successful completion + self.status.success() + } } #[derive(Debug, Clone, Default)] From d17ca3e5627d91b10ad3422d9273747ae7efd0ef Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 20 Feb 2024 20:55:23 +0100 Subject: [PATCH 034/174] CLI: Improve Report Information and Job Titles - Give a suitable title for each job depending on the target and the job itself. - Deliver the command in spawn result to use it in the report --- cli/src/main.rs | 20 ++++++++++++-------- cli/src/modules/mod.rs | 20 ++++++++++++++------ cli/src/spawner.rs | 12 ++++++++---- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 4d2845d17..3efb37894 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -209,25 +209,29 @@ fn get_report_option(report_argument: Option>) -> Result Result<(), io::Error> { +fn write_report(spawn_result: &SpawnResult, mut writer: impl io::Write) -> Result<(), io::Error> { assert!(!spawn_result.is_empty()); + writeln!(writer)?; + writeln!( + writer, + "====================================================" + )?; + writeln!(writer)?; + let status = if spawn_result.status.success() { "succeeded" } else { "failed" }; - writeln!(sink, "Job {} has {status}", spawn_result.job)?; - writeln!(sink, "Logs:")?; + writeln!(writer, "Job '{}' has {status}", spawn_result.job)?; + writeln!(writer, "Command: {}", spawn_result.cmd)?; + writeln!(writer, "Logs:")?; for line in spawn_result.report.iter() { - sink.write_all(line.as_bytes())?; + writer.write_all(line.as_bytes())?; } - writeln!(sink)?; - writeln!(sink, "====================================================")?; - writeln!(sink)?; - Ok(()) } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index fb461d361..44d81b255 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -100,7 +100,8 @@ pub trait Manager { self.kind().install_cmd(prod) }; if let Some(cmd) = cmd { - spawn(&cmd, Some(self.cwd()), Some(&cmd), None).await + let caption = format!("Install {}", self.owner()); + spawn(&cmd, Some(self.cwd()), caption, None).await } else { Ok(SpawnResult::empty()) } @@ -134,7 +135,8 @@ pub trait Manager { let cmd = self .build_cmd(prod) .unwrap_or_else(|| self.kind().build_cmd(prod)); - match spawn(&cmd, Some(path), Some(&cmd), None).await { + let caption = format!("Bulid {}", self.owner()); + match spawn(&cmd, Some(path), caption, None).await { Ok(status) => { if !status.status.success() { Ok(status) @@ -162,18 +164,23 @@ pub trait Manager { } async fn lint(&self) -> Result { let path = LOCATION.root.clone().join(self.cwd()); - let status = spawn("yarn run lint", Some(path.clone()), Some("linting"), None).await?; + let caption = format!("TS Lint {}", self.owner()); + let status = spawn("yarn run lint", Some(path.clone()), caption, None).await?; if !status.status.success() { return Ok(status); } - spawn("yarn run build", Some(path), Some("TS compilation"), None).await + + let caption = format!("Build {}", self.owner()); + spawn("yarn run build", Some(path), caption, None).await } async fn clippy(&self) -> Result { let path = LOCATION.root.clone().join(self.cwd()); + + let caption = format!("Clippy {}", self.owner()); spawn( "cargo clippy --color always --all --all-features -- -D warnings", Some(path), - Some("clippy"), + caption, None, ) .await @@ -189,11 +196,12 @@ pub trait Manager { return Ok(SpawnResult::empty()); } + let caption = format!("Test {}", self.owner()); let results = join_all(test_cmds.iter().map(|cmd| { spawn( &cmd.command, Some(cmd.cwd.to_owned()), - Some(&cmd.command), + caption.clone(), cmd.spawn_opts.clone(), ) })) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 7695f6941..214201194 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -20,6 +20,7 @@ pub struct SpawnResult { pub report: Vec, pub status: ExitStatus, pub job: String, + pub cmd: String, } impl SpawnResult { @@ -28,14 +29,16 @@ impl SpawnResult { report: Vec::default(), status: ExitStatus::default(), job: String::new(), + cmd: String::default(), } } pub fn is_empty(&self) -> bool { self.report.is_empty() && self.job.is_empty() && + self.cmd.is_empty() && // Default status indicates successful completion - self.status.success() + self.status.success() } } @@ -47,7 +50,7 @@ pub(crate) struct SpawnOptions { pub async fn spawn( command: &str, cwd: Option, - caption: Option<&str>, + caption: String, opts: Option, ) -> Result { let opts = opts.unwrap_or_default(); @@ -68,7 +71,7 @@ pub async fn spawn( .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; - let job_title = caption.unwrap_or(cmd); + let job_title = caption; let sequence = TRACKER .start( &format!("{}: {}", to_relative_path(&cwd).display(), job_title), @@ -139,7 +142,8 @@ pub async fn spawn( Ok(SpawnResult { report: report_lines, status, - job: job_title.to_string(), + job: job_title, + cmd: command.to_owned(), }) } else { TRACKER From a465a6954eae2a4e3821325237c5974ca2542660 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 20 Feb 2024 21:26:49 +0100 Subject: [PATCH 035/174] CLI: Fix & Improve Report for build command - Report Option wasn't defined in the build case - Implement method to merge two spawn results together for the case with calling after() after build to merge the results together --- cli/src/main.rs | 4 ++-- cli/src/modules/mod.rs | 8 ++++++-- cli/src/spawner.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 3efb37894..5e521f0ad 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -93,7 +93,7 @@ enum Command { async fn main() -> Result<(), Error> { let cli = Cli::parse(); let command = cli.command; - let mut report_opt = ReportOptions::None; + let report_opt: ReportOptions; let results = match command { Command::Lint { target, report } => { report_opt = get_report_option(report)?; @@ -111,7 +111,7 @@ async fn main() -> Result<(), Error> { production, report, } => { - dbg!(report); + report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 44d81b255..fd46df93e 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -137,7 +137,7 @@ pub trait Manager { .unwrap_or_else(|| self.kind().build_cmd(prod)); let caption = format!("Bulid {}", self.owner()); match spawn(&cmd, Some(path), caption, None).await { - Ok(status) => { + Ok(mut status) => { if !status.status.success() { Ok(status) } else { @@ -147,7 +147,11 @@ pub trait Manager { self.install(prod).await?; } - res.map_or(Ok(SpawnResult::empty()), Ok) + if let Some(res_after) = res { + status.merge_with(res_after); + } + + Ok(status) } } Err(err) => Err(err), diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 214201194..b31fd7255 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -40,6 +40,35 @@ impl SpawnResult { // Default status indicates successful completion self.status.success() } + + pub fn merge_with(&mut self, other: SpawnResult) { + if other.is_empty() { + return; + } + + if self.is_empty() { + *self = other; + return; + } + + self.report.push(String::from("-------------------------------------------")); + self.report.extend(other.report); + + self.job.push_str(" & "); + self.job.push_str(other.job.as_str()); + + self.cmd.push_str(" & "); + self.cmd.push_str(other.cmd.as_str()); + + // The failed status is the relevant one. + if !self.status.success() { + return; + } + + if !other.status.success() { + self.status = other.status; + } + } } #[derive(Debug, Clone, Default)] From d0ba969cc4f1b155bf8bfb55bf22da6fc5aace27 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 21 Feb 2024 11:58:39 +0100 Subject: [PATCH 036/174] CLI: Improve finding repo root directory Use Git to find the root directory and perform checks on the project structures. --- cli/Cargo.lock | 176 +++++++++++++++++++++++++++++++++++++++++++- cli/Cargo.toml | 1 + cli/src/location.rs | 35 ++++++--- 3 files changed, 201 insertions(+), 11 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 946505a72..590192524 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -103,6 +103,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + [[package]] name = "bytes" version = "1.5.0" @@ -115,6 +121,7 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -174,6 +181,7 @@ dependencies = [ "fs_extra", "futures", "futures-lite", + "git2", "indicatif", "lazy_static", "tokio", @@ -213,6 +221,15 @@ dependencies = [ "instant", ] +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -329,6 +346,21 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "git2" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" +dependencies = [ + "bitflags 2.4.2", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "heck" version = "0.4.1" @@ -341,6 +373,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indicatif" version = "0.17.7" @@ -363,6 +405,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -375,6 +426,46 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.11" @@ -385,6 +476,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + [[package]] name = "memchr" version = "2.6.4" @@ -436,6 +533,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae94056a791d0e1217d18b6cbdccb02c61e3054fc69893607f4067e3bb0b1fd1" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.0" @@ -465,6 +580,12 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -477,6 +598,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "portable-atomic" version = "1.5.1" @@ -507,7 +634,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -573,6 +700,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.36.0" @@ -603,24 +745,56 @@ dependencies = [ "syn", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "waker-fn" version = "1.1.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2eaa82c76..bfffadec2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,6 +12,7 @@ console = "0.15.7" fs_extra = "1.3.0" futures = "0.3.28" futures-lite = "1.13.0" +git2 = "0.18.2" indicatif = "0.17.7" lazy_static = "1.4.0" tokio = { version = "1.36.0", features = ["full"] } diff --git a/cli/src/location.rs b/cli/src/location.rs index da44b1214..e3245452e 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,3 +1,5 @@ +use git2::Repository; + use crate::LOCATION; use std::path::Path; use std::{ @@ -13,18 +15,31 @@ pub struct Location { impl Location { pub fn new() -> Result { - let mut root = current_dir()?; - // TODO: better compare folders stucts or some file, like some git config file - while !root.ends_with("chipmunk") && !root.ends_with("logviewer") { - if !root.pop() { - return Err(Error::new( - ErrorKind::NotFound, - "Fail to find project's root location", - )); + let current_dir = current_dir()?; + let root: PathBuf = match Repository::discover(current_dir) { + Ok(repo) => { + let Some(root) = repo.workdir() else { + return Err(Error::new( + ErrorKind::NotFound, + "Fail to find project's root location", + )); + }; + + root.into() } - } + Err(err) => return Err(Error::new(ErrorKind::NotFound, err)), + }; - Ok(Self { root }) + // Make sure we are in the chipmunk repository + // Note: This check will fail if the structure of the repo changes + if root.join("application").is_dir() && root.join("developing").is_dir() { + Ok(Self { root }) + } else { + Err(Error::new( + ErrorKind::NotFound, + "Fail to find project's root location", + )) + } } } From 39835cfcfaa6bd7c5dd0318120fed3445e133c96 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 21 Feb 2024 21:16:16 +0100 Subject: [PATCH 037/174] CLI: Replace lazy static with OnceCell for Location - tokio::OnceCell provides a way to return an error if the app is launched outside of chipmunk repo - New implementation saved us some clones for the root PathBuf --- cli/src/location.rs | 28 ++++++++++++++++++++++++++-- cli/src/main.rs | 7 ++++--- cli/src/modules/app.rs | 4 ++-- cli/src/modules/binding.rs | 4 ++-- cli/src/modules/cli.rs | 4 ++-- cli/src/modules/client.rs | 14 +++++++------- cli/src/modules/core.rs | 4 ++-- cli/src/modules/mod.rs | 9 +++++---- cli/src/modules/shared.rs | 4 ++-- cli/src/modules/wasm.rs | 4 ++-- cli/src/modules/wrapper.rs | 4 ++-- cli/src/spawner.rs | 6 +++--- 12 files changed, 59 insertions(+), 33 deletions(-) diff --git a/cli/src/location.rs b/cli/src/location.rs index e3245452e..d10bbce00 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,6 +1,5 @@ use git2::Repository; -use crate::LOCATION; use std::path::Path; use std::{ env::current_dir, @@ -8,6 +7,10 @@ use std::{ path::PathBuf, }; +use tokio::sync::OnceCell; + +pub static LOCATION: OnceCell = OnceCell::const_new(); + #[derive(Clone, Debug)] pub struct Location { pub root: PathBuf, @@ -43,6 +46,27 @@ impl Location { } } +/// Get the path of the root repository +pub fn get_root() -> &'static PathBuf { + &LOCATION + .get() + .expect("Location is initialized in main function") + .root +} + +/// Initial location instance to get the path of the root repository +/// return `Error` If the program isn't invoked inside chipmunk repository +pub fn init_location() -> Result<(), Error> { + assert!(LOCATION.get().is_none()); + + let location = Location::new()?; + LOCATION + .set(location) + .expect("init location can't be called more than once"); + Ok(()) +} + pub fn to_relative_path(path: &PathBuf) -> &Path { - path.strip_prefix(&LOCATION.root).unwrap_or(path) + let root = get_root(); + path.strip_prefix(root).unwrap_or(path) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 5e521f0ad..bde729eb7 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -7,7 +7,7 @@ mod tracker; use clap::{Parser, Subcommand}; use futures::future::join_all; -use location::Location; +use location::init_location; use modules::Manager; use spawner::SpawnResult; use std::{ @@ -22,8 +22,6 @@ use tracker::Tracker; extern crate lazy_static; lazy_static! { - static ref LOCATION: Location = - Location::new().expect("Fail to setup location of root of project"); static ref TRACKER: Tracker = Tracker::new(); } @@ -92,6 +90,9 @@ enum Command { #[tokio::main] async fn main() -> Result<(), Error> { let cli = Cli::parse(); + + init_location()?; + let command = cli.command; let report_opt: ReportOptions; let results = match command { diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index e1198b658..437ba7c97 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{fstools, spawner::SpawnResult, Target, LOCATION}; +use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use async_trait::async_trait; use std::{ fs, @@ -27,7 +27,7 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![Target::Shared, Target::Wrapper, Target::Client] diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 803366359..efed3ae72 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,7 +23,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index f0e16fbb1..3e67d9d82 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,7 +23,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 978791a1c..f798be369 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,16 +23,16 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![Target::Shared, Target::Wasm] } fn dist_path(&self, prod: bool) -> Option { - Some(LOCATION.root.clone().join(PATH).join(if prod { - "dist/release" - } else { - "dist/debug" - })) + Some( + get_root() + .join(PATH) + .join(if prod { "dist/release" } else { "dist/debug" }), + ) } } diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index 83b4ce6f2..aa9918826 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,7 +23,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index fd46df93e..48e74a795 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -9,8 +9,9 @@ pub mod wrapper; use crate::{ fstools, + location::get_root, spawner::{spawn, SpawnOptions, SpawnResult}, - Target, LOCATION, + Target, }; use async_trait::async_trait; use futures::future::join_all; @@ -131,7 +132,7 @@ pub trait Manager { return Ok(status); } } - let path = LOCATION.root.clone().join(self.cwd()); + let path = get_root().join(self.cwd()); let cmd = self .build_cmd(prod) .unwrap_or_else(|| self.kind().build_cmd(prod)); @@ -167,7 +168,7 @@ pub trait Manager { } } async fn lint(&self) -> Result { - let path = LOCATION.root.clone().join(self.cwd()); + let path = get_root().join(self.cwd()); let caption = format!("TS Lint {}", self.owner()); let status = spawn("yarn run lint", Some(path.clone()), caption, None).await?; if !status.status.success() { @@ -178,7 +179,7 @@ pub trait Manager { spawn("yarn run build", Some(path), caption, None).await } async fn clippy(&self) -> Result { - let path = LOCATION.root.clone().join(self.cwd()); + let path = get_root().join(self.cwd()); let caption = format!("Clippy {}", self.owner()); spawn( diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index 686c4f1d0..2d4394d86 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{Target, LOCATION}; +use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,7 +23,7 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 8676a707f..6c9ecca16 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager, TestCommand}; -use crate::{spawner::SpawnOptions, Target, LOCATION}; +use crate::{location::get_root, spawner::SpawnOptions, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -23,7 +23,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 024b71d69..e50729484 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{fstools, spawner::SpawnResult, Target, LOCATION}; +use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use async_trait::async_trait; use std::{ fs, @@ -27,7 +27,7 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - LOCATION.root.clone().join(PATH) + get_root().join(PATH) } fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index b31fd7255..da08bee92 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,7 +1,7 @@ // cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); use crate::{ - location::to_relative_path, - {LOCATION, TRACKER}, + location::{to_relative_path, get_root}, + TRACKER, }; use futures_lite::{future, FutureExt}; use std::{ @@ -83,7 +83,7 @@ pub async fn spawn( opts: Option, ) -> Result { let opts = opts.unwrap_or_default(); - let cwd = cwd.unwrap_or_else(|| LOCATION.root.clone()); + let cwd = cwd.unwrap_or_else(|| get_root().clone()); let mut parts = command.split(' ').collect::>(); let cmd = parts.remove(0); #[allow(clippy::useless_vec)] From 4d33763bee0fd6bbf7a540267a641dd416b4b4d4 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 21 Feb 2024 22:21:35 +0100 Subject: [PATCH 038/174] CLI: Remove Lazy-Statics and Replace it with OnceCell - tokio::OnceCell provides the same functionality of lazy static reducing the dependencies - Tracker was the last stuct depending on lazy-statics and it's replaced now with OnceCell from tokio --- cli/Cargo.lock | 1 - cli/Cargo.toml | 1 - cli/src/fstools.rs | 22 +++++++++++++--------- cli/src/main.rs | 13 ++++--------- cli/src/spawner.rs | 22 ++++++++++------------ cli/src/tracker.rs | 8 +++++++- 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 590192524..072df8422 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -183,7 +183,6 @@ dependencies = [ "futures-lite", "git2", "indicatif", - "lazy_static", "tokio", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bfffadec2..2eca226cb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -14,5 +14,4 @@ futures = "0.3.28" futures-lite = "1.13.0" git2 = "0.18.2" indicatif = "0.17.7" -lazy_static = "1.4.0" tokio = { version = "1.36.0", features = ["full"] } diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 8b0247592..134223211 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,13 +1,15 @@ extern crate fs_extra; -use crate::TRACKER; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; use std::sync::mpsc; use std::{fs, io::Error, path::PathBuf}; +use crate::tracker::get_tracker; + pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { - let sequence = TRACKER.start("copy file", None).await?; + let tracker = get_tracker().await; + let sequence = tracker.start("copy file", None).await?; fs::copy(&src, &dest)?; - TRACKER + tracker .success( sequence, &format!("copied: {} to {}", src.display(), dest.display()), @@ -17,7 +19,8 @@ pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { } pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { - let sequence = TRACKER.start("copy folder", None).await?; + let tracker = get_tracker().await; + let sequence = tracker.start("copy folder", None).await?; let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); let msg = format!("copied: {} to {}", src.display(), dest.display()); @@ -33,7 +36,7 @@ pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { }) .await; while let Ok(info) = rx.recv() { - TRACKER + tracker .msg( sequence, &format!( @@ -42,9 +45,9 @@ pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { ), ) .await; - TRACKER.progress(sequence, None).await; + tracker.progress(sequence, None).await; } - TRACKER.success(sequence, &msg).await; + tracker.success(sequence, &msg).await; Ok(()) } @@ -52,9 +55,10 @@ pub async fn rm_folder(path: PathBuf) -> Result<(), Error> { if !path.exists() { return Ok(()); } - let sequence = TRACKER.start("remove folder", None).await?; + let tracker = get_tracker().await; + let sequence = tracker.start("remove folder", None).await?; fs::remove_dir_all(&path)?; - TRACKER + tracker .success(sequence, &format!("removed: {}", path.display(),)) .await; Ok(()) diff --git a/cli/src/main.rs b/cli/src/main.rs index bde729eb7..ee1942178 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -16,14 +16,7 @@ use std::{ path::PathBuf, }; use target::Target; -use tracker::Tracker; - -#[macro_use] -extern crate lazy_static; - -lazy_static! { - static ref TRACKER: Tracker = Tracker::new(); -} +use tracker::get_tracker; static REPORT_HELP_TEXT: &str = "Write report from command logs to the given file or to stdout if no file is defined"; @@ -145,7 +138,9 @@ async fn main() -> Result<(), Error> { .await } }; - TRACKER.shutdown().await?; + + let tracker = get_tracker().await; + tracker.shutdown().await?; let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index da08bee92..8e8cb9252 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,8 +1,5 @@ // cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); -use crate::{ - location::{to_relative_path, get_root}, - TRACKER, -}; +use crate::{ location::{to_relative_path, get_root}, tracker::get_tracker}; use futures_lite::{future, FutureExt}; use std::{ env::vars, @@ -101,7 +98,8 @@ pub async fn spawn( .stderr(Stdio::piped()) .spawn()?; let job_title = caption; - let sequence = TRACKER + let tracker = get_tracker().await; + let sequence = tracker .start( &format!("{}: {}", to_relative_path(&cwd).display(), job_title), None, @@ -126,9 +124,9 @@ pub async fn spawn( break; } else { if !opts.suppress_msg { - TRACKER.msg(sequence, &stdout_line).await; + tracker.msg(sequence, &stdout_line).await; } - TRACKER.progress(sequence, None).await; + tracker.progress(sequence, None).await; storage_report.push(stdout_line); } } @@ -137,7 +135,7 @@ pub async fn spawn( if stderr_read_bytes == 0 { break; } else { - TRACKER.progress(sequence, None).await; + tracker.progress(sequence, None).await; if !stderr_line.trim().is_empty() { storage_report.push(stderr_line); } @@ -158,15 +156,15 @@ pub async fn spawn( { Ok(status) => status, Err(err) => { - TRACKER.fail(sequence, &err.to_string()).await; + tracker.fail(sequence, &err.to_string()).await; return Err(err); } }; if let Some(status) = status { if status.success() { - TRACKER.success(sequence, "").await; + tracker.success(sequence, "").await; } else { - TRACKER.fail(sequence, "finished with errors").await; + tracker.fail(sequence, "finished with errors").await; } Ok(SpawnResult { report: report_lines, @@ -175,7 +173,7 @@ pub async fn spawn( cmd: command.to_owned(), }) } else { - TRACKER + tracker .fail(sequence, "Fail to get exist status of spawned command") .await; Err(io::Error::new( diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 991eb7af2..657c6e2f1 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -7,7 +7,7 @@ use std::{ }; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, - oneshot, + oneshot, OnceCell, }; const TIME_BAR_WIDTH: usize = 5; @@ -67,6 +67,12 @@ impl JobBarState { } } +pub async fn get_tracker() -> &'static Tracker { + static TRACKER: OnceCell = OnceCell::const_new(); + + TRACKER.get_or_init(|| async { Tracker::new() }).await +} + impl Tracker { pub fn new() -> Self { let (tx, rx): (UnboundedSender, UnboundedReceiver) = unbounded_channel(); From 5f2c3a7474eff9581975e6dfa14634bbdeffa3db Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 24 Feb 2024 10:20:49 +0100 Subject: [PATCH 039/174] CLI: Replace io::Error with anyhow::Error - Error messages in Location were also improved --- cli/Cargo.lock | 7 +++++++ cli/Cargo.toml | 1 + cli/src/fstools.rs | 3 ++- cli/src/location.rs | 30 ++++++++---------------------- cli/src/main.rs | 5 +++-- cli/src/modules/app.rs | 28 +++++++++++----------------- cli/src/modules/mod.rs | 3 ++- cli/src/modules/wrapper.rs | 12 +++--------- cli/src/spawner.rs | 11 ++++------- cli/src/tracker.rs | 27 ++++++++++----------------- 10 files changed, 51 insertions(+), 76 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 072df8422..21260760e 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -65,6 +65,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "anyhow" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" + [[package]] name = "async-trait" version = "0.1.74" @@ -175,6 +181,7 @@ checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" name = "cli" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "clap", "console", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2eca226cb..4d8e7cfe6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.80" async-trait = "0.1.73" clap = { version = "4.4.4", features = ["derive"] } console = "0.15.7" diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 134223211..dba33f9e3 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,7 +1,8 @@ extern crate fs_extra; +use anyhow::Error; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; use std::sync::mpsc; -use std::{fs, io::Error, path::PathBuf}; +use std::{fs, path::PathBuf}; use crate::tracker::get_tracker; diff --git a/cli/src/location.rs b/cli/src/location.rs index d10bbce00..402415c46 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,11 +1,8 @@ +use anyhow::{bail, Context, Error}; use git2::Repository; use std::path::Path; -use std::{ - env::current_dir, - io::{Error, ErrorKind}, - path::PathBuf, -}; +use std::{env::current_dir, path::PathBuf}; use tokio::sync::OnceCell; @@ -19,29 +16,18 @@ pub struct Location { impl Location { pub fn new() -> Result { let current_dir = current_dir()?; - let root: PathBuf = match Repository::discover(current_dir) { - Ok(repo) => { - let Some(root) = repo.workdir() else { - return Err(Error::new( - ErrorKind::NotFound, - "Fail to find project's root location", - )); - }; - - root.into() - } - Err(err) => return Err(Error::new(ErrorKind::NotFound, err)), + let repo = + Repository::discover(current_dir).context("Fail to find chipmunk root directory")?; + let Some(root) = repo.workdir() else { + bail!("Fail to find chipmunk root directory") }; // Make sure we are in the chipmunk repository // Note: This check will fail if the structure of the repo changes if root.join("application").is_dir() && root.join("developing").is_dir() { - Ok(Self { root }) + Ok(Self { root: root.into() }) } else { - Err(Error::new( - ErrorKind::NotFound, - "Fail to find project's root location", - )) + bail!("Fail to find project's root location") } } } diff --git a/cli/src/main.rs b/cli/src/main.rs index ee1942178..a2bb99add 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -5,6 +5,7 @@ mod spawner; mod target; mod tracker; +use anyhow::{bail, Error}; use clap::{Parser, Subcommand}; use futures::future::join_all; use location::init_location; @@ -12,7 +13,7 @@ use modules::Manager; use spawner::SpawnResult; use std::{ fs::File, - io::{self, stdout, Error, ErrorKind, Stdout}, + io::{self, stdout, Stdout}, path::PathBuf, }; use target::Target; @@ -180,7 +181,7 @@ async fn main() -> Result<(), Error> { } } if !success { - return Err(Error::new(ErrorKind::Other, "Some task were failed")); + bail!("Some task were failed") } Ok(()) } diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 437ba7c97..24ddad339 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,11 +1,8 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::{ - fs, - io::{Error, ErrorKind}, - path::PathBuf, -}; +use std::{fs, path::PathBuf}; const PATH: &str = "application/holder"; @@ -37,16 +34,13 @@ impl Manager for Module { Some(String::from("yarn install")) } async fn after(&self, prod: bool) -> Result, Error> { - let src = Target::Client.get().dist_path(prod).ok_or(Error::new( - ErrorKind::NotFound, - "Fail to get client artifacts", - ))?; + let src = Target::Client + .get() + .dist_path(prod) + .context("Fail to get client artifacts")?; let dest = self.cwd().join("dist"); if !src.exists() { - return Err(Error::new( - ErrorKind::NotFound, - format!("Not found: {}", src.display()), - )); + bail!("Not found: {}", src.display()); } if !dest.exists() { fs::create_dir(&dest)?; @@ -57,10 +51,10 @@ impl Manager for Module { } fstools::cp_folder(src.clone(), dest.clone()).await?; std::fs::rename( - dest.join(src.file_name().ok_or(Error::new( - ErrorKind::NotFound, - "Fail to parse client artifacts path", - ))?), + dest.join( + src.file_name() + .context("Fail to parse client artifacts path")?, + ), dest.join("client"), )?; Ok(None) diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 48e74a795..3ee01995c 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -13,9 +13,10 @@ use crate::{ spawner::{spawn, SpawnOptions, SpawnResult}, Target, }; +use anyhow::Error; use async_trait::async_trait; use futures::future::join_all; -use std::{io::Error, path::PathBuf}; +use std::path::PathBuf; #[derive(Debug, Clone)] pub enum Kind { diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index e50729484..ffc5a1ba4 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,11 +1,8 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use anyhow::{bail, Error}; use async_trait::async_trait; -use std::{ - fs, - io::{Error, ErrorKind}, - path::PathBuf, -}; +use std::{fs, path::PathBuf}; const PATH: &str = "application/apps/rustcore/ts-bindings"; @@ -36,10 +33,7 @@ impl Manager for Module { let src = Target::Binding.get().cwd().join("dist/index.node"); let dest = self.cwd().join("dist/native"); if !src.exists() { - return Err(Error::new( - ErrorKind::NotFound, - format!("Not found: {}", src.to_string_lossy()), - )); + bail!("Not found: {}", src.to_string_lossy()); } if !dest.exists() { fs::create_dir(&dest)?; diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 8e8cb9252..126ab46a1 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,9 +1,9 @@ // cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); use crate::{ location::{to_relative_path, get_root}, tracker::get_tracker}; +use anyhow::bail; use futures_lite::{future, FutureExt}; use std::{ env::vars, - io, path::PathBuf, process::{ExitStatus, Stdio}, }; @@ -78,7 +78,7 @@ pub async fn spawn( cwd: Option, caption: String, opts: Option, -) -> Result { +) -> Result { let opts = opts.unwrap_or_default(); let cwd = cwd.unwrap_or_else(|| get_root().clone()); let mut parts = command.split(' ').collect::>(); @@ -146,7 +146,7 @@ pub async fn spawn( } future::pending::<()>().await; - Ok::, io::Error>(None) + Ok::, anyhow::Error>(None) } }; @@ -176,9 +176,6 @@ pub async fn spawn( tracker .fail(sequence, "Fail to get exist status of spawned command") .await; - Err(io::Error::new( - io::ErrorKind::Other, - "Fail to get exist status of spawned command", - )) + bail!( "Fail to get exist status of spawned command") } } diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 657c6e2f1..5492250d2 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -1,10 +1,7 @@ +use anyhow::{anyhow, Context, Error}; use console::style; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use std::{ - collections::HashMap, - io::{Error, ErrorKind}, - time::Instant, -}; +use std::{collections::HashMap, time::Instant}; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, oneshot, OnceCell, @@ -81,9 +78,9 @@ impl Tracker { } pub async fn run(mut rx: UnboundedReceiver) -> Result<(), Error> { - let spinner_style = ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}") - .map_err(|e| Error::new(ErrorKind::Other, e.to_string()))? - .tick_chars("▂▃▅▆▇▆▅▃▂ "); + let spinner_style = + ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}")? + .tick_chars("▂▃▅▆▇▆▅▃▂ "); async move { let mut sequence: usize = 0; let mut max_time_len = 0; @@ -223,10 +220,8 @@ impl Tracker { let (tx_response, rx_response) = oneshot::channel(); self.tx .send(Tick::Started(job.to_string(), max, tx_response)) - .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; - rx_response - .await - .map_err(|err| Error::new(ErrorKind::NotConnected, err.to_string())) + .context("Fail to send tick")?; + rx_response.await.context("Fail to receive tick") } pub async fn progress(&self, sequence: usize, pos: Option) { @@ -265,17 +260,15 @@ impl Tracker { let (tx_response, rx_response) = oneshot::channel(); self.tx .send(Tick::Shutdown(tx_response)) - .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}")))?; - rx_response - .await - .map_err(|err| Error::new(ErrorKind::NotConnected, err.to_string())) + .context("Fail to send tick")?; + rx_response.await.context("Fail to receive tick") } pub async fn _print(&self, msg: String) { if let Err(e) = self .tx .send(Tick::Print(msg)) - .map_err(|e| Error::new(ErrorKind::Other, format!("Fail to send tick: {e}"))) + .map_err(|e| anyhow!("Fail to send tick: {e}")) { eprintln!("Fail to communicate with tracker: {e}"); } From ab662563eaac537f2beb837a47e2548b7d5c2267 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 25 Feb 2024 11:37:15 +0100 Subject: [PATCH 040/174] CLI: Provide logs for artifacts operations after build The report now include the information for all file system operations after the main build command is done --- cli/src/main.rs | 3 ++- cli/src/modules/app.rs | 54 +++++++++++++++++++++++++++++++------- cli/src/modules/mod.rs | 8 +++--- cli/src/modules/wrapper.rs | 28 +++++++++++++++++--- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index a2bb99add..496a08af5 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -108,10 +108,11 @@ async fn main() -> Result<(), Error> { } => { report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); + let report = !matches!(report_opt, ReportOptions::None); join_all( targets .iter() - .map(|module| module.build(production)) + .map(|module| module.build(production, report)) .collect::>(), ) .await diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 24ddad339..e0c18e5c1 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -2,7 +2,7 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use std::{fs, path::PathBuf, process::ExitStatus}; const PATH: &str = "application/holder"; @@ -33,7 +33,8 @@ impl Manager for Module { // For app we don't need --production Some(String::from("yarn install")) } - async fn after(&self, prod: bool) -> Result, Error> { + async fn after(&self, prod: bool, report: bool) -> Result, Error> { + let mut report_logs = Vec::new(); let src = Target::Client .get() .dist_path(prod) @@ -43,20 +44,53 @@ impl Manager for Module { bail!("Not found: {}", src.display()); } if !dest.exists() { + if report { + let msg = format!("creating directory: {}", dest.display()); + report_logs.push(msg); + } fs::create_dir(&dest)?; } let prev = dest.join("client"); if prev.exists() { + if report { + let msg = format!("removig directory: {}", prev.display()); + report_logs.push(msg); + } fstools::rm_folder(prev).await?; } + + if report { + let msg = format!( + "copying directory: '{}' to '{}'", + src.display(), + dest.display() + ); + report_logs.push(msg); + } fstools::cp_folder(src.clone(), dest.clone()).await?; - std::fs::rename( - dest.join( - src.file_name() - .context("Fail to parse client artifacts path")?, - ), - dest.join("client"), - )?; - Ok(None) + + let rename_from = dest.join( + src.file_name() + .context("Fail to parse client artifacts path")?, + ); + let rename_to = dest.join("client"); + + if report { + let msg = format!( + "renaming '{}' to '{}'", + rename_from.display(), + rename_to.display() + ); + report_logs.push(msg); + } + + std::fs::rename(rename_from, rename_to)?; + + Ok(Some(SpawnResult { + report: report_logs, + status: ExitStatus::default(), + job: "Copy App Build Artifacts".into(), + cmd: "Multiple file system commands".into(), + })) } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 3ee01995c..8e8588949 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -120,15 +120,15 @@ pub trait Manager { Kind::Rs => Ok(SpawnResult::empty()), } } - async fn after(&self, _prod: bool) -> Result, Error> { + async fn after(&self, _prod: bool, _report: bool) -> Result, Error> { Ok(None) } - async fn build(&self, prod: bool) -> Result { + async fn build(&self, prod: bool, report: bool) -> Result { self.install(false).await?; let deps: Vec> = self.deps().iter().map(|target| target.get()).collect(); for module in deps { - let status = module.build(prod).await?; + let status = module.build(prod, report).await?; if !status.status.success() { return Ok(status); } @@ -143,7 +143,7 @@ pub trait Manager { if !status.status.success() { Ok(status) } else { - let res = self.after(prod).await?; + let res = self.after(prod, report).await?; if matches!(self.kind(), Kind::Ts) && prod { self.clean().await?; self.install(prod).await?; diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index ffc5a1ba4..f0391a74b 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -2,7 +2,7 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use anyhow::{bail, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use std::{fs, path::PathBuf, process::ExitStatus}; const PATH: &str = "application/apps/rustcore/ts-bindings"; @@ -29,16 +29,38 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] } - async fn after(&self, _prod: bool) -> Result, Error> { + async fn after(&self, _prod: bool, report: bool) -> Result, Error> { + let mut report_logs = Vec::new(); + let src = Target::Binding.get().cwd().join("dist/index.node"); let dest = self.cwd().join("dist/native"); if !src.exists() { bail!("Not found: {}", src.to_string_lossy()); } if !dest.exists() { + if report { + let msg = format!("creating directory: {}", dest.display()); + report_logs.push(msg); + } fs::create_dir(&dest)?; } + + if report { + let msg = format!( + "copying directory: '{}' to '{}'", + src.display(), + dest.display() + ); + report_logs.push(msg); + } + fstools::cp_file(src, dest.join("index.node")).await?; - Ok(None) + + Ok(Some(SpawnResult { + report: report_logs, + status: ExitStatus::default(), + job: "Copy Wrapper Build Artifacts".into(), + cmd: "Multiple file system commands".into(), + })) } } From f89993a5d72e7da666d8e591d007aba706e8491b Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 25 Feb 2024 11:38:37 +0100 Subject: [PATCH 041/174] CLI Report: Add new line when needed --- cli/src/main.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 496a08af5..8a54998da 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -229,6 +229,10 @@ fn write_report(spawn_result: &SpawnResult, mut writer: impl io::Write) -> Resul for line in spawn_result.report.iter() { writer.write_all(line.as_bytes())?; + + if !line.ends_with('\n') { + writeln!(writer)?; + } } Ok(()) From 803eb8fbd2cd3b0235785a430f1338e3fdfc59cb Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 25 Feb 2024 22:23:15 +0100 Subject: [PATCH 042/174] CLI: Changes to improve logs + Fix in test method - Main methods provides a collection of spawn results from all the involved jobs in the method - Fix in test method: install was called for jobs that doesn't have test commands yet. - Spawn results won't be merged anymore - Logs will always collected to improve source code readability --- cli/src/main.rs | 52 +++++++------- cli/src/modules/app.rs | 57 +++++++--------- cli/src/modules/mod.rs | 135 ++++++++++++++++++++++--------------- cli/src/modules/wrapper.rs | 35 +++++----- cli/src/spawner.rs | 45 ++++--------- 5 files changed, 160 insertions(+), 164 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 8a54998da..37570fa78 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -108,11 +108,10 @@ async fn main() -> Result<(), Error> { } => { report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let report = !matches!(report_opt, ReportOptions::None); join_all( targets .iter() - .map(|module| module.build(production, report)) + .map(|module| module.build(production)) .collect::>(), ) .await @@ -146,33 +145,36 @@ async fn main() -> Result<(), Error> { let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { - Ok(status) => { - let print_err = match &report_opt { - ReportOptions::None => true, - ReportOptions::Stdout(stdout) => { - if !status.is_empty() { - write_report(status, stdout)?; + Ok(statuses) => { + for status in statuses { + let print_err = match &report_opt { + ReportOptions::None => true, + ReportOptions::Stdout(stdout) => { + if !status.is_empty() { + write_report(status, stdout)?; + } + false } - false - } - ReportOptions::File(path, file) => { - if !status.is_empty() { - write_report(status, file)?; - } - if idx == results.len() - 1 { - let full_path = path.canonicalize().unwrap_or_else(|_| path.to_owned()); - println!("Report is written to '{}'", full_path.display()); + ReportOptions::File(path, file) => { + if !status.is_empty() { + write_report(status, file)?; + } + if idx == results.len() - 1 { + let full_path = + path.canonicalize().unwrap_or_else(|_| path.to_owned()); + println!("Report is written to '{}'", full_path.display()); + } + false } - false - } - }; + }; - if !status.status.success() { - if print_err { - eprintln!("Failed with errors"); - eprintln!("{}:\n{}", status.job, status.report.join("")); + if !status.status.success() { + if print_err { + eprintln!("Failed with errors"); + eprintln!("{}:\n{}", status.job, status.report.join("")); + } + success = false; } - success = false; } } Err(err) => { diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index e0c18e5c1..6a8347047 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -2,7 +2,7 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf, process::ExitStatus}; +use std::{fs, path::PathBuf}; const PATH: &str = "application/holder"; @@ -33,7 +33,7 @@ impl Manager for Module { // For app we don't need --production Some(String::from("yarn install")) } - async fn after(&self, prod: bool, report: bool) -> Result, Error> { + async fn after(&self, prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); let src = Target::Client .get() @@ -44,29 +44,26 @@ impl Manager for Module { bail!("Not found: {}", src.display()); } if !dest.exists() { - if report { - let msg = format!("creating directory: {}", dest.display()); - report_logs.push(msg); - } + let msg = format!("creating directory: {}", dest.display()); + report_logs.push(msg); + fs::create_dir(&dest)?; } let prev = dest.join("client"); if prev.exists() { - if report { - let msg = format!("removig directory: {}", prev.display()); - report_logs.push(msg); - } + let msg = format!("removig directory: {}", prev.display()); + report_logs.push(msg); + fstools::rm_folder(prev).await?; } - if report { - let msg = format!( - "copying directory: '{}' to '{}'", - src.display(), - dest.display() - ); - report_logs.push(msg); - } + let msg = format!( + "copying directory: '{}' to '{}'", + src.display(), + dest.display() + ); + report_logs.push(msg); + fstools::cp_folder(src.clone(), dest.clone()).await?; let rename_from = dest.join( @@ -75,22 +72,18 @@ impl Manager for Module { ); let rename_to = dest.join("client"); - if report { - let msg = format!( - "renaming '{}' to '{}'", - rename_from.display(), - rename_to.display() - ); - report_logs.push(msg); - } + let msg = format!( + "renaming '{}' to '{}'", + rename_from.display(), + rename_to.display() + ); + report_logs.push(msg); std::fs::rename(rename_from, rename_to)?; - Ok(Some(SpawnResult { - report: report_logs, - status: ExitStatus::default(), - job: "Copy App Build Artifacts".into(), - cmd: "Multiple file system commands".into(), - })) + Ok(Some(SpawnResult::create_for_fs( + "Copy App Build Artifacts".into(), + report_logs, + ))) } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 8e8588949..c384d5647 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -79,21 +79,38 @@ pub trait Manager { fn test_cmds(&self) -> Vec { Vec::new() } - async fn reset(&self) -> Result { - self.clean().await?; - fstools::rm_folder(self.cwd().join("dist")).await?; - Ok(SpawnResult::empty()) + async fn reset(&self) -> Result, Error> { + let mut results = Vec::new(); + let clean_result = self.clean().await?; + results.push(clean_result); + + let dist_path = self.cwd().join("dist"); + + let remove_log = format!("removing {}", dist_path.display()); + + fstools::rm_folder(dist_path).await?; + + let job = format!("Reset {}", self.owner()); + + results.push(SpawnResult::create_for_fs(job, vec![remove_log])); + + Ok(results) } - async fn clean(&self) -> Result<(), Error> { - match self.kind() { - Kind::Ts => { - fstools::rm_folder(self.cwd().join("node_modules")).await?; - } - Kind::Rs => { - fstools::rm_folder(self.cwd().join("target")).await?; - } - } - Ok(()) + async fn clean(&self) -> Result { + let mut logs = Vec::new(); + let path = match self.kind() { + Kind::Ts => self.cwd().join("node_modules"), + Kind::Rs => self.cwd().join("target"), + }; + + let remove_log = format!("removing directory {}", path.display()); + logs.push(remove_log); + + fstools::rm_folder(path).await?; + + let job = format!("Clean {}", self.owner()); + + Ok(SpawnResult::create_for_fs(job, logs)) } async fn install(&self, prod: bool) -> Result { let cmd = if self.install_cmd(prod).is_some() { @@ -120,17 +137,20 @@ pub trait Manager { Kind::Rs => Ok(SpawnResult::empty()), } } - async fn after(&self, _prod: bool, _report: bool) -> Result, Error> { + async fn after(&self, _prod: bool) -> Result, Error> { Ok(None) } - async fn build(&self, prod: bool, report: bool) -> Result { - self.install(false).await?; + async fn build(&self, prod: bool) -> Result, Error> { + let mut results = Vec::new(); + let install_result = self.install(false).await?; + results.push(install_result); let deps: Vec> = self.deps().iter().map(|target| target.get()).collect(); for module in deps { - let status = module.build(prod, report).await?; - if !status.status.success() { - return Ok(status); + let status = module.build(prod).await?; + results.extend(status); + if results.iter().any(|res| !res.status.success()) { + return Ok(results); } } let path = get_root().join(self.cwd()); @@ -139,34 +159,45 @@ pub trait Manager { .unwrap_or_else(|| self.kind().build_cmd(prod)); let caption = format!("Bulid {}", self.owner()); match spawn(&cmd, Some(path), caption, None).await { - Ok(mut status) => { + Ok(status) => { if !status.status.success() { - Ok(status) + results.push(status); + Ok(results) } else { - let res = self.after(prod, report).await?; - if matches!(self.kind(), Kind::Ts) && prod { - self.clean().await?; - self.install(prod).await?; + results.push(status); + let res = self.after(prod).await?; + if let Some(result) = res { + results.push(result); } - - if let Some(res_after) = res { - status.merge_with(res_after); + if matches!(self.kind(), Kind::Ts) && prod { + let clean_res = self.clean().await?; + results.push(clean_res); + let install_res = self.install(prod).await?; + results.push(install_res); } - Ok(status) + Ok(results) } } Err(err) => Err(err), } } - async fn check(&self) -> Result { + async fn check(&self) -> Result, Error> { + let mut results = Vec::new(); match self.kind() { Kind::Ts => { - self.install(false).await?; - self.lint().await + let install_result = self.install(false).await?; + let lint_restul = self.lint().await?; + results.push(install_result); + results.push(lint_restul); + } + Kind::Rs => { + let clippy_result = self.clippy().await?; + results.push(clippy_result); } - Kind::Rs => self.clippy().await, } + + Ok(results) } async fn lint(&self) -> Result { let path = get_root().join(self.cwd()); @@ -192,18 +223,19 @@ pub trait Manager { .await } - // flat_map in main() to get rid of the double join calls - async fn test(&self) -> Result { - self.install(false).await?; - // TODO: Check if we need to run the dependencies tests too - + async fn test(&self) -> Result, Error> { let test_cmds = self.test_cmds(); if test_cmds.is_empty() { - return Ok(SpawnResult::empty()); + return Ok(Vec::new()); } + let mut results = Vec::new(); + + let install_res = self.install(false).await?; + results.push(install_res); + let caption = format!("Test {}", self.owner()); - let results = join_all(test_cmds.iter().map(|cmd| { + let spawn_results = join_all(test_cmds.iter().map(|cmd| { spawn( &cmd.command, Some(cmd.cwd.to_owned()), @@ -213,18 +245,13 @@ pub trait Manager { })) .await; - // return the first failed result, or the first one if all was successful - let return_pos = results - .iter() - .position(|res| match res { - Ok(result) => !result.status.success(), - Err(_) => true, - }) - .unwrap_or(0); - - results - .into_iter() - .nth(return_pos) - .expect("Commands has been checked if they are empty before spawning tasks") + for res in spawn_results { + match res { + Ok(spawn_res) => results.push(spawn_res), + Err(err) => return Err(err), + } + } + + Ok(results) } } diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index f0391a74b..c5a398e4a 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -2,7 +2,7 @@ use super::{Kind, Manager}; use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; use anyhow::{bail, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf, process::ExitStatus}; +use std::{fs, path::PathBuf}; const PATH: &str = "application/apps/rustcore/ts-bindings"; @@ -29,7 +29,7 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] } - async fn after(&self, _prod: bool, report: bool) -> Result, Error> { + async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); let src = Target::Binding.get().cwd().join("dist/index.node"); @@ -38,29 +38,24 @@ impl Manager for Module { bail!("Not found: {}", src.to_string_lossy()); } if !dest.exists() { - if report { - let msg = format!("creating directory: {}", dest.display()); - report_logs.push(msg); - } + let msg = format!("creating directory: {}", dest.display()); + report_logs.push(msg); + fs::create_dir(&dest)?; } - if report { - let msg = format!( - "copying directory: '{}' to '{}'", - src.display(), - dest.display() - ); - report_logs.push(msg); - } + let msg = format!( + "copying directory: '{}' to '{}'", + src.display(), + dest.display() + ); + report_logs.push(msg); fstools::cp_file(src, dest.join("index.node")).await?; - Ok(Some(SpawnResult { - report: report_logs, - status: ExitStatus::default(), - job: "Copy Wrapper Build Artifacts".into(), - cmd: "Multiple file system commands".into(), - })) + Ok(Some(SpawnResult::create_for_fs( + "Copy Wrapper Build Artifacts".into(), + report_logs, + ))) } } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 126ab46a1..67b7bf390 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,5 +1,8 @@ // cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); -use crate::{ location::{to_relative_path, get_root}, tracker::get_tracker}; +use crate::{ + location::{get_root, to_relative_path}, + tracker::get_tracker, +}; use anyhow::bail; use futures_lite::{future, FutureExt}; use std::{ @@ -31,39 +34,15 @@ impl SpawnResult { } pub fn is_empty(&self) -> bool { - self.report.is_empty() && - self.job.is_empty() && - self.cmd.is_empty() && - // Default status indicates successful completion - self.status.success() + self.job.is_empty() && self.cmd.is_empty() } - pub fn merge_with(&mut self, other: SpawnResult) { - if other.is_empty() { - return; - } - - if self.is_empty() { - *self = other; - return; - } - - self.report.push(String::from("-------------------------------------------")); - self.report.extend(other.report); - - self.job.push_str(" & "); - self.job.push_str(other.job.as_str()); - - self.cmd.push_str(" & "); - self.cmd.push_str(other.cmd.as_str()); - - // The failed status is the relevant one. - if !self.status.success() { - return; - } - - if !other.status.success() { - self.status = other.status; + pub fn create_for_fs(job: String, report: Vec) -> Self { + SpawnResult { + report, + job, + status: ExitStatus::default(), + cmd: "Multiple file system commands".into(), } } } @@ -176,6 +155,6 @@ pub async fn spawn( tracker .fail(sequence, "Fail to get exist status of spawned command") .await; - bail!( "Fail to get exist status of spawned command") + bail!("Fail to get exist status of spawned command") } } From 7848d46ff36c2a4103d7eaafc7a02399492982a8 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sat, 2 Mar 2024 21:16:52 +0100 Subject: [PATCH 043/174] CLI: Provide test command for rust core (indexer) --- cli/src/modules/core.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index aa9918826..18bb684e6 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::{Kind, Manager, TestCommand}; use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -28,4 +28,12 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![] } + + fn test_cmds(&self) -> Vec { + vec![TestCommand::new( + "cargo +stable test --color always".into(), + self.cwd(), + None, + )] + } } From c8e5307ac2620850cf08482d91556b4dca27cafb Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 4 Mar 2024 22:55:47 +0100 Subject: [PATCH 044/174] CLI: Test command for TS bindings (in progress) - Test commands work for all test specs but "jobs" spec where the program gets interrupted from OS with signal SIGTRAP with out of memory error - test method is overridden in wrapper Module, because we need to run build specs command before them and we need to set environment variable there too. - Spawn method accepts environment variables now too. - Spawn method get the command as String instead of &str since it will be cloned inside the method anyway. --- cli/src/modules/mod.rs | 43 +++++++++++------ cli/src/modules/wrapper.rs | 94 ++++++++++++++++++++++++++++++++++++-- cli/src/spawner.rs | 20 ++++---- 3 files changed, 130 insertions(+), 27 deletions(-) diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index c384d5647..fdd73790d 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -16,7 +16,7 @@ use crate::{ use anyhow::Error; use async_trait::async_trait; use futures::future::join_all; -use std::path::PathBuf; +use std::{iter, path::PathBuf}; #[derive(Debug, Clone)] pub enum Kind { @@ -120,7 +120,7 @@ pub trait Manager { }; if let Some(cmd) = cmd { let caption = format!("Install {}", self.owner()); - spawn(&cmd, Some(self.cwd()), caption, None).await + spawn(cmd, Some(self.cwd()), caption, iter::empty(), None).await } else { Ok(SpawnResult::empty()) } @@ -157,8 +157,8 @@ pub trait Manager { let cmd = self .build_cmd(prod) .unwrap_or_else(|| self.kind().build_cmd(prod)); - let caption = format!("Bulid {}", self.owner()); - match spawn(&cmd, Some(path), caption, None).await { + let caption = format!("Build {}", self.owner()); + match spawn(cmd, Some(path), caption, iter::empty(), None).await { Ok(status) => { if !status.status.success() { results.push(status); @@ -202,22 +202,37 @@ pub trait Manager { async fn lint(&self) -> Result { let path = get_root().join(self.cwd()); let caption = format!("TS Lint {}", self.owner()); - let status = spawn("yarn run lint", Some(path.clone()), caption, None).await?; + let status = spawn( + "yarn run lint".into(), + Some(path.clone()), + caption, + iter::empty(), + None, + ) + .await?; if !status.status.success() { return Ok(status); } let caption = format!("Build {}", self.owner()); - spawn("yarn run build", Some(path), caption, None).await + spawn( + "yarn run build".into(), + Some(path), + caption, + iter::empty(), + None, + ) + .await } async fn clippy(&self) -> Result { let path = get_root().join(self.cwd()); let caption = format!("Clippy {}", self.owner()); spawn( - "cargo clippy --color always --all --all-features -- -D warnings", + "cargo clippy --color always --all --all-features -- -D warnings".into(), Some(path), caption, + iter::empty(), None, ) .await @@ -231,16 +246,18 @@ pub trait Manager { let mut results = Vec::new(); - let install_res = self.install(false).await?; - results.push(install_res); + // build method calls install + let build_results = self.build(false).await?; + results.extend(build_results); let caption = format!("Test {}", self.owner()); - let spawn_results = join_all(test_cmds.iter().map(|cmd| { + let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { spawn( - &cmd.command, - Some(cmd.cwd.to_owned()), + cmd.command, + Some(cmd.cwd), caption.clone(), - cmd.spawn_opts.clone(), + iter::empty(), + cmd.spawn_opts, ) })) .await; diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index c5a398e4a..4141f0758 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,8 +1,14 @@ -use super::{Kind, Manager}; -use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use super::{Kind, Manager, TestCommand}; +use crate::{ + fstools, + location::get_root, + spawner::{spawn, SpawnOptions, SpawnResult}, + Target, +}; use anyhow::{bail, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use futures::future::join_all; +use std::{fs, iter, path::PathBuf}; const PATH: &str = "application/apps/rustcore/ts-bindings"; @@ -58,4 +64,86 @@ impl Manager for Module { report_logs, ))) } + + async fn test(&self) -> Result, Error> { + let mut results = Vec::new(); + + let build_results = self.build(false).await?; + results.extend(build_results); + + let build_spec_path = self.cwd().join("spec"); + if !build_spec_path.join("build").exists() { + let build_spec_cmd = format!( + "{}/node_modules/.bin/tsc -p tsconfig.json", + self.cwd().to_string_lossy() + ); + + let spec_res = spawn( + build_spec_cmd, + Some(build_spec_path), + "bulid spec".into(), + iter::empty(), + None, + ) + .await?; + + results.push(spec_res); + } + + let test_cmds = self.test_cmds(); + + let caption = format!("Test {}", self.owner()); + let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { + spawn( + cmd.command, + Some(cmd.cwd), + caption.clone(), + vec![(String::from("ELECTRON_RUN_AS_NODE"), String::from("1"))], + cmd.spawn_opts.clone(), + ) + })) + .await; + + for res in spawn_results { + match res { + Ok(spawn_res) => results.push(spawn_res), + Err(err) => return Err(err), + } + } + + Ok(results) + } + + fn test_cmds(&self) -> Vec { + //TODO: this can be constant when "jobs" problem is solved + let test_specs: Vec<&'static str> = vec![ + // TODO: + // Running "jobs" here causes the program to receive SIGTRAP from OS because of an + // out-of-memory error in electron app, even if only this job was running (by + // commenting out the other specs) + // "jobs" is commented out temporally until the SIGTRAP problem is solved. + // "jobs", + "search", + "values", + "extract", + "ranges", + "exporting", + "map", + "observe", + "indexes", + "concat", + "cancel", + "errors", + "stream", + "promises", + ]; + + test_specs.iter().map(|spec| { + TestCommand::new( + format!("./node_modules/.bin/electron ./node_modules/jasmine/bin/jasmine.js spec/build/spec/session.{spec}.spec.js"), + self.cwd(), + Some(SpawnOptions{suppress_msg:true}), + ) + }).collect() + } } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 67b7bf390..a6a4ab027 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -4,9 +4,10 @@ use crate::{ tracker::get_tracker, }; use anyhow::bail; +use core::panic; use futures_lite::{future, FutureExt}; use std::{ - env::vars, + env, path::PathBuf, process::{ExitStatus, Stdio}, }; @@ -53,26 +54,23 @@ pub(crate) struct SpawnOptions { } pub async fn spawn( - command: &str, + command: String, cwd: Option, caption: String, + environment_vars: impl IntoIterator, opts: Option, ) -> Result { let opts = opts.unwrap_or_default(); let cwd = cwd.unwrap_or_else(|| get_root().clone()); let mut parts = command.split(' ').collect::>(); let cmd = parts.remove(0); - #[allow(clippy::useless_vec)] + let mut env_vars: Vec<_> = env::vars().chain(environment_vars).collect(); + env_vars.push((String::from("TERM"), String::from("xterm-256color"))); + let mut child = Command::new(cmd) .current_dir(&cwd) .args(parts) - .envs( - vec![ - vars().collect::>(), - vec![(String::from("TERM"), String::from("xterm-256color"))], - ] - .concat(), - ) + .envs(env_vars) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; @@ -149,7 +147,7 @@ pub async fn spawn( report: report_lines, status, job: job_title, - cmd: command.to_owned(), + cmd: command, }) } else { tracker From 21d47bd14a97797b048e80a14e13c02991c5aff6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 5 Mar 2024 20:07:21 +0100 Subject: [PATCH 045/174] CLI: Investigating test commands error with jobs sepcs - TODOs and comment are written in the corresponding places in source code for further investigations - Another TODO is written to clarify if we should build TS binding tests each time we run the tests - Typo is fixed --- .../rustcore/ts-bindings/spec/session.jobs.spec.ts | 1 + cli/src/modules/wrapper.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts index 1acb4a07c..b8294df95 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts @@ -133,6 +133,7 @@ describe('Jobs', function () { it(config.regular.list[4], function () { return runner(config.regular, 4, async (logger, done, collector) => { const jobs = collector(await Jobs.create()) as Jobs; + //TODO: If we comment out these lines, build CLI with Stdio::piped() works. const profiles = await jobs.getShellProfiles(); expect(profiles.length > 0).toBe(true); finish(jobs, done); diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 4141f0758..d3c3494b1 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -72,6 +72,8 @@ impl Manager for Module { results.extend(build_results); let build_spec_path = self.cwd().join("spec"); + //TODO: Make sure we should put this check here. This check exists in rake implementation + // but it could lead to wrong test values when the changes in the tests aren't built. if !build_spec_path.join("build").exists() { let build_spec_cmd = format!( "{}/node_modules/.bin/tsc -p tsconfig.json", @@ -81,7 +83,7 @@ impl Manager for Module { let spec_res = spawn( build_spec_cmd, Some(build_spec_path), - "bulid spec".into(), + "Build Specs".into(), iter::empty(), None, ) @@ -120,7 +122,13 @@ impl Manager for Module { // TODO: // Running "jobs" here causes the program to receive SIGTRAP from OS because of an // out-of-memory error in electron app, even if only this job was running (by - // commenting out the other specs) + // commenting out the other specs). + // + // The error happens while executing line 137 from the file `session.jobs.spec.ts` when + // we spawn the command using Stdio::piped() in the spawn command (line 74 in file + // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or + // using Stdio::inherit() in `spawner.rs` prevent this error from happening. + // // "jobs" is commented out temporally until the SIGTRAP problem is solved. // "jobs", "search", From 9adab696b47409f9b6ddd910872091a36dca2057 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 10 Mar 2024 12:53:47 +0100 Subject: [PATCH 046/174] CLI: Run TS bindings tests using Stdio::inherit Some TS tests fails when run inside a piped stdio. To work-around this problem a new method in the tracker is implemented that suspend the progress bars and run the blocking command. --- cli/src/modules/wrapper.rs | 98 ++++++++++++++++---------------------- cli/src/spawner.rs | 50 +++++++++++++++++++ cli/src/tracker.rs | 27 ++++++++++- 3 files changed, 117 insertions(+), 58 deletions(-) diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index d3c3494b1..754e5ff04 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,17 +1,45 @@ -use super::{Kind, Manager, TestCommand}; +use super::{Kind, Manager}; use crate::{ fstools, location::get_root, - spawner::{spawn, SpawnOptions, SpawnResult}, + spawner::{spawn, spawn_blocking, SpawnResult}, Target, }; use anyhow::{bail, Error}; use async_trait::async_trait; -use futures::future::join_all; use std::{fs, iter, path::PathBuf}; const PATH: &str = "application/apps/rustcore/ts-bindings"; +const TEST_SPECS: [&str; 14] = [ + // TODO: + // Running "jobs" here causes the program to receive SIGTRAP from OS because of an + // out-of-memory error in electron app, even if only this job was running (by + // commenting out the other specs). + // + // The error happens while executing line 137 from the file `session.jobs.spec.ts` when + // we spawn the command using Stdio::piped() in the spawn command (line 74 in file + // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or + // using Stdio::inherit() in `spawner.rs` prevent this error from happening. + // + // The current work-around to blocking run the all the test commands sequentially using inherit + // Stdio::inherit suspending the progress bars until all tests are done. + "jobs", + "search", + "values", + "extract", + "ranges", + "exporting", + "map", + "observe", + "indexes", + "concat", + "cancel", + "errors", + "stream", + "promises", +]; + #[derive(Clone, Debug)] pub struct Module {} @@ -92,66 +120,22 @@ impl Manager for Module { results.push(spec_res); } - let test_cmds = self.test_cmds(); + let cwd = self.cwd(); - let caption = format!("Test {}", self.owner()); - let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { - spawn( - cmd.command, - Some(cmd.cwd), + for spec in TEST_SPECS { + let caption = format!("Test {}: {}", self.owner(), spec); + let command = format!("./node_modules/.bin/electron ./node_modules/jasmine/bin/jasmine.js spec/build/spec/session.{spec}.spec.js"); + let res = spawn_blocking( + command, + Some(cwd.clone()), caption.clone(), vec![(String::from("ELECTRON_RUN_AS_NODE"), String::from("1"))], - cmd.spawn_opts.clone(), ) - })) - .await; - - for res in spawn_results { - match res { - Ok(spawn_res) => results.push(spawn_res), - Err(err) => return Err(err), - } + .await?; + + results.push(res); } Ok(results) } - - fn test_cmds(&self) -> Vec { - //TODO: this can be constant when "jobs" problem is solved - let test_specs: Vec<&'static str> = vec![ - // TODO: - // Running "jobs" here causes the program to receive SIGTRAP from OS because of an - // out-of-memory error in electron app, even if only this job was running (by - // commenting out the other specs). - // - // The error happens while executing line 137 from the file `session.jobs.spec.ts` when - // we spawn the command using Stdio::piped() in the spawn command (line 74 in file - // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or - // using Stdio::inherit() in `spawner.rs` prevent this error from happening. - // - // "jobs" is commented out temporally until the SIGTRAP problem is solved. - // "jobs", - "search", - "values", - "extract", - "ranges", - "exporting", - "map", - "observe", - "indexes", - "concat", - "cancel", - "errors", - "stream", - "promises", - ]; - - test_specs.iter().map(|spec| { - TestCommand::new( - format!("./node_modules/.bin/electron ./node_modules/jasmine/bin/jasmine.js spec/build/spec/session.{spec}.spec.js"), - self.cwd(), - Some(SpawnOptions{suppress_msg:true}), - ) - }).collect() - } } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index a6a4ab027..0a38f2d5a 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -156,3 +156,53 @@ pub async fn spawn( bail!("Fail to get exist status of spawned command") } } + +/// Suspend the progress bars and run the giving blocking command using `Stdio::inherit()` +/// This is used with commands that doesn't work with `Stdio::piped()` +pub async fn spawn_blocking( + command: String, + cwd: Option, + caption: String, + environment_vars: impl IntoIterator, +) -> Result { + let cwd = cwd.unwrap_or_else(|| get_root().clone()); + let mut parts = command.split(' ').collect::>(); + let cmd = parts.remove(0); + let mut env_vars: Vec<_> = env::vars().chain(environment_vars).collect(); + env_vars.push((String::from("TERM"), String::from("xterm-256color"))); + + let mut child = std::process::Command::new(cmd); + child.current_dir(&cwd); + child.args(parts); + child.envs(env_vars); + + let tracker = get_tracker().await; + + let sequence = tracker + .start( + &format!("{}: {}", to_relative_path(&cwd).display(), caption), + None, + ) + .await?; + + let status = match tracker.suspend_and_run(child).await { + Ok(status) => status, + Err(err) => { + tracker.fail(sequence, &err.to_string()).await; + return Err(err); + } + }; + + if status.success() { + tracker.success(sequence, "").await; + } else { + tracker.fail(sequence, "finished with errors").await; + } + + Ok(SpawnResult { + report: Vec::new(), + status, + job: caption, + cmd: command, + }) +} diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 5492250d2..2036a0756 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -1,7 +1,11 @@ use anyhow::{anyhow, Context, Error}; use console::style; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use std::{collections::HashMap, time::Instant}; +use std::{ + collections::HashMap, + process::{Command, ExitStatus}, + time::Instant, +}; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, oneshot, OnceCell, @@ -37,6 +41,8 @@ pub enum Tick { #[allow(dead_code)] Print(String), Shutdown(oneshot::Sender<()>), + /// Suspends the progress bars and execute the giving blocking command + SuspendAndRun(Command, oneshot::Sender>), } #[derive(Clone, Debug)] @@ -165,6 +171,12 @@ impl Tracker { } break; } + Tick::SuspendAndRun(mut command, tx_response ) => { + let status = mp.suspend(|| {command.status()}).context("Error while executing command"); + if tx_response.send(status).is_err() { + let _ = mp.println("Fail to send response"); + } + }, } } } @@ -273,4 +285,17 @@ impl Tracker { eprintln!("Fail to communicate with tracker: {e}"); } } + + /// Suspend the progress bars and run the giving blocking command returning its exit status + pub async fn suspend_and_run( + &self, + cmd: std::process::Command, + ) -> Result { + let (tx_response, rx_response) = oneshot::channel(); + self.tx + .send(Tick::SuspendAndRun(cmd, tx_response)) + .context("Fail to send tick")?; + + rx_response.await.context("Fail to receive tick")? + } } From 43eeda2983d6ff14cbd603d2bb2bebe523ca8490 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Sun, 10 Mar 2024 19:15:51 +0100 Subject: [PATCH 047/174] CLI: Replace hard-coded paths with PathBuf::join - `PathBuf::join` provides a cross-platform solution to define paths --- cli/src/modules/app.rs | 4 +--- cli/src/modules/binding.rs | 8 +++++--- cli/src/modules/cli.rs | 4 +--- cli/src/modules/client.rs | 10 ++++------ cli/src/modules/core.rs | 4 +--- cli/src/modules/shared.rs | 4 +--- cli/src/modules/wasm.rs | 8 +++++--- cli/src/modules/wrapper.rs | 8 +++++--- 8 files changed, 23 insertions(+), 27 deletions(-) diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 6a8347047..46be05b24 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -4,8 +4,6 @@ use anyhow::{bail, Context, Error}; use async_trait::async_trait; use std::{fs, path::PathBuf}; -const PATH: &str = "application/holder"; - #[derive(Clone, Debug)] pub struct Module {} @@ -24,7 +22,7 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root().join("application").join("holder") } fn deps(&self) -> Vec { vec![Target::Shared, Target::Wrapper, Target::Client] diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index efed3ae72..6901917e3 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "application/apps/rustcore/rs-bindings"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,7 +21,11 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root() + .join("application") + .join("apps") + .join("rustcore") + .join("rs-bindings") } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index 3e67d9d82..d5289f89c 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "cli"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,7 +21,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root().join("cli") } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index f798be369..56a588a9e 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "application/client"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,16 +21,16 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root().join("application").join("client") } fn deps(&self) -> Vec { vec![Target::Shared, Target::Wasm] } fn dist_path(&self, prod: bool) -> Option { Some( - get_root() - .join(PATH) - .join(if prod { "dist/release" } else { "dist/debug" }), + self.cwd() + .join("dist") + .join(if prod { "release" } else { "debug" }), ) } } diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index 18bb684e6..3a7ab62c1 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "application/apps/indexer"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,7 +21,7 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root().join("application").join("apps").join("indexer") } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index 2d4394d86..a1e2eced3 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "application/platform"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,7 +21,7 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root().join("application").join("platform") } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 6c9ecca16..eb91a08b5 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -3,8 +3,6 @@ use crate::{location::get_root, spawner::SpawnOptions, Target}; use async_trait::async_trait; use std::path::PathBuf; -const PATH: &str = "application/apps/rustcore/wasm-bindings"; - #[derive(Clone, Debug)] pub struct Module {} @@ -23,7 +21,11 @@ impl Manager for Module { Kind::Rs } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root() + .join("application") + .join("apps") + .join("rustcore") + .join("wasm-bindings") } fn deps(&self) -> Vec { diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 754e5ff04..09d14cf18 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -9,8 +9,6 @@ use anyhow::{bail, Error}; use async_trait::async_trait; use std::{fs, iter, path::PathBuf}; -const PATH: &str = "application/apps/rustcore/ts-bindings"; - const TEST_SPECS: [&str; 14] = [ // TODO: // Running "jobs" here causes the program to receive SIGTRAP from OS because of an @@ -58,7 +56,11 @@ impl Manager for Module { Kind::Ts } fn cwd(&self) -> PathBuf { - get_root().join(PATH) + get_root() + .join("application") + .join("apps") + .join("rustcore") + .join("ts-bindings") } fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] From 8b903882d0f35dd8f84fc35ab82374918b8c5393 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 13 Mar 2024 08:05:56 +0100 Subject: [PATCH 048/174] CLI: Use PathBuf paths in Wrapper & Update Comment - Replace hard-coded paths with PathBuf to make them cross-platform - TODO states how to check when to build the tests --- cli/src/modules/wrapper.rs | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 09d14cf18..0dc080c00 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -68,8 +68,8 @@ impl Manager for Module { async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); - let src = Target::Binding.get().cwd().join("dist/index.node"); - let dest = self.cwd().join("dist/native"); + let src = Target::Binding.get().cwd().join("dist").join("index.node"); + let dest = self.cwd().join("dist").join("native"); if !src.exists() { bail!("Not found: {}", src.to_string_lossy()); } @@ -102,13 +102,12 @@ impl Manager for Module { results.extend(build_results); let build_spec_path = self.cwd().join("spec"); - //TODO: Make sure we should put this check here. This check exists in rake implementation - // but it could lead to wrong test values when the changes in the tests aren't built. + //TODO: This check exists in rake implementation but it need to be improved. + // The check should cover if the test themselves or the code under the tests has been changed. if !build_spec_path.join("build").exists() { - let build_spec_cmd = format!( - "{}/node_modules/.bin/tsc -p tsconfig.json", - self.cwd().to_string_lossy() - ); + let test_builder_path = self.cwd().join("node_modules").join(".bin").join("tsc"); + let build_spec_cmd = + format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); let spec_res = spawn( build_spec_cmd, @@ -124,9 +123,24 @@ impl Manager for Module { let cwd = self.cwd(); + let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); + let electron_path = electron_path.to_string_lossy(); + + let jasmine_path: PathBuf = [".", "node_modules", "jasmine", "bin", "jasmine.js"] + .iter() + .collect(); + let jasmine_path = jasmine_path.to_string_lossy(); + + let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); + for spec in TEST_SPECS { let caption = format!("Test {}: {}", self.owner(), spec); - let command = format!("./node_modules/.bin/electron ./node_modules/jasmine/bin/jasmine.js spec/build/spec/session.{spec}.spec.js"); + let spec_file_name = format!("session.{spec}.spec.js"); + let spec_file_path = specs_dir_path.join(spec_file_name); + let command = format!( + "{electron_path} {jasmine_path} {}", + spec_file_path.to_string_lossy() + ); let res = spawn_blocking( command, Some(cwd.clone()), From 1a8bd73a141da864ef5bffe12d753a0b3e9dbf84 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 13 Mar 2024 22:15:06 +0100 Subject: [PATCH 049/174] CLI: Replace hard-coded path for multi-platform support --- cli/src/modules/binding.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 6901917e3..75dcfefb9 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -31,10 +31,11 @@ impl Manager for Module { vec![] } fn build_cmd(&self, prod: bool) -> Option { - let path = Target::Wrapper - .get() - .cwd() - .join("node_modules/.bin/electron-build-env"); + let mut path = Target::Wrapper.get().cwd(); + path.push("node_modules"); + path.push(".bin"); + path.push("electron-build-env"); + Some(format!( "{} nj-cli build{}", path.to_string_lossy(), From 3d74b3a882bdbfd395e82f1f8338952d08d65b06 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 16 Apr 2024 10:58:31 +0200 Subject: [PATCH 050/174] Build CLI: Add documentation & comments --- cli/src/main.rs | 2 ++ cli/src/modules/app.rs | 1 + cli/src/modules/binding.rs | 1 + cli/src/modules/cli.rs | 1 + cli/src/modules/client.rs | 1 + cli/src/modules/core.rs | 1 + cli/src/modules/mod.rs | 2 ++ cli/src/modules/shared.rs | 1 + cli/src/modules/wasm.rs | 1 + cli/src/modules/wrapper.rs | 1 + cli/src/target.rs | 8 ++++++++ 11 files changed, 20 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 37570fa78..75796fb8d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -87,6 +87,7 @@ async fn main() -> Result<(), Error> { init_location()?; + // Run the given command let command = cli.command; let report_opt: ReportOptions; let results = match command { @@ -140,6 +141,7 @@ async fn main() -> Result<(), Error> { } }; + // Shutdown and show results & report let tracker = get_tracker().await; tracker.shutdown().await?; let mut success: bool = true; diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 46be05b24..83b169590 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use std::{fs, path::PathBuf}; #[derive(Clone, Debug)] +/// Represents the path `application/holder` pub struct Module {} impl Module { diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 75dcfefb9..32462a620 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `application/apps/rustcore/rs-bindings` pub struct Module {} impl Module { diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index d5289f89c..c894baac7 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `cli` pub struct Module {} impl Module { diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 56a588a9e..75160dcc4 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `application/client` pub struct Module {} impl Module { diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index 3a7ab62c1..ef04563f3 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `application/apps/indexer` pub struct Module {} impl Module { diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index fdd73790d..df0f841ed 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -20,7 +20,9 @@ use std::{iter, path::PathBuf}; #[derive(Debug, Clone)] pub enum Kind { + /// TypeScript Ts, + /// Rust Rs, } diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index a1e2eced3..f7fde60cc 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `application/platform` pub struct Module {} impl Module { diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index eb91a08b5..9f5409e74 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use std::path::PathBuf; #[derive(Clone, Debug)] +/// Represents the path `application/apps/rustcore/wasm-bindings` pub struct Module {} impl Module { diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 0dc080c00..a69dbe79b 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -39,6 +39,7 @@ const TEST_SPECS: [&str; 14] = [ ]; #[derive(Clone, Debug)] +/// Represents the path `application/apps/rustcore/ts-bindings` pub struct Module {} impl Module { diff --git a/cli/src/target.rs b/cli/src/target.rs index 18449cc2c..26b327691 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -3,13 +3,21 @@ use clap::ValueEnum; #[derive(ValueEnum, Clone, Debug, Hash, PartialEq, Eq)] pub enum Target { + /// Represents the path `application/apps/indexer` Core, + /// Represents the path `application/apps/rustcore/rs-bindings` Binding, + /// Represents the path `application/apps/rustcore/ts-bindings` Wrapper, + /// Represents the path `application/client` Client, + /// Represents the path `application/platform` Shared, + /// Represents the path `application/holder` App, + /// Represents the path `cli` Cli, + /// Represents the path `application/apps/rustcore/wasm-bindings` Wasm, } From 22294f71634a7390a51a384adea326db5eff36ac Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 16 Apr 2024 14:27:36 +0200 Subject: [PATCH 051/174] Build CLI is now a cargo extension... CLI tool can be installed with `cargo install --path=cli` and after that it can be called as a cargo sub command `cargo chipmunk {args}` --- cli/Cargo.lock | 32 ++++++++++++++++---------------- cli/Cargo.toml | 3 ++- cli/src/main.rs | 10 ++++++++-- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 21260760e..6a29839c1 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -121,6 +121,22 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cargo-chipmunk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "console", + "fs_extra", + "futures", + "futures-lite", + "git2", + "indicatif", + "tokio", +] + [[package]] name = "cc" version = "1.0.83" @@ -177,22 +193,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" -[[package]] -name = "cli" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "clap", - "console", - "fs_extra", - "futures", - "futures-lite", - "git2", - "indicatif", - "tokio", -] - [[package]] name = "colorchoice" version = "1.0.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4d8e7cfe6..b276d7831 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,7 +1,8 @@ [package] -name = "cli" +name = "cargo-chipmunk" version = "0.1.0" edition = "2021" +description = "Tool for chipmunk application development" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/cli/src/main.rs b/cli/src/main.rs index 75796fb8d..91d6c0e6a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -23,7 +23,13 @@ static REPORT_HELP_TEXT: &str = "Write report from command logs to the given file or to stdout if no file is defined"; static REPORT_VALUE_NAME: &str = "FILE-PATH"; -#[derive(Parser, Debug)] +#[derive(Parser)] +#[command(name = "cargo", bin_name = "cargo")] +enum CargoCli { + Chipmunk(Cli), +} + +#[derive(clap::Args, Debug)] #[command(author, version, about, long_about = None)] struct Cli { #[command(subcommand)] @@ -83,7 +89,7 @@ enum Command { #[tokio::main] async fn main() -> Result<(), Error> { - let cli = Cli::parse(); + let CargoCli::Chipmunk(cli) = CargoCli::parse(); init_location()?; From 90492290ac6987a5a92145f536a0dd6c0f0aa8ce Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 16 Apr 2024 16:13:14 +0200 Subject: [PATCH 052/174] Build CLI: Add environment dependencies check. Check will take place before any other command runs --- cli/src/check_env.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++ cli/src/main.rs | 4 +++ 2 files changed, 72 insertions(+) create mode 100644 cli/src/check_env.rs diff --git a/cli/src/check_env.rs b/cli/src/check_env.rs new file mode 100644 index 000000000..54117395f --- /dev/null +++ b/cli/src/check_env.rs @@ -0,0 +1,68 @@ +use std::process::{Command, Stdio}; + +use anyhow::bail; + +const ENV_CHECKS: [EnvCheck; 7] = [ + EnvCheck::new("NodeJS", "node", "-v", None), + EnvCheck::new("npm", "npm", "-v", None), + EnvCheck::new("yarn", "yarn", "-v", Some("npm install --global yarn")), + EnvCheck::new("rust", "rustup", "-V", None), + EnvCheck::new("cargo", "cargo", "-V", None), + EnvCheck::new( + "wasm-pack", + "wasm-pack", + "--help", + Some("cargo install wasm-pack"), + ), + EnvCheck::new("nj-cli", "nj-cli", "-V", Some("cargo install nj-cli")), +]; + +struct EnvCheck { + app_name: &'static str, + command: &'static str, + arg: &'static str, + install_hint: Option<&'static str>, +} + +impl EnvCheck { + const fn new( + app_name: &'static str, + command: &'static str, + arg: &'static str, + install_hint: Option<&'static str>, + ) -> Self { + Self { + app_name, + command, + arg, + install_hint, + } + } +} + +/// Checks if all the needed dependencies for the build environment are available +pub fn check_env() -> anyhow::Result<()> { + for check in ENV_CHECKS.iter() { + let success = match Command::new(check.command) + .arg(check.arg) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + + if !success { + let mut err_msg = format!("Required dependency '{}' is not installed.", check.app_name); + if let Some(install_hint) = check.install_hint { + err_msg.push_str( + format!("\nConsider installing it using the command '{install_hint}'").as_str(), + ); + } + bail!(err_msg); + } + } + + Ok(()) +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 91d6c0e6a..d86f4264d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,4 @@ +mod check_env; mod fstools; mod location; mod modules; @@ -6,6 +7,7 @@ mod target; mod tracker; use anyhow::{bail, Error}; +use check_env::check_env; use clap::{Parser, Subcommand}; use futures::future::join_all; use location::init_location; @@ -91,6 +93,8 @@ enum Command { async fn main() -> Result<(), Error> { let CargoCli::Chipmunk(cli) = CargoCli::parse(); + check_env()?; + init_location()?; // Run the given command From 25310865159ba3824535d1454da1476455d33808 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 16 Apr 2024 16:56:09 +0200 Subject: [PATCH 053/174] Build CLI: Command for Environment & Improvements - Provide sub command to check the development environment - The check for environment report all the missing requirements --- cli/src/check_env.rs | 35 ++++++++++++++++++++++++++++------- cli/src/main.rs | 8 ++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/cli/src/check_env.rs b/cli/src/check_env.rs index 54117395f..1a77f7302 100644 --- a/cli/src/check_env.rs +++ b/cli/src/check_env.rs @@ -1,4 +1,7 @@ -use std::process::{Command, Stdio}; +use std::{ + fmt::Write, + process::{Command, Stdio}, +}; use anyhow::bail; @@ -42,6 +45,7 @@ impl EnvCheck { /// Checks if all the needed dependencies for the build environment are available pub fn check_env() -> anyhow::Result<()> { + let mut errors = None; for check in ENV_CHECKS.iter() { let success = match Command::new(check.command) .arg(check.arg) @@ -54,15 +58,32 @@ pub fn check_env() -> anyhow::Result<()> { }; if !success { - let mut err_msg = format!("Required dependency '{}' is not installed.", check.app_name); + let error_lines = + errors.get_or_insert(String::from("Following dependencies are missing:\n")); + writeln!( + error_lines, + "Required dependency '{}' is not installed.", + check.app_name + ) + .expect("Writing to string never fail"); if let Some(install_hint) = check.install_hint { - err_msg.push_str( - format!("\nConsider installing it using the command '{install_hint}'").as_str(), - ); + writeln!( + error_lines, + "Consider installing it using the command '{install_hint}'" + ) + .expect("Writing to string never fail"); } - bail!(err_msg); + + writeln!( + error_lines, + "------------------------------------------------------------------" + ) + .expect("Writing to string never fail"); } } - Ok(()) + match errors { + Some(err_text) => bail!("{}", err_text.trim()), + None => Ok(()), + } } diff --git a/cli/src/main.rs b/cli/src/main.rs index d86f4264d..742fd827c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -47,6 +47,9 @@ enum ReportOptions { #[derive(Subcommand, Debug, Clone)] enum Command { + /// Checks that all needed tools for the development are installed + #[clap(visible_alias = "env")] + Environment, /// Runs linting & clippy Lint { /// Target to lint, by default whole application will be linted @@ -101,6 +104,11 @@ async fn main() -> Result<(), Error> { let command = cli.command; let report_opt: ReportOptions; let results = match command { + Command::Environment => { + // Check for dependencies is already called before calling any command + println!("All needed tools for development are installed"); + return Ok(()); + } Command::Lint { target, report } => { report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); From 77498235167ac4265b5337b316e541baf79d798e Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 17 Apr 2024 11:10:41 +0200 Subject: [PATCH 054/174] Build CLI: Track Builds to prevent duplications - Build states will be stored in shared map. - When build is called the shared map will be checked as following: - If the build already done then return early with its result - If the build is running then wait for it to be done then return its result - If the build hasn't started then start it and update the build states map --- cli/src/build_state.rs | 31 ++++++++++++++++++ cli/src/main.rs | 1 + cli/src/modules/mod.rs | 73 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 cli/src/build_state.rs diff --git a/cli/src/build_state.rs b/cli/src/build_state.rs new file mode 100644 index 000000000..dc0aed3c4 --- /dev/null +++ b/cli/src/build_state.rs @@ -0,0 +1,31 @@ +use std::{collections::HashMap, sync::Mutex}; + +use tokio::sync::{oneshot, OnceCell}; + +use crate::{spawner::SpawnResult, target::Target}; + +type BuildResult = Result, anyhow::Error>; + +pub enum BuildState { + Running(Vec>), + Finished(BuildResult), +} + +pub struct BuildStatesTracker { + pub states_map: Mutex>, +} + +impl BuildStatesTracker { + fn new() -> Self { + let states_map = Mutex::new(HashMap::new()); + Self { states_map } + } + + pub async fn get() -> &'static BuildStatesTracker { + static BUILD_STATES: OnceCell = OnceCell::const_new(); + + BUILD_STATES + .get_or_init(|| async { BuildStatesTracker::new() }) + .await + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 742fd827c..641960dc2 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,4 @@ +mod build_state; mod check_env; mod fstools; mod location; diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index df0f841ed..0505ddec6 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -8,15 +8,17 @@ pub mod wasm; pub mod wrapper; use crate::{ + build_state::{BuildState, BuildStatesTracker}, fstools, location::get_root, spawner::{spawn, SpawnOptions, SpawnResult}, Target, }; -use anyhow::Error; +use anyhow::{bail, Context, Error}; use async_trait::async_trait; use futures::future::join_all; use std::{iter, path::PathBuf}; +use tokio::sync::oneshot; #[derive(Debug, Clone)] pub enum Kind { @@ -142,7 +144,75 @@ pub trait Manager { async fn after(&self, _prod: bool) -> Result, Error> { Ok(None) } + + /// Runs build considering the currently running builds and already finished ones as well. async fn build(&self, prod: bool) -> Result, Error> { + let build_states = BuildStatesTracker::get().await; + let (tx_result, rx_result) = oneshot::channel(); + + let target = self.owner(); + let mut run_build = false; + // Check the current build state + { + let mut states = build_states.states_map.lock().unwrap(); + match states.get_mut(&target) { + Some(BuildState::Running(senders)) => { + // If the build is currently running, add the sender to senders list so it get + // the results when the job is done; + senders.push(tx_result); + } + Some(BuildState::Finished(build_result)) => { + // If Build is already finished, then return the results directly + match build_result { + Ok(res) => return Ok(res.clone()), + Err(err) => bail!(err.to_string()), + } + } + None => { + // Run the build and add the sender to the list. + run_build = true; + states.insert(target.clone(), BuildState::Running(vec![tx_result])); + } + } + } + + if run_build { + // Run the build + let build_result = self.perform_build(prod).await; + // Update the build state and notify all the senders + { + let mut states = build_states.states_map.lock().unwrap(); + + let res_clone = match &build_result { + Ok(spawn_res) => Ok(spawn_res.clone()), + Err(err) => Err(anyhow::anyhow!("{}", err)), + }; + + let Some(BuildState::Running(senders)) = + states.insert(target, BuildState::Finished(build_result)) + else { + unreachable!("State after calling build must be renning"); + }; + + for sender in senders { + let res_clone = match &res_clone { + Ok(spawn_res) => Ok(spawn_res.clone()), + Err(err) => Err(anyhow::anyhow!("{}", err)), + }; + if sender.send(res_clone).is_err() { + bail!("Fail to communicate with builder"); + } + } + } + } + + rx_result + .await + .context("Fail to communicate with builder")? + } + + /// Performs build process without checking the current builds states + async fn perform_build(&self, prod: bool) -> Result, Error> { let mut results = Vec::new(); let install_result = self.install(false).await?; results.push(install_result); @@ -184,6 +254,7 @@ pub trait Manager { Err(err) => Err(err), } } + async fn check(&self) -> Result, Error> { let mut results = Vec::new(); match self.kind() { From 413356fd627e259e59eb0d1c591d38d299ca6241 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 17 Apr 2024 14:52:21 +0200 Subject: [PATCH 055/174] Build CLI: Add missing dependency rs-bindings -> Shared(Platform) --- cli/src/modules/binding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 32462a620..7e54c7e66 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -29,7 +29,7 @@ impl Manager for Module { .join("rs-bindings") } fn deps(&self) -> Vec { - vec![] + vec![Target::Shared] } fn build_cmd(&self, prod: bool) -> Option { let mut path = Target::Wrapper.get().cwd(); From 8421321ea394d4e338a4d8e2bbb9a2f8628c4fff Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 18 Apr 2024 17:47:30 +0200 Subject: [PATCH 056/174] Build CLI: Add Copy binding & platform to electron - Copying is taken from the method `copy_tsbindings_and_platform` in ruby - Copying will be invoked after calling build to ts-bindings only - TODO is left in the code to remain that the copying operation must be verified - Signatures of other methods in fstools is changes to avoid copying --- cli/src/fstools.rs | 56 ++++++++++++++++++++++- cli/src/modules/app.rs | 2 +- cli/src/modules/mod.rs | 4 +- cli/src/modules/wrapper.rs | 94 +++++++++++++++++++++++++++++++++++++- 4 files changed, 150 insertions(+), 6 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index dba33f9e3..15495688d 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,6 +1,8 @@ extern crate fs_extra; use anyhow::Error; +use fs_extra::copy_items_with_progress; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; +use std::fmt::Display; use std::sync::mpsc; use std::{fs, path::PathBuf}; @@ -52,13 +54,63 @@ pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { Ok(()) } -pub async fn rm_folder(path: PathBuf) -> Result<(), Error> { +/// Copy a collection of files and folders recursively. +pub async fn cp_many( + items: Vec, + dest: PathBuf, + general_source: impl Display, + logs: &mut Vec, +) -> Result<(), Error> { + let tracker = get_tracker().await; + let sequence = tracker.start("copy file and folders", None).await?; + let options = CopyOptions::new(); + let (tx, rx) = mpsc::channel(); + let msg = format!( + "copied files: from {} to {}", + general_source, + dest.display() + ); + + logs.extend( + items + .iter() + .map(|item| format!("Item: '{}' copied to '{}'", item.display(), dest.display())), + ); + + let _ = tokio::spawn(async move { + if let Err(e) = copy_items_with_progress(&items, dest, &options, |info| { + if tx.send(info).is_err() { + eprintln!("Fail to send copying progress"); + } + TransitProcessResult::ContinueOrAbort + }) { + panic!("Fail to copy: {e}") + } + }) + .await; + while let Ok(info) = rx.recv() { + tracker + .msg( + sequence, + &format!( + "copied: {} bytes; current: {}", + info.copied_bytes, info.file_name + ), + ) + .await; + tracker.progress(sequence, None).await; + } + tracker.success(sequence, &msg).await; + Ok(()) +} + +pub async fn rm_folder(path: &PathBuf) -> Result<(), Error> { if !path.exists() { return Ok(()); } let tracker = get_tracker().await; let sequence = tracker.start("remove folder", None).await?; - fs::remove_dir_all(&path)?; + fs::remove_dir_all(path)?; tracker .success(sequence, &format!("removed: {}", path.display(),)) .await; diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 83b169590..982f46f65 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -53,7 +53,7 @@ impl Manager for Module { let msg = format!("removig directory: {}", prev.display()); report_logs.push(msg); - fstools::rm_folder(prev).await?; + fstools::rm_folder(&prev).await?; } let msg = format!( diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 0505ddec6..6bde49eed 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -92,7 +92,7 @@ pub trait Manager { let remove_log = format!("removing {}", dist_path.display()); - fstools::rm_folder(dist_path).await?; + fstools::rm_folder(&dist_path).await?; let job = format!("Reset {}", self.owner()); @@ -110,7 +110,7 @@ pub trait Manager { let remove_log = format!("removing directory {}", path.display()); logs.push(remove_log); - fstools::rm_folder(path).await?; + fstools::rm_folder(&path).await?; let job = format!("Clean {}", self.owner()); diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index a69dbe79b..03d221542 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -90,8 +90,100 @@ impl Manager for Module { fstools::cp_file(src, dest.join("index.node")).await?; + // TODO: This part is taken from from `copy_tsbindings_and_platform` in ruby. + // - We still need to make sure that it fits well with the current implementation and + // remove the not needed parts and add the missing parts if any + + // *** Copying TS Bindings *** + report_logs.push(String::from("Copying ts-bindings to electron...")); + let rustcore_dest = Target::App + .get() + .cwd() + .join("node_modules") + .join("rustcore"); + + fstools::rm_folder(&rustcore_dest).await?; + + let msg = format!("Removing directory: '{}'", rustcore_dest.display()); + report_logs.push(msg); + + tokio::fs::create_dir_all(&rustcore_dest).await?; + + // This part to get all the needed files and folders to copy + let ts_source = self.cwd(); + let ts_entries_to_copy: Vec<_> = fs::read_dir(&ts_source)? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name().is_some_and(|file_name| { + !file_name.to_string_lossy().starts_with("node_modules") + }) + }) + .collect(); + + fstools::cp_many( + ts_entries_to_copy, + rustcore_dest.clone(), + ts_source.display(), + &mut report_logs, + ) + .await?; + + // *** Remove native folder *** + let native_dir_path = rustcore_dest.join("native"); + report_logs.push(format!( + "Removing the directory '{}'", + native_dir_path.display() + )); + + fstools::rm_folder(&native_dir_path).await?; + + // *** Copy Platform rustcore to electron *** + report_logs.push(String::from("Copying platform rustcore in to electron...")); + let platform_dest = rustcore_dest.join("node_modules").join("platform"); + + fstools::rm_folder(&platform_dest).await?; + tokio::fs::create_dir_all(&platform_dest).await?; + + let platform_src = Target::Shared.get().cwd(); + + let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src)? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name().is_some_and(|file_name| { + !file_name.to_string_lossy().starts_with("node_modules") + }) + }) + .collect(); + + fstools::cp_many( + platform_entries_to_copy.clone(), + platform_dest, + platform_src.display(), + &mut report_logs, + ) + .await?; + + // *** Copy Platform to electron *** + report_logs.push(String::from("Copying platform in to electron...")); + let platform_dest2 = Target::App + .get() + .cwd() + .join("node_modules") + .join("platform"); + + fstools::rm_folder(&platform_dest2).await?; + tokio::fs::create_dir_all(&platform_dest2).await?; + + fstools::cp_many( + platform_entries_to_copy, + platform_dest2, + platform_src.display(), + &mut report_logs, + ) + .await?; + Ok(Some(SpawnResult::create_for_fs( - "Copy Wrapper Build Artifacts".into(), + "Copy TS Bindings and Platform to Electron".into(), report_logs, ))) } From 46746418711d04a15ce521c4dcc8e5d98482d791 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 19 Apr 2024 08:03:44 +0200 Subject: [PATCH 057/174] Build CLI: Move writing logs to fstools methods Prevent repeating the code be centralizing writing the logs --- cli/src/fstools.rs | 25 +++++++++++++++++++++++-- cli/src/modules/app.rs | 9 +-------- cli/src/modules/wrapper.rs | 9 +-------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 15495688d..7c2eabfe6 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -8,9 +8,17 @@ use std::{fs, path::PathBuf}; use crate::tracker::get_tracker; -pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { +pub async fn cp_file( + src: PathBuf, + dest: PathBuf, + report_logs: &mut Vec, +) -> Result<(), Error> { let tracker = get_tracker().await; let sequence = tracker.start("copy file", None).await?; + + let msg = format!("copying file: '{}' to '{}'", src.display(), dest.display()); + report_logs.push(msg); + fs::copy(&src, &dest)?; tracker .success( @@ -21,12 +29,25 @@ pub async fn cp_file(src: PathBuf, dest: PathBuf) -> Result<(), Error> { Ok(()) } -pub async fn cp_folder(src: PathBuf, dest: PathBuf) -> Result<(), Error> { +pub async fn cp_folder( + src: PathBuf, + dest: PathBuf, + report_logs: &mut Vec, +) -> Result<(), Error> { let tracker = get_tracker().await; let sequence = tracker.start("copy folder", None).await?; let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); + + let msg = format!( + "copying directory: '{}' to '{}'", + src.display(), + dest.display() + ); + report_logs.push(msg); + let msg = format!("copied: {} to {}", src.display(), dest.display()); + let _ = tokio::spawn(async move { if let Err(e) = copy_with_progress(src, dest, &options, |info| { if tx.send(info).is_err() { diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 982f46f65..734586438 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -56,14 +56,7 @@ impl Manager for Module { fstools::rm_folder(&prev).await?; } - let msg = format!( - "copying directory: '{}' to '{}'", - src.display(), - dest.display() - ); - report_logs.push(msg); - - fstools::cp_folder(src.clone(), dest.clone()).await?; + fstools::cp_folder(src.clone(), dest.clone(), &mut report_logs).await?; let rename_from = dest.join( src.file_name() diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 03d221542..e61dc0d63 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -81,14 +81,7 @@ impl Manager for Module { fs::create_dir(&dest)?; } - let msg = format!( - "copying directory: '{}' to '{}'", - src.display(), - dest.display() - ); - report_logs.push(msg); - - fstools::cp_file(src, dest.join("index.node")).await?; + fstools::cp_file(src, dest.join("index.node"), &mut report_logs).await?; // TODO: This part is taken from from `copy_tsbindings_and_platform` in ruby. // - We still need to make sure that it fits well with the current implementation and From 0f995eb79cca4cffc4fd2b8d962d2a308c279f01 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 19 Apr 2024 09:47:51 +0200 Subject: [PATCH 058/174] Build CLI: Improve Error messages - Error messages have more context now - Copying many files return an Error instead of panicking --- cli/src/fstools.rs | 48 +++++++++++++++++++------------------- cli/src/modules/app.rs | 8 ++++--- cli/src/modules/wrapper.rs | 45 +++++++++++++++++++++++++++++------ cli/src/spawner.rs | 16 +++++++++---- 4 files changed, 78 insertions(+), 39 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 7c2eabfe6..aecb45591 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,5 +1,5 @@ extern crate fs_extra; -use anyhow::Error; +use anyhow::{Context, Error}; use fs_extra::copy_items_with_progress; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; use std::fmt::Display; @@ -19,7 +19,13 @@ pub async fn cp_file( let msg = format!("copying file: '{}' to '{}'", src.display(), dest.display()); report_logs.push(msg); - fs::copy(&src, &dest)?; + fs::copy(&src, &dest).with_context(|| { + format!( + "Error while copying file '{}' to '{}'", + src.display(), + dest.display() + ) + })?; tracker .success( sequence, @@ -39,26 +45,21 @@ pub async fn cp_folder( let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); - let msg = format!( - "copying directory: '{}' to '{}'", - src.display(), - dest.display() - ); - report_logs.push(msg); + let path_display = format!("'{}' to '{}'", src.display(), dest.display()); - let msg = format!("copied: {} to {}", src.display(), dest.display()); + let report_msg = format!("copying directory: {path_display}"); + report_logs.push(report_msg); let _ = tokio::spawn(async move { - if let Err(e) = copy_with_progress(src, dest, &options, |info| { + copy_with_progress(src, dest, &options, |info| { if tx.send(info).is_err() { eprintln!("Fail to send copying progress"); } TransitProcessResult::ContinueOrAbort - }) { - panic!("Fail to copy: {e}") - } + }) }) - .await; + .await + .with_context(|| format!("Error while copying directory: {path_display}"))?; while let Ok(info) = rx.recv() { tracker .msg( @@ -71,6 +72,8 @@ pub async fn cp_folder( .await; tracker.progress(sequence, None).await; } + + let msg = format!("copied: {path_display}"); tracker.success(sequence, &msg).await; Ok(()) } @@ -86,11 +89,7 @@ pub async fn cp_many( let sequence = tracker.start("copy file and folders", None).await?; let options = CopyOptions::new(); let (tx, rx) = mpsc::channel(); - let msg = format!( - "copied files: from {} to {}", - general_source, - dest.display() - ); + let path_display = format!("from '{}' to '{}'", general_source, dest.display()); logs.extend( items @@ -99,16 +98,15 @@ pub async fn cp_many( ); let _ = tokio::spawn(async move { - if let Err(e) = copy_items_with_progress(&items, dest, &options, |info| { + copy_items_with_progress(&items, dest, &options, |info| { if tx.send(info).is_err() { eprintln!("Fail to send copying progress"); } TransitProcessResult::ContinueOrAbort - }) { - panic!("Fail to copy: {e}") - } + }) }) - .await; + .await + .with_context(|| format!("Error while copying: {path_display}"))?; while let Ok(info) = rx.recv() { tracker .msg( @@ -121,6 +119,8 @@ pub async fn cp_many( .await; tracker.progress(sequence, None).await; } + + let msg = format!("copied files: {path_display}"); tracker.success(sequence, &msg).await; Ok(()) } diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 734586438..1fc48f11b 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -46,11 +46,12 @@ impl Manager for Module { let msg = format!("creating directory: {}", dest.display()); report_logs.push(msg); - fs::create_dir(&dest)?; + fs::create_dir(&dest) + .with_context(|| format!("Error while creating directory: {}", dest.display()))?; } let prev = dest.join("client"); if prev.exists() { - let msg = format!("removig directory: {}", prev.display()); + let msg = format!("removing directory: {}", prev.display()); report_logs.push(msg); fstools::rm_folder(&prev).await?; @@ -71,7 +72,8 @@ impl Manager for Module { ); report_logs.push(msg); - std::fs::rename(rename_from, rename_to)?; + std::fs::rename(&rename_from, &rename_to) + .with_context(|| format!("Error while renaming {}", rename_from.display()))?; Ok(Some(SpawnResult::create_for_fs( "Copy App Build Artifacts".into(), diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index e61dc0d63..8ee5553f5 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -5,7 +5,7 @@ use crate::{ spawner::{spawn, spawn_blocking, SpawnResult}, Target, }; -use anyhow::{bail, Error}; +use anyhow::{bail, Context, Error}; use async_trait::async_trait; use std::{fs, iter, path::PathBuf}; @@ -78,7 +78,8 @@ impl Manager for Module { let msg = format!("creating directory: {}", dest.display()); report_logs.push(msg); - fs::create_dir(&dest)?; + fs::create_dir(&dest) + .with_context(|| format!("Error while creating directory: {}", dest.display()))?; } fstools::cp_file(src, dest.join("index.node"), &mut report_logs).await?; @@ -100,11 +101,21 @@ impl Manager for Module { let msg = format!("Removing directory: '{}'", rustcore_dest.display()); report_logs.push(msg); - tokio::fs::create_dir_all(&rustcore_dest).await?; + tokio::fs::create_dir_all(&rustcore_dest) + .await + .with_context(|| { + format!("Error while creating directory {}", rustcore_dest.display()) + })?; // This part to get all the needed files and folders to copy let ts_source = self.cwd(); - let ts_entries_to_copy: Vec<_> = fs::read_dir(&ts_source)? + let ts_entries_to_copy: Vec<_> = fs::read_dir(&ts_source) + .with_context(|| { + format!( + "Error while reading directory content: {}", + ts_source.display() + ) + })? .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) .filter(|path| { path.file_name().is_some_and(|file_name| { @@ -135,11 +146,24 @@ impl Manager for Module { let platform_dest = rustcore_dest.join("node_modules").join("platform"); fstools::rm_folder(&platform_dest).await?; - tokio::fs::create_dir_all(&platform_dest).await?; + tokio::fs::create_dir_all(&platform_dest) + .await + .with_context(|| { + format!( + "Error while creating directory: {}", + platform_dest.display() + ) + })?; let platform_src = Target::Shared.get().cwd(); - let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src)? + let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src) + .with_context(|| { + format!( + "Error while reading directory content: {}", + platform_src.display() + ) + })? .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) .filter(|path| { path.file_name().is_some_and(|file_name| { @@ -165,7 +189,14 @@ impl Manager for Module { .join("platform"); fstools::rm_folder(&platform_dest2).await?; - tokio::fs::create_dir_all(&platform_dest2).await?; + tokio::fs::create_dir_all(&platform_dest2) + .await + .with_context(|| { + format!( + "Error while creating directory: {}", + platform_dest2.display() + ) + })?; fstools::cp_many( platform_entries_to_copy, diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 0a38f2d5a..7414bb431 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -3,7 +3,7 @@ use crate::{ location::{get_root, to_relative_path}, tracker::get_tracker, }; -use anyhow::bail; +use anyhow::{bail, Context}; use core::panic; use futures_lite::{future, FutureExt}; use std::{ @@ -69,11 +69,17 @@ pub async fn spawn( let mut child = Command::new(cmd) .current_dir(&cwd) - .args(parts) + .args(&parts) .envs(env_vars) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn()?; + .spawn() + .with_context(|| { + format!( + "Error While running the command '{cmd}'\nwith arguments: {parts:?}\ncwd: {}", + cwd.display() + ) + })?; let job_title = caption; let tracker = get_tracker().await; let sequence = tracker @@ -173,7 +179,7 @@ pub async fn spawn_blocking( let mut child = std::process::Command::new(cmd); child.current_dir(&cwd); - child.args(parts); + child.args(&parts); child.envs(env_vars); let tracker = get_tracker().await; @@ -189,7 +195,7 @@ pub async fn spawn_blocking( Ok(status) => status, Err(err) => { tracker.fail(sequence, &err.to_string()).await; - return Err(err); + bail!("Error While running the command '{cmd}'\nwith arguments: {parts:?}\ncwd: {}\n Error Info: {err}", cwd.display()); } }; From 233765c28e4e39f584d589c76c654d7a4044ab35 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 19 Apr 2024 10:57:14 +0200 Subject: [PATCH 059/174] Build CLI: Move CLI Arguments to their own module --- cli/src/cli_args.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++ cli/src/main.rs | 70 +++------------------------------------------ 2 files changed, 73 insertions(+), 66 deletions(-) create mode 100644 cli/src/cli_args.rs diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs new file mode 100644 index 000000000..98bd1e5f0 --- /dev/null +++ b/cli/src/cli_args.rs @@ -0,0 +1,69 @@ +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +use crate::target::Target; + +static REPORT_HELP_TEXT: &str = + "Write report from command logs to the given file or to stdout if no file is defined"; +static REPORT_VALUE_NAME: &str = "FILE-PATH"; + +#[derive(Parser)] +#[command(name = "cargo", bin_name = "cargo")] +pub enum CargoCli { + Chipmunk(Cli), +} + +#[derive(clap::Args, Debug)] +#[command(author, version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Checks that all needed tools for the development are installed + #[clap(visible_alias = "env")] + Environment, + /// Runs linting & clippy + Lint { + /// Target to lint, by default whole application will be linted + #[arg(index = 1)] + target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, + }, + /// Build + Build { + /// Target to build, by default whole application will be built + #[arg(index = 1)] + target: Option>, + + /// Build release version + #[arg(short, long, default_value_t = false)] + production: bool, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, + }, + /// Clean + Clean { + /// Target to clean, by default whole application will be cleaned + #[arg(index = 1)] + target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, + }, + /// Run tests + Test { + /// Target to test, by default whole application will be tested + #[arg(index = 1)] + target: Option>, + + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] + report: Option>, + }, +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 641960dc2..2df30b7f8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,5 +1,6 @@ mod build_state; mod check_env; +mod cli_args; mod fstools; mod location; mod modules; @@ -9,7 +10,8 @@ mod tracker; use anyhow::{bail, Error}; use check_env::check_env; -use clap::{Parser, Subcommand}; +use clap::Parser; +use cli_args::{CargoCli, Command}; use futures::future::join_all; use location::init_location; use modules::Manager; @@ -22,77 +24,13 @@ use std::{ use target::Target; use tracker::get_tracker; -static REPORT_HELP_TEXT: &str = - "Write report from command logs to the given file or to stdout if no file is defined"; -static REPORT_VALUE_NAME: &str = "FILE-PATH"; - -#[derive(Parser)] -#[command(name = "cargo", bin_name = "cargo")] -enum CargoCli { - Chipmunk(Cli), -} - -#[derive(clap::Args, Debug)] -#[command(author, version, about, long_about = None)] -struct Cli { - #[command(subcommand)] - command: Command, -} - #[derive(Debug)] -enum ReportOptions { +pub enum ReportOptions { None, Stdout(Stdout), File(PathBuf, File), } -#[derive(Subcommand, Debug, Clone)] -enum Command { - /// Checks that all needed tools for the development are installed - #[clap(visible_alias = "env")] - Environment, - /// Runs linting & clippy - Lint { - /// Target to lint, by default whole application will be linted - #[arg(index = 1)] - target: Option>, - - #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] - report: Option>, - }, - /// Build - Build { - /// Target to build, by default whole application will be built - #[arg(index = 1)] - target: Option>, - - /// Build release version - #[arg(short, long, default_value_t = false)] - production: bool, - - #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] - report: Option>, - }, - /// Clean - Clean { - /// Target to clean, by default whole application will be cleaned - #[arg(index = 1)] - target: Option>, - - #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] - report: Option>, - }, - /// Run tests - Test { - /// Target to test, by default whole application will be tested - #[arg(index = 1)] - target: Option>, - - #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] - report: Option>, - }, -} - #[tokio::main] async fn main() -> Result<(), Error> { let CargoCli::Chipmunk(cli) = CargoCli::parse(); From 79ca717267e1f4e9c14dab81240b64aa15187aca Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 19 Apr 2024 12:57:25 +0200 Subject: [PATCH 060/174] Build CLI: Run command added - Run Command will build the app then run it. --- cli/src/app_runner.rs | 15 +++++++++++++++ cli/src/cli_args.rs | 6 ++++++ cli/src/main.rs | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 cli/src/app_runner.rs diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs new file mode 100644 index 000000000..9f7c49189 --- /dev/null +++ b/cli/src/app_runner.rs @@ -0,0 +1,15 @@ +use std::{io, process::ExitStatus}; + +use tokio::process::Command; + +use crate::target::Target; + +pub async fn run_app() -> io::Result { + let electron_path = Target::App.get().cwd(); + + Command::new("yarn") + .current_dir(electron_path) + .args(["run", "electron"]) + .status() + .await +} diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 98bd1e5f0..1da023b79 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -66,4 +66,10 @@ pub enum Command { #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, + /// Build and Run the application + Run { + /// Run release version + #[arg(short, long, default_value_t = false)] + production: bool, + }, } diff --git a/cli/src/main.rs b/cli/src/main.rs index 2df30b7f8..75aae9843 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,4 @@ +mod app_runner; mod build_state; mod check_env; mod cli_args; @@ -42,6 +43,7 @@ async fn main() -> Result<(), Error> { // Run the given command let command = cli.command; let report_opt: ReportOptions; + let mut run_app = false; let results = match command { Command::Environment => { // Check for dependencies is already called before calling any command @@ -96,6 +98,17 @@ async fn main() -> Result<(), Error> { ) .await } + Command::Run { production } => { + report_opt = ReportOptions::None; + run_app = true; + join_all( + Target::all() + .iter() + .map(|module| module.build(production)) + .collect::>(), + ) + .await + } }; // Shutdown and show results & report @@ -144,6 +157,12 @@ async fn main() -> Result<(), Error> { } if !success { bail!("Some task were failed") + } else if run_app { + println!("Starting chipmunk..."); + let status = app_runner::run_app().await?; + if !status.success() { + bail!("Error: Chipmunk Exited with the Code {status}"); + } } Ok(()) } From 39489a9f54bf07299eb8f23c6d84887678118329 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Apr 2024 09:38:56 +0200 Subject: [PATCH 061/174] Build CLI: Remove TODO after checks & add comment --- cli/src/modules/wrapper.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 8ee5553f5..be178bc0d 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -69,6 +69,7 @@ impl Manager for Module { async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); + // *** Copy `index.node` from rs to ts bindings *** let src = Target::Binding.get().cwd().join("dist").join("index.node"); let dest = self.cwd().join("dist").join("native"); if !src.exists() { @@ -84,10 +85,6 @@ impl Manager for Module { fstools::cp_file(src, dest.join("index.node"), &mut report_logs).await?; - // TODO: This part is taken from from `copy_tsbindings_and_platform` in ruby. - // - We still need to make sure that it fits well with the current implementation and - // remove the not needed parts and add the missing parts if any - // *** Copying TS Bindings *** report_logs.push(String::from("Copying ts-bindings to electron...")); let rustcore_dest = Target::App From d98f4a92b12859e73319de1c11ae1a715bd210c6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Apr 2024 14:38:26 +0200 Subject: [PATCH 062/174] Build CLI: Add & Implement dir_checksum library The library walks the file tree in the given path recursively considering `gitignore` rules and calculates the checksums of these files returning either a list of the hashes or a combination of them --- cli/Cargo.lock | 263 ++++++++++++++++++++++- cli/Cargo.toml | 4 + cli/dir_checksum/Cargo.toml | 13 ++ cli/dir_checksum/src/file_hash_digest.rs | 17 ++ cli/dir_checksum/src/hash_digest.rs | 61 ++++++ cli/dir_checksum/src/hash_error.rs | 11 + cli/dir_checksum/src/input.rs | 108 ++++++++++ cli/dir_checksum/src/lib.rs | 108 ++++++++++ 8 files changed, 583 insertions(+), 2 deletions(-) create mode 100644 cli/dir_checksum/Cargo.toml create mode 100644 cli/dir_checksum/src/file_hash_digest.rs create mode 100644 cli/dir_checksum/src/hash_digest.rs create mode 100644 cli/dir_checksum/src/hash_error.rs create mode 100644 cli/dir_checksum/src/input.rs create mode 100644 cli/dir_checksum/src/lib.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 6a29839c1..a2c4669fd 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.4" @@ -71,6 +80,18 @@ version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "async-trait" version = "0.1.74" @@ -115,6 +136,30 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "blake3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "rayon", +] + +[[package]] +name = "bstr" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bytes" version = "1.5.0" @@ -129,6 +174,7 @@ dependencies = [ "async-trait", "clap", "console", + "dir_checksum", "fs_extra", "futures", "futures-lite", @@ -212,6 +258,54 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "dir_checksum" +version = "0.1.0" +dependencies = [ + "blake3", + "ignore", + "memmap2", + "rayon", + "thiserror", +] + +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -367,6 +461,19 @@ dependencies = [ "url", ] +[[package]] +name = "globset" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "heck" version = "0.4.1" @@ -389,6 +496,22 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indicatif" version = "0.17.7" @@ -428,9 +551,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libgit2-sys" @@ -494,6 +617,15 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memmap2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" +dependencies = [ + "libc", +] + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -634,6 +766,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -643,18 +795,64 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -706,6 +904,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3de26b0965292219b4287ff031fcba86837900fe9cd2b34ea8ad893c0953d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "268026685b2be38d7103e9e507c938a1fcb3d7e6eb15e87870b617bf37b6d581" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -807,12 +1025,53 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.45.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b276d7831..57e444ea2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" description = "Tool for chipmunk application development" +[workspace] +members = ["dir_checksum"] + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -17,3 +20,4 @@ futures-lite = "1.13.0" git2 = "0.18.2" indicatif = "0.17.7" tokio = { version = "1.36.0", features = ["full"] } +dir_checksum = { path = "./dir_checksum" } diff --git a/cli/dir_checksum/Cargo.toml b/cli/dir_checksum/Cargo.toml new file mode 100644 index 000000000..6ce35f0f6 --- /dev/null +++ b/cli/dir_checksum/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "dir_checksum" +version = "0.1.0" +authors = ["Ammar Abou Zor "] +edition = "2021" +description = "A Library which provides methods to calculate the checksum of the files in a given path recursively considering `gitignore` rules" + +[dependencies] +blake3 = { version = "1", features = ["rayon"] } +ignore = "0.4" +memmap2 = "0.9" +rayon = "1" +thiserror = "1" diff --git a/cli/dir_checksum/src/file_hash_digest.rs b/cli/dir_checksum/src/file_hash_digest.rs new file mode 100644 index 000000000..dc72aee00 --- /dev/null +++ b/cli/dir_checksum/src/file_hash_digest.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use crate::hash_digest::HashDigest; + +/// Holds the checksum for a file with its path +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FileHashDigest { + pub path: PathBuf, + /// The calculated checksum value + pub hash_digest: HashDigest, +} + +impl FileHashDigest { + pub fn new(path: PathBuf, hash_digest: HashDigest) -> Self { + Self { path, hash_digest } + } +} diff --git a/cli/dir_checksum/src/hash_digest.rs b/cli/dir_checksum/src/hash_digest.rs new file mode 100644 index 000000000..95cf54817 --- /dev/null +++ b/cli/dir_checksum/src/hash_digest.rs @@ -0,0 +1,61 @@ +use std::{fmt::Display, str::FromStr}; + +/// The number of bytes in a Hash, 32. +pub const OUT_LEN: usize = blake3::OUT_LEN; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +/// Represents the calculated checksum value +/// Provides different methods to represents the hash value +pub struct HashDigest { + hash: blake3::Hash, +} + +impl Display for HashDigest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.hash) + } +} + +impl FromStr for HashDigest { + type Err = String; + + fn from_str(s: &str) -> Result { + let hash = blake3::Hash::from_hex(s).map_err(|e| format!("Invalid Input. Error: {}", e))?; + + Ok(Self { hash }) + } +} + +impl From for HashDigest { + fn from(hash: blake3::Hash) -> Self { + Self { hash } + } +} + +impl From<[u8; OUT_LEN]> for HashDigest { + fn from(bytes: [u8; OUT_LEN]) -> Self { + blake3::Hash::from_bytes(bytes).into() + } +} + +impl From for [u8; OUT_LEN] { + fn from(hash: HashDigest) -> Self { + hash.to_bytes() + } +} + +impl<'a> From<&'a HashDigest> for &'a [u8; OUT_LEN] { + fn from(hash: &'a HashDigest) -> Self { + hash.as_bytes() + } +} + +impl HashDigest { + pub fn as_bytes(&self) -> &[u8; OUT_LEN] { + self.hash.as_bytes() + } + + pub fn to_bytes(self) -> [u8; OUT_LEN] { + self.hash.into() + } +} diff --git a/cli/dir_checksum/src/hash_error.rs b/cli/dir_checksum/src/hash_error.rs new file mode 100644 index 000000000..9a07999f9 --- /dev/null +++ b/cli/dir_checksum/src/hash_error.rs @@ -0,0 +1,11 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum HashError { + #[error("Unrecoverable error, cannot continue: {0}")] + Unrecoverable(String), + #[error("Entry could not be processed: {0}")] + Entry(String), + #[error("IO error: {0:?}")] + Io(#[from] std::io::Error), +} diff --git a/cli/dir_checksum/src/input.rs b/cli/dir_checksum/src/input.rs new file mode 100644 index 000000000..49d1c3c16 --- /dev/null +++ b/cli/dir_checksum/src/input.rs @@ -0,0 +1,108 @@ +use std::{ + fs::File, + io::{self, Read}, + path::Path, +}; + +use crate::HashError; + +pub(crate) enum Input { + Mmap(io::Cursor), + File(File), +} + +impl Input { + pub(crate) fn open(path: &Path) -> Result { + let file = File::open(path)?; + if let Some(mmap) = maybe_memmap_file(&file)? { + return Ok(Self::Mmap(io::Cursor::new(mmap))); + } + Ok(Self::File(file)) + } + + pub(crate) fn hash(&mut self, base_hasher: &blake3::Hasher) -> Result { + let mut hasher = base_hasher.clone(); + match self { + // The fast path: If we mmapped the file successfully, hash using + // multiple threads. + Self::Mmap(cursor) => { + hasher.update_rayon(cursor.get_ref()); + } + // The slower paths, for files we didn't/couldn't mmap. + // This is currently all single-threaded. Doing multi-threaded + // hashing without memory mapping is tricky, since all your worker + // threads have to stop every time you refill the buffer, and that + // ends up being a lot of overhead. To solve that, we need a more + // complicated double-buffering strategy where a background thread + // fills one buffer while the worker threads are hashing the other + // one. We might implement that in the future, but since this is + // the slow path anyway, it's not high priority. + Self::File(file) => { + copy_wide(file, &mut hasher)?; + } + } + Ok(hasher.finalize()) + } +} + +// Mmap a file, if it looks like a good idea. Return None in cases where we +// know mmap will fail, or if the file is short enough that mmapping isn't +// worth it. However, if we do try to mmap and it fails, return the error. +fn maybe_memmap_file(file: &File) -> Result, HashError> { + let metadata = file.metadata()?; + let file_size = metadata.len(); + let map = if !metadata.is_file() { + // Not a real file. + None + } else if file_size > isize::max_value() as u64 { + // Too long to safely map. + // https://github.com/danburkert/memmap-rs/issues/69 + None + } else if file_size < 16 * 1024 { + // Mapping small files is not worth it. + None + } else { + // Explicitly set the length of the memory map, so that file system + // changes can't race to violate the invariants we just checked. + // + // + // ## Safety + // + // All file-backed memory map constructors are marked `unsafe` because of the potential for + // *Undefined Behavior* using the map if the underlying file is subsequently modified, in or + // out of process. + // Memory map will be used here for a very short time only while calculating the hash. It's + // unlikely that the files will be changed at the same time, and in that case it's the + // responsibility of the user of this library to unsure that the files aren't changed from + // different process while calling this methods from this library. + let map = unsafe { + memmap2::MmapOptions::new() + .len(file_size as usize) + .map(file)? + }; + + Some(map) + }; + + Ok(map) +} + +// A 16 KiB buffer is enough to take advantage of all the SIMD instruction sets +// that we support, but `std::io::copy` currently uses 8 KiB. Most platforms +// can support at least 64 KiB, and there's some performance benefit to using +// bigger reads, so that's what we use here. +fn copy_wide(mut reader: impl Read, hasher: &mut blake3::Hasher) -> io::Result { + let mut buffer = [0; 65536]; + let mut total = 0; + loop { + match reader.read(&mut buffer) { + Ok(0) => return Ok(total), + Ok(n) => { + hasher.update(&buffer[..n]); + total += n as u64; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } +} diff --git a/cli/dir_checksum/src/lib.rs b/cli/dir_checksum/src/lib.rs new file mode 100644 index 000000000..da78b2999 --- /dev/null +++ b/cli/dir_checksum/src/lib.rs @@ -0,0 +1,108 @@ +use blake3::Hasher; +use ignore::Walk; +use input::Input; +use rayon::{ + iter::{IntoParallelIterator, ParallelIterator}, + ThreadPoolBuilder, +}; +use std::{ + io::{self, ErrorKind}, + path::{Path, PathBuf}, +}; + +mod file_hash_digest; +mod hash_digest; +mod hash_error; +mod input; + +pub use file_hash_digest::FileHashDigest; +pub use hash_digest::HashDigest; +pub use hash_digest::OUT_LEN; +pub use hash_error::HashError; + +/// Calculates the hash of each file in the given folder considering `gitignore` rules returning +/// the combination of their checksums +/// +/// * `dir_path`: Root directory to iterate through their files recursively +pub fn calc_combined_checksum

(dir_path: P) -> Result +where + P: AsRef + Send + Sync, +{ + run_intern(dir_path, |path, hasher| { + calc_files_hashes(path, hasher)?.iter().for_each(|entry| { + hasher.update(entry.hash_digest.as_bytes()); + }); + + Ok(hasher.finalize().into()) + }) +} + +/// Calculates the hash of each file in the given folder considering `gitignore` rules and returns +/// a list of the files with their checksums +/// +/// * `dir_path`: Root directory to iterate through their files recursively +pub fn calc_individual_checksum

(dir_path: P) -> Result, HashError> +where + P: AsRef + Send + Sync, +{ + run_intern(dir_path, |path, hasher| calc_files_hashes(path, hasher)) +} + +/// Validates the given path and prepares the run environment then calls the given function +/// returning its result +/// +/// * `calc_fn`: Function that will be called inside the function with the directory path and +/// the created hasher +fn run_intern(dir_path: P, calc_fn: F) -> Result +where + F: Fn(&Path, &mut blake3::Hasher) -> Result + Sync + Send, + P: AsRef + Send + Sync, + T: Send + Sync, +{ + let dir_path = dir_path.as_ref(); + if !dir_path.is_dir() { + return Err(io::Error::new( + ErrorKind::InvalidInput, + format!( + "Given path must be a directory. path: {}", + dir_path.display() + ), + ) + .into()); + } + + let mut hasher = blake3::Hasher::new(); + + let thread_pool = ThreadPoolBuilder::new() + .build() + .map_err(|e| HashError::Unrecoverable(format!("Threadpool can't be created: {e}")))?; + + thread_pool.install(|| calc_fn(dir_path, &mut hasher)) +} + +/// Walks through file trees calculate the checksum for each files of them +/// +/// * `dir_path`: Path of directory to walk the files from +fn calc_files_hashes>( + dir_path: P, + hasher: &blake3::Hasher, +) -> Result, HashError> { + let entries: Vec = Walk::new(dir_path) + .filter_map(|dir_entry| dir_entry.map(|entry| entry.into_path()).ok()) + .filter(|path| path.is_file()) + .collect(); + + entries + .into_par_iter() + .map(|path| calc_hash(&path, hasher).map(|hash| FileHashDigest::new(path, hash.into()))) + .collect() +} + +/// Calculates the hash for the given file path +fn calc_hash(file_path: &Path, base_hasher: &Hasher) -> Result { + let mut input = Input::open(file_path) + .map_err(|e| HashError::Entry(format!("Could not open file: {file_path:?} ({e})")))?; + input + .hash(base_hasher) + .map_err(|e| HashError::Entry(format!("Could not hash file: {file_path:?} ({e})"))) +} From c85177a5873c075779949851c3d9d4ab5094fb50 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Apr 2024 17:43:26 +0200 Subject: [PATCH 063/174] Build CLI: Integration tests for checksum library - Tests are created as a part of checksum library and referenced in the parent library (CLI) so it runs when we call `cargo test` on it - Both app and library shares the dependencies via workspaces --- cli/Cargo.lock | 65 +++++++ cli/Cargo.toml | 13 +- cli/dir_checksum/Cargo.toml | 4 + cli/dir_checksum/tests/integration_tests.rs | 178 ++++++++++++++++++++ 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 cli/dir_checksum/tests/integration_tests.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index a2c4669fd..f8baba0bc 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -180,6 +180,7 @@ dependencies = [ "futures-lite", "git2", "indicatif", + "tempdir", "tokio", ] @@ -293,10 +294,12 @@ checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" name = "dir_checksum" version = "0.1.0" dependencies = [ + "anyhow", "blake3", "ignore", "memmap2", "rayon", + "tempdir", "thiserror", ] @@ -336,6 +339,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.29" @@ -766,6 +775,34 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rayon" version = "1.10.0" @@ -786,6 +823,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -812,6 +858,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -904,6 +959,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand", + "remove_dir_all", +] + [[package]] name = "thiserror" version = "1.0.55" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 57e444ea2..aab4c3cd3 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -9,8 +9,12 @@ members = ["dir_checksum"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[dependencies] +[workspace.dependencies] +tempdir = "0.3" anyhow = "1.0.80" + +[dependencies] +anyhow.workspace = true async-trait = "0.1.73" clap = { version = "4.4.4", features = ["derive"] } console = "0.15.7" @@ -21,3 +25,10 @@ git2 = "0.18.2" indicatif = "0.17.7" tokio = { version = "1.36.0", features = ["full"] } dir_checksum = { path = "./dir_checksum" } + +[dev-dependencies] +tempdir.workspace = true + +[[test]] +name = "dir_checksum" +path = "dir_checksum/tests/integration_tests.rs" diff --git a/cli/dir_checksum/Cargo.toml b/cli/dir_checksum/Cargo.toml index 6ce35f0f6..fe82b9412 100644 --- a/cli/dir_checksum/Cargo.toml +++ b/cli/dir_checksum/Cargo.toml @@ -11,3 +11,7 @@ ignore = "0.4" memmap2 = "0.9" rayon = "1" thiserror = "1" + +[dev-dependencies] +tempdir.workspace = true +anyhow.workspace = true diff --git a/cli/dir_checksum/tests/integration_tests.rs b/cli/dir_checksum/tests/integration_tests.rs new file mode 100644 index 000000000..6511b16ba --- /dev/null +++ b/cli/dir_checksum/tests/integration_tests.rs @@ -0,0 +1,178 @@ +extern crate tempdir; + +use std::{ + fs::{self, File}, + path::PathBuf, +}; + +use dir_checksum::*; +use tempdir::TempDir; + +fn create_tmp_dir_with_file(dir_name: &'static str) -> anyhow::Result<(TempDir, PathBuf)> { + let tmp_dir = TempDir::new(dir_name)?; + let file_path = tmp_dir.path().join("file1.txt"); + fs::write(&file_path, "Initial text")?; + + Ok((tmp_dir, file_path)) +} + +#[test] +fn hash_combinations_file_identical() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("comb_file_identical")?; + + let original_hash = calc_combined_checksum(tmp_dir.path())?; + + assert_eq!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes of the same directory must be identical" + ); + + Ok(()) +} + +#[test] +fn hash_combinations_add_then_remove_file() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("comb_add_remove_file")?; + + let original_hash = calc_combined_checksum(tmp_dir.path())?; + + let file_path_2 = tmp_dir.path().join("file2.txt"); + fs::write(&file_path_2, "Initial text 2")?; + + assert_ne!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after adding one file can't be the same" + ); + + fs::remove_file(file_path_2)?; + + assert_eq!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after deleting the second file must be identical again" + ); + + Ok(()) +} + +#[test] +fn hash_combinations_change_file_content() -> anyhow::Result<()> { + let (tmp_dir, file_path_1) = create_tmp_dir_with_file("comg_change_contnet")?; + + let original_hash = calc_combined_checksum(tmp_dir.path())?; + + fs::write(&file_path_1, "changed text")?; + assert_ne!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after changing file content can't be the same" + ); + + Ok(()) +} + +#[test] +fn hash_combinations_empty_file() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("comb_empty_file")?; + + let original_hash = calc_combined_checksum(tmp_dir.path())?; + + // Create an empty file + let empty_file_path = tmp_dir.path().join("empty.txt"); + let empty_file = File::create(&empty_file_path)?; + drop(empty_file); + + assert_ne!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after creating an empty file can't be the same" + ); + + Ok(()) +} + +#[test] +fn hash_combinations_add_then_remove_sub_dir() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("comb_sub_dir")?; + + let original_hash = calc_combined_checksum(tmp_dir.path())?; + + let sub_dir = tmp_dir.path().join("sub_dir"); + fs::create_dir(&sub_dir)?; + + let file_path_2 = sub_dir.join("file2.txt"); + fs::write(&file_path_2, "Initial text 2")?; + + assert_ne!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after adding one file in sub directory can't be the same" + ); + + fs::remove_file(file_path_2)?; + + assert_eq!( + original_hash, + calc_combined_checksum(tmp_dir.path())?, + "Hashes after deleting the file in sub directory must be identical again" + ); + + Ok(()) +} + +#[test] +fn hash_individual_many_files() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("indiv_files")?; + + // Create non-empty file + let file2_path = tmp_dir.path().join("file2.txt"); + fs::write(file2_path, "file 2 content")?; + + // Create empty file + let empty_file_path = tmp_dir.path().join("empty.txt"); + let empty_file = File::create(&empty_file_path)?; + drop(empty_file); + + let items = calc_individual_checksum(tmp_dir.path())?; + + assert_eq!(items.len(), 3, "Hashes count must be 3"); + + assert_eq!( + &items, + &calc_individual_checksum(tmp_dir.path())?, + "Hash items must be identical" + ); + + Ok(()) +} + +#[test] +fn hash_individual_sub_directory() -> anyhow::Result<()> { + let (tmp_dir, _) = create_tmp_dir_with_file("indiv_sub_dir")?; + + let sub_dir = tmp_dir.path().join("sub_dir"); + fs::create_dir(&sub_dir)?; + + // Create non-empty file + let file2_path = &sub_dir.join("file2.txt"); + fs::write(file2_path, "file 2 content")?; + + // Create empty file + let empty_file_path = &sub_dir.join("empty.txt"); + let empty_file = File::create(&empty_file_path)?; + drop(empty_file); + + let items = calc_individual_checksum(tmp_dir.path())?; + + assert_eq!(items.len(), 3, "Hashes count must be 3"); + + assert_eq!( + &items, + &calc_individual_checksum(tmp_dir.path())?, + "Hash items must be identical" + ); + + Ok(()) +} From e3a67e2baa55683a94ed5ec22c61732e39c0ee2c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Apr 2024 08:52:45 +0200 Subject: [PATCH 064/174] Build CLI: Rename error type Environment is more descriptive here since we stop on all error types currently --- cli/dir_checksum/src/hash_error.rs | 4 ++-- cli/dir_checksum/src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/dir_checksum/src/hash_error.rs b/cli/dir_checksum/src/hash_error.rs index 9a07999f9..e90cec249 100644 --- a/cli/dir_checksum/src/hash_error.rs +++ b/cli/dir_checksum/src/hash_error.rs @@ -2,8 +2,8 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum HashError { - #[error("Unrecoverable error, cannot continue: {0}")] - Unrecoverable(String), + #[error("Environment error, cannot continue: {0}")] + Environment(String), #[error("Entry could not be processed: {0}")] Entry(String), #[error("IO error: {0:?}")] diff --git a/cli/dir_checksum/src/lib.rs b/cli/dir_checksum/src/lib.rs index da78b2999..412f630cd 100644 --- a/cli/dir_checksum/src/lib.rs +++ b/cli/dir_checksum/src/lib.rs @@ -75,7 +75,7 @@ where let thread_pool = ThreadPoolBuilder::new() .build() - .map_err(|e| HashError::Unrecoverable(format!("Threadpool can't be created: {e}")))?; + .map_err(|e| HashError::Environment(format!("Threadpool can't be created: {e}")))?; thread_pool.install(|| calc_fn(dir_path, &mut hasher)) } From 8710a091c1ff0c68e3ff9f8d743c80c146843eea Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Apr 2024 08:56:33 +0200 Subject: [PATCH 065/174] Build CLI: Add test command to CLI module --- cli/src/modules/cli.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index c894baac7..3f54cdb8e 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{location::get_root, Target}; +use crate::{location::get_root, modules::TestCommand, Target}; use async_trait::async_trait; use std::path::PathBuf; @@ -27,4 +27,11 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![] } + fn test_cmds(&self) -> Vec { + vec![TestCommand::new( + "cargo +stable test --color always".into(), + self.cwd(), + None, + )] + } } From c130717e4d1f5359f97cabf1147283f4df0e9ae5 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Apr 2024 13:23:43 +0200 Subject: [PATCH 066/174] Build CLI: Checksum Records (in progress) - Checksum Records is a thread safe singleton to track what changed in each target. - It can save/load the checksums for each target. - It provides methods to check if any file has changed in the given path considering `.gitignore` rules. - Add methods to target to parse them and to get all of them in vector --- cli/src/build_state.rs | 2 + cli/src/checksum_records.rs | 123 ++++++++++++++++++++++++++++++++++++ cli/src/main.rs | 1 + cli/src/target.rs | 52 ++++++++++++++- 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 cli/src/checksum_records.rs diff --git a/cli/src/build_state.rs b/cli/src/build_state.rs index dc0aed3c4..cd35ec16b 100644 --- a/cli/src/build_state.rs +++ b/cli/src/build_state.rs @@ -6,11 +6,13 @@ use crate::{spawner::SpawnResult, target::Target}; type BuildResult = Result, anyhow::Error>; +#[derive(Debug)] pub enum BuildState { Running(Vec>), Finished(BuildResult), } +#[derive(Debug)] pub struct BuildStatesTracker { pub states_map: Mutex>, } diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs new file mode 100644 index 000000000..9e27ab0ea --- /dev/null +++ b/cli/src/checksum_records.rs @@ -0,0 +1,123 @@ +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::Write, + path::PathBuf, + sync::Mutex, +}; + +use anyhow::{anyhow, Context}; +use dir_checksum::{calc_combined_checksum, HashDigest}; +use tokio::sync::OnceCell; + +use crate::{location::get_root, target::Target}; + +const FILE_NAME_DEV: &'static str = "build_chksum_dev.txt"; +const FILE_NAME_PROD: &'static str = "build_chksum_prod.txt"; + +#[derive(Debug)] +pub struct ChecksumRecords { + items: Mutex>, + production: bool, +} + +impl ChecksumRecords { + pub async fn get(production: bool) -> &'static anyhow::Result { + static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); + + CHECKSUM_RECORDS + .get_or_init(|| async { ChecksumRecords::load(production) }) + .await + } + + fn load(production: bool) -> anyhow::Result { + let file_path = Self::get_file_path(production); + + let items = if file_path.exists() { + let file_content = fs::read_to_string(file_path)?; + Self::parse_hashes(file_content)? + } else { + BTreeMap::new() + }; + + return Ok(Self { + items: Mutex::new(items), + production, + }); + } + + fn get_file_path(production: bool) -> PathBuf { + let root = get_root(); + if production { + root.join(FILE_NAME_PROD) + } else { + root.join(FILE_NAME_DEV) + } + } + + fn parse_hashes(text: String) -> anyhow::Result> { + let mut hashes = BTreeMap::new(); + + for (target, hash) in text.lines().filter_map(|line| line.split_once(':')) { + let target: Target = target.parse()?; + let hash: HashDigest = hash.parse().map_err(|e| anyhow!("{e}"))?; + + hashes.insert(target, hash); + } + + Ok(hashes) + } + + pub fn check_changed(&self, target: &Target) -> anyhow::Result { + let items = self.items.lock().unwrap(); + let saved_hash = match items.get(target) { + Some(hash) => hash, + None => return Ok(true), + }; + + let current_hash = Self::calc_hash_for_target(target)?; + + Ok(current_hash != *saved_hash) + } + + pub fn update_and_save(&self) -> anyhow::Result<()> { + self.calculate_hashes()?; + self.persist_hashes() + .context("Error while saving the updated hashes")?; + + Ok(()) + } + + fn calculate_hashes(&self) -> anyhow::Result<()> { + let mut calcuated_map = BTreeMap::new(); + for target in Target::all_enums() { + let hash = Self::calc_hash_for_target(&target)?; + calcuated_map.insert(target, hash); + } + + let mut items = self.items.lock().unwrap(); + *items = calcuated_map; + + Ok(()) + } + + fn persist_hashes(&self) -> anyhow::Result<()> { + let file_path = Self::get_file_path(self.production); + + let mut file = File::create(file_path)?; + let items = self.items.lock().unwrap(); + + for (target, hash) in items.iter() { + write!(file, "{}:{}", target, hash)?; + } + + Ok(()) + } + + fn calc_hash_for_target(target: &Target) -> anyhow::Result { + let path = target.get().cwd(); + calc_combined_checksum(path).with_context(|| { + format!("Error while calculating the current hash for target: {target}",) + }) + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 75aae9843..b003c1e4b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,6 +1,7 @@ mod app_runner; mod build_state; mod check_env; +mod checksum_records; mod cli_args; mod fstools; mod location; diff --git a/cli/src/target.rs b/cli/src/target.rs index 26b327691..5907c127a 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -1,7 +1,10 @@ +use std::str::FromStr; + use crate::{modules, modules::Manager}; +use anyhow::bail; use clap::ValueEnum; -#[derive(ValueEnum, Clone, Debug, Hash, PartialEq, Eq)] +#[derive(ValueEnum, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Target { /// Represents the path `application/apps/indexer` Core, @@ -40,7 +43,54 @@ impl std::fmt::Display for Target { } } +impl FromStr for Target { + type Err = anyhow::Error; + + fn from_str(input: &str) -> Result { + type T = Target; + + match input { + "Core" => Ok(T::Core), + "Wrapper" => Ok(T::Wrapper), + "Binding" => Ok(T::Binding), + "Cli" => Ok(T::Cli), + "Client" => Ok(T::Client), + "Shared" => Ok(T::Shared), + "App" => Ok(T::App), + "Wasm" => Ok(T::Wasm), + invalid => bail!("Invalid input: {invalid}"), + } + } +} + impl Target { + pub fn all_enums() -> Vec { + if cfg!(debug_assertions) { + // This check to remember to add the newly added enums to this function + let _ = match Target::App { + Target::Core => (), + Target::Binding => (), + Target::Wrapper => (), + Target::Client => (), + Target::Shared => (), + Target::App => (), + Target::Cli => (), + Target::Wasm => (), + }; + } + + vec![ + Target::Core, + Target::Binding, + Target::Wrapper, + Target::Client, + Target::Shared, + Target::App, + Target::Cli, + Target::Wasm, + ] + } + pub fn all() -> Vec> { vec![ Box::new(modules::binding::Module::new()), From 27f17ac7595a8d95a5b5cabb299387db10eb364a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Apr 2024 16:15:59 +0200 Subject: [PATCH 067/174] Build CLI: Checksum Records & Run tests Production - Tests can be run in production - Integrating the checksums in the main function (in progress): Currently we load and save the files without comparing the checksums in the build process - Clean command removes the corresponding checksums - Fix writing the checksum one on each line - Persist files have different names and are added to `.gitignore` --- .gitignore | 6 +++++- cli/src/checksum_records.rs | 31 +++++++++++++++++++------------ cli/src/cli_args.rs | 8 ++++++++ cli/src/main.rs | 31 +++++++++++++++++++++++++++---- cli/src/modules/cli.rs | 13 +++++++------ cli/src/modules/core.rs | 13 +++++++------ cli/src/modules/mod.rs | 11 +++++++---- cli/src/modules/wasm.rs | 2 +- cli/src/modules/wrapper.rs | 2 +- 9 files changed, 82 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 407a6234e..b6b34d256 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,8 @@ application/sandbox/serial/render/lib/views/dialog/port.available sandbox sandbox.backup **/holder/release -scripts/tools/file_checklists/*.* \ No newline at end of file +scripts/tools/file_checklists/*.* + +# CLI build tool files # +######################## +.build_chksum_* diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 9e27ab0ea..a58b2bf19 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -12,8 +12,8 @@ use tokio::sync::OnceCell; use crate::{location::get_root, target::Target}; -const FILE_NAME_DEV: &'static str = "build_chksum_dev.txt"; -const FILE_NAME_PROD: &'static str = "build_chksum_prod.txt"; +const FILE_NAME_DEV: &'static str = ".build_chksum_dev"; +const FILE_NAME_PROD: &'static str = ".build_chksum_prod"; #[derive(Debug)] pub struct ChecksumRecords { @@ -22,12 +22,14 @@ pub struct ChecksumRecords { } impl ChecksumRecords { - pub async fn get(production: bool) -> &'static anyhow::Result { + pub async fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); CHECKSUM_RECORDS .get_or_init(|| async { ChecksumRecords::load(production) }) .await + .as_ref() + .map_err(|err| anyhow!("{err}")) } fn load(production: bool) -> anyhow::Result { @@ -68,7 +70,7 @@ impl ChecksumRecords { Ok(hashes) } - pub fn check_changed(&self, target: &Target) -> anyhow::Result { + pub fn _check_changed(&self, target: &Target) -> anyhow::Result { let items = self.items.lock().unwrap(); let saved_hash = match items.get(target) { Some(hash) => hash, @@ -80,6 +82,13 @@ impl ChecksumRecords { Ok(current_hash != *saved_hash) } + fn calc_hash_for_target(target: &Target) -> anyhow::Result { + let path = target.get().cwd(); + calc_combined_checksum(path).with_context(|| { + format!("Error while calculating the current hash for target: {target}",) + }) + } + pub fn update_and_save(&self) -> anyhow::Result<()> { self.calculate_hashes()?; self.persist_hashes() @@ -88,6 +97,11 @@ impl ChecksumRecords { Ok(()) } + pub fn remove_hash_if_exist(&self, target: &Target) { + let mut items = self.items.lock().unwrap(); + _ = items.remove(target); + } + fn calculate_hashes(&self) -> anyhow::Result<()> { let mut calcuated_map = BTreeMap::new(); for target in Target::all_enums() { @@ -108,16 +122,9 @@ impl ChecksumRecords { let items = self.items.lock().unwrap(); for (target, hash) in items.iter() { - write!(file, "{}:{}", target, hash)?; + writeln!(file, "{}:{}", target, hash)?; } Ok(()) } - - fn calc_hash_for_target(target: &Target) -> anyhow::Result { - let path = target.get().cwd(); - calc_combined_checksum(path).with_context(|| { - format!("Error while calculating the current hash for target: {target}",) - }) - } } diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 1da023b79..4426165b5 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -54,6 +54,10 @@ pub enum Command { #[arg(index = 1)] target: Option>, + /// Clean release version + #[arg(short, long, default_value_t = false)] + production: bool, + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, @@ -63,6 +67,10 @@ pub enum Command { #[arg(index = 1)] target: Option>, + /// Test release version + #[arg(short, long, default_value_t = false)] + production: bool, + #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, diff --git a/cli/src/main.rs b/cli/src/main.rs index b003c1e4b..c4aead29e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -12,6 +12,7 @@ mod tracker; use anyhow::{bail, Error}; use check_env::check_env; +use checksum_records::ChecksumRecords; use clap::Parser; use cli_args::{CargoCli, Command}; use futures::future::join_all; @@ -33,6 +34,11 @@ pub enum ReportOptions { File(PathBuf, File), } +enum ChecksumOptions { + Update { production: bool }, + None, +} + #[tokio::main] async fn main() -> Result<(), Error> { let CargoCli::Chipmunk(cli) = CargoCli::parse(); @@ -44,6 +50,7 @@ async fn main() -> Result<(), Error> { // Run the given command let command = cli.command; let report_opt: ReportOptions; + let mut checksum_option = ChecksumOptions::None; let mut run_app = false; let results = match command { Command::Environment => { @@ -67,6 +74,7 @@ async fn main() -> Result<(), Error> { production, report, } => { + checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( @@ -77,29 +85,40 @@ async fn main() -> Result<(), Error> { ) .await } - Command::Clean { target, report } => { + Command::Clean { + target, + production, + report, + } => { + checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets .iter() - .map(|module| module.reset()) + .map(|module| module.reset(production)) .collect::>(), ) .await } - Command::Test { target, report } => { + Command::Test { + target, + production, + report, + } => { + checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); join_all( targets .iter() - .map(|module| module.test()) + .map(|module| module.test(production)) .collect::>(), ) .await } Command::Run { production } => { + checksum_option = ChecksumOptions::Update { production }; report_opt = ReportOptions::None; run_app = true; join_all( @@ -115,6 +134,10 @@ async fn main() -> Result<(), Error> { // Shutdown and show results & report let tracker = get_tracker().await; tracker.shutdown().await?; + if let ChecksumOptions::Update { production } = checksum_option { + let checksum = ChecksumRecords::get(production).await?; + checksum.update_and_save()?; + } let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index 3f54cdb8e..5d06a5f0c 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -27,11 +27,12 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![] } - fn test_cmds(&self) -> Vec { - vec![TestCommand::new( - "cargo +stable test --color always".into(), - self.cwd(), - None, - )] + fn test_cmds(&self, production: bool) -> Vec { + let cmd = format!( + "cargo +stable test{} --color always", + if production { " -r" } else { "" } + ); + + vec![TestCommand::new(cmd, self.cwd(), None)] } } diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index ef04563f3..d4e02a07b 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -28,11 +28,12 @@ impl Manager for Module { vec![] } - fn test_cmds(&self) -> Vec { - vec![TestCommand::new( - "cargo +stable test --color always".into(), - self.cwd(), - None, - )] + fn test_cmds(&self, production: bool) -> Vec { + let cmd = format!( + "cargo +stable test{} --color always", + if production { " -r" } else { "" } + ); + + vec![TestCommand::new(cmd, self.cwd(), None)] } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 6bde49eed..3386db37d 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -9,6 +9,7 @@ pub mod wrapper; use crate::{ build_state::{BuildState, BuildStatesTracker}, + checksum_records::ChecksumRecords, fstools, location::get_root, spawner::{spawn, SpawnOptions, SpawnResult}, @@ -80,10 +81,12 @@ pub trait Manager { fn install_cmd(&self, _prod: bool) -> Option { None } - fn test_cmds(&self) -> Vec { + fn test_cmds(&self, _production: bool) -> Vec { Vec::new() } - async fn reset(&self) -> Result, Error> { + async fn reset(&self, production: bool) -> anyhow::Result> { + let checksum = ChecksumRecords::get(production).await?; + checksum.remove_hash_if_exist(&self.owner()); let mut results = Vec::new(); let clean_result = self.clean().await?; results.push(clean_result); @@ -311,8 +314,8 @@ pub trait Manager { .await } - async fn test(&self) -> Result, Error> { - let test_cmds = self.test_cmds(); + async fn test(&self, production: bool) -> Result, Error> { + let test_cmds = self.test_cmds(production); if test_cmds.is_empty() { return Ok(Vec::new()); } diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 9f5409e74..87a923fb9 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -41,7 +41,7 @@ impl Manager for Module { )) } - fn test_cmds(&self) -> Vec { + fn test_cmds(&self, _production: bool) -> Vec { vec![ TestCommand::new( "wasm-pack test --node --color always".into(), diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index be178bc0d..b33b4f664 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -209,7 +209,7 @@ impl Manager for Module { ))) } - async fn test(&self) -> Result, Error> { + async fn test(&self, _production: bool) -> Result, Error> { let mut results = Vec::new(); let build_results = self.build(false).await?; From dd8d0f3c965842e86c7d169e5ddfd59b9118ba7f Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Apr 2024 16:18:07 +0200 Subject: [PATCH 068/174] Build CLI: Clippy fixes --- cli/src/checksum_records.rs | 8 ++++---- cli/src/target.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index a58b2bf19..077768e77 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -12,8 +12,8 @@ use tokio::sync::OnceCell; use crate::{location::get_root, target::Target}; -const FILE_NAME_DEV: &'static str = ".build_chksum_dev"; -const FILE_NAME_PROD: &'static str = ".build_chksum_prod"; +const FILE_NAME_DEV: &str = ".build_chksum_dev"; +const FILE_NAME_PROD: &str = ".build_chksum_prod"; #[derive(Debug)] pub struct ChecksumRecords { @@ -42,10 +42,10 @@ impl ChecksumRecords { BTreeMap::new() }; - return Ok(Self { + Ok(Self { items: Mutex::new(items), production, - }); + }) } fn get_file_path(production: bool) -> PathBuf { diff --git a/cli/src/target.rs b/cli/src/target.rs index 5907c127a..a7a3adda1 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -67,7 +67,7 @@ impl Target { pub fn all_enums() -> Vec { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function - let _ = match Target::App { + match Target::App { Target::Core => (), Target::Binding => (), Target::Wrapper => (), From 18449338aa6fdc1d6145f7a6dd6c49ea8f1d8d00 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 10:44:04 +0200 Subject: [PATCH 069/174] Build CLI: Changes & Checksums Integrations - Introduce JobType enum to encapsulate the job information - Call install method on each module after the dependencies build - Fixes in the checksum logic to cover clean and build for specific modules cases - Skipped tasks are added to the spawner and the report - Yet unused items are prefixed with underscore to please the linter --- cli/src/checksum_records.rs | 91 ++++++++++++++++++++++++------------- cli/src/job_type.rs | 21 +++++++++ cli/src/main.rs | 63 ++++++++++++------------- cli/src/modules/mod.rs | 49 +++++++++++++------- cli/src/spawner.rs | 38 ++++++++++++++++ cli/src/target.rs | 4 +- 6 files changed, 183 insertions(+), 83 deletions(-) create mode 100644 cli/src/job_type.rs diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 077768e77..64e486b46 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{btree_map, BTreeMap, BTreeSet}, fs::{self, File}, io::Write, path::PathBuf, @@ -10,41 +10,72 @@ use anyhow::{anyhow, Context}; use dir_checksum::{calc_combined_checksum, HashDigest}; use tokio::sync::OnceCell; -use crate::{location::get_root, target::Target}; +use crate::{job_type::JobType, location::get_root, target::Target}; const FILE_NAME_DEV: &str = ".build_chksum_dev"; const FILE_NAME_PROD: &str = ".build_chksum_prod"; #[derive(Debug)] pub struct ChecksumRecords { - items: Mutex>, - production: bool, + items: Mutex, + job_type: JobType, +} + +#[derive(Debug, Default)] +struct ChecksumItems { + map: BTreeMap, + involved_targets: BTreeSet, } impl ChecksumRecords { - pub async fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { + pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { + let calculate_involved = match &job_type { + JobType::_Environment | JobType::Lint => return Ok(()), + JobType::Build { production: _ } + | JobType::Run { production: _ } + | JobType::Test { production: _ } => true, + JobType::Clean { production: _ } => false, + }; + + let records = Self::get(job_type).await?; + + if calculate_involved { + records.calculate_involved_hashes()?; + } + records + .persist_hashes() + .context("Error while saving the updated hashes")?; + + Ok(()) + } + + pub async fn get(job_type: JobType) -> anyhow::Result<&'static ChecksumRecords> { static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); CHECKSUM_RECORDS - .get_or_init(|| async { ChecksumRecords::load(production) }) + .get_or_init(|| async { ChecksumRecords::load(job_type) }) .await .as_ref() .map_err(|err| anyhow!("{err}")) } - fn load(production: bool) -> anyhow::Result { - let file_path = Self::get_file_path(production); + fn load(job_type: JobType) -> anyhow::Result { + let file_path = Self::get_file_path(job_type.is_production().is_some_and(|prod| prod)); let items = if file_path.exists() { let file_content = fs::read_to_string(file_path)?; - Self::parse_hashes(file_content)? + let map = Self::parse_hashes(file_content)?; + ChecksumItems { + map, + involved_targets: BTreeSet::new(), + } } else { - BTreeMap::new() + ChecksumItems::default() }; Ok(Self { items: Mutex::new(items), - production, + job_type, }) } @@ -70,14 +101,15 @@ impl ChecksumRecords { Ok(hashes) } - pub fn _check_changed(&self, target: &Target) -> anyhow::Result { - let items = self.items.lock().unwrap(); - let saved_hash = match items.get(target) { + pub fn check_changed(&self, target: Target) -> anyhow::Result { + let mut items = self.items.lock().unwrap(); + items.involved_targets.insert(target); + let saved_hash = match items.map.get(&target) { Some(hash) => hash, None => return Ok(true), }; - let current_hash = Self::calc_hash_for_target(target)?; + let current_hash = Self::calc_hash_for_target(&target)?; Ok(current_hash != *saved_hash) } @@ -89,39 +121,34 @@ impl ChecksumRecords { }) } - pub fn update_and_save(&self) -> anyhow::Result<()> { - self.calculate_hashes()?; - self.persist_hashes() - .context("Error while saving the updated hashes")?; + pub fn remove_hash_if_exist(&self, target: Target) { + let mut items = self.items.lock().unwrap(); + items.involved_targets.insert(target); - Ok(()) + items.map.remove(&target); } - pub fn remove_hash_if_exist(&self, target: &Target) { + fn calculate_involved_hashes(&self) -> anyhow::Result<()> { let mut items = self.items.lock().unwrap(); - _ = items.remove(target); - } - fn calculate_hashes(&self) -> anyhow::Result<()> { - let mut calcuated_map = BTreeMap::new(); - for target in Target::all_enums() { + for target in items.involved_targets.clone() { let hash = Self::calc_hash_for_target(&target)?; - calcuated_map.insert(target, hash); + match items.map.entry(target) { + btree_map::Entry::Occupied(mut o) => *o.get_mut() = hash, + btree_map::Entry::Vacant(e) => _ = e.insert(hash), + }; } - let mut items = self.items.lock().unwrap(); - *items = calcuated_map; - Ok(()) } fn persist_hashes(&self) -> anyhow::Result<()> { - let file_path = Self::get_file_path(self.production); + let file_path = Self::get_file_path(self.job_type.is_production().is_some_and(|prod| prod)); let mut file = File::create(file_path)?; let items = self.items.lock().unwrap(); - for (target, hash) in items.iter() { + for (target, hash) in items.map.iter() { writeln!(file, "{}:{}", target, hash)?; } diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs new file mode 100644 index 000000000..4c3395b09 --- /dev/null +++ b/cli/src/job_type.rs @@ -0,0 +1,21 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobType { + _Environment, + Lint, + Build { production: bool }, + Clean { production: bool }, + Test { production: bool }, + Run { production: bool }, +} + +impl JobType { + pub fn is_production(&self) -> Option { + match self { + JobType::_Environment | JobType::Lint => None, + JobType::Build { production } + | JobType::Clean { production } + | JobType::Test { production } + | JobType::Run { production } => Some(*production), + } + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index c4aead29e..5d4f8e7fd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -4,6 +4,7 @@ mod check_env; mod checksum_records; mod cli_args; mod fstools; +mod job_type; mod location; mod modules; mod spawner; @@ -16,6 +17,7 @@ use checksum_records::ChecksumRecords; use clap::Parser; use cli_args::{CargoCli, Command}; use futures::future::join_all; +use job_type::JobType; use location::init_location; use modules::Manager; use spawner::SpawnResult; @@ -34,11 +36,6 @@ pub enum ReportOptions { File(PathBuf, File), } -enum ChecksumOptions { - Update { production: bool }, - None, -} - #[tokio::main] async fn main() -> Result<(), Error> { let CargoCli::Chipmunk(cli) = CargoCli::parse(); @@ -50,9 +47,7 @@ async fn main() -> Result<(), Error> { // Run the given command let command = cli.command; let report_opt: ReportOptions; - let mut checksum_option = ChecksumOptions::None; - let mut run_app = false; - let results = match command { + let (job_type, results) = match command { Command::Environment => { // Check for dependencies is already called before calling any command println!("All needed tools for development are installed"); @@ -61,83 +56,80 @@ async fn main() -> Result<(), Error> { Command::Lint { target, report } => { report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - join_all( + let results = join_all( targets .iter() .map(|module| module.check()) .collect::>(), ) - .await + .await; + (JobType::Lint, results) } Command::Build { target, production, report, } => { - checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - join_all( + let results = join_all( targets .iter() .map(|module| module.build(production)) .collect::>(), ) - .await + .await; + (JobType::Build { production }, results) } Command::Clean { target, production, report, } => { - checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - join_all( + let results = join_all( targets .iter() .map(|module| module.reset(production)) .collect::>(), ) - .await + .await; + (JobType::Clean { production }, results) } Command::Test { target, production, report, } => { - checksum_option = ChecksumOptions::Update { production }; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - join_all( + let results = join_all( targets .iter() .map(|module| module.test(production)) .collect::>(), ) - .await + .await; + (JobType::Test { production }, results) } Command::Run { production } => { - checksum_option = ChecksumOptions::Update { production }; report_opt = ReportOptions::None; - run_app = true; - join_all( + let results = join_all( Target::all() .iter() .map(|module| module.build(production)) .collect::>(), ) - .await + .await; + (JobType::Run { production }, results) } }; // Shutdown and show results & report let tracker = get_tracker().await; tracker.shutdown().await?; - if let ChecksumOptions::Update { production } = checksum_option { - let checksum = ChecksumRecords::get(production).await?; - checksum.update_and_save()?; - } + ChecksumRecords::update_and_save(job_type).await?; let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { @@ -181,7 +173,9 @@ async fn main() -> Result<(), Error> { } if !success { bail!("Some task were failed") - } else if run_app { + }; + + if matches!(job_type, JobType::Run { production: _ }) { println!("Starting chipmunk..."); let status = app_runner::run_app().await?; if !status.success() { @@ -221,16 +215,19 @@ fn write_report(spawn_result: &SpawnResult, mut writer: impl io::Write) -> Resul )?; writeln!(writer)?; - let status = if spawn_result.status.success() { - "succeeded" - } else { - "failed" + let status = match (spawn_result.skipped, spawn_result.status.success()) { + (true, _) => "been skipped", + (_, true) => "succeeded", + (_, false) => "failed", }; writeln!(writer, "Job '{}' has {status}", spawn_result.job)?; writeln!(writer, "Command: {}", spawn_result.cmd)?; - writeln!(writer, "Logs:")?; + if spawn_result.skipped { + return Ok(()); + } + writeln!(writer, "Logs:")?; for line in spawn_result.report.iter() { writer.write_all(line.as_bytes())?; diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 3386db37d..3cf9438da 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -11,8 +11,9 @@ use crate::{ build_state::{BuildState, BuildStatesTracker}, checksum_records::ChecksumRecords, fstools, + job_type::JobType, location::get_root, - spawner::{spawn, SpawnOptions, SpawnResult}, + spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, Target, }; use anyhow::{bail, Context, Error}; @@ -85,8 +86,8 @@ pub trait Manager { Vec::new() } async fn reset(&self, production: bool) -> anyhow::Result> { - let checksum = ChecksumRecords::get(production).await?; - checksum.remove_hash_if_exist(&self.owner()); + let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; + checksum.remove_hash_if_exist(self.owner()); let mut results = Vec::new(); let clean_result = self.clean().await?; results.push(clean_result); @@ -174,7 +175,7 @@ pub trait Manager { None => { // Run the build and add the sender to the list. run_build = true; - states.insert(target.clone(), BuildState::Running(vec![tx_result])); + states.insert(target, BuildState::Running(vec![tx_result])); } } } @@ -217,8 +218,6 @@ pub trait Manager { /// Performs build process without checking the current builds states async fn perform_build(&self, prod: bool) -> Result, Error> { let mut results = Vec::new(); - let install_result = self.install(false).await?; - results.push(install_result); let deps: Vec> = self.deps().iter().map(|target| target.get()).collect(); for module in deps { @@ -233,22 +232,40 @@ pub trait Manager { .build_cmd(prod) .unwrap_or_else(|| self.kind().build_cmd(prod)); let caption = format!("Build {}", self.owner()); - match spawn(cmd, Some(path), caption, iter::empty(), None).await { + + let mut skip_task = false; + let all_skipped = results.is_empty() || results.iter().all(|r| r.skipped); + if all_skipped { + let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; + skip_task = !checksum_rec.check_changed(self.owner())?; + } + + let spawn_reslt = if skip_task { + spawn_skip(cmd, Some(path), caption).await + } else { + let install_result = self.install(false).await?; + results.push(install_result); + spawn(cmd, Some(path), caption, iter::empty(), None).await + }; + + match spawn_reslt { Ok(status) => { if !status.status.success() { results.push(status); Ok(results) } else { results.push(status); - let res = self.after(prod).await?; - if let Some(result) = res { - results.push(result); - } - if matches!(self.kind(), Kind::Ts) && prod { - let clean_res = self.clean().await?; - results.push(clean_res); - let install_res = self.install(prod).await?; - results.push(install_res); + if !skip_task { + let res = self.after(prod).await?; + if let Some(result) = res { + results.push(result); + } + if matches!(self.kind(), Kind::Ts) && prod { + let clean_res = self.clean().await?; + results.push(clean_res); + let install_res = self.install(prod).await?; + results.push(install_res); + } } Ok(results) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 7414bb431..682970e82 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -22,6 +22,7 @@ pub struct SpawnResult { pub status: ExitStatus, pub job: String, pub cmd: String, + pub skipped: bool, } impl SpawnResult { @@ -31,6 +32,7 @@ impl SpawnResult { status: ExitStatus::default(), job: String::new(), cmd: String::default(), + skipped: false, } } @@ -38,12 +40,25 @@ impl SpawnResult { self.job.is_empty() && self.cmd.is_empty() } + /// Create spawn for multiple file system commands pub fn create_for_fs(job: String, report: Vec) -> Self { SpawnResult { report, job, status: ExitStatus::default(), cmd: "Multiple file system commands".into(), + skipped: false, + } + } + + /// Create spawn for jobs that has been skipped + pub fn create_for_skipped(job: String, cmd: String) -> Self { + SpawnResult { + report: Vec::new(), + job, + status: ExitStatus::default(), + cmd, + skipped: true, } } } @@ -154,6 +169,7 @@ pub async fn spawn( status, job: job_title, cmd: command, + skipped: false, }) } else { tracker @@ -210,5 +226,27 @@ pub async fn spawn_blocking( status, job: caption, cmd: command, + skipped: false, }) } + +/// This spawns a new task and return immediately showing that the job has been skipped +pub async fn spawn_skip( + command: String, + cwd: Option, + caption: String, +) -> anyhow::Result { + let cwd = cwd.unwrap_or_else(|| get_root().clone()); + + let tracker = get_tracker().await; + let sequence = tracker + .start( + &format!("{}: {}", to_relative_path(&cwd).display(), caption), + None, + ) + .await?; + + tracker.success(sequence, "skipped").await; + + Ok(SpawnResult::create_for_skipped(caption, command)) +} diff --git a/cli/src/target.rs b/cli/src/target.rs index a7a3adda1..4bf9bcffb 100644 --- a/cli/src/target.rs +++ b/cli/src/target.rs @@ -4,7 +4,7 @@ use crate::{modules, modules::Manager}; use anyhow::bail; use clap::ValueEnum; -#[derive(ValueEnum, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Target { /// Represents the path `application/apps/indexer` Core, @@ -64,7 +64,7 @@ impl FromStr for Target { } impl Target { - pub fn all_enums() -> Vec { + pub fn _all_enums() -> Vec { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function match Target::App { From f798dc5cf845bb98de1f011a24b7fd09d3c0bc06 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 11:25:45 +0200 Subject: [PATCH 070/174] Build CLI: Fixes for Checksum Integrations - Register each job in checksum records explicitly - Spawn Result: Skipped information will be provided for relevant jobs --- cli/src/checksum_records.rs | 7 ++++++- cli/src/main.rs | 4 ++-- cli/src/modules/mod.rs | 15 ++++++++++++--- cli/src/modules/wasm.rs | 5 ++++- cli/src/spawner.rs | 20 ++++++++++++++------ 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 64e486b46..c0d674a46 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -101,9 +101,14 @@ impl ChecksumRecords { Ok(hashes) } - pub fn check_changed(&self, target: Target) -> anyhow::Result { + pub fn register_job(&self, target: Target) { let mut items = self.items.lock().unwrap(); items.involved_targets.insert(target); + } + + pub fn check_changed(&self, target: Target) -> anyhow::Result { + let items = self.items.lock().unwrap(); + assert!(items.involved_targets.contains(&target)); let saved_hash = match items.map.get(&target) { Some(hash) => hash, None => return Ok(true), diff --git a/cli/src/main.rs b/cli/src/main.rs index 5d4f8e7fd..06c9af717 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -216,14 +216,14 @@ fn write_report(spawn_result: &SpawnResult, mut writer: impl io::Write) -> Resul writeln!(writer)?; let status = match (spawn_result.skipped, spawn_result.status.success()) { - (true, _) => "been skipped", + (Some(true), _) => "been skipped", (_, true) => "succeeded", (_, false) => "failed", }; writeln!(writer, "Job '{}' has {status}", spawn_result.job)?; writeln!(writer, "Command: {}", spawn_result.cmd)?; - if spawn_result.skipped { + if spawn_result.skipped.is_some_and(|skipped| skipped) { return Ok(()); } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 3cf9438da..c8fe9f5b3 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -217,6 +217,9 @@ pub trait Manager { /// Performs build process without checking the current builds states async fn perform_build(&self, prod: bool) -> Result, Error> { + let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; + checksum_rec.register_job(self.owner()); + let mut results = Vec::new(); let deps: Vec> = self.deps().iter().map(|target| target.get()).collect(); @@ -234,9 +237,11 @@ pub trait Manager { let caption = format!("Build {}", self.owner()); let mut skip_task = false; - let all_skipped = results.is_empty() || results.iter().all(|r| r.skipped); + let all_skipped = results.iter().all(|r| match r.skipped { + Some(skipped) => skipped, + None => true, // Tasks with no skip info are irrelevant + }); if all_skipped { - let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; skip_task = !checksum_rec.check_changed(self.owner())?; } @@ -245,7 +250,11 @@ pub trait Manager { } else { let install_result = self.install(false).await?; results.push(install_result); - spawn(cmd, Some(path), caption, iter::empty(), None).await + let spawn_opt = SpawnOptions { + has_skip_info: true, + ..Default::default() + }; + spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await }; match spawn_reslt { diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 87a923fb9..62eec41c5 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -51,7 +51,10 @@ impl Manager for Module { TestCommand::new( "npm run test".into(), self.cwd().join("spec"), - Some(SpawnOptions { suppress_msg: true }), + Some(SpawnOptions { + suppress_msg: true, + ..Default::default() + }), ), ] } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 682970e82..6dbe07855 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -22,7 +22,7 @@ pub struct SpawnResult { pub status: ExitStatus, pub job: String, pub cmd: String, - pub skipped: bool, + pub skipped: Option, } impl SpawnResult { @@ -32,7 +32,7 @@ impl SpawnResult { status: ExitStatus::default(), job: String::new(), cmd: String::default(), - skipped: false, + skipped: None, } } @@ -47,7 +47,7 @@ impl SpawnResult { job, status: ExitStatus::default(), cmd: "Multiple file system commands".into(), - skipped: false, + skipped: None, } } @@ -58,7 +58,7 @@ impl SpawnResult { job, status: ExitStatus::default(), cmd, - skipped: true, + skipped: Some(true), } } } @@ -66,6 +66,7 @@ impl SpawnResult { #[derive(Debug, Clone, Default)] pub(crate) struct SpawnOptions { pub suppress_msg: bool, + pub has_skip_info: bool, } pub async fn spawn( @@ -164,12 +165,19 @@ pub async fn spawn( } else { tracker.fail(sequence, "finished with errors").await; } + + let skipped = if opts.has_skip_info { + Some(false) + } else { + None + }; + Ok(SpawnResult { report: report_lines, status, job: job_title, cmd: command, - skipped: false, + skipped, }) } else { tracker @@ -226,7 +234,7 @@ pub async fn spawn_blocking( status, job: caption, cmd: command, - skipped: false, + skipped: None, }) } From b4045220bee47544ecc3082c10851fa30283ebab Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 12:55:52 +0200 Subject: [PATCH 071/174] Build CLI: Update Checksums with successful jobs only --- cli/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 06c9af717..12777d4a0 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -129,7 +129,6 @@ async fn main() -> Result<(), Error> { // Shutdown and show results & report let tracker = get_tracker().await; tracker.shutdown().await?; - ChecksumRecords::update_and_save(job_type).await?; let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { @@ -175,6 +174,8 @@ async fn main() -> Result<(), Error> { bail!("Some task were failed") }; + ChecksumRecords::update_and_save(job_type).await?; + if matches!(job_type, JobType::Run { production: _ }) { println!("Starting chipmunk..."); let status = app_runner::run_app().await?; From f974ec4daea670def557711e20044a763b094b39 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 12:58:43 +0200 Subject: [PATCH 072/174] Build: Improve error handling for failing commands - Show errors on the spawner - Print more context with the errors in both the spawner and the results --- cli/src/main.rs | 2 +- cli/src/modules/mod.rs | 43 ++++++++++++++++++++---------------------- cli/src/spawner.rs | 35 +++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 12777d4a0..43bcb8ba2 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -165,7 +165,7 @@ async fn main() -> Result<(), Error> { } } Err(err) => { - eprintln!("Builder error: {err}"); + eprintln!("Builder error: {:?}", err); success = false; } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index c8fe9f5b3..c9ec0e568 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -189,7 +189,7 @@ pub trait Manager { let res_clone = match &build_result { Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{}", err)), + Err(err) => Err(anyhow::anyhow!("{:?}", err)), }; let Some(BuildState::Running(senders)) = @@ -201,7 +201,7 @@ pub trait Manager { for sender in senders { let res_clone = match &res_clone { Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{}", err)), + Err(err) => Err(anyhow::anyhow!("{:?}", err)), }; if sender.send(res_clone).is_err() { bail!("Fail to communicate with builder"); @@ -257,30 +257,27 @@ pub trait Manager { spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await }; - match spawn_reslt { - Ok(status) => { - if !status.status.success() { - results.push(status); - Ok(results) - } else { - results.push(status); - if !skip_task { - let res = self.after(prod).await?; - if let Some(result) = res { - results.push(result); - } - if matches!(self.kind(), Kind::Ts) && prod { - let clean_res = self.clean().await?; - results.push(clean_res); - let install_res = self.install(prod).await?; - results.push(install_res); - } - } + let status = spawn_reslt?; - Ok(results) + if !status.status.success() { + results.push(status); + Ok(results) + } else { + results.push(status); + if !skip_task { + let res = self.after(prod).await?; + if let Some(result) = res { + results.push(result); + } + if matches!(self.kind(), Kind::Ts) && prod { + let clean_res = self.clean().await?; + results.push(clean_res); + let install_res = self.install(prod).await?; + results.push(install_res); } } - Err(err) => Err(err), + + Ok(results) } } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 6dbe07855..df293ebd1 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -83,7 +83,15 @@ pub async fn spawn( let mut env_vars: Vec<_> = env::vars().chain(environment_vars).collect(); env_vars.push((String::from("TERM"), String::from("xterm-256color"))); - let mut child = Command::new(cmd) + let tracker = get_tracker().await; + let sequence = tracker + .start( + &format!("{}: {}", to_relative_path(&cwd).display(), caption), + None, + ) + .await?; + + let command_result = Command::new(cmd) .current_dir(&cwd) .args(&parts) .envs(env_vars) @@ -91,19 +99,16 @@ pub async fn spawn( .stderr(Stdio::piped()) .spawn() .with_context(|| { - format!( - "Error While running the command '{cmd}'\nwith arguments: {parts:?}\ncwd: {}", - cwd.display() - ) - })?; - let job_title = caption; - let tracker = get_tracker().await; - let sequence = tracker - .start( - &format!("{}: {}", to_relative_path(&cwd).display(), job_title), - None, - ) - .await?; + format!("Error While running the command '{cmd}'\nwith arguments: {parts:?}") + }); + + let mut child = match command_result { + Ok(child) => child, + Err(err) => { + tracker.fail(sequence, format!("{err:#}").as_str()).await; + return Err(err); + } + }; let mut report_lines: Vec = vec![]; let drain_stdout_stderr = { @@ -175,7 +180,7 @@ pub async fn spawn( Ok(SpawnResult { report: report_lines, status, - job: job_title, + job: caption, cmd: command, skipped, }) From 4d0f0651d7939aa339d3ce4b2c7f95e37cc06f2c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 13:48:37 +0200 Subject: [PATCH 073/174] Build CLI: Improve error messages - Provide more context when nested fails - Add lines to separate the error on stderr --- cli/src/main.rs | 2 ++ cli/src/modules/mod.rs | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 43bcb8ba2..f27f3c036 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -159,6 +159,7 @@ async fn main() -> Result<(), Error> { if print_err { eprintln!("Failed with errors"); eprintln!("{}:\n{}", status.job, status.report.join("")); + eprintln!("---------------------------------------------------------------------"); } success = false; } @@ -166,6 +167,7 @@ async fn main() -> Result<(), Error> { } Err(err) => { eprintln!("Builder error: {:?}", err); + eprintln!("---------------------------------------------------------------------"); success = false; } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index c9ec0e568..9b476b90f 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -224,7 +224,13 @@ pub trait Manager { let deps: Vec> = self.deps().iter().map(|target| target.get()).collect(); for module in deps { - let status = module.build(prod).await?; + let status = module.build(prod).await.with_context(|| { + format!( + "Error while building the dependciy {} for target{}", + module.owner(), + self.owner() + ) + })?; results.extend(status); if results.iter().any(|res| !res.status.success()) { return Ok(results); From c7362ce08d496a3fe1e0c97c244e29d32601aaf6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 13:51:01 +0200 Subject: [PATCH 074/174] Build CLI: Clippy fix --- cli/src/modules/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 9b476b90f..a24bf12f7 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -243,10 +243,14 @@ pub trait Manager { let caption = format!("Build {}", self.owner()); let mut skip_task = false; - let all_skipped = results.iter().all(|r| match r.skipped { - Some(skipped) => skipped, - None => true, // Tasks with no skip info are irrelevant + + let all_skipped = results.iter().all(|r| { + r.skipped.unwrap_or({ + // Tasks with no skip info are irrelevant + true + }) }); + if all_skipped { skip_task = !checksum_rec.check_changed(self.owner())?; } From 8cca563d93e703383fe6a62f356883c88973db93 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 25 Apr 2024 17:19:20 +0200 Subject: [PATCH 075/174] Build CLI: Fixes for Copying files & Install - Move copying index to after building rs bindings + Added Copy index for mod-test - Add missing copy operations after building platform - Add work around to install ts bindings tools before calling build on rs bindings - TODOs and comments --- cli/src/modules/binding.rs | 61 ++++++++++++++++++++++++++++++++++++-- cli/src/modules/mod.rs | 2 +- cli/src/modules/shared.rs | 58 ++++++++++++++++++++++++++++++++++-- cli/src/modules/wrapper.rs | 18 +---------- 4 files changed, 117 insertions(+), 22 deletions(-) diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 7e54c7e66..73aa8dc33 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -1,7 +1,8 @@ use super::{Kind, Manager}; -use crate::{location::get_root, Target}; +use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::path::PathBuf; +use std::{fs, path::PathBuf}; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/rs-bindings` @@ -31,6 +32,12 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![Target::Shared] } + + //TODO: This is just a work around to solve installing ts bindings before biulding rs bindings + async fn install(&self, prod: bool) -> Result { + Target::Wrapper.get().install(prod).await + } + fn build_cmd(&self, prod: bool) -> Option { let mut path = Target::Wrapper.get().cwd(); path.push("node_modules"); @@ -40,7 +47,57 @@ impl Manager for Module { Some(format!( "{} nj-cli build{}", path.to_string_lossy(), + //TODO: Ruby code build always in release mode if prod { " --release" } else { "" } )) } + async fn after(&self, _prod: bool) -> Result, Error> { + let mut report_logs = Vec::new(); + + // *** Copy `index.node` from rs to ts bindings dist *** + report_logs.push(String::from("Copying `index.node` to ts-bindings dist...")); + + let src_file = self.cwd().join("dist").join("index.node"); + if !src_file.exists() { + bail!( + "Error while copying `rs-bindings`. Err: Not found: {}", + src_file.to_string_lossy() + ); + } + + let ts_dist_native_dir = Target::Wrapper.get().cwd().join("dist").join("native"); + if !ts_dist_native_dir.exists() { + let msg = format!("creating directory: {}", ts_dist_native_dir.display()); + report_logs.push(msg); + + fs::create_dir_all(&ts_dist_native_dir).with_context(|| { + format!( + "Error while creating directory: {}", + ts_dist_native_dir.display() + ) + })?; + } + + fstools::cp_file( + src_file.clone(), + ts_dist_native_dir.join("index.node"), + &mut report_logs, + ) + .await?; + + // *** Copy `index.node` from rs to ts bindings src native (dir-tests) *** + report_logs.push(String::from( + "Copying `index.node` to ts-bindings src native...", + )); + + let dir_tests = Target::Wrapper.get().cwd().join("src").join("native"); + let mod_file = dir_tests.join("index.node"); + + fstools::cp_file(src_file, mod_file, &mut report_logs).await?; + + Ok(Some(SpawnResult::create_for_fs( + "Copying `index.node` from rs to ts bindings".into(), + report_logs, + ))) + } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index a24bf12f7..da05a63ad 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -226,7 +226,7 @@ pub trait Manager { for module in deps { let status = module.build(prod).await.with_context(|| { format!( - "Error while building the dependciy {} for target{}", + "Error while building the dependciy {} for target {}", module.owner(), self.owner() ) diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index f7fde60cc..ed05232c5 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -1,7 +1,8 @@ use super::{Kind, Manager}; -use crate::{location::get_root, Target}; +use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use anyhow::{Context, Error}; use async_trait::async_trait; -use std::path::PathBuf; +use std::{fs, path::PathBuf}; #[derive(Clone, Debug)] /// Represents the path `application/platform` @@ -27,4 +28,57 @@ impl Manager for Module { fn deps(&self) -> Vec { vec![] } + async fn after(&self, _prod: bool) -> Result, Error> { + let mut report_logs = Vec::new(); + + report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); + + let platform_dest = Target::Wrapper + .get() + .cwd() + .join("node_modules") + .join("platform"); + + let msg = format!("Removing directory: '{}'", platform_dest.display()); + report_logs.push(msg); + + fstools::rm_folder(&platform_dest).await?; + + tokio::fs::create_dir_all(&platform_dest) + .await + .with_context(|| { + format!("Error while creating directory {}", platform_dest.display()) + })?; + + let source = self.cwd(); + + // This part to get all the needed files and folders to copy + let entries_to_copy: Vec<_> = fs::read_dir(&source) + .with_context(|| { + format!( + "Error while reading directory content: {}", + source.display() + ) + })? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name().is_some_and(|file_name| { + !file_name.to_string_lossy().starts_with("node_modules") + }) + }) + .collect(); + + fstools::cp_many( + entries_to_copy, + platform_dest, + source.display(), + &mut report_logs, + ) + .await?; + + Ok(Some(SpawnResult::create_for_fs( + "Copying Platform to Bindings".into(), + report_logs, + ))) + } } diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index b33b4f664..1640a5596 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -5,7 +5,7 @@ use crate::{ spawner::{spawn, spawn_blocking, SpawnResult}, Target, }; -use anyhow::{bail, Context, Error}; +use anyhow::{Context, Error}; use async_trait::async_trait; use std::{fs, iter, path::PathBuf}; @@ -69,22 +69,6 @@ impl Manager for Module { async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); - // *** Copy `index.node` from rs to ts bindings *** - let src = Target::Binding.get().cwd().join("dist").join("index.node"); - let dest = self.cwd().join("dist").join("native"); - if !src.exists() { - bail!("Not found: {}", src.to_string_lossy()); - } - if !dest.exists() { - let msg = format!("creating directory: {}", dest.display()); - report_logs.push(msg); - - fs::create_dir(&dest) - .with_context(|| format!("Error while creating directory: {}", dest.display()))?; - } - - fstools::cp_file(src, dest.join("index.node"), &mut report_logs).await?; - // *** Copying TS Bindings *** report_logs.push(String::from("Copying ts-bindings to electron...")); let rustcore_dest = Target::App From 29e541a7b70152a27bbda30385d83ff2525e4c87 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 08:53:26 +0200 Subject: [PATCH 076/174] Build CLI Refactoring: Replacing Trait with Enum - Move Target to its own folder since it'll contain much more information - Move cwd() to the enums and make the corresponding changes --- cli/src/app_runner.rs | 2 +- cli/src/checksum_records.rs | 2 +- cli/src/modules/app.rs | 9 +++---- cli/src/modules/binding.rs | 19 +++++---------- cli/src/modules/cli.rs | 8 ++----- cli/src/modules/client.rs | 9 ++++--- cli/src/modules/core.rs | 8 ++----- cli/src/modules/mod.rs | 17 +++++++------- cli/src/modules/shared.rs | 16 ++++--------- cli/src/modules/wasm.rs | 15 +++--------- cli/src/modules/wrapper.rs | 35 ++++++++++------------------ cli/src/{target.rs => target/mod.rs} | 24 +++++++++++++++++-- 12 files changed, 69 insertions(+), 95 deletions(-) rename cli/src/{target.rs => target/mod.rs} (80%) diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index 9f7c49189..990917bac 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -5,7 +5,7 @@ use tokio::process::Command; use crate::target::Target; pub async fn run_app() -> io::Result { - let electron_path = Target::App.get().cwd(); + let electron_path = Target::App.cwd(); Command::new("yarn") .current_dir(electron_path) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index c0d674a46..225e70235 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -120,7 +120,7 @@ impl ChecksumRecords { } fn calc_hash_for_target(target: &Target) -> anyhow::Result { - let path = target.get().cwd(); + let path = target.cwd(); calc_combined_checksum(path).with_context(|| { format!("Error while calculating the current hash for target: {target}",) }) diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 1fc48f11b..c5ffc305f 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,8 +1,8 @@ use super::{Kind, Manager}; -use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/holder` @@ -22,9 +22,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Ts } - fn cwd(&self) -> PathBuf { - get_root().join("application").join("holder") - } fn deps(&self) -> Vec { vec![Target::Shared, Target::Wrapper, Target::Client] } @@ -38,7 +35,7 @@ impl Manager for Module { .get() .dist_path(prod) .context("Fail to get client artifacts")?; - let dest = self.cwd().join("dist"); + let dest = self.owner().cwd().join("dist"); if !src.exists() { bail!("Not found: {}", src.display()); } diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 73aa8dc33..b14c5e04a 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -1,8 +1,8 @@ use super::{Kind, Manager}; -use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/rs-bindings` @@ -22,13 +22,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Rs } - fn cwd(&self) -> PathBuf { - get_root() - .join("application") - .join("apps") - .join("rustcore") - .join("rs-bindings") - } fn deps(&self) -> Vec { vec![Target::Shared] } @@ -39,7 +32,7 @@ impl Manager for Module { } fn build_cmd(&self, prod: bool) -> Option { - let mut path = Target::Wrapper.get().cwd(); + let mut path = Target::Wrapper.cwd(); path.push("node_modules"); path.push(".bin"); path.push("electron-build-env"); @@ -57,7 +50,7 @@ impl Manager for Module { // *** Copy `index.node` from rs to ts bindings dist *** report_logs.push(String::from("Copying `index.node` to ts-bindings dist...")); - let src_file = self.cwd().join("dist").join("index.node"); + let src_file = self.owner().cwd().join("dist").join("index.node"); if !src_file.exists() { bail!( "Error while copying `rs-bindings`. Err: Not found: {}", @@ -65,7 +58,7 @@ impl Manager for Module { ); } - let ts_dist_native_dir = Target::Wrapper.get().cwd().join("dist").join("native"); + let ts_dist_native_dir = Target::Wrapper.cwd().join("dist").join("native"); if !ts_dist_native_dir.exists() { let msg = format!("creating directory: {}", ts_dist_native_dir.display()); report_logs.push(msg); @@ -90,7 +83,7 @@ impl Manager for Module { "Copying `index.node` to ts-bindings src native...", )); - let dir_tests = Target::Wrapper.get().cwd().join("src").join("native"); + let dir_tests = Target::Wrapper.cwd().join("src").join("native"); let mod_file = dir_tests.join("index.node"); fstools::cp_file(src_file, mod_file, &mut report_logs).await?; diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index 5d06a5f0c..401861fcc 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -1,7 +1,6 @@ use super::{Kind, Manager}; -use crate::{location::get_root, modules::TestCommand, Target}; +use crate::{modules::TestCommand, Target}; use async_trait::async_trait; -use std::path::PathBuf; #[derive(Clone, Debug)] /// Represents the path `cli` @@ -21,9 +20,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Rs } - fn cwd(&self) -> PathBuf { - get_root().join("cli") - } fn deps(&self) -> Vec { vec![] } @@ -33,6 +29,6 @@ impl Manager for Module { if production { " -r" } else { "" } ); - vec![TestCommand::new(cmd, self.cwd(), None)] + vec![TestCommand::new(cmd, self.owner().cwd(), None)] } } diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 75160dcc4..471104540 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -1,5 +1,5 @@ use super::{Kind, Manager}; -use crate::{location::get_root, Target}; +use crate::Target; use async_trait::async_trait; use std::path::PathBuf; @@ -21,15 +21,14 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Ts } - fn cwd(&self) -> PathBuf { - get_root().join("application").join("client") - } + fn deps(&self) -> Vec { vec![Target::Shared, Target::Wasm] } fn dist_path(&self, prod: bool) -> Option { Some( - self.cwd() + self.owner() + .cwd() .join("dist") .join(if prod { "release" } else { "debug" }), ) diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index d4e02a07b..b0acd69bb 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -1,7 +1,6 @@ use super::{Kind, Manager, TestCommand}; -use crate::{location::get_root, Target}; +use crate::Target; use async_trait::async_trait; -use std::path::PathBuf; #[derive(Clone, Debug)] /// Represents the path `application/apps/indexer` @@ -21,9 +20,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Rs } - fn cwd(&self) -> PathBuf { - get_root().join("application").join("apps").join("indexer") - } fn deps(&self) -> Vec { vec![] } @@ -34,6 +30,6 @@ impl Manager for Module { if production { " -r" } else { "" } ); - vec![TestCommand::new(cmd, self.cwd(), None)] + vec![TestCommand::new(cmd, self.owner().cwd(), None)] } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index da05a63ad..50899d431 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -71,7 +71,6 @@ impl TestCommand { pub trait Manager { fn kind(&self) -> Kind; fn owner(&self) -> Target; - fn cwd(&self) -> PathBuf; fn dist_path(&self, _prod: bool) -> Option { None } @@ -92,7 +91,7 @@ pub trait Manager { let clean_result = self.clean().await?; results.push(clean_result); - let dist_path = self.cwd().join("dist"); + let dist_path = self.owner().cwd().join("dist"); let remove_log = format!("removing {}", dist_path.display()); @@ -107,8 +106,8 @@ pub trait Manager { async fn clean(&self) -> Result { let mut logs = Vec::new(); let path = match self.kind() { - Kind::Ts => self.cwd().join("node_modules"), - Kind::Rs => self.cwd().join("target"), + Kind::Ts => self.owner().cwd().join("node_modules"), + Kind::Rs => self.owner().cwd().join("target"), }; let remove_log = format!("removing directory {}", path.display()); @@ -128,7 +127,7 @@ pub trait Manager { }; if let Some(cmd) = cmd { let caption = format!("Install {}", self.owner()); - spawn(cmd, Some(self.cwd()), caption, iter::empty(), None).await + spawn(cmd, Some(self.owner().cwd()), caption, iter::empty(), None).await } else { Ok(SpawnResult::empty()) } @@ -136,7 +135,7 @@ pub trait Manager { async fn install_if_need(&self, prod: bool) -> Result { match self.kind() { Kind::Ts => { - if self.cwd().join("node_modules").exists() { + if self.owner().cwd().join("node_modules").exists() { Ok(SpawnResult::empty()) } else { self.install(prod).await @@ -236,7 +235,7 @@ pub trait Manager { return Ok(results); } } - let path = get_root().join(self.cwd()); + let path = get_root().join(self.owner().cwd()); let cmd = self .build_cmd(prod) .unwrap_or_else(|| self.kind().build_cmd(prod)); @@ -309,7 +308,7 @@ pub trait Manager { Ok(results) } async fn lint(&self) -> Result { - let path = get_root().join(self.cwd()); + let path = get_root().join(self.owner().cwd()); let caption = format!("TS Lint {}", self.owner()); let status = spawn( "yarn run lint".into(), @@ -334,7 +333,7 @@ pub trait Manager { .await } async fn clippy(&self) -> Result { - let path = get_root().join(self.cwd()); + let path = get_root().join(self.owner().cwd()); let caption = format!("Clippy {}", self.owner()); spawn( diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index ed05232c5..ea08bd046 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -1,8 +1,8 @@ use super::{Kind, Manager}; -use crate::{fstools, location::get_root, spawner::SpawnResult, Target}; +use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{Context, Error}; use async_trait::async_trait; -use std::{fs, path::PathBuf}; +use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/platform` @@ -22,9 +22,7 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Ts } - fn cwd(&self) -> PathBuf { - get_root().join("application").join("platform") - } + fn deps(&self) -> Vec { vec![] } @@ -33,11 +31,7 @@ impl Manager for Module { report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); - let platform_dest = Target::Wrapper - .get() - .cwd() - .join("node_modules") - .join("platform"); + let platform_dest = Target::Wrapper.cwd().join("node_modules").join("platform"); let msg = format!("Removing directory: '{}'", platform_dest.display()); report_logs.push(msg); @@ -50,7 +44,7 @@ impl Manager for Module { format!("Error while creating directory {}", platform_dest.display()) })?; - let source = self.cwd(); + let source = self.owner().cwd(); // This part to get all the needed files and folders to copy let entries_to_copy: Vec<_> = fs::read_dir(&source) diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 62eec41c5..14a45e459 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,7 +1,6 @@ use super::{Kind, Manager, TestCommand}; -use crate::{location::get_root, spawner::SpawnOptions, Target}; +use crate::{spawner::SpawnOptions, Target}; use async_trait::async_trait; -use std::path::PathBuf; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/wasm-bindings` @@ -21,14 +20,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Rs } - fn cwd(&self) -> PathBuf { - get_root() - .join("application") - .join("apps") - .join("rustcore") - .join("wasm-bindings") - } - fn deps(&self) -> Vec { vec![] } @@ -45,12 +36,12 @@ impl Manager for Module { vec![ TestCommand::new( "wasm-pack test --node --color always".into(), - self.cwd(), + self.owner().cwd(), None, ), TestCommand::new( "npm run test".into(), - self.cwd().join("spec"), + self.owner().cwd().join("spec"), Some(SpawnOptions { suppress_msg: true, ..Default::default() diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 1640a5596..4873f103b 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,7 +1,6 @@ use super::{Kind, Manager}; use crate::{ fstools, - location::get_root, spawner::{spawn, spawn_blocking, SpawnResult}, Target, }; @@ -56,13 +55,6 @@ impl Manager for Module { fn kind(&self) -> Kind { Kind::Ts } - fn cwd(&self) -> PathBuf { - get_root() - .join("application") - .join("apps") - .join("rustcore") - .join("ts-bindings") - } fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] } @@ -71,11 +63,7 @@ impl Manager for Module { // *** Copying TS Bindings *** report_logs.push(String::from("Copying ts-bindings to electron...")); - let rustcore_dest = Target::App - .get() - .cwd() - .join("node_modules") - .join("rustcore"); + let rustcore_dest = Target::App.cwd().join("node_modules").join("rustcore"); fstools::rm_folder(&rustcore_dest).await?; @@ -89,7 +77,7 @@ impl Manager for Module { })?; // This part to get all the needed files and folders to copy - let ts_source = self.cwd(); + let ts_source = self.owner().cwd(); let ts_entries_to_copy: Vec<_> = fs::read_dir(&ts_source) .with_context(|| { format!( @@ -136,7 +124,7 @@ impl Manager for Module { ) })?; - let platform_src = Target::Shared.get().cwd(); + let platform_src = Target::Shared.cwd(); let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src) .with_context(|| { @@ -163,11 +151,7 @@ impl Manager for Module { // *** Copy Platform to electron *** report_logs.push(String::from("Copying platform in to electron...")); - let platform_dest2 = Target::App - .get() - .cwd() - .join("node_modules") - .join("platform"); + let platform_dest2 = Target::App.cwd().join("node_modules").join("platform"); fstools::rm_folder(&platform_dest2).await?; tokio::fs::create_dir_all(&platform_dest2) @@ -199,11 +183,16 @@ impl Manager for Module { let build_results = self.build(false).await?; results.extend(build_results); - let build_spec_path = self.cwd().join("spec"); + let build_spec_path = self.owner().cwd().join("spec"); //TODO: This check exists in rake implementation but it need to be improved. // The check should cover if the test themselves or the code under the tests has been changed. if !build_spec_path.join("build").exists() { - let test_builder_path = self.cwd().join("node_modules").join(".bin").join("tsc"); + let test_builder_path = self + .owner() + .cwd() + .join("node_modules") + .join(".bin") + .join("tsc"); let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); @@ -219,7 +208,7 @@ impl Manager for Module { results.push(spec_res); } - let cwd = self.cwd(); + let cwd = self.owner().cwd(); let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); let electron_path = electron_path.to_string_lossy(); diff --git a/cli/src/target.rs b/cli/src/target/mod.rs similarity index 80% rename from cli/src/target.rs rename to cli/src/target/mod.rs index 4bf9bcffb..5df51e5c6 100644 --- a/cli/src/target.rs +++ b/cli/src/target/mod.rs @@ -1,6 +1,6 @@ -use std::str::FromStr; +use std::{path::PathBuf, str::FromStr}; -use crate::{modules, modules::Manager}; +use crate::{location::get_root, modules, modules::Manager}; use anyhow::bail; use clap::ValueEnum; @@ -91,6 +91,7 @@ impl Target { ] } + //TODO AAZ: Replace this with _all_enums pub fn all() -> Vec> { vec![ Box::new(modules::binding::Module::new()), @@ -103,6 +104,7 @@ impl Target { Box::new(modules::wasm::Module::new()), ] } + //TODO AAZ: Remove this pub fn get(&self) -> Box { match self { Target::Binding => Box::new(modules::binding::Module::new()), @@ -115,4 +117,22 @@ impl Target { Target::Wasm => Box::new(modules::wasm::Module::new()), } } + + pub fn cwd(&self) -> PathBuf { + let root = get_root(); + let sub_parts = match self { + Target::Core => ["application", "apps", "indexer"].iter(), + Target::Binding => ["application", "apps", "rustcore", "rs-bindings"].iter(), + Target::Wrapper => ["application", "apps", "rustcore", "ts-bindings"].iter(), + Target::Client => ["application", "client"].iter(), + Target::Shared => ["application", "platform"].iter(), + Target::App => ["application", "holder"].iter(), + Target::Cli => ["cli"].iter(), + Target::Wasm => ["application", "apps", "rustcore", "wasm-bindings"].iter(), + }; + + let sub_path: PathBuf = sub_parts.collect(); + + root.join(sub_path) + } } From 6b140e3891b0826a2f1457929bb0c23a4f560328 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 09:10:06 +0200 Subject: [PATCH 077/174] Build CLI Refactor: Replacing Trait with Enum... - Move kind() to the enums and make the corresponding changes --- cli/src/modules/app.rs | 6 ++---- cli/src/modules/binding.rs | 5 +---- cli/src/modules/cli.rs | 5 +---- cli/src/modules/client.rs | 6 +----- cli/src/modules/core.rs | 5 +---- cli/src/modules/mod.rs | 13 ++++++------- cli/src/modules/shared.rs | 5 +---- cli/src/modules/wasm.rs | 6 ++---- cli/src/modules/wrapper.rs | 5 +---- cli/src/target/mod.rs | 13 ++++++++++++- 10 files changed, 28 insertions(+), 41 deletions(-) diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index c5ffc305f..323199258 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; @@ -19,9 +19,7 @@ impl Manager for Module { fn owner(&self) -> Target { Target::App } - fn kind(&self) -> Kind { - Kind::Ts - } + fn deps(&self) -> Vec { vec![Target::Shared, Target::Wrapper, Target::Client] } diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index b14c5e04a..8eb60cbf1 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; @@ -19,9 +19,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Binding } - fn kind(&self) -> Kind { - Kind::Rs - } fn deps(&self) -> Vec { vec![Target::Shared] } diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index 401861fcc..3b6198928 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::{modules::TestCommand, Target}; use async_trait::async_trait; @@ -17,9 +17,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Cli } - fn kind(&self) -> Kind { - Kind::Rs - } fn deps(&self) -> Vec { vec![] } diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 471104540..8711e9385 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::Target; use async_trait::async_trait; use std::path::PathBuf; @@ -18,10 +18,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Client } - fn kind(&self) -> Kind { - Kind::Ts - } - fn deps(&self) -> Vec { vec![Target::Shared, Target::Wasm] } diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index b0acd69bb..a2b65889d 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager, TestCommand}; +use super::{Manager, TestCommand}; use crate::Target; use async_trait::async_trait; @@ -17,9 +17,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Core } - fn kind(&self) -> Kind { - Kind::Rs - } fn deps(&self) -> Vec { vec![] } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 50899d431..48ec77224 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -69,7 +69,6 @@ impl TestCommand { #[async_trait] pub trait Manager { - fn kind(&self) -> Kind; fn owner(&self) -> Target; fn dist_path(&self, _prod: bool) -> Option { None @@ -105,7 +104,7 @@ pub trait Manager { } async fn clean(&self) -> Result { let mut logs = Vec::new(); - let path = match self.kind() { + let path = match self.owner().kind() { Kind::Ts => self.owner().cwd().join("node_modules"), Kind::Rs => self.owner().cwd().join("target"), }; @@ -123,7 +122,7 @@ pub trait Manager { let cmd = if self.install_cmd(prod).is_some() { self.install_cmd(prod) } else { - self.kind().install_cmd(prod) + self.owner().kind().install_cmd(prod) }; if let Some(cmd) = cmd { let caption = format!("Install {}", self.owner()); @@ -133,7 +132,7 @@ pub trait Manager { } } async fn install_if_need(&self, prod: bool) -> Result { - match self.kind() { + match self.owner().kind() { Kind::Ts => { if self.owner().cwd().join("node_modules").exists() { Ok(SpawnResult::empty()) @@ -238,7 +237,7 @@ pub trait Manager { let path = get_root().join(self.owner().cwd()); let cmd = self .build_cmd(prod) - .unwrap_or_else(|| self.kind().build_cmd(prod)); + .unwrap_or_else(|| self.owner().kind().build_cmd(prod)); let caption = format!("Build {}", self.owner()); let mut skip_task = false; @@ -278,7 +277,7 @@ pub trait Manager { if let Some(result) = res { results.push(result); } - if matches!(self.kind(), Kind::Ts) && prod { + if matches!(self.owner().kind(), Kind::Ts) && prod { let clean_res = self.clean().await?; results.push(clean_res); let install_res = self.install(prod).await?; @@ -292,7 +291,7 @@ pub trait Manager { async fn check(&self) -> Result, Error> { let mut results = Vec::new(); - match self.kind() { + match self.owner().kind() { Kind::Ts => { let install_result = self.install(false).await?; let lint_restul = self.lint().await?; diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index ea08bd046..2e3d5cec3 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{Context, Error}; use async_trait::async_trait; @@ -19,9 +19,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Shared } - fn kind(&self) -> Kind { - Kind::Ts - } fn deps(&self) -> Vec { vec![] diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 14a45e459..d6860e394 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager, TestCommand}; +use super::{Manager, TestCommand}; use crate::{spawner::SpawnOptions, Target}; use async_trait::async_trait; @@ -17,9 +17,7 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Wasm } - fn kind(&self) -> Kind { - Kind::Rs - } + fn deps(&self) -> Vec { vec![] } diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 4873f103b..da977e9d3 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,4 +1,4 @@ -use super::{Kind, Manager}; +use super::Manager; use crate::{ fstools, spawner::{spawn, spawn_blocking, SpawnResult}, @@ -52,9 +52,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Wrapper } - fn kind(&self) -> Kind { - Kind::Ts - } fn deps(&self) -> Vec { vec![Target::Binding, Target::Shared] } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 5df51e5c6..4cd16b61c 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,6 +1,10 @@ use std::{path::PathBuf, str::FromStr}; -use crate::{location::get_root, modules, modules::Manager}; +use crate::{ + location::get_root, + modules::Manager, + modules::{self, Kind}, +}; use anyhow::bail; use clap::ValueEnum; @@ -135,4 +139,11 @@ impl Target { root.join(sub_path) } + + pub fn kind(&self) -> Kind { + match self { + Target::Binding | Target::Core | Target::Cli | Target::Wasm => Kind::Rs, + Target::Client | Target::Wrapper | Target::Shared | Target::App => Kind::Ts, + } + } } From aa2d8b6e865f758d9231c6a2988b4eec5a62802a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 09:35:23 +0200 Subject: [PATCH 078/174] Build CLI Refactor: Replacing Trait with Enum... - move dep() to enums & corresponding changes --- cli/src/modules/app.rs | 3 --- cli/src/modules/binding.rs | 4 ---- cli/src/modules/cli.rs | 3 --- cli/src/modules/client.rs | 3 --- cli/src/modules/core.rs | 3 --- cli/src/modules/mod.rs | 9 ++++++--- cli/src/modules/shared.rs | 3 --- cli/src/modules/wasm.rs | 4 ---- cli/src/modules/wrapper.rs | 3 --- cli/src/target/mod.rs | 10 ++++++++++ 10 files changed, 16 insertions(+), 29 deletions(-) diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index 323199258..ee05bdba3 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -20,9 +20,6 @@ impl Manager for Module { Target::App } - fn deps(&self) -> Vec { - vec![Target::Shared, Target::Wrapper, Target::Client] - } fn install_cmd(&self, _prod: bool) -> Option { // For app we don't need --production Some(String::from("yarn install")) diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 8eb60cbf1..85e4fa2aa 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -19,10 +19,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Binding } - fn deps(&self) -> Vec { - vec![Target::Shared] - } - //TODO: This is just a work around to solve installing ts bindings before biulding rs bindings async fn install(&self, prod: bool) -> Result { Target::Wrapper.get().install(prod).await diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index 3b6198928..e4c16e0a9 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -17,9 +17,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Cli } - fn deps(&self) -> Vec { - vec![] - } fn test_cmds(&self, production: bool) -> Vec { let cmd = format!( "cargo +stable test{} --color always", diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 8711e9385..3d41807fa 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -18,9 +18,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Client } - fn deps(&self) -> Vec { - vec![Target::Shared, Target::Wasm] - } fn dist_path(&self, prod: bool) -> Option { Some( self.owner() diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index a2b65889d..7508553d2 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -17,9 +17,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Core } - fn deps(&self) -> Vec { - vec![] - } fn test_cmds(&self, production: bool) -> Vec { let cmd = format!( diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 48ec77224..c67e23528 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -73,7 +73,6 @@ pub trait Manager { fn dist_path(&self, _prod: bool) -> Option { None } - fn deps(&self) -> Vec; fn build_cmd(&self, _prod: bool) -> Option { None } @@ -219,8 +218,12 @@ pub trait Manager { checksum_rec.register_job(self.owner()); let mut results = Vec::new(); - let deps: Vec> = - self.deps().iter().map(|target| target.get()).collect(); + let deps: Vec> = self + .owner() + .deps() + .iter() + .map(|target| target.get()) + .collect(); for module in deps { let status = module.build(prod).await.with_context(|| { format!( diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index 2e3d5cec3..ec9d7d32f 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -20,9 +20,6 @@ impl Manager for Module { Target::Shared } - fn deps(&self) -> Vec { - vec![] - } async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index d6860e394..acc295e63 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -18,10 +18,6 @@ impl Manager for Module { Target::Wasm } - fn deps(&self) -> Vec { - vec![] - } - fn build_cmd(&self, prod: bool) -> Option { let env = if prod { "--release" } else { "--dev" }; diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index da977e9d3..8f8c40d54 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -52,9 +52,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Wrapper } - fn deps(&self) -> Vec { - vec![Target::Binding, Target::Shared] - } async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 4cd16b61c..06957ef72 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -146,4 +146,14 @@ impl Target { Target::Client | Target::Wrapper | Target::Shared | Target::App => Kind::Ts, } } + + pub fn deps(&self) -> Vec { + match self { + Target::Core | Target::Cli | Target::Shared | Target::Wasm => Vec::new(), + Target::Binding => vec![Target::Shared], + Target::Wrapper => vec![Target::Binding, Target::Shared], + Target::Client => vec![Target::Shared, Target::Wasm], + Target::App => vec![Target::Shared, Target::Wrapper, Target::Client], + } + } } From 850ba0d358b416f0f03e33928a9689667bca0389 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 11:29:03 +0200 Subject: [PATCH 079/174] Build CLI Refactor: Replacing Trait with Enum... - Rename Kind to TargetKind & Move it to its own module - Replace dist_path with one method in the created client module --- cli/src/modules/app.rs | 7 ++---- cli/src/modules/client.rs | 9 ------- cli/src/modules/mod.rs | 47 ++++++----------------------------- cli/src/target/client.rs | 10 ++++++++ cli/src/target/mod.rs | 15 ++++++++--- cli/src/target/target_kind.rs | 28 +++++++++++++++++++++ 6 files changed, 59 insertions(+), 57 deletions(-) create mode 100644 cli/src/target/client.rs create mode 100644 cli/src/target/target_kind.rs diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index ee05bdba3..a2699dad1 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,5 +1,5 @@ use super::Manager; -use crate::{fstools, spawner::SpawnResult, Target}; +use crate::{fstools, spawner::SpawnResult, target, Target}; use anyhow::{bail, Context, Error}; use async_trait::async_trait; use std::fs; @@ -26,10 +26,7 @@ impl Manager for Module { } async fn after(&self, prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); - let src = Target::Client - .get() - .dist_path(prod) - .context("Fail to get client artifacts")?; + let src = target::client::get_dist_path(prod); let dest = self.owner().cwd().join("dist"); if !src.exists() { bail!("Not found: {}", src.display()); diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs index 3d41807fa..29b2608f7 100644 --- a/cli/src/modules/client.rs +++ b/cli/src/modules/client.rs @@ -1,7 +1,6 @@ use super::Manager; use crate::Target; use async_trait::async_trait; -use std::path::PathBuf; #[derive(Clone, Debug)] /// Represents the path `application/client` @@ -18,12 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Client } - fn dist_path(&self, prod: bool) -> Option { - Some( - self.owner() - .cwd() - .join("dist") - .join(if prod { "release" } else { "debug" }), - ) - } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index c67e23528..4e06097f1 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -14,6 +14,7 @@ use crate::{ job_type::JobType, location::get_root, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, + target::TargetKind, Target, }; use anyhow::{bail, Context, Error}; @@ -22,35 +23,6 @@ use futures::future::join_all; use std::{iter, path::PathBuf}; use tokio::sync::oneshot; -#[derive(Debug, Clone)] -pub enum Kind { - /// TypeScript - Ts, - /// Rust - Rs, -} - -impl Kind { - pub fn build_cmd(&self, prod: bool) -> String { - match self { - Kind::Ts => format!("yarn run {}", if prod { "prod" } else { "build" }), - Kind::Rs => format!( - "cargo build --color always{}", - if prod { " --release" } else { "" } - ), - } - } - pub fn install_cmd(&self, prod: bool) -> Option { - match self { - Kind::Ts => Some(format!( - "yarn install{}", - if prod { " --production" } else { "" } - )), - Kind::Rs => None, - } - } -} - pub(crate) struct TestCommand { command: String, cwd: PathBuf, @@ -70,9 +42,6 @@ impl TestCommand { #[async_trait] pub trait Manager { fn owner(&self) -> Target; - fn dist_path(&self, _prod: bool) -> Option { - None - } fn build_cmd(&self, _prod: bool) -> Option { None } @@ -104,8 +73,8 @@ pub trait Manager { async fn clean(&self) -> Result { let mut logs = Vec::new(); let path = match self.owner().kind() { - Kind::Ts => self.owner().cwd().join("node_modules"), - Kind::Rs => self.owner().cwd().join("target"), + TargetKind::Ts => self.owner().cwd().join("node_modules"), + TargetKind::Rs => self.owner().cwd().join("target"), }; let remove_log = format!("removing directory {}", path.display()); @@ -132,14 +101,14 @@ pub trait Manager { } async fn install_if_need(&self, prod: bool) -> Result { match self.owner().kind() { - Kind::Ts => { + TargetKind::Ts => { if self.owner().cwd().join("node_modules").exists() { Ok(SpawnResult::empty()) } else { self.install(prod).await } } - Kind::Rs => Ok(SpawnResult::empty()), + TargetKind::Rs => Ok(SpawnResult::empty()), } } async fn after(&self, _prod: bool) -> Result, Error> { @@ -280,7 +249,7 @@ pub trait Manager { if let Some(result) = res { results.push(result); } - if matches!(self.owner().kind(), Kind::Ts) && prod { + if matches!(self.owner().kind(), TargetKind::Ts) && prod { let clean_res = self.clean().await?; results.push(clean_res); let install_res = self.install(prod).await?; @@ -295,13 +264,13 @@ pub trait Manager { async fn check(&self) -> Result, Error> { let mut results = Vec::new(); match self.owner().kind() { - Kind::Ts => { + TargetKind::Ts => { let install_result = self.install(false).await?; let lint_restul = self.lint().await?; results.push(install_result); results.push(lint_restul); } - Kind::Rs => { + TargetKind::Rs => { let clippy_result = self.clippy().await?; results.push(clippy_result); } diff --git a/cli/src/target/client.rs b/cli/src/target/client.rs new file mode 100644 index 000000000..f80047ff2 --- /dev/null +++ b/cli/src/target/client.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +use super::Target; + +pub fn get_dist_path(prod: bool) -> PathBuf { + Target::Client + .cwd() + .join("dist") + .join(if prod { "release" } else { "debug" }) +} diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 06957ef72..8009de840 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -3,11 +3,18 @@ use std::{path::PathBuf, str::FromStr}; use crate::{ location::get_root, modules::Manager, - modules::{self, Kind}, + modules::{self}, }; use anyhow::bail; use clap::ValueEnum; +//TODO AAZ: Conisder which module should be pub after teh refactoring is done +pub mod client; +mod target_kind; + +//TODO AAZ: Conisder removing this when refactoring is done +pub use target_kind::TargetKind; + #[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Target { /// Represents the path `application/apps/indexer` @@ -140,10 +147,10 @@ impl Target { root.join(sub_path) } - pub fn kind(&self) -> Kind { + pub fn kind(&self) -> TargetKind { match self { - Target::Binding | Target::Core | Target::Cli | Target::Wasm => Kind::Rs, - Target::Client | Target::Wrapper | Target::Shared | Target::App => Kind::Ts, + Target::Binding | Target::Core | Target::Cli | Target::Wasm => TargetKind::Rs, + Target::Client | Target::Wrapper | Target::Shared | Target::App => TargetKind::Ts, } } diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs new file mode 100644 index 000000000..9989ee55a --- /dev/null +++ b/cli/src/target/target_kind.rs @@ -0,0 +1,28 @@ +#[derive(Debug, Clone)] +pub enum TargetKind { + /// TypeScript + Ts, + /// Rust + Rs, +} + +impl TargetKind { + pub fn build_cmd(&self, prod: bool) -> String { + match self { + TargetKind::Ts => format!("yarn run {}", if prod { "prod" } else { "build" }), + TargetKind::Rs => format!( + "cargo build --color always{}", + if prod { " --release" } else { "" } + ), + } + } + pub fn install_cmd(&self, prod: bool) -> Option { + match self { + TargetKind::Ts => Some(format!( + "yarn install{}", + if prod { " --production" } else { "" } + )), + TargetKind::Rs => None, + } + } +} From e597e0ad10619c774af176be6b27c4b1aa38e2a8 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 11:41:26 +0200 Subject: [PATCH 080/174] Build CLI Refactor: Replacing Trait with Enum... - Replace build_cmd method and make it return the value explicitly - Create target modules for binding and wasm to place build_cmd methods within --- cli/src/modules/binding.rs | 13 ------------- cli/src/modules/mod.rs | 7 +------ cli/src/modules/wasm.rs | 8 -------- cli/src/target/binding.rs | 15 +++++++++++++++ cli/src/target/mod.rs | 10 ++++++++++ cli/src/target/wasm.rs | 5 +++++ 6 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 cli/src/target/binding.rs create mode 100644 cli/src/target/wasm.rs diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 85e4fa2aa..4bdd7f207 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -24,19 +24,6 @@ impl Manager for Module { Target::Wrapper.get().install(prod).await } - fn build_cmd(&self, prod: bool) -> Option { - let mut path = Target::Wrapper.cwd(); - path.push("node_modules"); - path.push(".bin"); - path.push("electron-build-env"); - - Some(format!( - "{} nj-cli build{}", - path.to_string_lossy(), - //TODO: Ruby code build always in release mode - if prod { " --release" } else { "" } - )) - } async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 4e06097f1..e03ca16bb 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -42,9 +42,6 @@ impl TestCommand { #[async_trait] pub trait Manager { fn owner(&self) -> Target; - fn build_cmd(&self, _prod: bool) -> Option { - None - } fn install_cmd(&self, _prod: bool) -> Option { None } @@ -207,9 +204,7 @@ pub trait Manager { } } let path = get_root().join(self.owner().cwd()); - let cmd = self - .build_cmd(prod) - .unwrap_or_else(|| self.owner().kind().build_cmd(prod)); + let cmd = self.owner().build_cmd(prod); let caption = format!("Build {}", self.owner()); let mut skip_task = false; diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index acc295e63..27fde74a0 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -18,14 +18,6 @@ impl Manager for Module { Target::Wasm } - fn build_cmd(&self, prod: bool) -> Option { - let env = if prod { "--release" } else { "--dev" }; - - Some(format!( - "wasm-pack build {env} --target bundler --color always" - )) - } - fn test_cmds(&self, _production: bool) -> Vec { vec![ TestCommand::new( diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs new file mode 100644 index 000000000..d4d12e5f0 --- /dev/null +++ b/cli/src/target/binding.rs @@ -0,0 +1,15 @@ +use super::Target; + +pub fn get_build_cmd(prod: bool) -> String { + let mut path = Target::Wrapper.cwd(); + path.push("node_modules"); + path.push(".bin"); + path.push("electron-build-env"); + + format!( + "{} nj-cli build{}", + path.to_string_lossy(), + //TODO: Ruby code build always in release mode + if prod { " --release" } else { "" } + ) +} diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8009de840..13fdf28cd 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -9,8 +9,10 @@ use anyhow::bail; use clap::ValueEnum; //TODO AAZ: Conisder which module should be pub after teh refactoring is done +mod binding; pub mod client; mod target_kind; +mod wasm; //TODO AAZ: Conisder removing this when refactoring is done pub use target_kind::TargetKind; @@ -163,4 +165,12 @@ impl Target { Target::App => vec![Target::Shared, Target::Wrapper, Target::Client], } } + + pub fn build_cmd(&self, prod: bool) -> String { + match self { + Target::Binding => binding::get_build_cmd(prod), + Target::Wasm => wasm::get_build_cmd(prod), + rest_targets => rest_targets.kind().build_cmd(prod), + } + } } diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs new file mode 100644 index 000000000..1f1226526 --- /dev/null +++ b/cli/src/target/wasm.rs @@ -0,0 +1,5 @@ +pub fn get_build_cmd(prod: bool) -> String { + let env = if prod { "--release" } else { "--dev" }; + + format!("wasm-pack build {env} --target bundler --color always") +} From 220f29a9cc1d773afec731ae7b65fc397b7378cb Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 12:17:26 +0200 Subject: [PATCH 081/174] Build CLI Refactor: Replacing Trait with Enum... - Reimplement install methods to provide a simpler way to deal with the special install cases for rs and ts bindings - Remove unused install if needed method - Add comments and TODOs --- cli/src/modules/app.rs | 4 ---- cli/src/modules/binding.rs | 4 ---- cli/src/modules/mod.rs | 35 ++++------------------------------- cli/src/target/mod.rs | 28 +++++++++++++++++++++++++++- 4 files changed, 31 insertions(+), 40 deletions(-) diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index a2699dad1..fbe473976 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -20,10 +20,6 @@ impl Manager for Module { Target::App } - fn install_cmd(&self, _prod: bool) -> Option { - // For app we don't need --production - Some(String::from("yarn install")) - } async fn after(&self, prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); let src = target::client::get_dist_path(prod); diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 4bdd7f207..5289e290d 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -19,10 +19,6 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Binding } - //TODO: This is just a work around to solve installing ts bindings before biulding rs bindings - async fn install(&self, prod: bool) -> Result { - Target::Wrapper.get().install(prod).await - } async fn after(&self, _prod: bool) -> Result, Error> { let mut report_logs = Vec::new(); diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index e03ca16bb..541fab2bf 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -42,9 +42,6 @@ impl TestCommand { #[async_trait] pub trait Manager { fn owner(&self) -> Target; - fn install_cmd(&self, _prod: bool) -> Option { - None - } fn test_cmds(&self, _production: bool) -> Vec { Vec::new() } @@ -83,31 +80,7 @@ pub trait Manager { Ok(SpawnResult::create_for_fs(job, logs)) } - async fn install(&self, prod: bool) -> Result { - let cmd = if self.install_cmd(prod).is_some() { - self.install_cmd(prod) - } else { - self.owner().kind().install_cmd(prod) - }; - if let Some(cmd) = cmd { - let caption = format!("Install {}", self.owner()); - spawn(cmd, Some(self.owner().cwd()), caption, iter::empty(), None).await - } else { - Ok(SpawnResult::empty()) - } - } - async fn install_if_need(&self, prod: bool) -> Result { - match self.owner().kind() { - TargetKind::Ts => { - if self.owner().cwd().join("node_modules").exists() { - Ok(SpawnResult::empty()) - } else { - self.install(prod).await - } - } - TargetKind::Rs => Ok(SpawnResult::empty()), - } - } + async fn after(&self, _prod: bool) -> Result, Error> { Ok(None) } @@ -223,7 +196,7 @@ pub trait Manager { let spawn_reslt = if skip_task { spawn_skip(cmd, Some(path), caption).await } else { - let install_result = self.install(false).await?; + let install_result = self.owner().install(false).await?; results.push(install_result); let spawn_opt = SpawnOptions { has_skip_info: true, @@ -247,7 +220,7 @@ pub trait Manager { if matches!(self.owner().kind(), TargetKind::Ts) && prod { let clean_res = self.clean().await?; results.push(clean_res); - let install_res = self.install(prod).await?; + let install_res = self.owner().install(prod).await?; results.push(install_res); } } @@ -260,7 +233,7 @@ pub trait Manager { let mut results = Vec::new(); match self.owner().kind() { TargetKind::Ts => { - let install_result = self.install(false).await?; + let install_result = self.owner().install(false).await?; let lint_restul = self.lint().await?; results.push(install_result); results.push(lint_restul); diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 13fdf28cd..241b29d91 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,9 +1,10 @@ -use std::{path::PathBuf, str::FromStr}; +use std::{iter, path::PathBuf, str::FromStr}; use crate::{ location::get_root, modules::Manager, modules::{self}, + spawner::{spawn, SpawnResult}, }; use anyhow::bail; use clap::ValueEnum; @@ -173,4 +174,29 @@ impl Target { rest_targets => rest_targets.kind().build_cmd(prod), } } + + pub async fn install(&self, prod: bool) -> Result { + match self { + // We must install ts binding tools before running rs bindings, therefore we call + // wrapper (ts-bindings) install in the rs bindings install. + // TODO AAZ: Make sure the following statement is correct: + // Since rs bindings is a dependency for ts bindings, we don't need to call to install + // on ts bindings again. + Target::Binding => install_general(&Target::Wrapper, prod).await, + Target::Wrapper => Ok(SpawnResult::empty()), + // For app we don't need --production + Target::App => install_general(&Target::App, false).await, + rest_targets => install_general(rest_targets, prod).await, + } + } +} +/// run install using the general routine for the given target +async fn install_general(target: &Target, prod: bool) -> Result { + let cmd = target.kind().install_cmd(prod); + if let Some(cmd) = cmd { + let caption = format!("Install {}", target); + spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await + } else { + Ok(SpawnResult::empty()) + } } From baaedb530d6a74a614e0b87767aebb1d460f07c9 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 15:00:09 +0200 Subject: [PATCH 082/174] Build CLI Refactor: Replacing Trait with Enum... - Move test commands to the enums - Change the function signature to return an option instead of empty vector when no test commands are available instead of the hidden contract with the empty instead of the hidden contract with the empty vector --- cli/src/modules/cli.rs | 10 +--------- cli/src/modules/core.rs | 11 +---------- cli/src/modules/mod.rs | 28 +++++----------------------- cli/src/modules/wasm.rs | 22 ++-------------------- cli/src/target/cli.rs | 12 ++++++++++++ cli/src/target/core.rs | 12 ++++++++++++ cli/src/target/mod.rs | 29 ++++++++++++++++++++++++++++- cli/src/target/wasm.rs | 23 +++++++++++++++++++++++ 8 files changed, 84 insertions(+), 63 deletions(-) create mode 100644 cli/src/target/cli.rs create mode 100644 cli/src/target/core.rs diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs index e4c16e0a9..4655950ea 100644 --- a/cli/src/modules/cli.rs +++ b/cli/src/modules/cli.rs @@ -1,5 +1,5 @@ use super::Manager; -use crate::{modules::TestCommand, Target}; +use crate::Target; use async_trait::async_trait; #[derive(Clone, Debug)] @@ -17,12 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Cli } - fn test_cmds(&self, production: bool) -> Vec { - let cmd = format!( - "cargo +stable test{} --color always", - if production { " -r" } else { "" } - ); - - vec![TestCommand::new(cmd, self.owner().cwd(), None)] - } } diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs index 7508553d2..31a7999f1 100644 --- a/cli/src/modules/core.rs +++ b/cli/src/modules/core.rs @@ -1,4 +1,4 @@ -use super::{Manager, TestCommand}; +use super::Manager; use crate::Target; use async_trait::async_trait; @@ -17,13 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Core } - - fn test_cmds(&self, production: bool) -> Vec { - let cmd = format!( - "cargo +stable test{} --color always", - if production { " -r" } else { "" } - ); - - vec![TestCommand::new(cmd, self.owner().cwd(), None)] - } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 541fab2bf..cdb08e9a2 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -20,31 +20,12 @@ use crate::{ use anyhow::{bail, Context, Error}; use async_trait::async_trait; use futures::future::join_all; -use std::{iter, path::PathBuf}; +use std::iter; use tokio::sync::oneshot; -pub(crate) struct TestCommand { - command: String, - cwd: PathBuf, - spawn_opts: Option, -} - -impl TestCommand { - pub(crate) fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { - Self { - command, - cwd, - spawn_opts, - } - } -} - #[async_trait] pub trait Manager { fn owner(&self) -> Target; - fn test_cmds(&self, _production: bool) -> Vec { - Vec::new() - } async fn reset(&self, production: bool) -> anyhow::Result> { let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; checksum.remove_hash_if_exist(self.owner()); @@ -286,10 +267,11 @@ pub trait Manager { } async fn test(&self, production: bool) -> Result, Error> { - let test_cmds = self.test_cmds(production); - if test_cmds.is_empty() { + let Some(test_cmds) = self.owner().test_cmds(production) else { return Ok(Vec::new()); - } + }; + + debug_assert!(!test_cmds.is_empty()); let mut results = Vec::new(); diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs index 27fde74a0..433e5104d 100644 --- a/cli/src/modules/wasm.rs +++ b/cli/src/modules/wasm.rs @@ -1,5 +1,5 @@ -use super::{Manager, TestCommand}; -use crate::{spawner::SpawnOptions, Target}; +use super::Manager; +use crate::Target; use async_trait::async_trait; #[derive(Clone, Debug)] @@ -17,22 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Wasm } - - fn test_cmds(&self, _production: bool) -> Vec { - vec![ - TestCommand::new( - "wasm-pack test --node --color always".into(), - self.owner().cwd(), - None, - ), - TestCommand::new( - "npm run test".into(), - self.owner().cwd().join("spec"), - Some(SpawnOptions { - suppress_msg: true, - ..Default::default() - }), - ), - ] - } } diff --git a/cli/src/target/cli.rs b/cli/src/target/cli.rs new file mode 100644 index 000000000..33fd87a96 --- /dev/null +++ b/cli/src/target/cli.rs @@ -0,0 +1,12 @@ +use crate::target::Target; + +use super::TestCommand; + +pub fn gettest_cmds(production: bool) -> Vec { + let cmd = format!( + "cargo +stable test{} --color always", + if production { " -r" } else { "" } + ); + + vec![TestCommand::new(cmd, Target::Cli.cwd(), None)] +} diff --git a/cli/src/target/core.rs b/cli/src/target/core.rs new file mode 100644 index 000000000..8633dc4f2 --- /dev/null +++ b/cli/src/target/core.rs @@ -0,0 +1,12 @@ +use crate::target::Target; + +use super::TestCommand; + +pub fn get_test_cmds(production: bool) -> Vec { + let cmd = format!( + "cargo +stable test{} --color always", + if production { " -r" } else { "" } + ); + + vec![TestCommand::new(cmd, Target::Core.cwd(), None)] +} diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 241b29d91..23bf4f4f0 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -4,14 +4,16 @@ use crate::{ location::get_root, modules::Manager, modules::{self}, - spawner::{spawn, SpawnResult}, + spawner::{spawn, SpawnOptions, SpawnResult}, }; use anyhow::bail; use clap::ValueEnum; //TODO AAZ: Conisder which module should be pub after teh refactoring is done mod binding; +mod cli; pub mod client; +mod core; mod target_kind; mod wasm; @@ -38,6 +40,22 @@ pub enum Target { Wasm, } +pub struct TestCommand { + pub command: String, + pub cwd: PathBuf, + pub spawn_opts: Option, +} + +impl TestCommand { + pub(crate) fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { + Self { + command, + cwd, + spawn_opts, + } + } +} + impl std::fmt::Display for Target { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -189,6 +207,15 @@ impl Target { rest_targets => install_general(rest_targets, prod).await, } } + + pub fn test_cmds(&self, production: bool) -> Option> { + match self { + Target::Core => Some(core::get_test_cmds(production)), + Target::Cli => Some(cli::gettest_cmds(production)), + Target::Wasm => Some(wasm::get_test_cmds()), + _ => None, + } + } } /// run install using the general routine for the given target async fn install_general(target: &Target, prod: bool) -> Result { diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index 1f1226526..798f55f0f 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -1,5 +1,28 @@ +use crate::{spawner::SpawnOptions, target::Target}; + +use super::TestCommand; + pub fn get_build_cmd(prod: bool) -> String { let env = if prod { "--release" } else { "--dev" }; format!("wasm-pack build {env} --target bundler --color always") } + +pub fn get_test_cmds() -> Vec { + let cwd = Target::Wasm.cwd(); + vec![ + TestCommand::new( + "wasm-pack test --node --color always".into(), + cwd.clone(), + None, + ), + TestCommand::new( + "npm run test".into(), + cwd.join("spec"), + Some(SpawnOptions { + suppress_msg: true, + ..Default::default() + }), + ), + ] +} From 66c0bc10277c5dd118ba6a995b2de6632e587d31 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 15:18:46 +0200 Subject: [PATCH 083/174] Build CLI Refactor: Replacing Trait with Enum... - Move test method to the enmus & Create wrapper module --- cli/src/main.rs | 6 ++- cli/src/modules/mod.rs | 36 ------------- cli/src/modules/wrapper.rs | 102 +------------------------------------ cli/src/target/mod.rs | 50 +++++++++++++++++- cli/src/target/wrapper.rs | 93 +++++++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 cli/src/target/wrapper.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index f27f3c036..5ab94e040 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -103,7 +103,11 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - let targets = get_targets_or_default(target); + //TODO AAZ: Work around until manager trait is removed + let targets: Vec<_> = get_targets_or_default(target) + .into_iter() + .map(|t| t.owner()) + .collect(); let results = join_all( targets .iter() diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index cdb08e9a2..0c2506304 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -19,7 +19,6 @@ use crate::{ }; use anyhow::{bail, Context, Error}; use async_trait::async_trait; -use futures::future::join_all; use std::iter; use tokio::sync::oneshot; @@ -265,39 +264,4 @@ pub trait Manager { ) .await } - - async fn test(&self, production: bool) -> Result, Error> { - let Some(test_cmds) = self.owner().test_cmds(production) else { - return Ok(Vec::new()); - }; - - debug_assert!(!test_cmds.is_empty()); - - let mut results = Vec::new(); - - // build method calls install - let build_results = self.build(false).await?; - results.extend(build_results); - - let caption = format!("Test {}", self.owner()); - let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { - spawn( - cmd.command, - Some(cmd.cwd), - caption.clone(), - iter::empty(), - cmd.spawn_opts, - ) - })) - .await; - - for res in spawn_results { - match res { - Ok(spawn_res) => results.push(spawn_res), - Err(err) => return Err(err), - } - } - - Ok(results) - } } diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 8f8c40d54..8870241a0 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,41 +1,8 @@ use super::Manager; -use crate::{ - fstools, - spawner::{spawn, spawn_blocking, SpawnResult}, - Target, -}; +use crate::{fstools, spawner::SpawnResult, Target}; use anyhow::{Context, Error}; use async_trait::async_trait; -use std::{fs, iter, path::PathBuf}; - -const TEST_SPECS: [&str; 14] = [ - // TODO: - // Running "jobs" here causes the program to receive SIGTRAP from OS because of an - // out-of-memory error in electron app, even if only this job was running (by - // commenting out the other specs). - // - // The error happens while executing line 137 from the file `session.jobs.spec.ts` when - // we spawn the command using Stdio::piped() in the spawn command (line 74 in file - // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or - // using Stdio::inherit() in `spawner.rs` prevent this error from happening. - // - // The current work-around to blocking run the all the test commands sequentially using inherit - // Stdio::inherit suspending the progress bars until all tests are done. - "jobs", - "search", - "values", - "extract", - "ranges", - "exporting", - "map", - "observe", - "indexes", - "concat", - "cancel", - "errors", - "stream", - "promises", -]; +use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/ts-bindings` @@ -170,69 +137,4 @@ impl Manager for Module { report_logs, ))) } - - async fn test(&self, _production: bool) -> Result, Error> { - let mut results = Vec::new(); - - let build_results = self.build(false).await?; - results.extend(build_results); - - let build_spec_path = self.owner().cwd().join("spec"); - //TODO: This check exists in rake implementation but it need to be improved. - // The check should cover if the test themselves or the code under the tests has been changed. - if !build_spec_path.join("build").exists() { - let test_builder_path = self - .owner() - .cwd() - .join("node_modules") - .join(".bin") - .join("tsc"); - let build_spec_cmd = - format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); - - let spec_res = spawn( - build_spec_cmd, - Some(build_spec_path), - "Build Specs".into(), - iter::empty(), - None, - ) - .await?; - - results.push(spec_res); - } - - let cwd = self.owner().cwd(); - - let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); - let electron_path = electron_path.to_string_lossy(); - - let jasmine_path: PathBuf = [".", "node_modules", "jasmine", "bin", "jasmine.js"] - .iter() - .collect(); - let jasmine_path = jasmine_path.to_string_lossy(); - - let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); - - for spec in TEST_SPECS { - let caption = format!("Test {}: {}", self.owner(), spec); - let spec_file_name = format!("session.{spec}.spec.js"); - let spec_file_path = specs_dir_path.join(spec_file_name); - let command = format!( - "{electron_path} {jasmine_path} {}", - spec_file_path.to_string_lossy() - ); - let res = spawn_blocking( - command, - Some(cwd.clone()), - caption.clone(), - vec![(String::from("ELECTRON_RUN_AS_NODE"), String::from("1"))], - ) - .await?; - - results.push(res); - } - - Ok(results) - } } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 23bf4f4f0..c404e3293 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -2,8 +2,8 @@ use std::{iter, path::PathBuf, str::FromStr}; use crate::{ location::get_root, + modules, modules::Manager, - modules::{self}, spawner::{spawn, SpawnOptions, SpawnResult}, }; use anyhow::bail; @@ -16,7 +16,9 @@ pub mod client; mod core; mod target_kind; mod wasm; +mod wrapper; +use futures::future::join_all; //TODO AAZ: Conisder removing this when refactoring is done pub use target_kind::TargetKind; @@ -208,7 +210,14 @@ impl Target { } } - pub fn test_cmds(&self, production: bool) -> Option> { + pub async fn test(&self, production: bool) -> Result, anyhow::Error> { + match self { + Target::Wrapper => wrapper::run_test().await, + rest_targets => rest_targets.run_test_general(production).await, + } + } + + fn test_cmds(&self, production: bool) -> Option> { match self { Target::Core => Some(core::get_test_cmds(production)), Target::Cli => Some(cli::gettest_cmds(production)), @@ -216,7 +225,44 @@ impl Target { _ => None, } } + + /// run test using the general routine with `test_cmds()` method + async fn run_test_general(&self, production: bool) -> Result, anyhow::Error> { + let Some(test_cmds) = self.test_cmds(production) else { + return Ok(Vec::new()); + }; + + debug_assert!(!test_cmds.is_empty()); + + let mut results = Vec::new(); + + // build method calls install + let build_results = self.get().build(false).await?; + results.extend(build_results); + + let caption = format!("Test {}", self); + let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { + spawn( + cmd.command, + Some(cmd.cwd), + caption.clone(), + iter::empty(), + cmd.spawn_opts, + ) + })) + .await; + + for res in spawn_results { + match res { + Ok(spawn_res) => results.push(spawn_res), + Err(err) => return Err(err), + } + } + + Ok(results) + } } + /// run install using the general routine for the given target async fn install_general(target: &Target, prod: bool) -> Result { let cmd = target.kind().install_cmd(prod); diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs new file mode 100644 index 000000000..3c7a1910e --- /dev/null +++ b/cli/src/target/wrapper.rs @@ -0,0 +1,93 @@ +use std::{iter, path::PathBuf}; + +use crate::spawner::{spawn, spawn_blocking, SpawnResult}; + +use super::Target; + +const TEST_SPECS: [&str; 14] = [ + // TODO: + // Running "jobs" here causes the program to receive SIGTRAP from OS because of an + // out-of-memory error in electron app, even if only this job was running (by + // commenting out the other specs). + // + // The error happens while executing line 137 from the file `session.jobs.spec.ts` when + // we spawn the command using Stdio::piped() in the spawn command (line 74 in file + // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or + // using Stdio::inherit() in `spawner.rs` prevent this error from happening. + // + // The current work-around to blocking run the all the test commands sequentially using inherit + // Stdio::inherit suspending the progress bars until all tests are done. + "jobs", + "search", + "values", + "extract", + "ranges", + "exporting", + "map", + "observe", + "indexes", + "concat", + "cancel", + "errors", + "stream", + "promises", +]; + +pub async fn run_test() -> Result, anyhow::Error> { + let mut results = Vec::new(); + + let build_results = Target::Wrapper.get().build(false).await?; + results.extend(build_results); + + let cwd = Target::Wrapper.cwd(); + + let build_spec_path = cwd.join("spec"); + //TODO: This check exists in rake implementation but it need to be improved. + // The check should cover if the test themselves or the code under the tests has been changed. + if !build_spec_path.join("build").exists() { + let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); + let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); + + let spec_res = spawn( + build_spec_cmd, + Some(build_spec_path), + "Build Specs".into(), + iter::empty(), + None, + ) + .await?; + + results.push(spec_res); + } + + let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); + let electron_path = electron_path.to_string_lossy(); + + let jasmine_path: PathBuf = [".", "node_modules", "jasmine", "bin", "jasmine.js"] + .iter() + .collect(); + let jasmine_path = jasmine_path.to_string_lossy(); + + let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); + + for spec in TEST_SPECS { + let caption = format!("Test {}: {}", Target::Wrapper, spec); + let spec_file_name = format!("session.{spec}.spec.js"); + let spec_file_path = specs_dir_path.join(spec_file_name); + let command = format!( + "{electron_path} {jasmine_path} {}", + spec_file_path.to_string_lossy() + ); + let res = spawn_blocking( + command, + Some(cwd.clone()), + caption.clone(), + vec![(String::from("ELECTRON_RUN_AS_NODE"), String::from("1"))], + ) + .await?; + + results.push(res); + } + + Ok(results) +} From 5806a6720104e68aff4b4a362521871030e24357 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 15:35:28 +0200 Subject: [PATCH 084/174] Build CLI Refactor: Replacing Trait with Enum... Move linting functions to the Enum --- cli/src/main.rs | 5 +++- cli/src/modules/mod.rs | 56 --------------------------------------- cli/src/target/mod.rs | 60 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 5ab94e040..cb2a69630 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -55,7 +55,10 @@ async fn main() -> Result<(), Error> { } Command::Lint { target, report } => { report_opt = get_report_option(report)?; - let targets = get_targets_or_default(target); + let targets: Vec<_> = get_targets_or_default(target) + .into_iter() + .map(|t| t.owner()) + .collect(); let results = join_all( targets .iter() diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 0c2506304..34a160d4a 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -208,60 +208,4 @@ pub trait Manager { Ok(results) } } - - async fn check(&self) -> Result, Error> { - let mut results = Vec::new(); - match self.owner().kind() { - TargetKind::Ts => { - let install_result = self.owner().install(false).await?; - let lint_restul = self.lint().await?; - results.push(install_result); - results.push(lint_restul); - } - TargetKind::Rs => { - let clippy_result = self.clippy().await?; - results.push(clippy_result); - } - } - - Ok(results) - } - async fn lint(&self) -> Result { - let path = get_root().join(self.owner().cwd()); - let caption = format!("TS Lint {}", self.owner()); - let status = spawn( - "yarn run lint".into(), - Some(path.clone()), - caption, - iter::empty(), - None, - ) - .await?; - if !status.status.success() { - return Ok(status); - } - - let caption = format!("Build {}", self.owner()); - spawn( - "yarn run build".into(), - Some(path), - caption, - iter::empty(), - None, - ) - .await - } - async fn clippy(&self) -> Result { - let path = get_root().join(self.owner().cwd()); - - let caption = format!("Clippy {}", self.owner()); - spawn( - "cargo clippy --color always --all --all-features -- -D warnings".into(), - Some(path), - caption, - iter::empty(), - None, - ) - .await - } } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index c404e3293..aedeaa3d2 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -9,7 +9,7 @@ use crate::{ use anyhow::bail; use clap::ValueEnum; -//TODO AAZ: Conisder which module should be pub after teh refactoring is done +//TODO AAZ: Conisder which module should be pub after the refactoring is done mod binding; mod cli; pub mod client; @@ -261,6 +261,64 @@ impl Target { Ok(results) } + + pub async fn check(&self) -> Result, anyhow::Error> { + let mut results = Vec::new(); + match self.kind() { + TargetKind::Ts => { + let install_result = self.install(false).await?; + let lint_restul = self.ts_lint().await?; + results.push(install_result); + results.push(lint_restul); + } + TargetKind::Rs => { + let clippy_result = self.clippy().await?; + results.push(clippy_result); + } + } + + Ok(results) + } + + async fn ts_lint(&self) -> Result { + let path = get_root().join(self.cwd()); + let caption = format!("TS Lint {}", self); + let status = spawn( + "yarn run lint".into(), + Some(path.clone()), + caption, + iter::empty(), + None, + ) + .await?; + if !status.status.success() { + return Ok(status); + } + + let caption = format!("Build {}", self); + spawn( + "yarn run build".into(), + Some(path), + caption, + iter::empty(), + None, + ) + .await + } + + async fn clippy(&self) -> Result { + let path = get_root().join(self.cwd()); + + let caption = format!("Clippy {}", self); + spawn( + "cargo clippy --color always --all --all-features -- -D warnings".into(), + Some(path), + caption, + iter::empty(), + None, + ) + .await + } } /// run install using the general routine for the given target From eead200e612f9536f5dd796a401a0e89e1a7f9f6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 16:27:41 +0200 Subject: [PATCH 085/174] Build CLI Refactor: Replacing Trait with Enum... - Move Build & Clean & Copying to Enums System - Copying names are now more descriptive - Box Future is used for recursive async calls since async_trait isn't used with build anymore --- cli/src/main.rs | 12 ++- cli/src/modules/app.rs | 50 +-------- cli/src/modules/binding.rs | 54 +--------- cli/src/modules/mod.rs | 202 +----------------------------------- cli/src/modules/shared.rs | 54 +--------- cli/src/modules/wrapper.rs | 122 +--------------------- cli/src/target/app.rs | 53 ++++++++++ cli/src/target/binding.rs | 56 ++++++++++ cli/src/target/mod.rs | 206 ++++++++++++++++++++++++++++++++++++- cli/src/target/shared.rs | 54 ++++++++++ cli/src/target/wrapper.rs | 120 ++++++++++++++++++++- 11 files changed, 500 insertions(+), 483 deletions(-) create mode 100644 cli/src/target/app.rs create mode 100644 cli/src/target/shared.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index cb2a69630..e0d6fc87b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -74,7 +74,10 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - let targets = get_targets_or_default(target); + let targets: Vec<_> = get_targets_or_default(target) + .into_iter() + .map(|t| t.owner()) + .collect(); let results = join_all( targets .iter() @@ -90,7 +93,10 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - let targets = get_targets_or_default(target); + let targets: Vec<_> = get_targets_or_default(target) + .into_iter() + .map(|t| t.owner()) + .collect(); let results = join_all( targets .iter() @@ -123,7 +129,7 @@ async fn main() -> Result<(), Error> { Command::Run { production } => { report_opt = ReportOptions::None; let results = join_all( - Target::all() + Target::_all_enums() .iter() .map(|module| module.build(production)) .collect::>(), diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs index fbe473976..5528872e9 100644 --- a/cli/src/modules/app.rs +++ b/cli/src/modules/app.rs @@ -1,8 +1,6 @@ use super::Manager; -use crate::{fstools, spawner::SpawnResult, target, Target}; -use anyhow::{bail, Context, Error}; +use crate::Target; use async_trait::async_trait; -use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/holder` @@ -19,50 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::App } - - async fn after(&self, prod: bool) -> Result, Error> { - let mut report_logs = Vec::new(); - let src = target::client::get_dist_path(prod); - let dest = self.owner().cwd().join("dist"); - if !src.exists() { - bail!("Not found: {}", src.display()); - } - if !dest.exists() { - let msg = format!("creating directory: {}", dest.display()); - report_logs.push(msg); - - fs::create_dir(&dest) - .with_context(|| format!("Error while creating directory: {}", dest.display()))?; - } - let prev = dest.join("client"); - if prev.exists() { - let msg = format!("removing directory: {}", prev.display()); - report_logs.push(msg); - - fstools::rm_folder(&prev).await?; - } - - fstools::cp_folder(src.clone(), dest.clone(), &mut report_logs).await?; - - let rename_from = dest.join( - src.file_name() - .context("Fail to parse client artifacts path")?, - ); - let rename_to = dest.join("client"); - - let msg = format!( - "renaming '{}' to '{}'", - rename_from.display(), - rename_to.display() - ); - report_logs.push(msg); - - std::fs::rename(&rename_from, &rename_to) - .with_context(|| format!("Error while renaming {}", rename_from.display()))?; - - Ok(Some(SpawnResult::create_for_fs( - "Copy App Build Artifacts".into(), - report_logs, - ))) - } } diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs index 5289e290d..89a4c37e0 100644 --- a/cli/src/modules/binding.rs +++ b/cli/src/modules/binding.rs @@ -1,8 +1,6 @@ use super::Manager; -use crate::{fstools, spawner::SpawnResult, Target}; -use anyhow::{bail, Context, Error}; +use crate::target::Target; use async_trait::async_trait; -use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/rs-bindings` @@ -19,54 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Binding } - - async fn after(&self, _prod: bool) -> Result, Error> { - let mut report_logs = Vec::new(); - - // *** Copy `index.node` from rs to ts bindings dist *** - report_logs.push(String::from("Copying `index.node` to ts-bindings dist...")); - - let src_file = self.owner().cwd().join("dist").join("index.node"); - if !src_file.exists() { - bail!( - "Error while copying `rs-bindings`. Err: Not found: {}", - src_file.to_string_lossy() - ); - } - - let ts_dist_native_dir = Target::Wrapper.cwd().join("dist").join("native"); - if !ts_dist_native_dir.exists() { - let msg = format!("creating directory: {}", ts_dist_native_dir.display()); - report_logs.push(msg); - - fs::create_dir_all(&ts_dist_native_dir).with_context(|| { - format!( - "Error while creating directory: {}", - ts_dist_native_dir.display() - ) - })?; - } - - fstools::cp_file( - src_file.clone(), - ts_dist_native_dir.join("index.node"), - &mut report_logs, - ) - .await?; - - // *** Copy `index.node` from rs to ts bindings src native (dir-tests) *** - report_logs.push(String::from( - "Copying `index.node` to ts-bindings src native...", - )); - - let dir_tests = Target::Wrapper.cwd().join("src").join("native"); - let mod_file = dir_tests.join("index.node"); - - fstools::cp_file(src_file, mod_file, &mut report_logs).await?; - - Ok(Some(SpawnResult::create_for_fs( - "Copying `index.node` from rs to ts bindings".into(), - report_logs, - ))) - } } diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 34a160d4a..f7589b037 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -1,3 +1,7 @@ +use async_trait::async_trait; + +use crate::target::Target; + pub mod app; pub mod binding; pub mod cli; @@ -7,205 +11,7 @@ pub mod shared; pub mod wasm; pub mod wrapper; -use crate::{ - build_state::{BuildState, BuildStatesTracker}, - checksum_records::ChecksumRecords, - fstools, - job_type::JobType, - location::get_root, - spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, - target::TargetKind, - Target, -}; -use anyhow::{bail, Context, Error}; -use async_trait::async_trait; -use std::iter; -use tokio::sync::oneshot; - #[async_trait] pub trait Manager { fn owner(&self) -> Target; - async fn reset(&self, production: bool) -> anyhow::Result> { - let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; - checksum.remove_hash_if_exist(self.owner()); - let mut results = Vec::new(); - let clean_result = self.clean().await?; - results.push(clean_result); - - let dist_path = self.owner().cwd().join("dist"); - - let remove_log = format!("removing {}", dist_path.display()); - - fstools::rm_folder(&dist_path).await?; - - let job = format!("Reset {}", self.owner()); - - results.push(SpawnResult::create_for_fs(job, vec![remove_log])); - - Ok(results) - } - async fn clean(&self) -> Result { - let mut logs = Vec::new(); - let path = match self.owner().kind() { - TargetKind::Ts => self.owner().cwd().join("node_modules"), - TargetKind::Rs => self.owner().cwd().join("target"), - }; - - let remove_log = format!("removing directory {}", path.display()); - logs.push(remove_log); - - fstools::rm_folder(&path).await?; - - let job = format!("Clean {}", self.owner()); - - Ok(SpawnResult::create_for_fs(job, logs)) - } - - async fn after(&self, _prod: bool) -> Result, Error> { - Ok(None) - } - - /// Runs build considering the currently running builds and already finished ones as well. - async fn build(&self, prod: bool) -> Result, Error> { - let build_states = BuildStatesTracker::get().await; - let (tx_result, rx_result) = oneshot::channel(); - - let target = self.owner(); - let mut run_build = false; - // Check the current build state - { - let mut states = build_states.states_map.lock().unwrap(); - match states.get_mut(&target) { - Some(BuildState::Running(senders)) => { - // If the build is currently running, add the sender to senders list so it get - // the results when the job is done; - senders.push(tx_result); - } - Some(BuildState::Finished(build_result)) => { - // If Build is already finished, then return the results directly - match build_result { - Ok(res) => return Ok(res.clone()), - Err(err) => bail!(err.to_string()), - } - } - None => { - // Run the build and add the sender to the list. - run_build = true; - states.insert(target, BuildState::Running(vec![tx_result])); - } - } - } - - if run_build { - // Run the build - let build_result = self.perform_build(prod).await; - // Update the build state and notify all the senders - { - let mut states = build_states.states_map.lock().unwrap(); - - let res_clone = match &build_result { - Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{:?}", err)), - }; - - let Some(BuildState::Running(senders)) = - states.insert(target, BuildState::Finished(build_result)) - else { - unreachable!("State after calling build must be renning"); - }; - - for sender in senders { - let res_clone = match &res_clone { - Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{:?}", err)), - }; - if sender.send(res_clone).is_err() { - bail!("Fail to communicate with builder"); - } - } - } - } - - rx_result - .await - .context("Fail to communicate with builder")? - } - - /// Performs build process without checking the current builds states - async fn perform_build(&self, prod: bool) -> Result, Error> { - let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; - checksum_rec.register_job(self.owner()); - - let mut results = Vec::new(); - let deps: Vec> = self - .owner() - .deps() - .iter() - .map(|target| target.get()) - .collect(); - for module in deps { - let status = module.build(prod).await.with_context(|| { - format!( - "Error while building the dependciy {} for target {}", - module.owner(), - self.owner() - ) - })?; - results.extend(status); - if results.iter().any(|res| !res.status.success()) { - return Ok(results); - } - } - let path = get_root().join(self.owner().cwd()); - let cmd = self.owner().build_cmd(prod); - let caption = format!("Build {}", self.owner()); - - let mut skip_task = false; - - let all_skipped = results.iter().all(|r| { - r.skipped.unwrap_or({ - // Tasks with no skip info are irrelevant - true - }) - }); - - if all_skipped { - skip_task = !checksum_rec.check_changed(self.owner())?; - } - - let spawn_reslt = if skip_task { - spawn_skip(cmd, Some(path), caption).await - } else { - let install_result = self.owner().install(false).await?; - results.push(install_result); - let spawn_opt = SpawnOptions { - has_skip_info: true, - ..Default::default() - }; - spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await - }; - - let status = spawn_reslt?; - - if !status.status.success() { - results.push(status); - Ok(results) - } else { - results.push(status); - if !skip_task { - let res = self.after(prod).await?; - if let Some(result) = res { - results.push(result); - } - if matches!(self.owner().kind(), TargetKind::Ts) && prod { - let clean_res = self.clean().await?; - results.push(clean_res); - let install_res = self.owner().install(prod).await?; - results.push(install_res); - } - } - - Ok(results) - } - } } diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs index ec9d7d32f..66489a2fd 100644 --- a/cli/src/modules/shared.rs +++ b/cli/src/modules/shared.rs @@ -1,8 +1,6 @@ use super::Manager; -use crate::{fstools, spawner::SpawnResult, Target}; -use anyhow::{Context, Error}; +use crate::Target; use async_trait::async_trait; -use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/platform` @@ -19,54 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Shared } - - async fn after(&self, _prod: bool) -> Result, Error> { - let mut report_logs = Vec::new(); - - report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); - - let platform_dest = Target::Wrapper.cwd().join("node_modules").join("platform"); - - let msg = format!("Removing directory: '{}'", platform_dest.display()); - report_logs.push(msg); - - fstools::rm_folder(&platform_dest).await?; - - tokio::fs::create_dir_all(&platform_dest) - .await - .with_context(|| { - format!("Error while creating directory {}", platform_dest.display()) - })?; - - let source = self.owner().cwd(); - - // This part to get all the needed files and folders to copy - let entries_to_copy: Vec<_> = fs::read_dir(&source) - .with_context(|| { - format!( - "Error while reading directory content: {}", - source.display() - ) - })? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name().is_some_and(|file_name| { - !file_name.to_string_lossy().starts_with("node_modules") - }) - }) - .collect(); - - fstools::cp_many( - entries_to_copy, - platform_dest, - source.display(), - &mut report_logs, - ) - .await?; - - Ok(Some(SpawnResult::create_for_fs( - "Copying Platform to Bindings".into(), - report_logs, - ))) - } } diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs index 8870241a0..8791907cb 100644 --- a/cli/src/modules/wrapper.rs +++ b/cli/src/modules/wrapper.rs @@ -1,8 +1,6 @@ use super::Manager; -use crate::{fstools, spawner::SpawnResult, Target}; -use anyhow::{Context, Error}; +use crate::Target; use async_trait::async_trait; -use std::fs; #[derive(Clone, Debug)] /// Represents the path `application/apps/rustcore/ts-bindings` @@ -19,122 +17,4 @@ impl Manager for Module { fn owner(&self) -> Target { Target::Wrapper } - async fn after(&self, _prod: bool) -> Result, Error> { - let mut report_logs = Vec::new(); - - // *** Copying TS Bindings *** - report_logs.push(String::from("Copying ts-bindings to electron...")); - let rustcore_dest = Target::App.cwd().join("node_modules").join("rustcore"); - - fstools::rm_folder(&rustcore_dest).await?; - - let msg = format!("Removing directory: '{}'", rustcore_dest.display()); - report_logs.push(msg); - - tokio::fs::create_dir_all(&rustcore_dest) - .await - .with_context(|| { - format!("Error while creating directory {}", rustcore_dest.display()) - })?; - - // This part to get all the needed files and folders to copy - let ts_source = self.owner().cwd(); - let ts_entries_to_copy: Vec<_> = fs::read_dir(&ts_source) - .with_context(|| { - format!( - "Error while reading directory content: {}", - ts_source.display() - ) - })? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name().is_some_and(|file_name| { - !file_name.to_string_lossy().starts_with("node_modules") - }) - }) - .collect(); - - fstools::cp_many( - ts_entries_to_copy, - rustcore_dest.clone(), - ts_source.display(), - &mut report_logs, - ) - .await?; - - // *** Remove native folder *** - let native_dir_path = rustcore_dest.join("native"); - report_logs.push(format!( - "Removing the directory '{}'", - native_dir_path.display() - )); - - fstools::rm_folder(&native_dir_path).await?; - - // *** Copy Platform rustcore to electron *** - report_logs.push(String::from("Copying platform rustcore in to electron...")); - let platform_dest = rustcore_dest.join("node_modules").join("platform"); - - fstools::rm_folder(&platform_dest).await?; - tokio::fs::create_dir_all(&platform_dest) - .await - .with_context(|| { - format!( - "Error while creating directory: {}", - platform_dest.display() - ) - })?; - - let platform_src = Target::Shared.cwd(); - - let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src) - .with_context(|| { - format!( - "Error while reading directory content: {}", - platform_src.display() - ) - })? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name().is_some_and(|file_name| { - !file_name.to_string_lossy().starts_with("node_modules") - }) - }) - .collect(); - - fstools::cp_many( - platform_entries_to_copy.clone(), - platform_dest, - platform_src.display(), - &mut report_logs, - ) - .await?; - - // *** Copy Platform to electron *** - report_logs.push(String::from("Copying platform in to electron...")); - let platform_dest2 = Target::App.cwd().join("node_modules").join("platform"); - - fstools::rm_folder(&platform_dest2).await?; - tokio::fs::create_dir_all(&platform_dest2) - .await - .with_context(|| { - format!( - "Error while creating directory: {}", - platform_dest2.display() - ) - })?; - - fstools::cp_many( - platform_entries_to_copy, - platform_dest2, - platform_src.display(), - &mut report_logs, - ) - .await?; - - Ok(Some(SpawnResult::create_for_fs( - "Copy TS Bindings and Platform to Electron".into(), - report_logs, - ))) - } } diff --git a/cli/src/target/app.rs b/cli/src/target/app.rs new file mode 100644 index 000000000..ac845e02a --- /dev/null +++ b/cli/src/target/app.rs @@ -0,0 +1,53 @@ +use std::fs; + +use anyhow::{bail, Context}; + +use crate::{fstools, spawner::SpawnResult}; + +use super::{client::get_dist_path, Target}; + +pub async fn copy_client_to_app(prod: bool) -> Result, anyhow::Error> { + let mut report_logs = Vec::new(); + let src = get_dist_path(prod); + let dest = Target::App.cwd().join("dist"); + if !src.exists() { + bail!("Not found: {}", src.display()); + } + if !dest.exists() { + let msg = format!("creating directory: {}", dest.display()); + report_logs.push(msg); + + fs::create_dir(&dest) + .with_context(|| format!("Error while creating directory: {}", dest.display()))?; + } + let prev = dest.join("client"); + if prev.exists() { + let msg = format!("removing directory: {}", prev.display()); + report_logs.push(msg); + + fstools::rm_folder(&prev).await?; + } + + fstools::cp_folder(src.clone(), dest.clone(), &mut report_logs).await?; + + let rename_from = dest.join( + src.file_name() + .context("Fail to parse client artifacts path")?, + ); + let rename_to = dest.join("client"); + + let msg = format!( + "renaming '{}' to '{}'", + rename_from.display(), + rename_to.display() + ); + report_logs.push(msg); + + std::fs::rename(&rename_from, &rename_to) + .with_context(|| format!("Error while renaming {}", rename_from.display()))?; + + Ok(Some(SpawnResult::create_for_fs( + "Copy App Build Artifacts".into(), + report_logs, + ))) +} diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index d4d12e5f0..e0081d776 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -1,3 +1,9 @@ +use std::fs; + +use anyhow::{bail, Context}; + +use crate::{fstools, spawner::SpawnResult}; + use super::Target; pub fn get_build_cmd(prod: bool) -> String { @@ -13,3 +19,53 @@ pub fn get_build_cmd(prod: bool) -> String { if prod { " --release" } else { "" } ) } + +pub async fn copy_index_node() -> Result, anyhow::Error> { + let mut report_logs = Vec::new(); + + // *** Copy `index.node` from rs to ts bindings dist *** + report_logs.push(String::from("Copying `index.node` to ts-bindings dist...")); + + let src_file = Target::Binding.cwd().join("dist").join("index.node"); + if !src_file.exists() { + bail!( + "Error while copying `rs-bindings`. Err: Not found: {}", + src_file.to_string_lossy() + ); + } + + let ts_dist_native_dir = Target::Wrapper.cwd().join("dist").join("native"); + if !ts_dist_native_dir.exists() { + let msg = format!("creating directory: {}", ts_dist_native_dir.display()); + report_logs.push(msg); + + fs::create_dir_all(&ts_dist_native_dir).with_context(|| { + format!( + "Error while creating directory: {}", + ts_dist_native_dir.display() + ) + })?; + } + + fstools::cp_file( + src_file.clone(), + ts_dist_native_dir.join("index.node"), + &mut report_logs, + ) + .await?; + + // *** Copy `index.node` from rs to ts bindings src native (dir-tests) *** + report_logs.push(String::from( + "Copying `index.node` to ts-bindings src native...", + )); + + let dir_tests = Target::Wrapper.cwd().join("src").join("native"); + let mod_file = dir_tests.join("index.node"); + + fstools::cp_file(src_file, mod_file, &mut report_logs).await?; + + Ok(Some(SpawnResult::create_for_fs( + "Copying `index.node` from rs to ts bindings".into(), + report_logs, + ))) +} diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index aedeaa3d2..688b6cd46 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,26 +1,36 @@ use std::{iter, path::PathBuf, str::FromStr}; use crate::{ + build_state::{BuildState, BuildStatesTracker}, + checksum_records::ChecksumRecords, + fstools, + job_type::JobType, location::get_root, modules, modules::Manager, - spawner::{spawn, SpawnOptions, SpawnResult}, + spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; -use anyhow::bail; +use anyhow::{bail, Context}; use clap::ValueEnum; //TODO AAZ: Conisder which module should be pub after the refactoring is done +mod app; mod binding; mod cli; pub mod client; mod core; +mod shared; mod target_kind; mod wasm; mod wrapper; -use futures::future::join_all; +use futures::{ + future::{join_all, BoxFuture}, + FutureExt, +}; //TODO AAZ: Conisder removing this when refactoring is done pub use target_kind::TargetKind; +use tokio::sync::oneshot; #[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Target { @@ -237,7 +247,7 @@ impl Target { let mut results = Vec::new(); // build method calls install - let build_results = self.get().build(false).await?; + let build_results = self.build(false).await?; results.extend(build_results); let caption = format!("Test {}", self); @@ -319,6 +329,194 @@ impl Target { ) .await } + + pub async fn reset(&self, production: bool) -> anyhow::Result> { + let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; + checksum.remove_hash_if_exist(*self); + let mut results = Vec::new(); + let clean_result = self.clean().await?; + results.push(clean_result); + + let dist_path = self.cwd().join("dist"); + + let remove_log = format!("removing {}", dist_path.display()); + + fstools::rm_folder(&dist_path).await?; + + let job = format!("Reset {}", self); + + results.push(SpawnResult::create_for_fs(job, vec![remove_log])); + + Ok(results) + } + + async fn clean(&self) -> Result { + let mut logs = Vec::new(); + let path = match self.kind() { + TargetKind::Ts => self.cwd().join("node_modules"), + TargetKind::Rs => self.cwd().join("target"), + }; + + let remove_log = format!("removing directory {}", path.display()); + logs.push(remove_log); + + fstools::rm_folder(&path).await?; + + let job = format!("Clean {}", self); + + Ok(SpawnResult::create_for_fs(job, logs)) + } + + /// Runs build considering the currently running builds and already finished ones as well. + pub async fn build(&self, prod: bool) -> Result, anyhow::Error> { + let build_states = BuildStatesTracker::get().await; + let (tx_result, rx_result) = oneshot::channel(); + + let mut run_build = false; + // Check the current build state + { + let mut states = build_states.states_map.lock().unwrap(); + match states.get_mut(self) { + Some(BuildState::Running(senders)) => { + // If the build is currently running, add the sender to senders list so it get + // the results when the job is done; + senders.push(tx_result); + } + Some(BuildState::Finished(build_result)) => { + // If Build is already finished, then return the results directly + match build_result { + Ok(res) => return Ok(res.clone()), + Err(err) => bail!(err.to_string()), + } + } + None => { + // Run the build and add the sender to the list. + run_build = true; + states.insert(*self, BuildState::Running(vec![tx_result])); + } + } + } + + if run_build { + // Run the build + let build_result = self.perform_build(prod).await; + // Update the build state and notify all the senders + { + let mut states = build_states.states_map.lock().unwrap(); + + let res_clone = match &build_result { + Ok(spawn_res) => Ok(spawn_res.clone()), + Err(err) => Err(anyhow::anyhow!("{:?}", err)), + }; + + let Some(BuildState::Running(senders)) = + states.insert(*self, BuildState::Finished(build_result)) + else { + unreachable!("State after calling build must be renning"); + }; + + for sender in senders { + let res_clone = match &res_clone { + Ok(spawn_res) => Ok(spawn_res.clone()), + Err(err) => Err(anyhow::anyhow!("{:?}", err)), + }; + if sender.send(res_clone).is_err() { + bail!("Fail to communicate with builder"); + } + } + } + } + + rx_result + .await + .context("Fail to communicate with builder")? + } + + /// Performs build process without checking the current builds states + fn perform_build(&self, prod: bool) -> BoxFuture, anyhow::Error>> { + // BoxFuture is needed because recursion isn't supported with async rust yet + async move { + let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; + checksum_rec.register_job(*self); + + let mut results = Vec::new(); + let deps: Vec = self.deps(); + for module in deps { + let status = module.build(prod).await.with_context(|| { + format!( + "Error while building the dependciy {} for target {}", + module, self + ) + })?; + results.extend(status); + if results.iter().any(|res| !res.status.success()) { + return Ok(results); + } + } + let path = get_root().join(self.cwd()); + let cmd = self.build_cmd(prod); + let caption = format!("Build {}", self); + + let mut skip_task = false; + + let all_skipped = results.iter().all(|r| { + r.skipped.unwrap_or({ + // Tasks with no skip info are irrelevant + true + }) + }); + + if all_skipped { + skip_task = !checksum_rec.check_changed(*self)?; + } + + let spawn_reslt = if skip_task { + spawn_skip(cmd, Some(path), caption).await + } else { + let install_result = self.install(false).await?; + results.push(install_result); + let spawn_opt = SpawnOptions { + has_skip_info: true, + ..Default::default() + }; + spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await + }; + + let status = spawn_reslt?; + + if !status.status.success() { + results.push(status); + Ok(results) + } else { + results.push(status); + if !skip_task { + let res = self.after_build(prod).await?; + if let Some(result) = res { + results.push(result); + } + if matches!(self.kind(), TargetKind::Ts) && prod { + let clean_res = self.clean().await?; + results.push(clean_res); + let install_res = self.install(prod).await?; + results.push(install_res); + } + } + + Ok(results) + } + } + .boxed() + } + + async fn after_build(&self, prod: bool) -> Result, anyhow::Error> { + match self { + Target::Binding => binding::copy_index_node().await, + Target::Wrapper => wrapper::copy_binding_to_app().await, + Target::Shared => shared::copy_platform_to_binding().await, + Target::App => app::copy_client_to_app(prod).await, + _ => Ok(None), + } + } } /// run install using the general routine for the given target diff --git a/cli/src/target/shared.rs b/cli/src/target/shared.rs new file mode 100644 index 000000000..62ddad15c --- /dev/null +++ b/cli/src/target/shared.rs @@ -0,0 +1,54 @@ +use std::fs; + +use anyhow::Context; + +use crate::{fstools, spawner::SpawnResult}; + +use super::Target; + +pub async fn copy_platform_to_binding() -> Result, anyhow::Error> { + let mut report_logs = Vec::new(); + + report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); + + let platform_dest = Target::Wrapper.cwd().join("node_modules").join("platform"); + + let msg = format!("Removing directory: '{}'", platform_dest.display()); + report_logs.push(msg); + + fstools::rm_folder(&platform_dest).await?; + + tokio::fs::create_dir_all(&platform_dest) + .await + .with_context(|| format!("Error while creating directory {}", platform_dest.display()))?; + + let source = Target::Shared.cwd(); + + // This part to get all the needed files and folders to copy + let entries_to_copy: Vec<_> = fs::read_dir(&source) + .with_context(|| { + format!( + "Error while reading directory content: {}", + source.display() + ) + })? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) + }) + .collect(); + + fstools::cp_many( + entries_to_copy, + platform_dest, + source.display(), + &mut report_logs, + ) + .await?; + + Ok(Some(SpawnResult::create_for_fs( + "Copying Platform to Bindings".into(), + report_logs, + ))) +} diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 3c7a1910e..c4ae6ef76 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,6 +1,12 @@ use std::{iter, path::PathBuf}; -use crate::spawner::{spawn, spawn_blocking, SpawnResult}; +use anyhow::Context; +use std::fs; + +use crate::{ + fstools, + spawner::{spawn, spawn_blocking, SpawnResult}, +}; use super::Target; @@ -36,7 +42,7 @@ const TEST_SPECS: [&str; 14] = [ pub async fn run_test() -> Result, anyhow::Error> { let mut results = Vec::new(); - let build_results = Target::Wrapper.get().build(false).await?; + let build_results = Target::Wrapper.build(false).await?; results.extend(build_results); let cwd = Target::Wrapper.cwd(); @@ -91,3 +97,113 @@ pub async fn run_test() -> Result, anyhow::Error> { Ok(results) } + +pub async fn copy_binding_to_app() -> Result, anyhow::Error> { + let mut report_logs = Vec::new(); + + // *** Copying TS Bindings *** + report_logs.push(String::from("Copying ts-bindings to electron...")); + let rustcore_dest = Target::App.cwd().join("node_modules").join("rustcore"); + + fstools::rm_folder(&rustcore_dest).await?; + + let msg = format!("Removing directory: '{}'", rustcore_dest.display()); + report_logs.push(msg); + + fs::create_dir_all(&rustcore_dest) + .with_context(|| format!("Error while creating directory {}", rustcore_dest.display()))?; + + // This part to get all the needed files and folders to copy + let ts_source = Target::Wrapper.cwd(); + let with_context = fs::read_dir(&ts_source).with_context(|| { + format!( + "Error while reading directory content: {}", + ts_source.display() + ) + }); + let ts_entries_to_copy: Vec<_> = with_context? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) + }) + .collect(); + + fstools::cp_many( + ts_entries_to_copy, + rustcore_dest.clone(), + ts_source.display(), + &mut report_logs, + ) + .await?; + + // *** Remove native folder *** + let native_dir_path = rustcore_dest.join("native"); + report_logs.push(format!( + "Removing the directory '{}'", + native_dir_path.display() + )); + + fstools::rm_folder(&native_dir_path).await?; + + // *** Copy Platform rustcore to electron *** + report_logs.push(String::from("Copying platform rustcore in to electron...")); + let platform_dest = rustcore_dest.join("node_modules").join("platform"); + + fstools::rm_folder(&platform_dest).await?; + fs::create_dir_all(&platform_dest).with_context(|| { + format!( + "Error while creating directory: {}", + platform_dest.display() + ) + })?; + + let platform_src = Target::Shared.cwd(); + + let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src) + .with_context(|| { + format!( + "Error while reading directory content: {}", + platform_src.display() + ) + })? + .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) + }) + .collect(); + + fstools::cp_many( + platform_entries_to_copy.clone(), + platform_dest, + platform_src.display(), + &mut report_logs, + ) + .await?; + + // *** Copy Platform to electron *** + report_logs.push(String::from("Copying platform in to electron...")); + let platform_dest2 = Target::App.cwd().join("node_modules").join("platform"); + + fstools::rm_folder(&platform_dest2).await?; + fs::create_dir_all(&platform_dest2).with_context(|| { + format!( + "Error while creating directory: {}", + platform_dest2.display() + ) + })?; + + fstools::cp_many( + platform_entries_to_copy, + platform_dest2, + platform_src.display(), + &mut report_logs, + ) + .await?; + + Ok(Some(SpawnResult::create_for_fs( + "Copy TS Bindings and Platform to Electron".into(), + report_logs, + ))) +} From ab945afd509f9187925ba07a6c364536f17e16af Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 16:39:57 +0200 Subject: [PATCH 086/174] Build CLI Refactor: Replacing Trait with Enum... - Remove Manager Trait and all its modules since they are not used anymore - Replace all Managers with Target Enums --- cli/src/main.rs | 29 +++++++--------------------- cli/src/modules/app.rs | 20 ------------------- cli/src/modules/binding.rs | 20 ------------------- cli/src/modules/cli.rs | 20 ------------------- cli/src/modules/client.rs | 20 ------------------- cli/src/modules/core.rs | 20 ------------------- cli/src/modules/mod.rs | 17 ----------------- cli/src/modules/shared.rs | 20 ------------------- cli/src/modules/wasm.rs | 20 ------------------- cli/src/modules/wrapper.rs | 20 ------------------- cli/src/target/mod.rs | 39 +++++--------------------------------- 11 files changed, 12 insertions(+), 233 deletions(-) delete mode 100644 cli/src/modules/app.rs delete mode 100644 cli/src/modules/binding.rs delete mode 100644 cli/src/modules/cli.rs delete mode 100644 cli/src/modules/client.rs delete mode 100644 cli/src/modules/core.rs delete mode 100644 cli/src/modules/mod.rs delete mode 100644 cli/src/modules/shared.rs delete mode 100644 cli/src/modules/wasm.rs delete mode 100644 cli/src/modules/wrapper.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index e0d6fc87b..ee991eb18 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,7 +6,6 @@ mod cli_args; mod fstools; mod job_type; mod location; -mod modules; mod spawner; mod target; mod tracker; @@ -19,7 +18,6 @@ use cli_args::{CargoCli, Command}; use futures::future::join_all; use job_type::JobType; use location::init_location; -use modules::Manager; use spawner::SpawnResult; use std::{ fs::File, @@ -55,10 +53,7 @@ async fn main() -> Result<(), Error> { } Command::Lint { target, report } => { report_opt = get_report_option(report)?; - let targets: Vec<_> = get_targets_or_default(target) - .into_iter() - .map(|t| t.owner()) - .collect(); + let targets = get_targets_or_default(target); let results = join_all( targets .iter() @@ -74,10 +69,7 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - let targets: Vec<_> = get_targets_or_default(target) - .into_iter() - .map(|t| t.owner()) - .collect(); + let targets = get_targets_or_default(target); let results = join_all( targets .iter() @@ -93,10 +85,7 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - let targets: Vec<_> = get_targets_or_default(target) - .into_iter() - .map(|t| t.owner()) - .collect(); + let targets = get_targets_or_default(target); let results = join_all( targets .iter() @@ -112,11 +101,7 @@ async fn main() -> Result<(), Error> { report, } => { report_opt = get_report_option(report)?; - //TODO AAZ: Work around until manager trait is removed - let targets: Vec<_> = get_targets_or_default(target) - .into_iter() - .map(|t| t.owner()) - .collect(); + let targets = get_targets_or_default(target); let results = join_all( targets .iter() @@ -129,7 +114,7 @@ async fn main() -> Result<(), Error> { Command::Run { production } => { report_opt = ReportOptions::None; let results = join_all( - Target::_all_enums() + Target::all() .iter() .map(|module| module.build(production)) .collect::>(), @@ -201,10 +186,10 @@ async fn main() -> Result<(), Error> { Ok(()) } -fn get_targets_or_default(targets: Option>) -> Vec> { +fn get_targets_or_default(targets: Option>) -> Vec { if let Some(mut list) = targets { list.dedup(); - list.iter().map(|target| target.get()).collect() + list } else { Target::all() } diff --git a/cli/src/modules/app.rs b/cli/src/modules/app.rs deleted file mode 100644 index 5528872e9..000000000 --- a/cli/src/modules/app.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/holder` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::App - } -} diff --git a/cli/src/modules/binding.rs b/cli/src/modules/binding.rs deleted file mode 100644 index 89a4c37e0..000000000 --- a/cli/src/modules/binding.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::target::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/apps/rustcore/rs-bindings` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Binding - } -} diff --git a/cli/src/modules/cli.rs b/cli/src/modules/cli.rs deleted file mode 100644 index 4655950ea..000000000 --- a/cli/src/modules/cli.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `cli` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Cli - } -} diff --git a/cli/src/modules/client.rs b/cli/src/modules/client.rs deleted file mode 100644 index 29b2608f7..000000000 --- a/cli/src/modules/client.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/client` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Client - } -} diff --git a/cli/src/modules/core.rs b/cli/src/modules/core.rs deleted file mode 100644 index 31a7999f1..000000000 --- a/cli/src/modules/core.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/apps/indexer` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Core - } -} diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs deleted file mode 100644 index f7589b037..000000000 --- a/cli/src/modules/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use async_trait::async_trait; - -use crate::target::Target; - -pub mod app; -pub mod binding; -pub mod cli; -pub mod client; -pub mod core; -pub mod shared; -pub mod wasm; -pub mod wrapper; - -#[async_trait] -pub trait Manager { - fn owner(&self) -> Target; -} diff --git a/cli/src/modules/shared.rs b/cli/src/modules/shared.rs deleted file mode 100644 index 66489a2fd..000000000 --- a/cli/src/modules/shared.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/platform` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Shared - } -} diff --git a/cli/src/modules/wasm.rs b/cli/src/modules/wasm.rs deleted file mode 100644 index 433e5104d..000000000 --- a/cli/src/modules/wasm.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/apps/rustcore/wasm-bindings` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Wasm - } -} diff --git a/cli/src/modules/wrapper.rs b/cli/src/modules/wrapper.rs deleted file mode 100644 index 8791907cb..000000000 --- a/cli/src/modules/wrapper.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::Manager; -use crate::Target; -use async_trait::async_trait; - -#[derive(Clone, Debug)] -/// Represents the path `application/apps/rustcore/ts-bindings` -pub struct Module {} - -impl Module { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl Manager for Module { - fn owner(&self) -> Target { - Target::Wrapper - } -} diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 688b6cd46..caedbf89d 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -6,8 +6,6 @@ use crate::{ fstools, job_type::JobType, location::get_root, - modules, - modules::Manager, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; use anyhow::{bail, Context}; @@ -108,7 +106,7 @@ impl FromStr for Target { } impl Target { - pub fn _all_enums() -> Vec { + pub fn all() -> Vec { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function match Target::App { @@ -124,44 +122,17 @@ impl Target { } vec![ - Target::Core, Target::Binding, + Target::Cli, + Target::App, + Target::Core, Target::Wrapper, - Target::Client, Target::Shared, - Target::App, - Target::Cli, + Target::Client, Target::Wasm, ] } - //TODO AAZ: Replace this with _all_enums - pub fn all() -> Vec> { - vec![ - Box::new(modules::binding::Module::new()), - Box::new(modules::cli::Module::new()), - Box::new(modules::app::Module::new()), - Box::new(modules::core::Module::new()), - Box::new(modules::wrapper::Module::new()), - Box::new(modules::shared::Module::new()), - Box::new(modules::client::Module::new()), - Box::new(modules::wasm::Module::new()), - ] - } - //TODO AAZ: Remove this - pub fn get(&self) -> Box { - match self { - Target::Binding => Box::new(modules::binding::Module::new()), - Target::Cli => Box::new(modules::cli::Module::new()), - Target::Client => Box::new(modules::client::Module::new()), - Target::Core => Box::new(modules::core::Module::new()), - Target::Wrapper => Box::new(modules::wrapper::Module::new()), - Target::Shared => Box::new(modules::shared::Module::new()), - Target::App => Box::new(modules::app::Module::new()), - Target::Wasm => Box::new(modules::wasm::Module::new()), - } - } - pub fn cwd(&self) -> PathBuf { let root = get_root(); let sub_parts = match self { From 8f2b595847478670087727a0397c56809960cf2a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 16:44:02 +0200 Subject: [PATCH 087/174] Build CLI Refactor: Replacing Trait with Enum... - Check & Remove TODOs - Remove public accessors for private items - Tidy up using statements --- cli/src/target/mod.rs | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index caedbf89d..2919a2c79 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,4 +1,11 @@ +use anyhow::{bail, Context}; +use clap::ValueEnum; +use futures::{ + future::{join_all, BoxFuture}, + FutureExt, +}; use std::{iter, path::PathBuf, str::FromStr}; +use tokio::sync::oneshot; use crate::{ build_state::{BuildState, BuildStatesTracker}, @@ -8,28 +15,19 @@ use crate::{ location::get_root, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; -use anyhow::{bail, Context}; -use clap::ValueEnum; -//TODO AAZ: Conisder which module should be pub after the refactoring is done +use target_kind::TargetKind; + mod app; mod binding; mod cli; -pub mod client; +mod client; mod core; mod shared; mod target_kind; mod wasm; mod wrapper; -use futures::{ - future::{join_all, BoxFuture}, - FutureExt, -}; -//TODO AAZ: Conisder removing this when refactoring is done -pub use target_kind::TargetKind; -use tokio::sync::oneshot; - #[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Target { /// Represents the path `application/apps/indexer` @@ -51,13 +49,13 @@ pub enum Target { } pub struct TestCommand { - pub command: String, - pub cwd: PathBuf, - pub spawn_opts: Option, + command: String, + cwd: PathBuf, + spawn_opts: Option, } impl TestCommand { - pub(crate) fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { + fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { Self { command, cwd, @@ -180,7 +178,6 @@ impl Target { match self { // We must install ts binding tools before running rs bindings, therefore we call // wrapper (ts-bindings) install in the rs bindings install. - // TODO AAZ: Make sure the following statement is correct: // Since rs bindings is a dependency for ts bindings, we don't need to call to install // on ts bindings again. Target::Binding => install_general(&Target::Wrapper, prod).await, From 8ea54469fad9c092b2758abfe98d38135d152805 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Apr 2024 16:49:23 +0200 Subject: [PATCH 088/174] Build CLI Refactor: Replace Trait with Enum (done) - Remove the crate `async-trait` since it's not used anymore --- cli/Cargo.lock | 12 ------------ cli/Cargo.toml | 1 - 2 files changed, 13 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index f8baba0bc..5796926f7 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -92,17 +92,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "async-trait" -version = "0.1.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -171,7 +160,6 @@ name = "cargo-chipmunk" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", "clap", "console", "dir_checksum", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index aab4c3cd3..79ccd4b13 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -15,7 +15,6 @@ anyhow = "1.0.80" [dependencies] anyhow.workspace = true -async-trait = "0.1.73" clap = { version = "4.4.4", features = ["derive"] } console = "0.15.7" fs_extra = "1.3.0" From 3d915bf0c514acc4f96968f31846fd6234bae190 Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Tue, 23 Apr 2024 17:19:07 +0200 Subject: [PATCH 089/174] Add changes for arm64 build and CI --- .github/workflows/release.yml | 6 ++++-- application/holder/src/env/os/platform.ts | 15 ++++++++++++++- scripts/elements/release.rb | 1 + scripts/env/paths.rb | 12 ++++++++++++ scripts/tools/os.rb | 5 +++++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 703806944..937553dd7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: CSC_FOR_PULL_REQUEST: true strategy: matrix: - os: [ubuntu-20.04, windows-latest, macOS-latest] + os: [ubuntu-20.04, windows-latest, macos-13, macOS-latest] rust: [stable] steps: - name: Checkout @@ -88,7 +88,9 @@ jobs: "$KEYCHAIN_NAME" - name: build release working-directory: ./application/holder - run: rake release:prod --trace + run: | + uname -m + rake release:prod --trace env: KEYCHAIN_NAME: "build.keychain" CSC_LINK: ${{ secrets.CSC_LINK }} diff --git a/application/holder/src/env/os/platform.ts b/application/holder/src/env/os/platform.ts index 1ef9fb9ce..d72e0dc4f 100644 --- a/application/holder/src/env/os/platform.ts +++ b/application/holder/src/env/os/platform.ts @@ -3,8 +3,10 @@ import * as os from 'os'; export enum Platform { aix = 'aix', darwin = 'darwin', + darwinaarch64 = 'darwin-arm64', freebsd = 'freebsd', linux = 'linux', + linuxaarch64 = 'linux-arm64', openbsd = 'openbsd', sunos = 'sunos', win32 = 'win32', @@ -18,10 +20,21 @@ export function getPlatform(win32Only = false): Platform { case Platform.aix: case Platform.freebsd: case Platform.linux: + if (os.arch() === 'arm64') { + return Platform.linuxaarch64; + } else { + return Platform.linux; + } + break; case Platform.openbsd: return Platform.linux; case Platform.darwin: - return Platform.darwin; + if (os.arch() === 'arm64') { + return Platform.darwinaarch64; + } else { + return Platform.darwin; + } + break; case Platform.win32: if (win32Only) { return Platform.win32; diff --git a/scripts/elements/release.rb b/scripts/elements/release.rb index 75acf4a09..a79b5c4b8 100644 --- a/scripts/elements/release.rb +++ b/scripts/elements/release.rb @@ -67,6 +67,7 @@ def self.version def release_file_name prefix = OS.prefix prefix += '64' if prefix == 'win' + prefix += '-arm64' if OS.arm64? "chipmunk@#{Release.version}-#{prefix}-portable" end diff --git a/scripts/env/paths.rb b/scripts/env/paths.rb index 3d2c7fca7..153a47f39 100644 --- a/scripts/env/paths.rb +++ b/scripts/env/paths.rb @@ -7,8 +7,12 @@ module Paths def self.release_build_folder if OS.windows? 'win-unpacked' + elsif OS.linux? && OS.arm64? + 'linux-arm64-unpacked' elsif OS.linux? 'linux-unpacked' + elsif OS.arm64? + 'mac-arm64' else 'mac' end @@ -17,8 +21,12 @@ def self.release_build_folder def self.release_bin_folder if OS.windows? 'win-unpacked' + elsif OS.linux? && OS.arm64? + 'linux-arm64-unpacked' elsif OS.linux? 'linux-unpacked' + elsif OS.arm64? + 'mac-arm64/chipmunk.app/Contents/MacOS' else 'mac/chipmunk.app/Contents/MacOS' end @@ -27,8 +35,12 @@ def self.release_bin_folder def self.release_resources_folder if OS.windows? 'win-unpacked/Resources' + elsif OS.linux? && OS.arm64? + 'linux-arm64-unpacked' elsif OS.linux? 'linux-unpacked/Resources' + elsif OS.arm64? + 'mac-arm64/chipmunk.app/Contents/Resources' else 'mac/chipmunk.app/Contents/Resources' end diff --git a/scripts/tools/os.rb b/scripts/tools/os.rb index 291165742..dc4d36d04 100644 --- a/scripts/tools/os.rb +++ b/scripts/tools/os.rb @@ -18,6 +18,11 @@ def self.linux? OS.unix? && !OS.mac? end + def self.arm64? + arch = `uname -m`.chomp + arch=='arm64' || arch=='aarch64' + end + def self.executable(filename) exe = if OS.windows? '.exe' From 99df73105ba9f59a1eb78f459c2fe1b0b14ad463 Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Wed, 24 Apr 2024 14:05:57 +0200 Subject: [PATCH 090/174] Bypass fetching the architecture on Windows machine --- .github/workflows/release.yml | 1 - scripts/tools/os.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 937553dd7..082aea626 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,7 +89,6 @@ jobs: - name: build release working-directory: ./application/holder run: | - uname -m rake release:prod --trace env: KEYCHAIN_NAME: "build.keychain" diff --git a/scripts/tools/os.rb b/scripts/tools/os.rb index dc4d36d04..f5f15d4bf 100644 --- a/scripts/tools/os.rb +++ b/scripts/tools/os.rb @@ -19,7 +19,7 @@ def self.linux? end def self.arm64? - arch = `uname -m`.chomp + arch = (OS.unix? || OS.mac?) ? `uname -m`.chomp : "" arch=='arm64' || arch=='aarch64' end From 9c9a0e0daae7002399ba56f6a7a81075d808501d Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Thu, 25 Apr 2024 10:38:54 +0200 Subject: [PATCH 091/174] Resolve PR comments --- scripts/env/paths.rb | 2 +- scripts/tools/os.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/env/paths.rb b/scripts/env/paths.rb index 153a47f39..5541f0386 100644 --- a/scripts/env/paths.rb +++ b/scripts/env/paths.rb @@ -36,7 +36,7 @@ def self.release_resources_folder if OS.windows? 'win-unpacked/Resources' elsif OS.linux? && OS.arm64? - 'linux-arm64-unpacked' + 'linux-arm64-unpacked/Resources' elsif OS.linux? 'linux-unpacked/Resources' elsif OS.arm64? diff --git a/scripts/tools/os.rb b/scripts/tools/os.rb index f5f15d4bf..c2fa11b47 100644 --- a/scripts/tools/os.rb +++ b/scripts/tools/os.rb @@ -19,7 +19,8 @@ def self.linux? end def self.arm64? - arch = (OS.unix? || OS.mac?) ? `uname -m`.chomp : "" + arch = (OS.unix? || OS.mac?) ? `uname -m` : `echo %PROCESSOR_ARCHITECTURE%` + arch.chomp!.downcase! arch=='arm64' || arch=='aarch64' end From 38d401773a412ffa7b39cba72d72032a459be66f Mon Sep 17 00:00:00 2001 From: DmitryAstafyev Date: Fri, 26 Apr 2024 18:25:45 +0200 Subject: [PATCH 092/174] Up to 3.12.5 --- application/client/package.json | 2 +- application/holder/package.json | 2 +- changelog.md | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/application/client/package.json b/application/client/package.json index b36d818aa..6ebb8e81f 100644 --- a/application/client/package.json +++ b/application/client/package.json @@ -1,6 +1,6 @@ { "name": "chipmunk", - "version": "3.12.4", + "version": "3.12.5", "description": "Logs analyzer tool", "author": "Dmitry Astafyev", "scripts": { diff --git a/application/holder/package.json b/application/holder/package.json index 3f70a38da..93e4b35e6 100644 --- a/application/holder/package.json +++ b/application/holder/package.json @@ -1,6 +1,6 @@ { "name": "chipmunk", - "version": "3.12.4", + "version": "3.12.5", "chipmunk": { "versions": {} }, diff --git a/changelog.md b/changelog.md index 4f8238012..e163e0d97 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 3.12.5 (26.04.2024) + +## Added +- Support ARM64 for MacOS + # 3.12.4 (19.04.2024) ## Fixes From 23cfcada3435a7f92e51b93054734cfcb831fe8c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 09:09:15 +0200 Subject: [PATCH 093/174] Build CLI: Print env dependencies infos - Environment command now has two subcommands one for checks only and the other to print the information - Use the argument `-V` for `wasm_pack` instead of printing help - Remove `Environment` job_type since it's not used --- cli/src/checksum_records.rs | 2 +- cli/src/cli_args.rs | 13 +++++++-- cli/src/{check_env.rs => dev_environment.rs} | 14 +++++++++- cli/src/job_type.rs | 3 +- cli/src/main.rs | 29 ++++++++++++++------ 5 files changed, 46 insertions(+), 15 deletions(-) rename cli/src/{check_env.rs => dev_environment.rs} (82%) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 225e70235..b709f980a 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -30,7 +30,7 @@ struct ChecksumItems { impl ChecksumRecords { pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { let calculate_involved = match &job_type { - JobType::_Environment | JobType::Lint => return Ok(()), + JobType::Lint => return Ok(()), JobType::Build { production: _ } | JobType::Run { production: _ } | JobType::Test { production: _ } => true, diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 4426165b5..e14f60593 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -23,9 +23,10 @@ pub struct Cli { #[derive(Subcommand, Debug, Clone)] pub enum Command { - /// Checks that all needed tools for the development are installed + /// Provides commands for the needed tools for the development #[clap(visible_alias = "env")] - Environment, + #[command(subcommand)] + Environment(EnvironmentCommand), /// Runs linting & clippy Lint { /// Target to lint, by default whole application will be linted @@ -81,3 +82,11 @@ pub enum Command { production: bool, }, } + +#[derive(Subcommand, Debug, Clone)] +pub enum EnvironmentCommand { + /// Checks that all needed tools for the development are installed + Check, + /// Prints the information of the needed tools for the development + Print, +} diff --git a/cli/src/check_env.rs b/cli/src/dev_environment.rs similarity index 82% rename from cli/src/check_env.rs rename to cli/src/dev_environment.rs index 1a77f7302..b54f6a716 100644 --- a/cli/src/check_env.rs +++ b/cli/src/dev_environment.rs @@ -14,7 +14,7 @@ const ENV_CHECKS: [EnvCheck; 7] = [ EnvCheck::new( "wasm-pack", "wasm-pack", - "--help", + "-V", Some("cargo install wasm-pack"), ), EnvCheck::new("nj-cli", "nj-cli", "-V", Some("cargo install nj-cli")), @@ -87,3 +87,15 @@ pub fn check_env() -> anyhow::Result<()> { None => Ok(()), } } + +/// Prints the information of the needed tools for the development if available, otherwise prints +/// error information to `stderr` +pub fn print_env_info() { + for check in ENV_CHECKS.iter() { + println!("{} Info:", check.app_name); + if let Err(err) = Command::new(check.command).arg(check.arg).status() { + eprintln!("Error while retrieving dependency's information: {err}"); + } + println!("------------------------------------------------------------------"); + } +} diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 4c3395b09..13dead40a 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -1,6 +1,5 @@ #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum JobType { - _Environment, Lint, Build { production: bool }, Clean { production: bool }, @@ -11,7 +10,7 @@ pub enum JobType { impl JobType { pub fn is_production(&self) -> Option { match self { - JobType::_Environment | JobType::Lint => None, + JobType::Lint => None, JobType::Build { production } | JobType::Clean { production } | JobType::Test { production } diff --git a/cli/src/main.rs b/cli/src/main.rs index ee991eb18..4f66fc2c8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,8 +1,8 @@ mod app_runner; mod build_state; -mod check_env; mod checksum_records; mod cli_args; +mod dev_environment; mod fstools; mod job_type; mod location; @@ -11,10 +11,10 @@ mod target; mod tracker; use anyhow::{bail, Error}; -use check_env::check_env; use checksum_records::ChecksumRecords; use clap::Parser; use cli_args::{CargoCli, Command}; +use dev_environment::{check_env, print_env_info}; use futures::future::join_all; use job_type::JobType; use location::init_location; @@ -27,6 +27,8 @@ use std::{ use target::Target; use tracker::get_tracker; +use crate::cli_args::EnvironmentCommand; + #[derive(Debug)] pub enum ReportOptions { None, @@ -38,20 +40,25 @@ pub enum ReportOptions { async fn main() -> Result<(), Error> { let CargoCli::Chipmunk(cli) = CargoCli::parse(); - check_env()?; - init_location()?; // Run the given command let command = cli.command; let report_opt: ReportOptions; let (job_type, results) = match command { - Command::Environment => { - // Check for dependencies is already called before calling any command - println!("All needed tools for development are installed"); - return Ok(()); - } + Command::Environment(sub_command) => match sub_command { + EnvironmentCommand::Check => { + check_env()?; + println!("All needed tools for development are installed"); + return Ok(()); + } + EnvironmentCommand::Print => { + print_env_info(); + return Ok(()); + } + }, Command::Lint { target, report } => { + check_env()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -68,6 +75,7 @@ async fn main() -> Result<(), Error> { production, report, } => { + check_env()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -84,6 +92,7 @@ async fn main() -> Result<(), Error> { production, report, } => { + check_env()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -100,6 +109,7 @@ async fn main() -> Result<(), Error> { production, report, } => { + check_env()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -112,6 +122,7 @@ async fn main() -> Result<(), Error> { (JobType::Test { production }, results) } Command::Run { production } => { + check_env()?; report_opt = ReportOptions::None; let results = join_all( Target::all() From 3ad2190846035edb4ef990945f5cf422c65f9b3f Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 10:33:20 +0200 Subject: [PATCH 094/174] Build CLI: Add Updater to Targets - Updater should be built in release only with stable channel (Taken from rake solution) --- cli/src/target/mod.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 2919a2c79..c7f4e7bdd 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -46,6 +46,8 @@ pub enum Target { Cli, /// Represents the path `application/apps/rustcore/wasm-bindings` Wasm, + /// Represents the path `application/apps/precompiled/updater + Updater, } pub struct TestCommand { @@ -78,6 +80,7 @@ impl std::fmt::Display for Target { Target::Shared => "Shared", Target::App => "App", Target::Wasm => "Wasm", + Target::Updater => "Updater", } ) } @@ -89,6 +92,21 @@ impl FromStr for Target { fn from_str(input: &str) -> Result { type T = Target; + if cfg!(debug_assertions) { + // This check to remember to add the newly added enums to this function + match T::App { + T::Core => (), + T::Binding => (), + T::Wrapper => (), + T::Client => (), + T::Shared => (), + T::App => (), + T::Cli => (), + T::Wasm => (), + T::Updater => (), + }; + } + match input { "Core" => Ok(T::Core), "Wrapper" => Ok(T::Wrapper), @@ -98,7 +116,8 @@ impl FromStr for Target { "Shared" => Ok(T::Shared), "App" => Ok(T::App), "Wasm" => Ok(T::Wasm), - invalid => bail!("Invalid input: {invalid}"), + "Updater" => Ok(T::Updater), + invalid => bail!("Invalid input for Parsing Target: {invalid}"), } } } @@ -116,6 +135,7 @@ impl Target { Target::App => (), Target::Cli => (), Target::Wasm => (), + Target::Updater => (), }; } @@ -128,6 +148,7 @@ impl Target { Target::Shared, Target::Client, Target::Wasm, + Target::Updater, ] } @@ -142,6 +163,7 @@ impl Target { Target::App => ["application", "holder"].iter(), Target::Cli => ["cli"].iter(), Target::Wasm => ["application", "apps", "rustcore", "wasm-bindings"].iter(), + Target::Updater => ["application", "apps", "precompiled", "updater"].iter(), }; let sub_path: PathBuf = sub_parts.collect(); @@ -151,14 +173,18 @@ impl Target { pub fn kind(&self) -> TargetKind { match self { - Target::Binding | Target::Core | Target::Cli | Target::Wasm => TargetKind::Rs, + Target::Binding | Target::Core | Target::Cli | Target::Wasm | Target::Updater => { + TargetKind::Rs + } Target::Client | Target::Wrapper | Target::Shared | Target::App => TargetKind::Ts, } } pub fn deps(&self) -> Vec { match self { - Target::Core | Target::Cli | Target::Shared | Target::Wasm => Vec::new(), + Target::Core | Target::Cli | Target::Shared | Target::Wasm | Target::Updater => { + Vec::new() + } Target::Binding => vec![Target::Shared], Target::Wrapper => vec![Target::Binding, Target::Shared], Target::Client => vec![Target::Shared, Target::Wasm], @@ -170,6 +196,7 @@ impl Target { match self { Target::Binding => binding::get_build_cmd(prod), Target::Wasm => wasm::get_build_cmd(prod), + Target::Updater => "cargo +stable build --color always --release".into(), rest_targets => rest_targets.kind().build_cmd(prod), } } From dd437df9aa6d77ed13c99d80ef330ab8d5a353f0 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 11:07:52 +0200 Subject: [PATCH 095/174] Build CLI: `Print Dot` Command for dependencies graph - The commands prints the dependencies between the targets in dot format to be visualized with Graphviz --- cli/src/cli_args.rs | 3 +++ cli/src/main.rs | 5 +++++ cli/src/print_dot.rs | 14 ++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 cli/src/print_dot.rs diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index e14f60593..94724b811 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -27,6 +27,9 @@ pub enum Command { #[clap(visible_alias = "env")] #[command(subcommand)] Environment(EnvironmentCommand), + /// Prints an overview of targets dependencies in print-dot format for `Graphviz` + #[clap(visible_alias = "dot")] + PrintDot, /// Runs linting & clippy Lint { /// Target to lint, by default whole application will be linted diff --git a/cli/src/main.rs b/cli/src/main.rs index 4f66fc2c8..80bb097e3 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,6 +6,7 @@ mod dev_environment; mod fstools; mod job_type; mod location; +mod print_dot; mod spawner; mod target; mod tracker; @@ -57,6 +58,10 @@ async fn main() -> Result<(), Error> { return Ok(()); } }, + Command::PrintDot => { + print_dot::print_dependencies(); + return Ok(()); + } Command::Lint { target, report } => { check_env()?; report_opt = get_report_option(report)?; diff --git a/cli/src/print_dot.rs b/cli/src/print_dot.rs new file mode 100644 index 000000000..20dd92de4 --- /dev/null +++ b/cli/src/print_dot.rs @@ -0,0 +1,14 @@ +use crate::target::Target; + +/// Prints an overview of targets dependencies in print-dot format for `Graphviz` +pub fn print_dependencies() { + println!("digraph dependencies {{"); + + for target in Target::all() { + for dep in target.deps() { + println!(r#" "{target}" -> "{dep}""#); + } + } + + println!("}}"); +} From 1a288266d233976019251b110b700a04f967d89a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 13:42:13 +0200 Subject: [PATCH 096/174] Build CLI: Add README & Link it in Contribution.md --- cli/Cargo.toml | 2 +- cli/README.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ contribution.md | 9 +++++- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 cli/README.md diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 79ccd4b13..7ae043d6f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ name = "cargo-chipmunk" version = "0.1.0" edition = "2021" -description = "Tool for chipmunk application development" +description = "CLI Tool for chipmunk application development" [workspace] members = ["dir_checksum"] diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000..d30859700 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,86 @@ +[![LICENSE](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE.txt) + +# Chipmunk Development CLI Tool + +This CLI Tool provides an easier way to manage various development tasks for Chipmunk. +Chipmunk consists of multiple modules with complex dependencies on each other, and this tool helps streamline the development process by coordinating these tasks seamlessly. +This tool acts as a cargo extension. Once installed, you can access it by running `cargo chipmunk ` from anywhere within the repository. + +## Build/Installation + +### Prerequisites + +Before installing the Chipmunk CLI tool, ensure that Rust is installed on your system. If Rust is not yet installed, follow the official installation instructions for your platform: + +- **Install Rust:** Visit [rustup.rs](https://rustup.rs/) and follow the instructions to install Rust. + +### Install Chipmunk CLI + +Navigate to the root directory of the Chipmunk repository in your terminal and run the following command to install the Chipmunk CLI tool: + +```bash +cargo install --path cli +``` + +This command installs this tool as a cargo extension, allowing you to use `cargo chipmunk ` to execute various development tasks for Chipmunk. + + +## Usage + +This CLI tool provides multiple sub-commands for different tasks, with each sub-command offering various arguments. + +### General Commands Overview + +```bash +Tool for chipmunk application development + +Usage: cargo chipmunk + +Commands: + environment Provides commands for the needed tools for the development [aliases: env] + print-dot Prints an overview of targets dependencies in print-dot format for `Graphviz` [aliases: dot] + lint Runs linting & clippy + build Build + clean Clean + test Run tests + run Build and Run the application + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help + -V, --version Print version +``` + +### Build Command + +```bash +Usage: cargo chipmunk build [OPTIONS] [TARGET]... + +Arguments: + [TARGET]... + Target to build, by default whole application will be built + + Possible values: + - core: Represents the path `application/apps/indexer` + - binding: Represents the path `application/apps/rustcore/rs-bindings` + - wrapper: Represents the path `application/apps/rustcore/ts-bindings` + - client: Represents the path `application/client` + - shared: Represents the path `application/platform` + - app: Represents the path `application/holder` + - cli: Represents the path `cli` + - wasm: Represents the path `application/apps/rustcore/wasm-bindings` + - updater: Represents the path `application/apps/precompiled/updater + +Options: + -p, --production + Build release version + + -r, --report [] + Write report from command logs to the given file or to stdout if no file is defined + + -h, --help + Print help (see a summary with '-h') +``` + +## Contributing +See our [contribution](contribution.md) guide for details diff --git a/contribution.md b/contribution.md index ab102de6d..6642bb71c 100644 --- a/contribution.md +++ b/contribution.md @@ -1,13 +1,16 @@ # Contributing Chipmunk uses [Rust](https://www.rust-lang.org/) for processing the log files while frontend application is built using the [ElectronJS](https://www.electronjs.org/) + +## Chipmunk Development CLI Tool +With the new [Chipmunk Development CLI Tool](cli/README.md), you can perform various Chipmunk development tasks without the need to install Ruby. ## Pre-requisite To build and run chipmunk on local you will need following languages installed on your system. 1. Rust 2. NodeJS -3. Ruby +3. Ruby (Not needed with the new [Chipmunk Development CLI Tool](cli/README.md)) To check if you have all the pre-requisite installed or not, chipmunk provides the shell script for this purpose. After cloning the repo run following command in your preferred terminal. @@ -52,6 +55,10 @@ node -v which will print the installed NodeJS version on terminal. + +> [!TIP] +> You can use the new [Chipmunk Development CLI Tool](cli/README.md) instead of Ruby & `rake` + ### Installing Ruby You must have latest Ruby installed on your system. Prefer your choice of version manager either [RBENV](https://github.com/rbenv/rbenv) or [RVM](https://rvm.io/). From 619c341a71e50d953f1b1e3a93a6da808974c790 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 14:43:52 +0200 Subject: [PATCH 097/174] Build CLI: Add 'list' alias env print command - These two commands seem to be used interchangeably --- cli/src/cli_args.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 94724b811..81a93d810 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -91,5 +91,6 @@ pub enum EnvironmentCommand { /// Checks that all needed tools for the development are installed Check, /// Prints the information of the needed tools for the development + #[clap(visible_alias = "list")] Print, } From 862bee634db84d473837bf17dedcbcc670220f1a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 29 Apr 2024 17:04:48 +0200 Subject: [PATCH 098/174] Build CLI: Provide different node cmds on Windows - `npm` and `yarn` are saved as `*.cmd` files on windows which must be run with the extension from outside windows cmd or power shell. --- cli/src/app_runner.rs | 4 ++-- cli/src/dev_environment.rs | 11 +++++++++-- cli/src/main.rs | 1 + cli/src/node_cmd.rs | 13 +++++++++++++ cli/src/target/mod.rs | 5 +++-- cli/src/target/target_kind.rs | 11 +++++++++-- cli/src/target/wasm.rs | 4 ++-- 7 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 cli/src/node_cmd.rs diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index 990917bac..fc45942ee 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -2,12 +2,12 @@ use std::{io, process::ExitStatus}; use tokio::process::Command; -use crate::target::Target; +use crate::{node_cmd, target::Target}; pub async fn run_app() -> io::Result { let electron_path = Target::App.cwd(); - Command::new("yarn") + Command::new(node_cmd::YARN) .current_dir(electron_path) .args(["run", "electron"]) .status() diff --git a/cli/src/dev_environment.rs b/cli/src/dev_environment.rs index b54f6a716..b247f6c66 100644 --- a/cli/src/dev_environment.rs +++ b/cli/src/dev_environment.rs @@ -5,10 +5,17 @@ use std::{ use anyhow::bail; +use crate::node_cmd; + const ENV_CHECKS: [EnvCheck; 7] = [ EnvCheck::new("NodeJS", "node", "-v", None), - EnvCheck::new("npm", "npm", "-v", None), - EnvCheck::new("yarn", "yarn", "-v", Some("npm install --global yarn")), + EnvCheck::new("npm", node_cmd::NPM, "-v", None), + EnvCheck::new( + "yarn", + node_cmd::YARN, + "-v", + Some("npm install --global yarn"), + ), EnvCheck::new("rust", "rustup", "-V", None), EnvCheck::new("cargo", "cargo", "-V", None), EnvCheck::new( diff --git a/cli/src/main.rs b/cli/src/main.rs index 80bb097e3..645763cba 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,6 +6,7 @@ mod dev_environment; mod fstools; mod job_type; mod location; +mod node_cmd; mod print_dot; mod spawner; mod target; diff --git a/cli/src/node_cmd.rs b/cli/src/node_cmd.rs new file mode 100644 index 000000000..d715480de --- /dev/null +++ b/cli/src/node_cmd.rs @@ -0,0 +1,13 @@ +#[cfg(windows)] +mod commands { + pub const NPM: &str = "npm.cmd"; + pub const YARN: &str = "yarn.cmd"; +} + +#[cfg(not(windows))] +mod commands { + pub const NPM: &str = "npm"; + pub const YARN: &str = "yarn"; +} + +pub use commands::*; diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index c7f4e7bdd..d643c2821 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -13,6 +13,7 @@ use crate::{ fstools, job_type::JobType, location::get_root, + node_cmd, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; @@ -289,7 +290,7 @@ impl Target { let path = get_root().join(self.cwd()); let caption = format!("TS Lint {}", self); let status = spawn( - "yarn run lint".into(), + format!("{} run lint", node_cmd::YARN), Some(path.clone()), caption, iter::empty(), @@ -302,7 +303,7 @@ impl Target { let caption = format!("Build {}", self); spawn( - "yarn run build".into(), + format!("{} run build", node_cmd::YARN), Some(path), caption, iter::empty(), diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index 9989ee55a..c4148d650 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -1,3 +1,5 @@ +use crate::node_cmd; + #[derive(Debug, Clone)] pub enum TargetKind { /// TypeScript @@ -9,7 +11,11 @@ pub enum TargetKind { impl TargetKind { pub fn build_cmd(&self, prod: bool) -> String { match self { - TargetKind::Ts => format!("yarn run {}", if prod { "prod" } else { "build" }), + TargetKind::Ts => format!( + "{} run {}", + node_cmd::YARN, + if prod { "prod" } else { "build" } + ), TargetKind::Rs => format!( "cargo build --color always{}", if prod { " --release" } else { "" } @@ -19,7 +25,8 @@ impl TargetKind { pub fn install_cmd(&self, prod: bool) -> Option { match self { TargetKind::Ts => Some(format!( - "yarn install{}", + "{} install{}", + node_cmd::YARN, if prod { " --production" } else { "" } )), TargetKind::Rs => None, diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index 798f55f0f..e9559d0e3 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -1,4 +1,4 @@ -use crate::{spawner::SpawnOptions, target::Target}; +use crate::{node_cmd, spawner::SpawnOptions, target::Target}; use super::TestCommand; @@ -17,7 +17,7 @@ pub fn get_test_cmds() -> Vec { None, ), TestCommand::new( - "npm run test".into(), + format!("{} run test", node_cmd::NPM), cwd.join("spec"), Some(SpawnOptions { suppress_msg: true, From 5c593a20db8817780b44abeada9c381249969db8 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 11:43:43 +0200 Subject: [PATCH 099/174] Bulid CLI: Define & Resolve Development Tools - Development tools are defined as enums which get resolved using the crate `which` to handle the edge cases in Windows --- cli/Cargo.lock | 130 ++++++++++++++++++++++++++++ cli/Cargo.toml | 1 + cli/src/app_runner.rs | 6 +- cli/src/dev_environment.rs | 119 ++++++++------------------ cli/src/dev_tools.rs | 155 ++++++++++++++++++++++++++++++++++ cli/src/main.rs | 18 ++-- cli/src/node_cmd.rs | 13 --- cli/src/target/cli.rs | 8 +- cli/src/target/core.rs | 8 +- cli/src/target/mod.rs | 36 ++++---- cli/src/target/target_kind.rs | 44 ++++++---- cli/src/target/updater.rs | 10 +++ cli/src/target/wasm.rs | 23 +++-- 13 files changed, 421 insertions(+), 150 deletions(-) create mode 100644 cli/src/dev_tools.rs delete mode 100644 cli/src/node_cmd.rs create mode 100644 cli/src/target/updater.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 5796926f7..76d7fc943 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -170,6 +170,7 @@ dependencies = [ "indicatif", "tempdir", "tokio", + "which", ] [[package]] @@ -303,6 +304,16 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -483,6 +494,15 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "idna" version = "0.5.0" @@ -592,6 +612,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + [[package]] name = "lock_api" version = "0.4.11" @@ -861,6 +887,19 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1094,6 +1133,18 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "which" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8211e4f58a2b2805adfbefbc07bab82958fc91e3836339b1ab7ae32465dce0d7" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1143,6 +1194,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -1173,6 +1233,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -1185,6 +1261,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -1197,6 +1279,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -1209,6 +1297,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -1221,6 +1321,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -1233,6 +1339,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -1245,6 +1357,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -1256,3 +1374,15 @@ name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 7ae043d6f..9d24e2536 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -24,6 +24,7 @@ git2 = "0.18.2" indicatif = "0.17.7" tokio = { version = "1.36.0", features = ["full"] } dir_checksum = { path = "./dir_checksum" } +which = "6.0" [dev-dependencies] tempdir.workspace = true diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index fc45942ee..121de1733 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -2,12 +2,14 @@ use std::{io, process::ExitStatus}; use tokio::process::Command; -use crate::{node_cmd, target::Target}; +use crate::{dev_tools::DevTool, target::Target}; pub async fn run_app() -> io::Result { let electron_path = Target::App.cwd(); - Command::new(node_cmd::YARN) + let yarn_path = DevTool::Yarn.path().await; + + Command::new(yarn_path) .current_dir(electron_path) .args(["run", "electron"]) .status() diff --git a/cli/src/dev_environment.rs b/cli/src/dev_environment.rs index b247f6c66..03bf11362 100644 --- a/cli/src/dev_environment.rs +++ b/cli/src/dev_environment.rs @@ -1,92 +1,42 @@ -use std::{ - fmt::Write, - process::{Command, Stdio}, -}; +use std::{fmt::Write, process::Command}; use anyhow::bail; -use crate::node_cmd; +use crate::dev_tools::DevTool; -const ENV_CHECKS: [EnvCheck; 7] = [ - EnvCheck::new("NodeJS", "node", "-v", None), - EnvCheck::new("npm", node_cmd::NPM, "-v", None), - EnvCheck::new( - "yarn", - node_cmd::YARN, - "-v", - Some("npm install --global yarn"), - ), - EnvCheck::new("rust", "rustup", "-V", None), - EnvCheck::new("cargo", "cargo", "-V", None), - EnvCheck::new( - "wasm-pack", - "wasm-pack", - "-V", - Some("cargo install wasm-pack"), - ), - EnvCheck::new("nj-cli", "nj-cli", "-V", Some("cargo install nj-cli")), -]; - -struct EnvCheck { - app_name: &'static str, - command: &'static str, - arg: &'static str, - install_hint: Option<&'static str>, -} - -impl EnvCheck { - const fn new( - app_name: &'static str, - command: &'static str, - arg: &'static str, - install_hint: Option<&'static str>, - ) -> Self { - Self { - app_name, - command, - arg, - install_hint, - } - } -} - -/// Checks if all the needed dependencies for the build environment are available -pub fn check_env() -> anyhow::Result<()> { +/// Resolve the paths for all development tool returning an Error if any of them can't be resolved +pub async fn resolve_dev_tools() -> anyhow::Result<()> { let mut errors = None; - for check in ENV_CHECKS.iter() { - let success = match Command::new(check.command) - .arg(check.arg) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(status) => status.success(), - Err(_) => false, + for tool in DevTool::all() { + let Err(err) = tool.resolve().await else { + continue; }; - if !success { - let error_lines = - errors.get_or_insert(String::from("Following dependencies are missing:\n")); - writeln!( - error_lines, - "Required dependency '{}' is not installed.", - check.app_name - ) - .expect("Writing to string never fail"); - if let Some(install_hint) = check.install_hint { - writeln!( - error_lines, - "Consider installing it using the command '{install_hint}'" - ) - .expect("Writing to string never fail"); - } + let error_lines = + errors.get_or_insert(String::from("Following dependencies are missing:\n")); + + // Writing to string never fails + writeln!( + error_lines, + "Required dependency '{tool}' is not installed.", + ) + .unwrap(); + writeln!(error_lines, "Resolve Error Info:{err}",).unwrap(); + + if let Some(install_hint) = tool.install_hint() { writeln!( error_lines, - "------------------------------------------------------------------" + "Consider installing it using the command '{install_hint}'" ) - .expect("Writing to string never fail"); + .unwrap(); } + + writeln!( + error_lines, + "------------------------------------------------------------------" + ) + .expect("Writing to string never fail"); } match errors { @@ -97,11 +47,16 @@ pub fn check_env() -> anyhow::Result<()> { /// Prints the information of the needed tools for the development if available, otherwise prints /// error information to `stderr` -pub fn print_env_info() { - for check in ENV_CHECKS.iter() { - println!("{} Info:", check.app_name); - if let Err(err) = Command::new(check.command).arg(check.arg).status() { - eprintln!("Error while retrieving dependency's information: {err}"); +pub async fn print_env_info() { + for tool in DevTool::all() { + println!("{tool} Info:"); + match tool.resolve().await { + Ok(cmd) => { + if let Err(err) = Command::new(cmd).arg(tool.version_args()).status() { + eprintln!("Error while retrieving dependency's information: {err}"); + } + } + Err(err) => eprintln!("Error while resolving tool '{tool}': {err}"), } println!("------------------------------------------------------------------"); } diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs new file mode 100644 index 000000000..7350e11b1 --- /dev/null +++ b/cli/src/dev_tools.rs @@ -0,0 +1,155 @@ +use anyhow::{anyhow, Result}; +use std::{fmt::Display, path::PathBuf}; + +use tokio::sync::OnceCell; +use which::{which_all_global, which_global}; + +#[derive(Debug, Clone, Copy)] +/// Represents the development tools which used to build & test the app +pub enum DevTool { + Node, + Npm, + Yarn, + RustUp, + Cargo, + WasmPack, + NjCli, +} + +impl Display for DevTool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DevTool::Node => write!(f, "NodeJS"), + DevTool::Npm => write!(f, "npm"), + DevTool::Yarn => write!(f, "yarn"), + DevTool::RustUp => write!(f, "Rust"), + DevTool::Cargo => write!(f, "cargo"), + DevTool::WasmPack => write!(f, "wasm-pack"), + DevTool::NjCli => write!(f, "nj-cl"), + } + } +} + +impl DevTool { + pub fn all() -> &'static [DevTool] { + if cfg!(debug_assertions) { + // This check to remember to add the newly added enums to this function + match DevTool::Node { + DevTool::Node => (), + DevTool::Npm => (), + DevTool::Yarn => (), + DevTool::RustUp => (), + DevTool::Cargo => (), + DevTool::WasmPack => (), + DevTool::NjCli => (), + }; + } + + [ + DevTool::Node, + DevTool::Npm, + DevTool::Yarn, + DevTool::RustUp, + DevTool::Cargo, + DevTool::WasmPack, + DevTool::NjCli, + ] + .as_slice() + } + + pub fn install_hint(&self) -> Option<&'static str> { + match self { + DevTool::Node | DevTool::Npm | DevTool::RustUp | DevTool::Cargo => None, + DevTool::Yarn => Some("npm install --global yarn"), + DevTool::WasmPack => Some("cargo install wasm-pack"), + DevTool::NjCli => Some("cargo install nj-cli"), + } + } + + pub fn version_args(&self) -> &'static str { + match self { + DevTool::Node | DevTool::Npm | DevTool::Yarn => "-v", + DevTool::RustUp | DevTool::Cargo | DevTool::WasmPack | DevTool::NjCli => "-V", + } + } + + pub async fn resolve(&self) -> &'static Result { + match self { + DevTool::Node => resolve_node().await, + DevTool::Npm => resolve_npm().await, + DevTool::Yarn => resolve_yarn().await, + DevTool::RustUp => resolve_rustup().await, + DevTool::Cargo => resolve_cargo().await, + DevTool::WasmPack => resolve_wasm_pack().await, + DevTool::NjCli => resolve_nj_cli().await, + } + } + + pub async fn path(&self) -> &'static PathBuf { + self.resolve() + .await + .as_ref() + .expect("Cmd has already been resolved") + } +} + +async fn resolve_node() -> &'static Result { + static NODE: OnceCell> = OnceCell::const_new(); + + NODE.get_or_init(|| async { find_cmd("node") }).await +} + +fn find_cmd(cmd: &str) -> Result { + which_global(cmd).map_err(|err| anyhow!("Command `{cmd}` couldn't be resolved. Err: {err}")) +} + +async fn resolve_npm() -> &'static Result { + static NPM: OnceCell> = OnceCell::const_new(); + + NPM.get_or_init(|| async { find_cmd("npm") }).await +} + +async fn resolve_yarn() -> &'static Result { + static YARN: OnceCell> = OnceCell::const_new(); + + YARN.get_or_init(|| async { find_cmd("yarn") }).await +} + +async fn resolve_rustup() -> &'static Result { + static RUSTUP: OnceCell> = OnceCell::const_new(); + + RUSTUP.get_or_init(|| async { find_cmd("rustup") }).await +} + +async fn resolve_cargo() -> &'static Result { + static CARGO: OnceCell> = OnceCell::const_new(); + + if cfg!(windows) { + // Rust adds its toolchain to PATH in windows which must be filtered out + CARGO + .get_or_init(|| async { + let mut paths = which_all_global("cargo")?; + + paths + .find(|p| p.components().any(|c| c.as_os_str() == ".cargo")) + .ok_or_else(|| anyhow!("The command 'cargo' can't be found")) + }) + .await + } else { + CARGO.get_or_init(|| async { find_cmd("cargo") }).await + } +} + +async fn resolve_wasm_pack() -> &'static Result { + static WASM_PACK: OnceCell> = OnceCell::const_new(); + + WASM_PACK + .get_or_init(|| async { find_cmd("wasm-pack") }) + .await +} + +async fn resolve_nj_cli() -> &'static Result { + static NJ_CLI: OnceCell> = OnceCell::const_new(); + + NJ_CLI.get_or_init(|| async { find_cmd("nj-cli") }).await +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 645763cba..8da970eaa 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,10 +3,10 @@ mod build_state; mod checksum_records; mod cli_args; mod dev_environment; +mod dev_tools; mod fstools; mod job_type; mod location; -mod node_cmd; mod print_dot; mod spawner; mod target; @@ -16,7 +16,7 @@ use anyhow::{bail, Error}; use checksum_records::ChecksumRecords; use clap::Parser; use cli_args::{CargoCli, Command}; -use dev_environment::{check_env, print_env_info}; +use dev_environment::{print_env_info, resolve_dev_tools}; use futures::future::join_all; use job_type::JobType; use location::init_location; @@ -50,12 +50,12 @@ async fn main() -> Result<(), Error> { let (job_type, results) = match command { Command::Environment(sub_command) => match sub_command { EnvironmentCommand::Check => { - check_env()?; + resolve_dev_tools().await?; println!("All needed tools for development are installed"); return Ok(()); } EnvironmentCommand::Print => { - print_env_info(); + print_env_info().await; return Ok(()); } }, @@ -64,7 +64,7 @@ async fn main() -> Result<(), Error> { return Ok(()); } Command::Lint { target, report } => { - check_env()?; + resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -81,7 +81,7 @@ async fn main() -> Result<(), Error> { production, report, } => { - check_env()?; + resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -98,7 +98,7 @@ async fn main() -> Result<(), Error> { production, report, } => { - check_env()?; + resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -115,7 +115,7 @@ async fn main() -> Result<(), Error> { production, report, } => { - check_env()?; + resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( @@ -128,7 +128,7 @@ async fn main() -> Result<(), Error> { (JobType::Test { production }, results) } Command::Run { production } => { - check_env()?; + resolve_dev_tools().await?; report_opt = ReportOptions::None; let results = join_all( Target::all() diff --git a/cli/src/node_cmd.rs b/cli/src/node_cmd.rs deleted file mode 100644 index d715480de..000000000 --- a/cli/src/node_cmd.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[cfg(windows)] -mod commands { - pub const NPM: &str = "npm.cmd"; - pub const YARN: &str = "yarn.cmd"; -} - -#[cfg(not(windows))] -mod commands { - pub const NPM: &str = "npm"; - pub const YARN: &str = "yarn"; -} - -pub use commands::*; diff --git a/cli/src/target/cli.rs b/cli/src/target/cli.rs index 33fd87a96..401b894c0 100644 --- a/cli/src/target/cli.rs +++ b/cli/src/target/cli.rs @@ -1,10 +1,12 @@ -use crate::target::Target; +use crate::{dev_tools::DevTool, target::Target}; use super::TestCommand; -pub fn gettest_cmds(production: bool) -> Vec { +pub async fn get_test_cmds(production: bool) -> Vec { + let cargo_path = DevTool::Cargo.path().await; let cmd = format!( - "cargo +stable test{} --color always", + "{} +stable test{} --color always", + cargo_path.to_string_lossy(), if production { " -r" } else { "" } ); diff --git a/cli/src/target/core.rs b/cli/src/target/core.rs index 8633dc4f2..31f7f165f 100644 --- a/cli/src/target/core.rs +++ b/cli/src/target/core.rs @@ -1,10 +1,12 @@ -use crate::target::Target; +use crate::{dev_tools::DevTool, target::Target}; use super::TestCommand; -pub fn get_test_cmds(production: bool) -> Vec { +pub async fn get_test_cmds(production: bool) -> Vec { + let cargo_path = DevTool::Cargo.path().await; let cmd = format!( - "cargo +stable test{} --color always", + "{} +stable test{} --color always", + cargo_path.to_string_lossy(), if production { " -r" } else { "" } ); diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index d643c2821..4784b99f3 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -10,10 +10,10 @@ use tokio::sync::oneshot; use crate::{ build_state::{BuildState, BuildStatesTracker}, checksum_records::ChecksumRecords, + dev_tools::DevTool, fstools, job_type::JobType, location::get_root, - node_cmd, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; @@ -26,6 +26,7 @@ mod client; mod core; mod shared; mod target_kind; +mod updater; mod wasm; mod wrapper; @@ -193,12 +194,12 @@ impl Target { } } - pub fn build_cmd(&self, prod: bool) -> String { + pub async fn build_cmd(&self, prod: bool) -> String { match self { Target::Binding => binding::get_build_cmd(prod), - Target::Wasm => wasm::get_build_cmd(prod), - Target::Updater => "cargo +stable build --color always --release".into(), - rest_targets => rest_targets.kind().build_cmd(prod), + Target::Wasm => wasm::get_build_cmd(prod).await, + Target::Updater => updater::get_build_cmd().await, + rest_targets => rest_targets.kind().build_cmd(prod).await, } } @@ -223,18 +224,18 @@ impl Target { } } - fn test_cmds(&self, production: bool) -> Option> { + async fn test_cmds(&self, production: bool) -> Option> { match self { - Target::Core => Some(core::get_test_cmds(production)), - Target::Cli => Some(cli::gettest_cmds(production)), - Target::Wasm => Some(wasm::get_test_cmds()), + Target::Core => Some(core::get_test_cmds(production).await), + Target::Cli => Some(cli::get_test_cmds(production).await), + Target::Wasm => Some(wasm::get_test_cmds().await), _ => None, } } /// run test using the general routine with `test_cmds()` method async fn run_test_general(&self, production: bool) -> Result, anyhow::Error> { - let Some(test_cmds) = self.test_cmds(production) else { + let Some(test_cmds) = self.test_cmds(production).await else { return Ok(Vec::new()); }; @@ -289,8 +290,9 @@ impl Target { async fn ts_lint(&self) -> Result { let path = get_root().join(self.cwd()); let caption = format!("TS Lint {}", self); + let yarn_cmd = DevTool::Yarn.path().await.to_string_lossy(); let status = spawn( - format!("{} run lint", node_cmd::YARN), + format!("{} run lint", yarn_cmd), Some(path.clone()), caption, iter::empty(), @@ -303,7 +305,7 @@ impl Target { let caption = format!("Build {}", self); spawn( - format!("{} run build", node_cmd::YARN), + format!("{} run build", yarn_cmd), Some(path), caption, iter::empty(), @@ -315,9 +317,13 @@ impl Target { async fn clippy(&self) -> Result { let path = get_root().join(self.cwd()); + let cargo_path = DevTool::Cargo.path().await; let caption = format!("Clippy {}", self); spawn( - "cargo clippy --color always --all --all-features -- -D warnings".into(), + format!( + "{} clippy --color always --all --all-features -- -D warnings", + cargo_path.to_string_lossy() + ), Some(path), caption, iter::empty(), @@ -450,7 +456,7 @@ impl Target { } } let path = get_root().join(self.cwd()); - let cmd = self.build_cmd(prod); + let cmd = self.build_cmd(prod).await; let caption = format!("Build {}", self); let mut skip_task = false; @@ -517,7 +523,7 @@ impl Target { /// run install using the general routine for the given target async fn install_general(target: &Target, prod: bool) -> Result { - let cmd = target.kind().install_cmd(prod); + let cmd = target.kind().install_cmd(prod).await; if let Some(cmd) = cmd { let caption = format!("Install {}", target); spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index c4148d650..cb061c825 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -1,4 +1,4 @@ -use crate::node_cmd; +use crate::dev_tools::DevTool; #[derive(Debug, Clone)] pub enum TargetKind { @@ -9,26 +9,36 @@ pub enum TargetKind { } impl TargetKind { - pub fn build_cmd(&self, prod: bool) -> String { + pub async fn build_cmd(&self, prod: bool) -> String { match self { - TargetKind::Ts => format!( - "{} run {}", - node_cmd::YARN, - if prod { "prod" } else { "build" } - ), - TargetKind::Rs => format!( - "cargo build --color always{}", - if prod { " --release" } else { "" } - ), + TargetKind::Ts => { + let yarn_path = DevTool::Yarn.path().await; + format!( + "{} run {}", + yarn_path.to_string_lossy(), + if prod { "prod" } else { "build" } + ) + } + TargetKind::Rs => { + let cargo_path = DevTool::Cargo.path().await; + format!( + "{} build --color always{}", + cargo_path.to_string_lossy(), + if prod { " --release" } else { "" } + ) + } } } - pub fn install_cmd(&self, prod: bool) -> Option { + pub async fn install_cmd(&self, prod: bool) -> Option { match self { - TargetKind::Ts => Some(format!( - "{} install{}", - node_cmd::YARN, - if prod { " --production" } else { "" } - )), + TargetKind::Ts => { + let yarn_path = DevTool::Yarn.path().await; + Some(format!( + "{} install{}", + yarn_path.to_string_lossy(), + if prod { " --production" } else { "" } + )) + } TargetKind::Rs => None, } } diff --git a/cli/src/target/updater.rs b/cli/src/target/updater.rs new file mode 100644 index 000000000..334606d3d --- /dev/null +++ b/cli/src/target/updater.rs @@ -0,0 +1,10 @@ +use crate::dev_tools::DevTool; + +pub async fn get_build_cmd() -> String { + let cargo_path = DevTool::Cargo.path().await; + + format!( + "{} +stable build --color always --release", + cargo_path.to_string_lossy() + ) +} diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index e9559d0e3..95691566f 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -1,23 +1,34 @@ -use crate::{node_cmd, spawner::SpawnOptions, target::Target}; +use crate::{dev_tools::DevTool, spawner::SpawnOptions, target::Target}; use super::TestCommand; -pub fn get_build_cmd(prod: bool) -> String { +pub async fn get_build_cmd(prod: bool) -> String { + let wasm_pack_path = DevTool::WasmPack.path().await; let env = if prod { "--release" } else { "--dev" }; - format!("wasm-pack build {env} --target bundler --color always") + format!( + "{} build {env} --target bundler --color always", + wasm_pack_path.to_string_lossy() + ) } -pub fn get_test_cmds() -> Vec { +pub async fn get_test_cmds() -> Vec { let cwd = Target::Wasm.cwd(); + + let npm_path = DevTool::Npm.path().await; + let wasm_pack_path = DevTool::WasmPack.path().await; + vec![ TestCommand::new( - "wasm-pack test --node --color always".into(), + format!( + "{} test --node --color always", + wasm_pack_path.to_string_lossy() + ), cwd.clone(), None, ), TestCommand::new( - format!("{} run test", node_cmd::NPM), + format!("{} run test", npm_path.to_string_lossy()), cwd.join("spec"), Some(SpawnOptions { suppress_msg: true, From 21703edcb54b2b26f9569f529183e4e190b8c767 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 12:57:49 +0200 Subject: [PATCH 100/174] Build CLI: Fix script file name on Windows Script file to build electron can get the extension '*.cmd' on Windows --- cli/src/target/binding.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index e0081d776..9f0fefa9a 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -10,7 +10,15 @@ pub fn get_build_cmd(prod: bool) -> String { let mut path = Target::Wrapper.cwd(); path.push("node_modules"); path.push(".bin"); - path.push("electron-build-env"); + + if cfg!(windows) { + // The script files can get the extension '*.cmd' on Windows + let electron_build_env_path = + which::which_in("electron-build-env", Some(&path), &path).unwrap(); + path = electron_build_env_path; + } else { + path.push("electron-build-env"); + } format!( "{} nj-cli build{}", From 13b65e74ca81400534464db9e9afccc73b10a09a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 13:27:52 +0200 Subject: [PATCH 101/174] Build CLI: Fix run app command on Windows To run electron on windows with yarn we must use the argument `electron-win` instead of `electron` --- cli/src/app_runner.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index 121de1733..73cb39cdb 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -9,9 +9,15 @@ pub async fn run_app() -> io::Result { let yarn_path = DevTool::Yarn.path().await; + let electron_arg = if cfg!(windows) { + "electron-win" + } else { + "electron" + }; + Command::new(yarn_path) .current_dir(electron_path) - .args(["run", "electron"]) + .args(["run", electron_arg]) .status() .await } From 4eee7a428f12ef9dad6b7ed65869ffeefd005d57 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 15:12:27 +0200 Subject: [PATCH 102/174] Build CLI: Add Documentation & Comments --- cli/src/checksum_records.rs | 7 +++++++ cli/src/cli_args.rs | 8 ++++---- cli/src/dev_tools.rs | 4 ++++ cli/src/fstools.rs | 6 +++++- cli/src/location.rs | 1 + cli/src/spawner.rs | 1 + cli/src/target/mod.rs | 13 +++++++++++++ cli/src/target/target_kind.rs | 2 ++ cli/src/tracker.rs | 14 ++++++++++++++ 9 files changed, 51 insertions(+), 5 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index b709f980a..a9d892edf 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -29,6 +29,7 @@ struct ChecksumItems { impl ChecksumRecords { pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { + // calculate should be involved when build is called at some point of the job let calculate_involved = match &job_type { JobType::Lint => return Ok(()), JobType::Build { production: _ } @@ -59,6 +60,7 @@ impl ChecksumRecords { .map_err(|err| anyhow!("{err}")) } + /// Loads the persisted records from checksums file if exist fn load(job_type: JobType) -> anyhow::Result { let file_path = Self::get_file_path(job_type.is_production().is_some_and(|prod| prod)); @@ -79,6 +81,7 @@ impl ChecksumRecords { }) } + /// Gets the path of the file where the checksums are saved fn get_file_path(production: bool) -> PathBuf { let root = get_root(); if production { @@ -101,11 +104,14 @@ impl ChecksumRecords { Ok(hashes) } + /// Marks the job is involved in the record tracker pub fn register_job(&self, target: Target) { let mut items = self.items.lock().unwrap(); items.involved_targets.insert(target); } + /// Calculate the current checksum for the given target and compare it to the saved one. + /// This method panics if the provided target isn't registered pub fn check_changed(&self, target: Target) -> anyhow::Result { let items = self.items.lock().unwrap(); assert!(items.involved_targets.contains(&target)); @@ -126,6 +132,7 @@ impl ChecksumRecords { }) } + /// Remove the target from the checksum records pub fn remove_hash_if_exist(&self, target: Target) { let mut items = self.items.lock().unwrap(); items.involved_targets.insert(target); diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 81a93d810..f5409608d 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -30,7 +30,7 @@ pub enum Command { /// Prints an overview of targets dependencies in print-dot format for `Graphviz` #[clap(visible_alias = "dot")] PrintDot, - /// Runs linting & clippy + /// Runs linting & clippy for all or the specified targets Lint { /// Target to lint, by default whole application will be linted #[arg(index = 1)] @@ -39,7 +39,7 @@ pub enum Command { #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, - /// Build + /// Build all or the specified targets Build { /// Target to build, by default whole application will be built #[arg(index = 1)] @@ -52,7 +52,7 @@ pub enum Command { #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, - /// Clean + /// Clean all or the specified targets Clean { /// Target to clean, by default whole application will be cleaned #[arg(index = 1)] @@ -65,7 +65,7 @@ pub enum Command { #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, - /// Run tests + /// Run tests for all or the specified targets Test { /// Target to test, by default whole application will be tested #[arg(index = 1)] diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index 7350e11b1..57830e29b 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -57,6 +57,7 @@ impl DevTool { .as_slice() } + /// Provide the suggested way to install the tool pub fn install_hint(&self) -> Option<&'static str> { match self { DevTool::Node | DevTool::Npm | DevTool::RustUp | DevTool::Cargo => None, @@ -66,6 +67,7 @@ impl DevTool { } } + /// Provide the command line argument to get the version of the installed tool pub fn version_args(&self) -> &'static str { match self { DevTool::Node | DevTool::Npm | DevTool::Yarn => "-v", @@ -73,6 +75,7 @@ impl DevTool { } } + /// Resolve the path of the tool if exists. Returning an Error when not possible pub async fn resolve(&self) -> &'static Result { match self { DevTool::Node => resolve_node().await, @@ -85,6 +88,7 @@ impl DevTool { } } + /// Get the path of the resolved tool. Panics if the tool can't be resolved pub async fn path(&self) -> &'static PathBuf { self.resolve() .await diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index aecb45591..1eb664ce3 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -8,6 +8,7 @@ use std::{fs, path::PathBuf}; use crate::tracker::get_tracker; +/// Spawn a job to copy a file, adding the info the report logs pub async fn cp_file( src: PathBuf, dest: PathBuf, @@ -35,6 +36,7 @@ pub async fn cp_file( Ok(()) } +/// Spawn a job to copy a directory, adding the info the report logs pub async fn cp_folder( src: PathBuf, dest: PathBuf, @@ -78,7 +80,8 @@ pub async fn cp_folder( Ok(()) } -/// Copy a collection of files and folders recursively. +/// Spawn a job to Copy a collection of files and folders recursively, adding copying info to the +/// log records pub async fn cp_many( items: Vec, dest: PathBuf, @@ -125,6 +128,7 @@ pub async fn cp_many( Ok(()) } +/// Spawn a job to remove a directory recursively, adding the info the report logs pub async fn rm_folder(path: &PathBuf) -> Result<(), Error> { if !path.exists() { return Ok(()); diff --git a/cli/src/location.rs b/cli/src/location.rs index 402415c46..60bad9900 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -52,6 +52,7 @@ pub fn init_location() -> Result<(), Error> { Ok(()) } +/// Returns the path relative to the repository root pub fn to_relative_path(path: &PathBuf) -> &Path { let root = get_root(); path.strip_prefix(root).unwrap_or(path) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index df293ebd1..261b838cd 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -69,6 +69,7 @@ pub(crate) struct SpawnOptions { pub has_skip_info: bool, } +/// Spawns and runs a job asynchronously, updating the bar when job infos are available pub async fn spawn( command: String, cwd: Option, diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 4784b99f3..23b111b2b 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -154,6 +154,7 @@ impl Target { ] } + /// Provides the absolute path to the target code pub fn cwd(&self) -> PathBuf { let root = get_root(); let sub_parts = match self { @@ -173,6 +174,7 @@ impl Target { root.join(sub_path) } + /// Provide the kind of the target between Rust or Type-Script pub fn kind(&self) -> TargetKind { match self { Target::Binding | Target::Core | Target::Cli | Target::Wasm | Target::Updater => { @@ -182,6 +184,7 @@ impl Target { } } + /// Provides the target which this target depend on pub fn deps(&self) -> Vec { match self { Target::Core | Target::Cli | Target::Shared | Target::Wasm | Target::Updater => { @@ -194,6 +197,7 @@ impl Target { } } + /// Provide the command that should be used in to build the target pub async fn build_cmd(&self, prod: bool) -> String { match self { Target::Binding => binding::get_build_cmd(prod), @@ -203,6 +207,7 @@ impl Target { } } + /// Installs the needed module to perform the development task pub async fn install(&self, prod: bool) -> Result { match self { // We must install ts binding tools before running rs bindings, therefore we call @@ -217,6 +222,7 @@ impl Target { } } + /// Run tests for the giving the target pub async fn test(&self, production: bool) -> Result, anyhow::Error> { match self { Target::Wrapper => wrapper::run_test().await, @@ -224,6 +230,7 @@ impl Target { } } + /// Provides the test commands for the given target if available async fn test_cmds(&self, production: bool) -> Option> { match self { Target::Core => Some(core::get_test_cmds(production).await), @@ -269,6 +276,7 @@ impl Target { Ok(results) } + /// Perform Linting Checks on the giving target pub async fn check(&self) -> Result, anyhow::Error> { let mut results = Vec::new(); match self.kind() { @@ -287,6 +295,8 @@ impl Target { Ok(results) } + /// Perform Linting the Building the giving target since linting Type-Script doesn't check for + /// compiling errors async fn ts_lint(&self) -> Result { let path = get_root().join(self.cwd()); let caption = format!("TS Lint {}", self); @@ -314,6 +324,7 @@ impl Target { .await } + /// Runs Clippy for the given rust target async fn clippy(&self) -> Result { let path = get_root().join(self.cwd()); @@ -332,6 +343,7 @@ impl Target { .await } + /// Clean the given target, removing it from the checksum tracker as well. pub async fn reset(&self, production: bool) -> anyhow::Result> { let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; checksum.remove_hash_if_exist(*self); @@ -510,6 +522,7 @@ impl Target { .boxed() } + /// Perform any needed copy operation after the build is done async fn after_build(&self, prod: bool) -> Result, anyhow::Error> { match self { Target::Binding => binding::copy_index_node().await, diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index cb061c825..cfc6b2e71 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -9,6 +9,7 @@ pub enum TargetKind { } impl TargetKind { + /// Provide the general build command for each target type pub async fn build_cmd(&self, prod: bool) -> String { match self { TargetKind::Ts => { @@ -29,6 +30,7 @@ impl TargetKind { } } } + /// Provide the general install command for each target type pub async fn install_cmd(&self, prod: bool) -> Option { match self { TargetKind::Ts => { diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 2036a0756..8b4fa113a 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -33,13 +33,20 @@ impl std::fmt::Display for OperationResult { } #[derive(Debug)] +/// Represents tasks information that can be sent to and from the tracker pub enum Tick { + /// Start a job giving the job name and the sender to return the job number. Started(String, Option, oneshot::Sender), + /// Update the job with the given id providing an optional progress value. Progress(usize, Option), + /// Send a message to the job with the giving id Message(usize, String), + /// Sets the job with the given id as finished providing the job result and a message Finished(usize, OperationResult, String), #[allow(dead_code)] + /// Prints the given text outside the progress bar Print(String), + /// Close all the jobs and shutdown the progress bars Shutdown(oneshot::Sender<()>), /// Suspends the progress bars and execute the giving blocking command SuspendAndRun(Command, oneshot::Sender>), @@ -228,6 +235,7 @@ impl Tracker { count } + /// Start a job giving the job name and the sender to return the job number. pub async fn start(&self, job: &str, max: Option) -> Result { let (tx_response, rx_response) = oneshot::channel(); self.tx @@ -236,18 +244,21 @@ impl Tracker { rx_response.await.context("Fail to receive tick") } + /// Update the job with the given id providing an optional progress value. pub async fn progress(&self, sequence: usize, pos: Option) { if let Err(e) = self.tx.send(Tick::Progress(sequence, pos)) { eprintln!("Fail to communicate with tracker: {e}"); } } + /// Send a message to the job with the giving id pub async fn msg(&self, sequence: usize, log: &str) { if let Err(e) = self.tx.send(Tick::Message(sequence, log.to_string())) { eprintln!("Fail to communicate with tracker: {e}"); } } + /// Sets the job with the given id as finished providing successful result and a message pub async fn success(&self, sequence: usize, msg: &str) { if let Err(e) = self.tx.send(Tick::Finished( sequence, @@ -258,6 +269,7 @@ impl Tracker { } } + /// Sets the job with the given id as finished providing failed result and a message pub async fn fail(&self, sequence: usize, msg: &str) { if let Err(e) = self.tx.send(Tick::Finished( sequence, @@ -268,6 +280,7 @@ impl Tracker { } } + /// Close all the jobs and shutdown the progress bars pub async fn shutdown(&self) -> Result<(), Error> { let (tx_response, rx_response) = oneshot::channel(); self.tx @@ -276,6 +289,7 @@ impl Tracker { rx_response.await.context("Fail to receive tick") } + /// Prints the given text outside the progress bar pub async fn _print(&self, msg: String) { if let Err(e) = self .tx From c75532cf3881e03096fa80b412fc4318feb0ca0c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 15:19:38 +0200 Subject: [PATCH 103/174] Build CLI: Remove unused argument in tracker - The optional value `max` for the tracker was never assigned in the code and can be removed --- cli/src/fstools.rs | 8 ++++---- cli/src/spawner.rs | 27 +++++++++++++++------------ cli/src/tracker.rs | 10 +++++----- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 1eb664ce3..c9e8ddcb1 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -15,7 +15,7 @@ pub async fn cp_file( report_logs: &mut Vec, ) -> Result<(), Error> { let tracker = get_tracker().await; - let sequence = tracker.start("copy file", None).await?; + let sequence = tracker.start("copy file").await?; let msg = format!("copying file: '{}' to '{}'", src.display(), dest.display()); report_logs.push(msg); @@ -43,7 +43,7 @@ pub async fn cp_folder( report_logs: &mut Vec, ) -> Result<(), Error> { let tracker = get_tracker().await; - let sequence = tracker.start("copy folder", None).await?; + let sequence = tracker.start("copy folder").await?; let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); @@ -89,7 +89,7 @@ pub async fn cp_many( logs: &mut Vec, ) -> Result<(), Error> { let tracker = get_tracker().await; - let sequence = tracker.start("copy file and folders", None).await?; + let sequence = tracker.start("copy file and folders").await?; let options = CopyOptions::new(); let (tx, rx) = mpsc::channel(); let path_display = format!("from '{}' to '{}'", general_source, dest.display()); @@ -134,7 +134,7 @@ pub async fn rm_folder(path: &PathBuf) -> Result<(), Error> { return Ok(()); } let tracker = get_tracker().await; - let sequence = tracker.start("remove folder", None).await?; + let sequence = tracker.start("remove folder").await?; fs::remove_dir_all(path)?; tracker .success(sequence, &format!("removed: {}", path.display(),)) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 261b838cd..9b68672c8 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -86,10 +86,11 @@ pub async fn spawn( let tracker = get_tracker().await; let sequence = tracker - .start( - &format!("{}: {}", to_relative_path(&cwd).display(), caption), - None, - ) + .start(&format!( + "{}: {}", + to_relative_path(&cwd).display(), + caption + )) .await?; let command_result = Command::new(cmd) @@ -215,10 +216,11 @@ pub async fn spawn_blocking( let tracker = get_tracker().await; let sequence = tracker - .start( - &format!("{}: {}", to_relative_path(&cwd).display(), caption), - None, - ) + .start(&format!( + "{}: {}", + to_relative_path(&cwd).display(), + caption + )) .await?; let status = match tracker.suspend_and_run(child).await { @@ -254,10 +256,11 @@ pub async fn spawn_skip( let tracker = get_tracker().await; let sequence = tracker - .start( - &format!("{}: {}", to_relative_path(&cwd).display(), caption), - None, - ) + .start(&format!( + "{}: {}", + to_relative_path(&cwd).display(), + caption + )) .await?; tracker.success(sequence, "skipped").await; diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 8b4fa113a..0ddbbdf4a 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -36,7 +36,7 @@ impl std::fmt::Display for OperationResult { /// Represents tasks information that can be sent to and from the tracker pub enum Tick { /// Start a job giving the job name and the sender to return the job number. - Started(String, Option, oneshot::Sender), + Started(String, oneshot::Sender), /// Update the job with the given id providing an optional progress value. Progress(usize, Option), /// Send a message to the job with the giving id @@ -103,9 +103,9 @@ impl Tracker { let start_time = Instant::now(); while let Some(tick) = rx.recv().await { match tick { - Tick::Started(job, len, tx_response) => { + Tick::Started(job, tx_response) => { sequence += 1; - let bar = mp.add(ProgressBar::new(len.unwrap_or(max))); + let bar = mp.add(ProgressBar::new(max)); bar.set_style(spinner_style.clone()); let job_bar = JobBarState::start_job(job, bar); bars.insert(sequence, job_bar); @@ -236,10 +236,10 @@ impl Tracker { } /// Start a job giving the job name and the sender to return the job number. - pub async fn start(&self, job: &str, max: Option) -> Result { + pub async fn start(&self, job: &str) -> Result { let (tx_response, rx_response) = oneshot::channel(); self.tx - .send(Tick::Started(job.to_string(), max, tx_response)) + .send(Tick::Started(job.to_string(), tx_response)) .context("Fail to send tick")?; rx_response.await.context("Fail to receive tick") } From c040de8f51bbca8bf5acba3f78c16aee08f7b473 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Apr 2024 16:53:21 +0200 Subject: [PATCH 104/174] Build CLI Refactor: Return slice instead of vector - All Targets can be provided as a static slice instead of allocating a new vector each time it's called --- cli/src/main.rs | 2 +- cli/src/target/mod.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 8da970eaa..db532a021 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -208,7 +208,7 @@ fn get_targets_or_default(targets: Option>) -> Vec { list.dedup(); list } else { - Target::all() + Target::all().to_vec() } } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 23b111b2b..33e76160a 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -125,7 +125,8 @@ impl FromStr for Target { } impl Target { - pub fn all() -> Vec { + /// Return all the available targets + pub fn all() -> &'static [Target] { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function match Target::App { @@ -141,7 +142,7 @@ impl Target { }; } - vec![ + [ Target::Binding, Target::Cli, Target::App, @@ -152,6 +153,7 @@ impl Target { Target::Wasm, Target::Updater, ] + .as_slice() } /// Provides the absolute path to the target code From ad419267b792bc66956a19819c3cd1723c24fc11 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 1 May 2024 08:04:26 +0200 Subject: [PATCH 105/174] Build CLI: Add needed env vars to commands only Only provided environment variables should be added/modified since the rest will be loaded in the command anyway --- cli/src/spawner.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 9b68672c8..3a03ee35f 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -7,7 +7,6 @@ use anyhow::{bail, Context}; use core::panic; use futures_lite::{future, FutureExt}; use std::{ - env, path::PathBuf, process::{ExitStatus, Stdio}, }; @@ -81,8 +80,8 @@ pub async fn spawn( let cwd = cwd.unwrap_or_else(|| get_root().clone()); let mut parts = command.split(' ').collect::>(); let cmd = parts.remove(0); - let mut env_vars: Vec<_> = env::vars().chain(environment_vars).collect(); - env_vars.push((String::from("TERM"), String::from("xterm-256color"))); + let mut combined_env_vars = vec![(String::from("TERM"), String::from("xterm-256color"))]; + combined_env_vars.extend(environment_vars); let tracker = get_tracker().await; let sequence = tracker @@ -96,7 +95,7 @@ pub async fn spawn( let command_result = Command::new(cmd) .current_dir(&cwd) .args(&parts) - .envs(env_vars) + .envs(combined_env_vars) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -205,13 +204,14 @@ pub async fn spawn_blocking( let cwd = cwd.unwrap_or_else(|| get_root().clone()); let mut parts = command.split(' ').collect::>(); let cmd = parts.remove(0); - let mut env_vars: Vec<_> = env::vars().chain(environment_vars).collect(); - env_vars.push((String::from("TERM"), String::from("xterm-256color"))); + + let mut combined_env_vars = vec![(String::from("TERM"), String::from("xterm-256color"))]; + combined_env_vars.extend(environment_vars); let mut child = std::process::Command::new(cmd); child.current_dir(&cwd); child.args(&parts); - child.envs(env_vars); + child.envs(combined_env_vars); let tracker = get_tracker().await; From d77af1632f2754773eab29564b13612ca56e9aea Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 2 May 2024 08:27:18 +0200 Subject: [PATCH 106/174] Build CLI: Add command to reset checksum records --- cli/src/checksum_records.rs | 15 +++++++++++++++ cli/src/cli_args.rs | 8 ++++++++ cli/src/main.rs | 13 +++++++++++++ 3 files changed, 36 insertions(+) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index a9d892edf..bc5e47e22 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -91,6 +91,21 @@ impl ChecksumRecords { } } + /// Removes the records file for the given environment + pub fn remove_records_file(production: bool) -> anyhow::Result<()> { + let file_path = Self::get_file_path(production); + if file_path.exists() { + std::fs::remove_file(&file_path).with_context(|| { + format!( + "Error while removing the file {} to reset checksum records", + file_path.display() + ) + })?; + } + + Ok(()) + } + fn parse_hashes(text: String) -> anyhow::Result> { let mut hashes = BTreeMap::new(); diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index f5409608d..e58b9c323 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -84,6 +84,14 @@ pub enum Command { #[arg(short, long, default_value_t = false)] production: bool, }, + /// Resets the checksums records what is used to check if there were any code changes for + /// each target + #[clap(visible_alias = "reset")] + ResetChecksum { + /// Reset release records + #[arg(short, long, default_value_t = false)] + production: bool, + }, } #[derive(Subcommand, Debug, Clone)] diff --git a/cli/src/main.rs b/cli/src/main.rs index db532a021..8988b4fc3 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -139,6 +139,19 @@ async fn main() -> Result<(), Error> { .await; (JobType::Run { production }, results) } + Command::ResetChecksum { production } => { + ChecksumRecords::remove_records_file(production)?; + println!( + "Checksum-Records for {} has been reset", + if production { + "production" + } else { + "development" + } + ); + + return Ok(()); + } }; // Shutdown and show results & report From 870494d07b9dd50daef23ef68001531fc9b7c6e0 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 2 May 2024 09:49:06 +0200 Subject: [PATCH 107/174] Build CLI: Provide shell completion command - Shell completion will write the completion for the given shell to stdout --- cli/Cargo.lock | 10 ++++++++++ cli/Cargo.toml | 1 + cli/src/cli_args.rs | 10 +++++++++- cli/src/main.rs | 6 ++++++ cli/src/shell_completion.rs | 14 ++++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 cli/src/shell_completion.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 76d7fc943..0be655565 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -161,6 +161,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "clap_complete", "console", "dir_checksum", "fs_extra", @@ -211,6 +212,15 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_complete" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79504325bf38b10165b02e89b4347300f855f273c4cb30c4a3209e6583275e" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" version = "4.4.7" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 9d24e2536..c45618efe 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,6 +25,7 @@ indicatif = "0.17.7" tokio = { version = "1.36.0", features = ["full"] } dir_checksum = { path = "./dir_checksum" } which = "6.0" +clap_complete = "4.5" [dev-dependencies] tempdir.workspace = true diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index e58b9c323..70a287159 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -85,13 +85,21 @@ pub enum Command { production: bool, }, /// Resets the checksums records what is used to check if there were any code changes for - /// each target + /// each target. #[clap(visible_alias = "reset")] ResetChecksum { /// Reset release records #[arg(short, long, default_value_t = false)] production: bool, }, + /// Generate shell completion for the commands of this tool in the given shell, + /// printing them to stdout. + #[clap(visible_alias = "compl")] + ShellCompletion { + /// Shell to generate the completion for + #[arg(value_enum)] + shell: clap_complete::Shell, + }, } #[derive(Subcommand, Debug, Clone)] diff --git a/cli/src/main.rs b/cli/src/main.rs index 8988b4fc3..244b8aa49 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -8,6 +8,7 @@ mod fstools; mod job_type; mod location; mod print_dot; +mod shell_completion; mod spawner; mod target; mod tracker; @@ -150,6 +151,11 @@ async fn main() -> Result<(), Error> { } ); + return Ok(()); + } + Command::ShellCompletion { shell } => { + shell_completion::generate_completion(shell); + return Ok(()); } }; diff --git a/cli/src/shell_completion.rs b/cli/src/shell_completion.rs new file mode 100644 index 000000000..b20471d7b --- /dev/null +++ b/cli/src/shell_completion.rs @@ -0,0 +1,14 @@ +use std::io; + +use clap::CommandFactory; +use clap_complete::{generate, Shell}; + +use crate::cli_args::CargoCli; + +/// Generates shell complition for the given shell printing them to stdout +pub fn generate_completion(shell: Shell) { + let mut cmd = CargoCli::command(); + let bin_name = cmd.get_bin_name().unwrap().to_owned(); + + generate(shell, &mut cmd, bin_name, &mut io::stdout()); +} From a5919387bb379422e91938ff320863fca7871e62 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 2 May 2024 10:26:23 +0200 Subject: [PATCH 108/174] Build CLI: Add shell completion to README --- cli/README.md | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/cli/README.md b/cli/README.md index d30859700..509515e6c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -32,19 +32,21 @@ This CLI tool provides multiple sub-commands for different tasks, with each sub- ### General Commands Overview ```bash -Tool for chipmunk application development +CLI Tool for chipmunk application development Usage: cargo chipmunk Commands: - environment Provides commands for the needed tools for the development [aliases: env] - print-dot Prints an overview of targets dependencies in print-dot format for `Graphviz` [aliases: dot] - lint Runs linting & clippy - build Build - clean Clean - test Run tests - run Build and Run the application - help Print this message or the help of the given subcommand(s) + environment Provides commands for the needed tools for the development [aliases: env] + print-dot Prints an overview of targets dependencies in print-dot format for `Graphviz` [aliases: dot] + lint Runs linting & clippy for all or the specified targets + build Build all or the specified targets + clean Clean all or the specified targets + test Run tests for all or the specified targets + run Build and Run the application + reset-checksum Resets the checksums records what is used to check if there were any code changes for each target [aliases: reset] + shell-completion Generate shell completion for the commands of this tool in the given shell, printing them to stdout [aliases: compl] + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help @@ -82,5 +84,28 @@ Options: Print help (see a summary with '-h') ``` +## Shell Completion + +The Chipmunk CLI tool supports shell completion for various shells. You can generate shell completions and print them to `stdout` using the following command: + +```bash +cargo chipmunk shell-completion +``` +Replace with the name of your shell (e.g., bash, zsh, fish, powershell). + +To use shell completion, you can redirect the output of the completion command to a file and save the file to the appropriate shell completion directory. + +After installing the completion script, restart your shell session or source the completion file to enable shell completion for the Chipmunk CLI tool. + + +### Example: Bash Shell +To enable bash shell completion, run the following command to generate the completion script and save it to a file: + +```bash +cargo chipmunk shell-completion bash > chipmunk-completion.bash +``` +Next, copy the chipmunk-completion.bash file to your bash completion directory (typically ~/.bash_completion.d/ or /etc/bash_completion.d/). + + ## Contributing See our [contribution](contribution.md) guide for details From 1eb6be310d7d552d3a0f8677d34d9091c2e06386 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 6 May 2024 10:55:50 +0200 Subject: [PATCH 109/174] Build CLI: Call reinstall TS in prod before copy Reinstall dependencies in production should be called before copying the files in order to copy the only necessary files for production only --- cli/src/target/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 33e76160a..c9980e8c6 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -506,16 +506,22 @@ impl Target { } else { results.push(status); if !skip_task { - let res = self.after_build(prod).await?; - if let Some(result) = res { - results.push(result); - } + // Taken from a discussion on GitHub: + // To build an npm package you would need (in most cases) to be in dev-mode - install dev-dependencies + dependencies. + // But to prepare a package for production, you have to remove dev-dependencies. + // That's not an issue, if npm-package is published in npmjs; but we are coping packages manually in a right destination + // and before copy it, we have to reinstall it to get rid of dev-dependencies. if matches!(self.kind(), TargetKind::Ts) && prod { let clean_res = self.clean().await?; results.push(clean_res); let install_res = self.install(prod).await?; results.push(install_res); } + + let res = self.after_build(prod).await?; + if let Some(result) = res { + results.push(result); + } } Ok(results) From 426b0a9318938cf88333d1d789d1443c5b923306 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 28 May 2024 16:28:02 +0200 Subject: [PATCH 110/174] Build CLI: Replace Unwrap & implicit expect message - Extend except error where the error happens because of developer error only to state that this is a developer error. - unwrap is replace with results everywhere in the code base --- cli/src/checksum_records.rs | 34 +++++++++++++++++++++++++++------- cli/src/dev_environment.rs | 11 ++++------- cli/src/dev_tools.rs | 2 +- cli/src/location.rs | 4 ++-- cli/src/main.rs | 2 +- cli/src/shell_completion.rs | 9 +++++++-- cli/src/spawner.rs | 8 ++++++-- cli/src/target/binding.rs | 10 +++++----- cli/src/target/mod.rs | 27 +++++++++++++++++---------- 9 files changed, 70 insertions(+), 37 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index bc5e47e22..7028d8e00 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -120,15 +120,23 @@ impl ChecksumRecords { } /// Marks the job is involved in the record tracker - pub fn register_job(&self, target: Target) { - let mut items = self.items.lock().unwrap(); + pub fn register_job(&self, target: Target) -> anyhow::Result<()> { + let mut items = self + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; items.involved_targets.insert(target); + Ok(()) } /// Calculate the current checksum for the given target and compare it to the saved one. /// This method panics if the provided target isn't registered pub fn check_changed(&self, target: Target) -> anyhow::Result { - let items = self.items.lock().unwrap(); + let items = self + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; + assert!(items.involved_targets.contains(&target)); let saved_hash = match items.map.get(&target) { Some(hash) => hash, @@ -148,15 +156,24 @@ impl ChecksumRecords { } /// Remove the target from the checksum records - pub fn remove_hash_if_exist(&self, target: Target) { - let mut items = self.items.lock().unwrap(); + pub fn remove_hash_if_exist(&self, target: Target) -> anyhow::Result<()> { + let mut items = self + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; + items.involved_targets.insert(target); items.map.remove(&target); + + Ok(()) } fn calculate_involved_hashes(&self) -> anyhow::Result<()> { - let mut items = self.items.lock().unwrap(); + let mut items = self + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; for target in items.involved_targets.clone() { let hash = Self::calc_hash_for_target(&target)?; @@ -173,7 +190,10 @@ impl ChecksumRecords { let file_path = Self::get_file_path(self.job_type.is_production().is_some_and(|prod| prod)); let mut file = File::create(file_path)?; - let items = self.items.lock().unwrap(); + let items = self + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; for (target, hash) in items.map.iter() { writeln!(file, "{}:{}", target, hash)?; diff --git a/cli/src/dev_environment.rs b/cli/src/dev_environment.rs index 03bf11362..dbf0c18e3 100644 --- a/cli/src/dev_environment.rs +++ b/cli/src/dev_environment.rs @@ -19,24 +19,21 @@ pub async fn resolve_dev_tools() -> anyhow::Result<()> { writeln!( error_lines, "Required dependency '{tool}' is not installed.", - ) - .unwrap(); + )?; - writeln!(error_lines, "Resolve Error Info:{err}",).unwrap(); + writeln!(error_lines, "Resolve Error Info:{err}",)?; if let Some(install_hint) = tool.install_hint() { writeln!( error_lines, "Consider installing it using the command '{install_hint}'" - ) - .unwrap(); + )?; } writeln!( error_lines, "------------------------------------------------------------------" - ) - .expect("Writing to string never fail"); + )?; } match errors { diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index 57830e29b..d1db69387 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -93,7 +93,7 @@ impl DevTool { self.resolve() .await .as_ref() - .expect("Cmd has already been resolved") + .expect("Developer Error: Cmd has already been resolved") } } diff --git a/cli/src/location.rs b/cli/src/location.rs index 60bad9900..bd000beb8 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -36,7 +36,7 @@ impl Location { pub fn get_root() -> &'static PathBuf { &LOCATION .get() - .expect("Location is initialized in main function") + .expect("Developer Error: Location is initialized in main function") .root } @@ -48,7 +48,7 @@ pub fn init_location() -> Result<(), Error> { let location = Location::new()?; LOCATION .set(location) - .expect("init location can't be called more than once"); + .expect("Developer Error: init location can't be called more than once"); Ok(()) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 244b8aa49..94b5c23ec 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -154,7 +154,7 @@ async fn main() -> Result<(), Error> { return Ok(()); } Command::ShellCompletion { shell } => { - shell_completion::generate_completion(shell); + shell_completion::generate_completion(shell)?; return Ok(()); } diff --git a/cli/src/shell_completion.rs b/cli/src/shell_completion.rs index b20471d7b..d807a0636 100644 --- a/cli/src/shell_completion.rs +++ b/cli/src/shell_completion.rs @@ -1,14 +1,19 @@ use std::io; +use anyhow::Context; use clap::CommandFactory; use clap_complete::{generate, Shell}; use crate::cli_args::CargoCli; /// Generates shell complition for the given shell printing them to stdout -pub fn generate_completion(shell: Shell) { +pub fn generate_completion(shell: Shell) -> anyhow::Result<()> { let mut cmd = CargoCli::command(); - let bin_name = cmd.get_bin_name().unwrap().to_owned(); + let bin_name = cmd + .get_bin_name() + .context("Error while getting bin name")? + .to_owned(); generate(shell, &mut cmd, bin_name, &mut io::stdout()); + Ok(()) } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 3a03ee35f..81fd5bf45 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -113,8 +113,12 @@ pub async fn spawn( let mut report_lines: Vec = vec![]; let drain_stdout_stderr = { - let stdout = child.stdout.take().unwrap(); - let stderr = child.stderr.take().unwrap(); + let stdout = child.stdout.take().expect( + "Developer Error: Stdout is implicity set in command definition from which the child is spawn", + ); + let stderr = child.stderr.take().expect( + "Developer Error: Stderr is implicity set in command definition from which the child is spawn", + ); let storage_report = &mut report_lines; async move { let mut stdout_buf = BufReader::new(stdout); diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index 9f0fefa9a..3ec953d93 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -6,26 +6,26 @@ use crate::{fstools, spawner::SpawnResult}; use super::Target; -pub fn get_build_cmd(prod: bool) -> String { +pub fn get_build_cmd(prod: bool) -> anyhow::Result { let mut path = Target::Wrapper.cwd(); path.push("node_modules"); path.push(".bin"); if cfg!(windows) { // The script files can get the extension '*.cmd' on Windows - let electron_build_env_path = - which::which_in("electron-build-env", Some(&path), &path).unwrap(); + let electron_build_env_path = which::which_in("electron-build-env", Some(&path), &path) + .context("Error while resolving electron bin path on Windows")?; path = electron_build_env_path; } else { path.push("electron-build-env"); } - format!( + Ok(format!( "{} nj-cli build{}", path.to_string_lossy(), //TODO: Ruby code build always in release mode if prod { " --release" } else { "" } - ) + )) } pub async fn copy_index_node() -> Result, anyhow::Error> { diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index c9980e8c6..9e014a5be 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context}; +use anyhow::{anyhow, bail, Context}; use clap::ValueEnum; use futures::{ future::{join_all, BoxFuture}, @@ -200,13 +200,15 @@ impl Target { } /// Provide the command that should be used in to build the target - pub async fn build_cmd(&self, prod: bool) -> String { - match self { - Target::Binding => binding::get_build_cmd(prod), + pub async fn build_cmd(&self, prod: bool) -> anyhow::Result { + let build_cmd = match self { + Target::Binding => binding::get_build_cmd(prod)?, Target::Wasm => wasm::get_build_cmd(prod).await, Target::Updater => updater::get_build_cmd().await, rest_targets => rest_targets.kind().build_cmd(prod).await, - } + }; + + Ok(build_cmd) } /// Installs the needed module to perform the development task @@ -348,7 +350,7 @@ impl Target { /// Clean the given target, removing it from the checksum tracker as well. pub async fn reset(&self, production: bool) -> anyhow::Result> { let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; - checksum.remove_hash_if_exist(*self); + checksum.remove_hash_if_exist(*self)?; let mut results = Vec::new(); let clean_result = self.clean().await?; results.push(clean_result); @@ -391,7 +393,10 @@ impl Target { let mut run_build = false; // Check the current build state { - let mut states = build_states.states_map.lock().unwrap(); + let mut states = build_states + .states_map + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; match states.get_mut(self) { Some(BuildState::Running(senders)) => { // If the build is currently running, add the sender to senders list so it get @@ -418,7 +423,9 @@ impl Target { let build_result = self.perform_build(prod).await; // Update the build state and notify all the senders { - let mut states = build_states.states_map.lock().unwrap(); + let mut states = build_states.states_map.lock().map_err(|err| { + anyhow!("Error while acquiring items jobs mutex: Error {err}") + })?; let res_clone = match &build_result { Ok(spawn_res) => Ok(spawn_res.clone()), @@ -453,7 +460,7 @@ impl Target { // BoxFuture is needed because recursion isn't supported with async rust yet async move { let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; - checksum_rec.register_job(*self); + checksum_rec.register_job(*self)?; let mut results = Vec::new(); let deps: Vec = self.deps(); @@ -470,7 +477,7 @@ impl Target { } } let path = get_root().join(self.cwd()); - let cmd = self.build_cmd(prod).await; + let cmd = self.build_cmd(prod).await?; let caption = format!("Build {}", self); let mut skip_task = false; From 9b27f1c508250c09cd697dd6a0434bc1ec2f75c5 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 28 May 2024 16:34:16 +0200 Subject: [PATCH 111/174] Build CLI: Use Rayon default thread pool - There is no need to use a custom thread pool for rayon. --- cli/dir_checksum/src/lib.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cli/dir_checksum/src/lib.rs b/cli/dir_checksum/src/lib.rs index 412f630cd..ca4e7d7fb 100644 --- a/cli/dir_checksum/src/lib.rs +++ b/cli/dir_checksum/src/lib.rs @@ -1,10 +1,7 @@ use blake3::Hasher; use ignore::Walk; use input::Input; -use rayon::{ - iter::{IntoParallelIterator, ParallelIterator}, - ThreadPoolBuilder, -}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::{ io::{self, ErrorKind}, path::{Path, PathBuf}, @@ -73,11 +70,7 @@ where let mut hasher = blake3::Hasher::new(); - let thread_pool = ThreadPoolBuilder::new() - .build() - .map_err(|e| HashError::Environment(format!("Threadpool can't be created: {e}")))?; - - thread_pool.install(|| calc_fn(dir_path, &mut hasher)) + calc_fn(dir_path, &mut hasher) } /// Walks through file trees calculate the checksum for each files of them From 5557bcec008c375755ad8f94548f0cb83f4fddad Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 29 May 2024 08:20:55 +0200 Subject: [PATCH 112/174] Build CLI: Replace spawn empty result with option - Empty spawn results can be expressed much better with rust type system using options --- cli/src/main.rs | 10 ++-------- cli/src/spawner.rs | 14 -------------- cli/src/target/mod.rs | 29 ++++++++++++++++++----------- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 94b5c23ec..c5b57397e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -171,15 +171,11 @@ async fn main() -> Result<(), Error> { let print_err = match &report_opt { ReportOptions::None => true, ReportOptions::Stdout(stdout) => { - if !status.is_empty() { - write_report(status, stdout)?; - } + write_report(status, stdout)?; false } ReportOptions::File(path, file) => { - if !status.is_empty() { - write_report(status, file)?; - } + write_report(status, file)?; if idx == results.len() - 1 { let full_path = path.canonicalize().unwrap_or_else(|_| path.to_owned()); @@ -243,8 +239,6 @@ fn get_report_option(report_argument: Option>) -> Result Result<(), io::Error> { - assert!(!spawn_result.is_empty()); - writeln!(writer)?; writeln!( writer, diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 81fd5bf45..94c9ab602 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -25,20 +25,6 @@ pub struct SpawnResult { } impl SpawnResult { - pub fn empty() -> Self { - SpawnResult { - report: Vec::default(), - status: ExitStatus::default(), - job: String::new(), - cmd: String::default(), - skipped: None, - } - } - - pub fn is_empty(&self) -> bool { - self.job.is_empty() && self.cmd.is_empty() - } - /// Create spawn for multiple file system commands pub fn create_for_fs(job: String, report: Vec) -> Self { SpawnResult { diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 9e014a5be..8417bf70f 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -212,14 +212,14 @@ impl Target { } /// Installs the needed module to perform the development task - pub async fn install(&self, prod: bool) -> Result { + pub async fn install(&self, prod: bool) -> Result, anyhow::Error> { match self { // We must install ts binding tools before running rs bindings, therefore we call // wrapper (ts-bindings) install in the rs bindings install. // Since rs bindings is a dependency for ts bindings, we don't need to call to install // on ts bindings again. Target::Binding => install_general(&Target::Wrapper, prod).await, - Target::Wrapper => Ok(SpawnResult::empty()), + Target::Wrapper => Ok(None), // For app we don't need --production Target::App => install_general(&Target::App, false).await, rest_targets => install_general(rest_targets, prod).await, @@ -285,9 +285,10 @@ impl Target { let mut results = Vec::new(); match self.kind() { TargetKind::Ts => { - let install_result = self.install(false).await?; + if let Some(install_result) = self.install(false).await? { + results.push(install_result); + } let lint_restul = self.ts_lint().await?; - results.push(install_result); results.push(lint_restul); } TargetKind::Rs => { @@ -496,8 +497,9 @@ impl Target { let spawn_reslt = if skip_task { spawn_skip(cmd, Some(path), caption).await } else { - let install_result = self.install(false).await?; - results.push(install_result); + if let Some(install_result) = self.install(false).await? { + results.push(install_result); + } let spawn_opt = SpawnOptions { has_skip_info: true, ..Default::default() @@ -521,8 +523,9 @@ impl Target { if matches!(self.kind(), TargetKind::Ts) && prod { let clean_res = self.clean().await?; results.push(clean_res); - let install_res = self.install(prod).await?; - results.push(install_res); + if let Some(install_res) = self.install(prod).await? { + results.push(install_res); + } } let res = self.after_build(prod).await?; @@ -550,12 +553,16 @@ impl Target { } /// run install using the general routine for the given target -async fn install_general(target: &Target, prod: bool) -> Result { +async fn install_general( + target: &Target, + prod: bool, +) -> Result, anyhow::Error> { let cmd = target.kind().install_cmd(prod).await; if let Some(cmd) = cmd { let caption = format!("Install {}", target); - spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await + let res = spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await?; + Ok(Some(res)) } else { - Ok(SpawnResult::empty()) + Ok(None) } } From 16589427149b527286d445eecb6774d40fbe9ddc Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 29 May 2024 09:54:21 +0200 Subject: [PATCH 113/174] Build CLI Tracker: Replace HashMap with Vector Bars can be tracked with a vector since their key in the hasmap was the their index anyway --- cli/src/tracker.rs | 71 +++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 0ddbbdf4a..16669ffe6 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -2,7 +2,6 @@ use anyhow::{anyhow, Context, Error}; use console::style; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use std::{ - collections::HashMap, process::{Command, ExitStatus}, time::Instant, }; @@ -95,32 +94,33 @@ impl Tracker { ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}")? .tick_chars("▂▃▅▆▇▆▅▃▂ "); async move { - let mut sequence: usize = 0; let mut max_time_len = 0; let max = u64::MAX; - let mut bars: HashMap = HashMap::new(); + let mut bars: Vec = Vec::new(); let mp = MultiProgress::new(); let start_time = Instant::now(); while let Some(tick) = rx.recv().await { match tick { Tick::Started(job, tx_response) => { - sequence += 1; let bar = mp.add(ProgressBar::new(max)); bar.set_style(spinner_style.clone()); let job_bar = JobBarState::start_job(job, bar); - bars.insert(sequence, job_bar); - Self::refresh_all_bars(&mut bars, sequence, max_time_len, None); - if let Err(e) = tx_response.send(sequence) { + bars.push(job_bar); + Self::refresh_all_bars(&mut bars, max_time_len, None); + let job_number = bars.len(); + if let Err(e) = tx_response.send(job_number) { let _ = mp.println(format!("Fail to send response: {e}")); } } - Tick::Message(sequence, log) => { - if let Some(job_bar) = bars.get(&sequence) { + Tick::Message(job_number, log) => { + let idx = job_number.checked_sub(1).expect("Job number can't be zero"); + if let Some(job_bar) = bars.get(idx) { job_bar.bar.set_message(log); } } - Tick::Progress(sequence, pos) => { - if let Some(job_bar) = bars.get(&sequence) { + Tick::Progress(job_number, pos) => { + let idx = job_number.checked_sub(1).expect("Job number can't be zero"); + if let Some(job_bar) = bars.get(idx) { if let Some(pos) = pos { job_bar.bar.set_position(pos); } else { @@ -128,30 +128,31 @@ impl Tracker { } } } - Tick::Finished(seq, result, msg) => { - if let Some(job_bar) = bars.get_mut(&seq) { - let sequence_txt = sequence.to_string(); + Tick::Finished(job_number, result, msg) => { + let idx = job_number.checked_sub(1).expect("Job number can't be zero"); + let jobs_count_txt = bars.len().to_string(); + if let Some(job_bar) = bars.get_mut(idx) { // It doesn't make sense to show that a job is done in 0 seconds let time = job_bar.start_time.elapsed().as_secs().max(1); max_time_len = max_time_len.max(Self::count_digits(time)); - let seq_width = sequence_txt.len(); + let seq_width = jobs_count_txt.len(); let job = job_bar.name.as_str(); job_bar.bar.set_prefix(format!( - "[{seq:seq_width$}/{sequence_txt}][{result}][{time:max_time_len$}s][{job}].", + "[{job_number:seq_width$}/{jobs_count_txt}][{result}][{time:max_time_len$}s][{job}].", )); job_bar.bar.finish_with_message(msg); job_bar.result.replace((result, time)); - Self::refresh_all_bars(&mut bars, sequence, max_time_len, None); + Self::refresh_all_bars(&mut bars, max_time_len, None); } } Tick::Print(msg) => { let _ = mp.println(msg); } Tick::Shutdown(tx_response) => { - bars.iter_mut().for_each(|(_, job_bar)| { + bars.iter_mut().for_each(|job_bar| { if !job_bar.bar.is_finished() { let time = job_bar.start_time.elapsed().as_secs().max(1); job_bar.result.replace((OperationResult::Success, time)); @@ -163,7 +164,7 @@ impl Tracker { // Insert graphic bar for the running duration of each bars let total_time = start_time.elapsed().as_secs().max(1) as usize; - Self::refresh_all_bars(&mut bars, sequence, max_time_len, Some(total_time)); + Self::refresh_all_bars(&mut bars, max_time_len, Some(total_time)); // Insert total time bar let total_bar = mp.add(ProgressBar::new((bars.len() + 1) as u64)); @@ -192,27 +193,27 @@ impl Tracker { } fn refresh_all_bars( - bars: &mut HashMap, - sequence: usize, + bars: &mut Vec, max_time_len: usize, total_time: Option, ) { - let sequence_txt = sequence.to_string(); + let jobs_count_txt = bars.len().to_string(); - bars.iter_mut().for_each(|(k, job_bar)| { - let seq_width = sequence_txt.len(); + bars.iter_mut().enumerate().for_each(|(idx, job_bar)| { + let job_number = idx + 1; + let seq_width = jobs_count_txt.len(); let job = job_bar.name.as_str(); let line_prefix = match job_bar.result.as_ref() { None => { - format!("[{k:seq_width$}/{sequence_txt}][....][{job}]") + format!("[{job_number:seq_width$}/{jobs_count_txt}][....][{job}]") } Some((res, time)) => { if let Some(total_time) = total_time { let finish_limit = (*time as usize * TIME_BAR_WIDTH) / total_time; let time_bar: String = (0..TIME_BAR_WIDTH).map(|idx| if idx <= finish_limit {'█'}else {'░'}).collect(); - format!("[{k:seq_width$}/{sequence_txt}][{res}][{time_bar} {time:max_time_len$}s][{job}].") + format!("[{job_number:seq_width$}/{jobs_count_txt}][{res}][{time_bar} {time:max_time_len$}s][{job}].") }else { - format!("[{k:seq_width$}/{sequence_txt}][{res}][{time:max_time_len$}s][{job}].") + format!("[{job_number:seq_width$}/{jobs_count_txt}][{res}][{time:max_time_len$}s][{job}].") } } }; @@ -245,23 +246,23 @@ impl Tracker { } /// Update the job with the given id providing an optional progress value. - pub async fn progress(&self, sequence: usize, pos: Option) { - if let Err(e) = self.tx.send(Tick::Progress(sequence, pos)) { + pub async fn progress(&self, job_number: usize, pos: Option) { + if let Err(e) = self.tx.send(Tick::Progress(job_number, pos)) { eprintln!("Fail to communicate with tracker: {e}"); } } /// Send a message to the job with the giving id - pub async fn msg(&self, sequence: usize, log: &str) { - if let Err(e) = self.tx.send(Tick::Message(sequence, log.to_string())) { + pub async fn msg(&self, job_number: usize, log: &str) { + if let Err(e) = self.tx.send(Tick::Message(job_number, log.to_string())) { eprintln!("Fail to communicate with tracker: {e}"); } } /// Sets the job with the given id as finished providing successful result and a message - pub async fn success(&self, sequence: usize, msg: &str) { + pub async fn success(&self, job_number: usize, msg: &str) { if let Err(e) = self.tx.send(Tick::Finished( - sequence, + job_number, OperationResult::Success, msg.to_string(), )) { @@ -270,9 +271,9 @@ impl Tracker { } /// Sets the job with the given id as finished providing failed result and a message - pub async fn fail(&self, sequence: usize, msg: &str) { + pub async fn fail(&self, job_number: usize, msg: &str) { if let Err(e) = self.tx.send(Tick::Finished( - sequence, + job_number, OperationResult::Failed, msg.to_string(), )) { From 423c4a8744a405447c38c23f8402d9195e6c790b Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 29 May 2024 16:44:01 +0200 Subject: [PATCH 114/174] CLI: Rewriting To Solve Dependencies upfront.. - Create JobsRunner and JobDefinitions structs to manage running all the jobs. Calling jobs methods should be managed form JobsRunner - Add Install and After Build to build types since they'll be used in the jobs runner - Remove Production from clean because it wasn't used. - Unify Jobs methods signatures to return an optional result with one job only. - SpawnResults can be combined together because each job should return on result only. - Combine test results together to form one result for each target - Implement and Unit Tests for has_job method to determine if a certain target has a certain job to run - Comment out many position in code + TODOs and comments --- cli/src/build_state.rs | 1 + cli/src/checksum_records.rs | 4 +- cli/src/cli_args.rs | 4 - cli/src/fstools.rs | 3 + cli/src/job_type.rs | 56 ++++- cli/src/jobs_runner/job_definition.rs | 55 +++++ cli/src/jobs_runner/mod.rs | 26 ++ cli/src/main.rs | 62 +++-- cli/src/spawner.rs | 28 +++ cli/src/target/app.rs | 6 +- cli/src/target/binding.rs | 6 +- cli/src/target/mod.rs | 341 +++++++++++--------------- cli/src/target/shared.rs | 6 +- cli/src/target/wrapper.rs | 62 ++--- 14 files changed, 381 insertions(+), 279 deletions(-) create mode 100644 cli/src/jobs_runner/job_definition.rs create mode 100644 cli/src/jobs_runner/mod.rs diff --git a/cli/src/build_state.rs b/cli/src/build_state.rs index cd35ec16b..e3a78f2d4 100644 --- a/cli/src/build_state.rs +++ b/cli/src/build_state.rs @@ -13,6 +13,7 @@ pub enum BuildState { } #[derive(Debug)] +//TODO AAZ: Delete the whole struct pub struct BuildStatesTracker { pub states_map: Mutex>, } diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 7028d8e00..d07e82da1 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -35,7 +35,9 @@ impl ChecksumRecords { JobType::Build { production: _ } | JobType::Run { production: _ } | JobType::Test { production: _ } => true, - JobType::Clean { production: _ } => false, + JobType::Clean + | JobType::Install { production: _ } + | JobType::AfterBuild { production: _ } => false, }; let records = Self::get(job_type).await?; diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index 70a287159..a4b5e7501 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -58,10 +58,6 @@ pub enum Command { #[arg(index = 1)] target: Option>, - /// Clean release version - #[arg(short, long, default_value_t = false)] - production: bool, - #[arg(short, long, value_name = REPORT_VALUE_NAME, help = REPORT_HELP_TEXT)] report: Option>, }, diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index c9e8ddcb1..3cf83afe4 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -8,6 +8,9 @@ use std::{fs, path::PathBuf}; use crate::tracker::get_tracker; +//TODO AAZ: Tools here shouldn't spawn new trackers. This should be able to send updates to their +// corresponding tracker only. + /// Spawn a job to copy a file, adding the info the report logs pub async fn cp_file( src: PathBuf, diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 13dead40a..12702ede5 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -1,20 +1,68 @@ -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +use std::fmt::Display; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum JobType { Lint, + Clean, Build { production: bool }, - Clean { production: bool }, + Install { production: bool }, + AfterBuild { production: bool }, Test { production: bool }, Run { production: bool }, } +impl Display for JobType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + JobType::Lint => write!(f, "Lint"), + JobType::Clean => write!(f, "Clean"), + JobType::Build { production: _ } => write!(f, "Build"), + JobType::Install { production: _ } => write!(f, "Install"), + JobType::AfterBuild { production: _ } => write!(f, "After Bulid"), + JobType::Test { production: _ } => write!(f, "Test"), + JobType::Run { production: _ } => write!(f, "Run"), + } + } +} + impl JobType { pub fn is_production(&self) -> Option { match self { - JobType::Lint => None, + JobType::Lint | JobType::Clean => None, JobType::Build { production } - | JobType::Clean { production } + | JobType::Install { production } + | JobType::AfterBuild { production } | JobType::Test { production } | JobType::Run { production } => Some(*production), } } } + +#[cfg(test)] +impl JobType { + pub fn all() -> &'static [JobType] { + if cfg!(debug_assertions) { + // This check to remember to add the newly added enums to this function + match JobType::Lint { + JobType::Lint => (), + JobType::Clean => (), + JobType::Build { production: _ } => (), + JobType::Install { production: _ } => (), + JobType::AfterBuild { production: _ } => (), + JobType::Test { production: _ } => (), + JobType::Run { production: _ } => (), + }; + } + + [ + JobType::Lint, + JobType::Clean, + JobType::Build { production: false }, + JobType::Install { production: false }, + JobType::AfterBuild { production: false }, + JobType::Test { production: false }, + JobType::Run { production: false }, + ] + .as_slice() + } +} diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs new file mode 100644 index 000000000..6d9fd26dc --- /dev/null +++ b/cli/src/jobs_runner/job_definition.rs @@ -0,0 +1,55 @@ +use crate::{job_type::JobType, spawner::SpawnResult, target::Target}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct JobDefinition { + target: Target, + job_type: JobType, +} + +impl JobDefinition { + pub fn new(target: Target, job_type: JobType) -> Self { + Self { target, job_type } + } + + pub fn job_title(&self) -> String { + format!("{} {}", self.target, self.job_type) + } + + async fn run(&self) -> Option> { + let res = match self.job_type { + JobType::Lint => self.target.check().await, + JobType::Build { production } => self.target.build(production).await, + JobType::Install { production } => return self.target.install(production).await, + JobType::AfterBuild { production } => return self.target.after_build(production).await, + JobType::Clean => self.target.reset().await, + JobType::Test { production } => return self.target.test(production).await, + JobType::Run { production: _ } => return None, + }; + + Some(res) + } +} + +#[cfg(test)] +mod tests { + use crate::{job_type::JobType, jobs_runner::JobDefinition}; + + use super::Target; + + #[tokio::test] + async fn target_has_job() { + for target in Target::all() { + for job_type in JobType::all() { + if !target.has_job(&job_type) { + let job_def = JobDefinition::new(*target, job_type.clone()); + assert!( + job_def.run().await.is_none(), + "'{}' has no job for '{}' but it returns Some when calling run", + target, + job_type + ) + } + } + } + } +} diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs new file mode 100644 index 000000000..2de4305e9 --- /dev/null +++ b/cli/src/jobs_runner/mod.rs @@ -0,0 +1,26 @@ +mod job_definition; + +use std::collections::BTreeMap; + +pub use job_definition::JobDefinition; + +use crate::{spawner::SpawnResult, target::Target}; + +enum JobPhase { + Awaiting(Vec), + Running, + //TODO AAZ: Errors on Spawn calls should terminate the execution. Make sure results aren't + // returned if a command fails or for expect reasons + Done(SpawnResult), +} + +struct JobState { + phase: JobPhase, + job_number: usize, +} + +pub struct JobsRunner { + // BTreeMap keeps the jobs in logical ordering + jobs: BTreeMap, + resolved_targets: Vec, +} diff --git a/cli/src/main.rs b/cli/src/main.rs index c5b57397e..33010e849 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,6 +6,7 @@ mod dev_environment; mod dev_tools; mod fstools; mod job_type; +mod jobs_runner; mod location; mod print_dot; mod shell_completion; @@ -94,22 +95,18 @@ async fn main() -> Result<(), Error> { .await; (JobType::Build { production }, results) } - Command::Clean { - target, - production, - report, - } => { + Command::Clean { target, report } => { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = join_all( targets .iter() - .map(|module| module.reset(production)) + .map(|module| module.reset()) .collect::>(), ) .await; - (JobType::Clean { production }, results) + (JobType::Clean, results) } Command::Test { target, @@ -126,7 +123,9 @@ async fn main() -> Result<(), Error> { .collect::>(), ) .await; - (JobType::Test { production }, results) + todo!() + //TODO AAZ: This will replaced with the new logic + // (JobType::Test { production }, results) } Command::Run { production } => { resolve_dev_tools().await?; @@ -166,33 +165,32 @@ async fn main() -> Result<(), Error> { let mut success: bool = true; for (idx, res) in results.iter().enumerate() { match res { - Ok(statuses) => { - for status in statuses { - let print_err = match &report_opt { - ReportOptions::None => true, - ReportOptions::Stdout(stdout) => { - write_report(status, stdout)?; - false - } - ReportOptions::File(path, file) => { - write_report(status, file)?; - if idx == results.len() - 1 { - let full_path = - path.canonicalize().unwrap_or_else(|_| path.to_owned()); - println!("Report is written to '{}'", full_path.display()); - } - false + Ok(status) => { + let print_err = match &report_opt { + ReportOptions::None => true, + ReportOptions::Stdout(stdout) => { + write_report(status, stdout)?; + false + } + ReportOptions::File(path, file) => { + write_report(status, file)?; + if idx == results.len() - 1 { + let full_path = path.canonicalize().unwrap_or_else(|_| path.to_owned()); + println!("Report is written to '{}'", full_path.display()); } - }; + false + } + }; - if !status.status.success() { - if print_err { - eprintln!("Failed with errors"); - eprintln!("{}:\n{}", status.job, status.report.join("")); - eprintln!("---------------------------------------------------------------------"); - } - success = false; + if !status.status.success() { + if print_err { + eprintln!("Failed with errors"); + eprintln!("{}:\n{}", status.job, status.report.join("")); + eprintln!( + "---------------------------------------------------------------------" + ); } + success = false; } } Err(err) => { diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 94c9ab602..2022800e8 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -16,6 +16,7 @@ use tokio::{ }; #[derive(Clone, Debug)] +//TODO AAZ: Move this to its own file pub struct SpawnResult { pub report: Vec, pub status: ExitStatus, @@ -46,6 +47,33 @@ impl SpawnResult { skipped: Some(true), } } + + /// Append other result to the current one producing a combined results form them + pub fn append(&mut self, other: SpawnResult) { + if !other.report.is_empty() { + self.report.extend( + [ + String::default(), + String::from("-------------------------------------------------------------------------------"), + String::default() + ]); + self.report.extend(other.report); + } + + self.job = format!("{} & {}", self.job, other.job); + self.status = match (self.status.success(), other.status.success()) { + (_, true) => self.status, + (false, false) => self.status, + (_, false) => other.status, + }; + + self.cmd = format!("{} \n {}", self.cmd, other.cmd); + self.skipped = match (self.skipped, other.skipped) { + (Some(false), _) | (_, Some(false)) => Some(false), + (Some(true), _) | (_, Some(true)) => Some(true), + _ => None, + }; + } } #[derive(Debug, Clone, Default)] diff --git a/cli/src/target/app.rs b/cli/src/target/app.rs index ac845e02a..1ea0c5f17 100644 --- a/cli/src/target/app.rs +++ b/cli/src/target/app.rs @@ -6,7 +6,7 @@ use crate::{fstools, spawner::SpawnResult}; use super::{client::get_dist_path, Target}; -pub async fn copy_client_to_app(prod: bool) -> Result, anyhow::Error> { +pub async fn copy_client_to_app(prod: bool) -> Result { let mut report_logs = Vec::new(); let src = get_dist_path(prod); let dest = Target::App.cwd().join("dist"); @@ -46,8 +46,8 @@ pub async fn copy_client_to_app(prod: bool) -> Result, anyho std::fs::rename(&rename_from, &rename_to) .with_context(|| format!("Error while renaming {}", rename_from.display()))?; - Ok(Some(SpawnResult::create_for_fs( + Ok(SpawnResult::create_for_fs( "Copy App Build Artifacts".into(), report_logs, - ))) + )) } diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index 3ec953d93..afd30afbc 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -28,7 +28,7 @@ pub fn get_build_cmd(prod: bool) -> anyhow::Result { )) } -pub async fn copy_index_node() -> Result, anyhow::Error> { +pub async fn copy_index_node() -> Result { let mut report_logs = Vec::new(); // *** Copy `index.node` from rs to ts bindings dist *** @@ -72,8 +72,8 @@ pub async fn copy_index_node() -> Result, anyhow::Error> { fstools::cp_file(src_file, mod_file, &mut report_logs).await?; - Ok(Some(SpawnResult::create_for_fs( + Ok(SpawnResult::create_for_fs( "Copying `index.node` from rs to ts bindings".into(), report_logs, - ))) + )) } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8417bf70f..0ee352d4a 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -199,6 +199,27 @@ impl Target { } } + pub fn has_job(&self, job_type: &JobType) -> bool { + match job_type { + JobType::Lint | JobType::Clean | JobType::Build { production: _ } => true, + JobType::Install { production: _ } => { + matches!( + self, + Target::Binding | Target::Client | Target::Shared | Target::App + ) + } + JobType::AfterBuild { production: _ } => matches!( + self, + Target::Shared | Target::Binding | Target::Wrapper | Target::App + ), + JobType::Test { production: _ } => matches!( + self, + Target::Wrapper | Target::Core | Target::Cli | Target::Wasm + ), + JobType::Run { production: _ } => false, + } + } + /// Provide the command that should be used in to build the target pub async fn build_cmd(&self, prod: bool) -> anyhow::Result { let build_cmd = match self { @@ -212,14 +233,14 @@ impl Target { } /// Installs the needed module to perform the development task - pub async fn install(&self, prod: bool) -> Result, anyhow::Error> { + pub async fn install(&self, prod: bool) -> Option> { match self { // We must install ts binding tools before running rs bindings, therefore we call // wrapper (ts-bindings) install in the rs bindings install. // Since rs bindings is a dependency for ts bindings, we don't need to call to install // on ts bindings again. Target::Binding => install_general(&Target::Wrapper, prod).await, - Target::Wrapper => Ok(None), + Target::Wrapper => None, // For app we don't need --production Target::App => install_general(&Target::App, false).await, rest_targets => install_general(rest_targets, prod).await, @@ -227,9 +248,9 @@ impl Target { } /// Run tests for the giving the target - pub async fn test(&self, production: bool) -> Result, anyhow::Error> { + pub async fn test(&self, production: bool) -> Option> { match self { - Target::Wrapper => wrapper::run_test().await, + Target::Wrapper => Some(wrapper::run_test().await), rest_targets => rest_targets.run_test_general(production).await, } } @@ -245,18 +266,18 @@ impl Target { } /// run test using the general routine with `test_cmds()` method - async fn run_test_general(&self, production: bool) -> Result, anyhow::Error> { - let Some(test_cmds) = self.test_cmds(production).await else { - return Ok(Vec::new()); - }; + async fn run_test_general( + &self, + production: bool, + ) -> Option> { + let test_cmds = self.test_cmds(production).await?; debug_assert!(!test_cmds.is_empty()); - let mut results = Vec::new(); - + // TODO AAZ: Call build outside // build method calls install - let build_results = self.build(false).await?; - results.extend(build_results); + // let build_results = self.build(false).await?; + // results.extend(build_results); let caption = format!("Test {}", self); let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { @@ -270,34 +291,35 @@ impl Target { })) .await; - for res in spawn_results { - match res { - Ok(spawn_res) => results.push(spawn_res), - Err(err) => return Err(err), + let mut spawn_results = spawn_results.into_iter(); + + let mut result = match spawn_results.next()? { + Ok(result) => result, + Err(err) => return Some(Err(err)), + }; + + while let Some(next_result) = spawn_results.next() { + match next_result { + Ok(next_res) => result.append(next_res), + Err(err) => return Some(Err(err)), } } - Ok(results) + Some(Ok(result)) } /// Perform Linting Checks on the giving target - pub async fn check(&self) -> Result, anyhow::Error> { - let mut results = Vec::new(); + pub async fn check(&self) -> Result { match self.kind() { TargetKind::Ts => { - if let Some(install_result) = self.install(false).await? { - results.push(install_result); - } - let lint_restul = self.ts_lint().await?; - results.push(lint_restul); - } - TargetKind::Rs => { - let clippy_result = self.clippy().await?; - results.push(clippy_result); + //TODO AAZ: Check needs install on TS + // if let Some(install_result) = self.install(false).await? { + // todo!() + // } + self.ts_lint().await } + TargetKind::Rs => self.clippy().await, } - - Ok(results) } /// Perform Linting the Building the giving target since linting Type-Script doesn't check for @@ -349,24 +371,18 @@ impl Target { } /// Clean the given target, removing it from the checksum tracker as well. - pub async fn reset(&self, production: bool) -> anyhow::Result> { - let checksum = ChecksumRecords::get(JobType::Clean { production }).await?; + pub async fn reset(&self) -> anyhow::Result { + let checksum = ChecksumRecords::get(JobType::Clean).await?; checksum.remove_hash_if_exist(*self)?; - let mut results = Vec::new(); - let clean_result = self.clean().await?; - results.push(clean_result); - - let dist_path = self.cwd().join("dist"); - - let remove_log = format!("removing {}", dist_path.display()); - - fstools::rm_folder(&dist_path).await?; - let job = format!("Reset {}", self); + self.clean().await + //TODO AAZ: - results.push(SpawnResult::create_for_fs(job, vec![remove_log])); - - Ok(results) + // let dist_path = self.cwd().join("dist"); + // let remove_log = format!("removing {}", dist_path.display()); + // fstools::rm_folder(&dist_path).await?; + // + // Ok(results) } async fn clean(&self) -> Result { @@ -387,168 +403,93 @@ impl Target { } /// Runs build considering the currently running builds and already finished ones as well. - pub async fn build(&self, prod: bool) -> Result, anyhow::Error> { - let build_states = BuildStatesTracker::get().await; - let (tx_result, rx_result) = oneshot::channel(); - - let mut run_build = false; - // Check the current build state - { - let mut states = build_states - .states_map - .lock() - .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; - match states.get_mut(self) { - Some(BuildState::Running(senders)) => { - // If the build is currently running, add the sender to senders list so it get - // the results when the job is done; - senders.push(tx_result); - } - Some(BuildState::Finished(build_result)) => { - // If Build is already finished, then return the results directly - match build_result { - Ok(res) => return Ok(res.clone()), - Err(err) => bail!(err.to_string()), - } - } - None => { - // Run the build and add the sender to the list. - run_build = true; - states.insert(*self, BuildState::Running(vec![tx_result])); - } - } - } + pub async fn build(&self, prod: bool) -> Result { + let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; + checksum_rec.register_job(*self)?; + + //TODO AAZ: Dependencies should be resolved before running the jobs + // let deps: Vec = self.deps(); + // for module in deps { + // let status = module.build(prod).await.with_context(|| { + // format!( + // "Error while building the dependciy {} for target {}", + // module, self + // ) + // })?; + // results.extend(status); + // if results.iter().any(|res| !res.status.success()) { + // return Ok(results); + // } + // } + let path = get_root().join(self.cwd()); + let cmd = self.build_cmd(prod).await?; + let caption = format!("Build {}", self); - if run_build { - // Run the build - let build_result = self.perform_build(prod).await; - // Update the build state and notify all the senders - { - let mut states = build_states.states_map.lock().map_err(|err| { - anyhow!("Error while acquiring items jobs mutex: Error {err}") - })?; - - let res_clone = match &build_result { - Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{:?}", err)), - }; - - let Some(BuildState::Running(senders)) = - states.insert(*self, BuildState::Finished(build_result)) - else { - unreachable!("State after calling build must be renning"); - }; - - for sender in senders { - let res_clone = match &res_clone { - Ok(spawn_res) => Ok(spawn_res.clone()), - Err(err) => Err(anyhow::anyhow!("{:?}", err)), - }; - if sender.send(res_clone).is_err() { - bail!("Fail to communicate with builder"); - } - } - } - } + //TODO AAZ: Skipping jobs should be resolved before running the jobs + // let mut skip_task = false; + // + // let all_skipped = results.iter().all(|r| { + // r.skipped.unwrap_or({ + // // Tasks with no skip info are irrelevant + // true + // }) + // }); + // + // if all_skipped { + // skip_task = !checksum_rec.check_changed(*self)?; + // } + // + + //TODO AAZ: Install Run outside of build + // if let Some(install_result) = self.install(false).await? { + // results.push(install_result); + // } + let spawn_opt = SpawnOptions { + has_skip_info: true, + ..Default::default() + }; + let status = spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await?; - rx_result - .await - .context("Fail to communicate with builder")? + if !status.status.success() { + Ok(status) + } else { + //TODO AAZ: This should be solved before running the tasks + // Taken from a discussion on GitHub: + // To build an npm package you would need (in most cases) to be in dev-mode - install dev-dependencies + dependencies. + // But to prepare a package for production, you have to remove dev-dependencies. + // That's not an issue, if npm-package is published in npmjs; but we are coping packages manually in a right destination + // and before copy it, we have to reinstall it to get rid of dev-dependencies. + // if matches!(self.kind(), TargetKind::Ts) && prod { + // let clean_res = self.clean().await?; + // results.push(clean_res); + // if let Some(install_res) = self.install(prod).await? { + // results.push(install_res); + // } + // } + + // TODO AAZ: After Build should be called separately + // let res = self.after_build(prod).await?; + // if let Some(result) = res { + // results.push(result); + // } + + Ok(status) + } } /// Performs build process without checking the current builds states - fn perform_build(&self, prod: bool) -> BoxFuture, anyhow::Error>> { - // BoxFuture is needed because recursion isn't supported with async rust yet - async move { - let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; - checksum_rec.register_job(*self)?; - - let mut results = Vec::new(); - let deps: Vec = self.deps(); - for module in deps { - let status = module.build(prod).await.with_context(|| { - format!( - "Error while building the dependciy {} for target {}", - module, self - ) - })?; - results.extend(status); - if results.iter().any(|res| !res.status.success()) { - return Ok(results); - } - } - let path = get_root().join(self.cwd()); - let cmd = self.build_cmd(prod).await?; - let caption = format!("Build {}", self); - - let mut skip_task = false; - - let all_skipped = results.iter().all(|r| { - r.skipped.unwrap_or({ - // Tasks with no skip info are irrelevant - true - }) - }); - - if all_skipped { - skip_task = !checksum_rec.check_changed(*self)?; - } - - let spawn_reslt = if skip_task { - spawn_skip(cmd, Some(path), caption).await - } else { - if let Some(install_result) = self.install(false).await? { - results.push(install_result); - } - let spawn_opt = SpawnOptions { - has_skip_info: true, - ..Default::default() - }; - spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await - }; - - let status = spawn_reslt?; - - if !status.status.success() { - results.push(status); - Ok(results) - } else { - results.push(status); - if !skip_task { - // Taken from a discussion on GitHub: - // To build an npm package you would need (in most cases) to be in dev-mode - install dev-dependencies + dependencies. - // But to prepare a package for production, you have to remove dev-dependencies. - // That's not an issue, if npm-package is published in npmjs; but we are coping packages manually in a right destination - // and before copy it, we have to reinstall it to get rid of dev-dependencies. - if matches!(self.kind(), TargetKind::Ts) && prod { - let clean_res = self.clean().await?; - results.push(clean_res); - if let Some(install_res) = self.install(prod).await? { - results.push(install_res); - } - } - - let res = self.after_build(prod).await?; - if let Some(result) = res { - results.push(result); - } - } - - Ok(results) - } - } - .boxed() - } /// Perform any needed copy operation after the build is done - async fn after_build(&self, prod: bool) -> Result, anyhow::Error> { - match self { + pub async fn after_build(&self, prod: bool) -> Option> { + let res = match self { Target::Binding => binding::copy_index_node().await, Target::Wrapper => wrapper::copy_binding_to_app().await, Target::Shared => shared::copy_platform_to_binding().await, Target::App => app::copy_client_to_app(prod).await, - _ => Ok(None), - } + _ => return None, + }; + + Some(res) } } @@ -556,13 +497,13 @@ impl Target { async fn install_general( target: &Target, prod: bool, -) -> Result, anyhow::Error> { +) -> Option> { let cmd = target.kind().install_cmd(prod).await; if let Some(cmd) = cmd { let caption = format!("Install {}", target); - let res = spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await?; - Ok(Some(res)) + let res = spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await; + Some(res) } else { - Ok(None) + None } } diff --git a/cli/src/target/shared.rs b/cli/src/target/shared.rs index 62ddad15c..67c865379 100644 --- a/cli/src/target/shared.rs +++ b/cli/src/target/shared.rs @@ -6,7 +6,7 @@ use crate::{fstools, spawner::SpawnResult}; use super::Target; -pub async fn copy_platform_to_binding() -> Result, anyhow::Error> { +pub async fn copy_platform_to_binding() -> Result { let mut report_logs = Vec::new(); report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); @@ -47,8 +47,8 @@ pub async fn copy_platform_to_binding() -> Result, anyhow::E ) .await?; - Ok(Some(SpawnResult::create_for_fs( + Ok(SpawnResult::create_for_fs( "Copying Platform to Bindings".into(), report_logs, - ))) + )) } diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index c4ae6ef76..091e85f79 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,6 +1,6 @@ use std::{iter, path::PathBuf}; -use anyhow::Context; +use anyhow::{anyhow, Context}; use std::fs; use crate::{ @@ -39,32 +39,32 @@ const TEST_SPECS: [&str; 14] = [ "promises", ]; -pub async fn run_test() -> Result, anyhow::Error> { - let mut results = Vec::new(); - - let build_results = Target::Wrapper.build(false).await?; - results.extend(build_results); +pub async fn run_test() -> Result { + //TODO AAZ: Build should happen outside of test + // let build_results = Target::Wrapper.build(false).await?; + // results.extend(build_results); let cwd = Target::Wrapper.cwd(); - let build_spec_path = cwd.join("spec"); - //TODO: This check exists in rake implementation but it need to be improved. - // The check should cover if the test themselves or the code under the tests has been changed. - if !build_spec_path.join("build").exists() { - let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); - let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); - - let spec_res = spawn( - build_spec_cmd, - Some(build_spec_path), - "Build Specs".into(), - iter::empty(), - None, - ) - .await?; - - results.push(spec_res); - } + //TODO AAZ: Append build_spec_path to SpawnResult + // let build_spec_path = cwd.join("spec"); + // //TODO: This check exists in rake implementation but it need to be improved. + // // The check should cover if the test themselves or the code under the tests has been changed. + // if !build_spec_path.join("build").exists() { + // let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); + // let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); + // + // let spec_res = spawn( + // build_spec_cmd, + // Some(build_spec_path), + // "Build Specs".into(), + // iter::empty(), + // None, + // ) + // .await?; + // + // results.push(spec_res); + // } let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); let electron_path = electron_path.to_string_lossy(); @@ -76,6 +76,7 @@ pub async fn run_test() -> Result, anyhow::Error> { let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); + let mut final_result: Option = None; for spec in TEST_SPECS { let caption = format!("Test {}: {}", Target::Wrapper, spec); let spec_file_name = format!("session.{spec}.spec.js"); @@ -92,13 +93,16 @@ pub async fn run_test() -> Result, anyhow::Error> { ) .await?; - results.push(res); + match final_result.as_mut() { + Some(acc) => acc.append(res), + None => final_result = Some(res), + }; } - Ok(results) + final_result.ok_or_else(|| anyhow!("Wrapper doesn't have test specs")) } -pub async fn copy_binding_to_app() -> Result, anyhow::Error> { +pub async fn copy_binding_to_app() -> Result { let mut report_logs = Vec::new(); // *** Copying TS Bindings *** @@ -202,8 +206,8 @@ pub async fn copy_binding_to_app() -> Result, anyhow::Error> ) .await?; - Ok(Some(SpawnResult::create_for_fs( + Ok(SpawnResult::create_for_fs( "Copy TS Bindings and Platform to Electron".into(), report_logs, - ))) + )) } From 3cda884c07efdd2cde96c3718647c15657ff06cb Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 31 May 2024 13:53:05 +0200 Subject: [PATCH 115/174] CLI: Rewriting To Solve Dependencies upfront.. - Implement Jobs Resolver to resolve all dependencies for all jobs upfront - Write Unit tests for flatten and resolving jobs and targets - Change the order of the targets and job-type enums to match their dependencies order to be used on resolving dependencies algorithm - Ignore Warning while rewriting the app logic - App panics after printing the dependencies tree --- cli/src/job_type.rs | 25 +- cli/src/jobs_runner/job_definition.rs | 4 +- cli/src/jobs_runner/jobs_resolver.rs | 350 ++++++++++++++++++++++++++ cli/src/jobs_runner/mod.rs | 14 +- cli/src/main.rs | 8 + cli/src/target/mod.rs | 10 +- 6 files changed, 402 insertions(+), 9 deletions(-) create mode 100644 cli/src/jobs_runner/jobs_resolver.rs diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 12702ede5..b66b89c5c 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -1,11 +1,13 @@ use std::fmt::Display; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +//NOTE: The order of job types must match the runnig-order of them because it's used by +// solving their dependencies-graph using BTreeMap pub enum JobType { Lint, Clean, - Build { production: bool }, Install { production: bool }, + Build { production: bool }, AfterBuild { production: bool }, Test { production: bool }, Run { production: bool }, @@ -36,6 +38,27 @@ impl JobType { | JobType::Run { production } => Some(*production), } } + + /// Returns job types that are involved with this job and should run with it. + pub fn get_involved_jobs(&self) -> Vec { + match self { + JobType::Lint => vec![JobType::Install { production: false }], + JobType::Build { production } => vec![ + JobType::Install { + production: *production, + }, + JobType::AfterBuild { + production: *production, + }, + ], + JobType::Run { production } | JobType::Test { production } => vec![JobType::Build { + production: *production, + }], + JobType::Clean + | JobType::Install { production: _ } + | JobType::AfterBuild { production: _ } => Vec::new(), + } + } } #[cfg(test)] diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 6d9fd26dc..9671206a3 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -2,8 +2,8 @@ use crate::{job_type::JobType, spawner::SpawnResult, target::Target}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct JobDefinition { - target: Target, - job_type: JobType, + pub target: Target, + pub job_type: JobType, } impl JobDefinition { diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs new file mode 100644 index 000000000..19c3c1939 --- /dev/null +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -0,0 +1,350 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::{job_type::JobType, target::Target}; + +use super::JobDefinition; + +pub fn resolve( + targets: &[Target], + main_job: JobType, +) -> BTreeMap> { + let involved_jobs = flatten_jobs(main_job); + + let has_build_deps = involved_jobs + .iter() + .any(|job| matches!(job, JobType::Build { production: _ })); + + let involved_targets = if has_build_deps { + flatten_targets_for_build(targets) + } else { + BTreeSet::from_iter(targets.to_owned()) + }; + + let mut jobs_tree: BTreeMap> = BTreeMap::new(); + + for target in involved_targets { + for job in involved_jobs.iter().filter(|j| target.has_job(j)) { + // Start with dependencies from other targets (Applies for Build & Install jobs only) + // Install jobs are involved here too because copying the files in the after build + // process could delete the current files. + let mut dep_jobs = if matches!( + job, + JobType::Build { production: _ } | JobType::Install { production: _ } + ) { + let deps = flatten_targets_for_build(target.deps().as_slice()); + + jobs_tree + .keys() + .filter(|job_def| deps.contains(&job_def.target)) + .cloned() + .collect() + } else { + Vec::new() + }; + + // Add dependencies from the same target + // NOTE: This relays on that JobType enums are listed in the current order + dep_jobs.extend( + jobs_tree + .keys() + .filter(|job_d| job_d.target == target) + .cloned(), + ); + + let job_def = JobDefinition::new(target, *job); + + assert!( + jobs_tree.insert(job_def, dep_jobs).is_none(), + "JobDefinition is added to tree more than once. Target: {}, Job: {}", + target, + job + ); + } + } + + jobs_tree +} + +fn flatten_jobs(main_job: JobType) -> BTreeSet { + fn flatten_rec(job: JobType, involved_jobs: &mut BTreeSet) { + if !involved_jobs.insert(job) { + return; + } + for involved_job in job.get_involved_jobs() { + flatten_rec(involved_job, involved_jobs); + } + } + + let mut jobs = BTreeSet::new(); + + flatten_rec(main_job, &mut jobs); + + jobs +} + +fn flatten_targets_for_build(targets: &[Target]) -> BTreeSet { + fn flatten_rec(target: Target, involved_targets: &mut BTreeSet) { + if !involved_targets.insert(target) { + return; + } + for involved_target in target.deps() { + flatten_rec(involved_target, involved_targets); + } + } + + let mut resolved_targets = BTreeSet::new(); + + for target in targets { + flatten_rec(*target, &mut resolved_targets); + } + + resolved_targets +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flatten_clean_job() { + let expected_clean = BTreeSet::from([JobType::Clean]); + assert_eq!(flatten_jobs(JobType::Clean), expected_clean); + } + + #[test] + fn flatten_lint_job() { + let production = false; + let expected_lint = BTreeSet::from([JobType::Lint, JobType::Install { production }]); + assert_eq!(flatten_jobs(JobType::Lint), expected_lint); + } + + #[test] + fn flatten_install_job() { + let production = false; + let expected_install = BTreeSet::from([JobType::Install { production }]); + assert_eq!( + flatten_jobs(JobType::Install { production }), + expected_install + ); + } + + #[test] + fn flatten_build_job() { + let production = false; + let expected_build = BTreeSet::from([ + JobType::Build { production }, + JobType::Install { production }, + JobType::AfterBuild { production }, + ]); + assert_eq!(flatten_jobs(JobType::Build { production }), expected_build); + } + + #[test] + fn flatten_test_job() { + let production = false; + let expected_test = BTreeSet::from([ + JobType::Build { production }, + JobType::Install { production }, + JobType::AfterBuild { production }, + JobType::Test { production }, + ]); + assert_eq!(flatten_jobs(JobType::Test { production }), expected_test); + } + + #[test] + fn flatten_core_target() { + let expected = BTreeSet::from([Target::Core]); + assert_eq!(flatten_targets_for_build(&[Target::Core]), expected); + } + + #[test] + fn flatten_wrapper_target() { + let expected = BTreeSet::from([Target::Shared, Target::Binding, Target::Wrapper]); + assert_eq!(flatten_targets_for_build(&[Target::Wrapper]), expected); + } + + #[test] + fn flatten_app_target() { + let expected = BTreeSet::from([ + Target::Shared, + Target::Binding, + Target::Wrapper, + Target::Client, + Target::Wasm, + Target::App, + ]); + assert_eq!(flatten_targets_for_build(&[Target::App]), expected); + } + + #[test] + fn flatten_all_target() { + let expected = BTreeSet::from_iter(Target::all().to_owned()); + assert_eq!(flatten_targets_for_build(&Target::all()), expected); + } + + #[test] + fn flatten_core_client_target() { + let expected = + BTreeSet::from_iter([Target::Core, Target::Shared, Target::Wasm, Target::Client]); + assert_eq!( + flatten_targets_for_build(&[Target::Core, Target::Client]), + expected + ); + } + + #[test] + fn resolve_lint_core_cli() { + let expected = BTreeMap::from([ + (JobDefinition::new(Target::Core, JobType::Lint), Vec::new()), + (JobDefinition::new(Target::Cli, JobType::Lint), Vec::new()), + ]); + + assert_eq!( + expected, + resolve(&[Target::Core, Target::Cli], JobType::Lint) + ); + } + + #[test] + fn resolve_test_core() { + let production = false; + let expected = BTreeMap::from([ + ( + JobDefinition::new(Target::Core, JobType::Build { production }), + vec![], + ), + ( + JobDefinition::new(Target::Core, JobType::Test { production }), + vec![JobDefinition::new( + Target::Core, + JobType::Build { production: false }, + )], + ), + ]); + + assert_eq!( + expected, + resolve(&[Target::Core], JobType::Test { production }) + ); + } + + #[test] + fn resolve_build_binding() { + let production = false; + let expected = BTreeMap::from([ + ( + JobDefinition::new(Target::Shared, JobType::Install { production }), + vec![], + ), + ( + JobDefinition::new(Target::Shared, JobType::Build { production }), + vec![JobDefinition::new( + Target::Shared, + JobType::Install { production }, + )], + ), + ( + JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), + vec![ + JobDefinition::new(Target::Shared, JobType::Install { production }), + JobDefinition::new(Target::Shared, JobType::Build { production }), + ], + ), + ( + JobDefinition::new(Target::Binding, JobType::Install { production }), + vec![ + JobDefinition::new(Target::Shared, JobType::Install { production }), + JobDefinition::new(Target::Shared, JobType::Build { production }), + JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), + ], + ), + ( + JobDefinition::new(Target::Binding, JobType::Build { production }), + vec![ + JobDefinition::new(Target::Shared, JobType::Install { production }), + JobDefinition::new(Target::Shared, JobType::Build { production }), + JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), + JobDefinition::new(Target::Binding, JobType::Install { production }), + ], + ), + ( + JobDefinition::new(Target::Binding, JobType::AfterBuild { production }), + vec![ + JobDefinition::new(Target::Binding, JobType::Install { production }), + JobDefinition::new(Target::Binding, JobType::Build { production }), + ], + ), + ]); + + assert_eq!( + expected, + resolve(&[Target::Binding], JobType::Build { production }) + ); + } + + #[test] + /// Resolves build for all targets and checks some cases in the dependencies-tree since the + /// tree is too huge to be tested one by one. + fn resolve_build_all_fuzzy() { + let production = false; + + let tree = resolve(&Target::all(), JobType::Build { production }); + + assert!( + tree.get(&JobDefinition::new( + Target::Cli, + JobType::Build { production } + )) + .is_some_and(|dep| dep.is_empty()), + "Build CLI should have no dependencies" + ); + + assert!( + tree.get(&JobDefinition::new( + Target::App, + JobType::Build { production } + )) + .is_some_and(|dep| dep.contains(&JobDefinition::new( + Target::Shared, + JobType::Build { production } + ))), + "Build App should have dependency on shared build" + ); + + assert!( + tree.get(&JobDefinition::new( + Target::Wrapper, + JobType::Build { production } + )) + .is_some_and(|dep| dep.contains(&JobDefinition::new( + Target::Binding, + JobType::AfterBuild { production } + ))), + "Build Wrapper should have dependency on Binding AfterBuild" + ); + + assert!( + tree.get(&JobDefinition::new( + Target::Wrapper, + JobType::Build { production } + )) + .is_some_and(|dep| dep.contains(&JobDefinition::new( + Target::Binding, + JobType::AfterBuild { production } + ))), + "Build Wrapper should have dependency on Binding AfterBuild" + ); + + assert!( + tree.get(&JobDefinition::new( + Target::App, + JobType::Install { production } + )) + .is_some_and(|dep| dep.contains(&JobDefinition::new( + Target::Wasm, + JobType::Build { production } + ))), + "Install App should have dependency on Wasm Build" + ); + } +} diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 2de4305e9..04a57be69 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -1,10 +1,11 @@ mod job_definition; +mod jobs_resolver; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; pub use job_definition::JobDefinition; -use crate::{spawner::SpawnResult, target::Target}; +use crate::{job_type::JobType, spawner::SpawnResult, target::Target}; enum JobPhase { Awaiting(Vec), @@ -12,6 +13,7 @@ enum JobPhase { //TODO AAZ: Errors on Spawn calls should terminate the execution. Make sure results aren't // returned if a command fails or for expect reasons Done(SpawnResult), + Skipped, } struct JobState { @@ -24,3 +26,11 @@ pub struct JobsRunner { jobs: BTreeMap, resolved_targets: Vec, } + +impl JobsRunner { + pub fn print_deps(targets: &[Target], main_job: JobType) { + let jobs_tree = jobs_resolver::resolve(targets, main_job); + dbg!(jobs_tree); + todo!() + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 33010e849..84d505f2c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,6 @@ +//TODO AAZ: Remove this when done prototyping +#![allow(dead_code, unused_imports, unused)] + mod app_runner; mod build_state; mod checksum_records; @@ -21,6 +24,7 @@ use cli_args::{CargoCli, Command}; use dev_environment::{print_env_info, resolve_dev_tools}; use futures::future::join_all; use job_type::JobType; +use jobs_runner::JobsRunner; use location::init_location; use spawner::SpawnResult; use std::{ @@ -69,6 +73,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); + JobsRunner::print_deps(&targets, JobType::Lint); let results = join_all( targets .iter() @@ -86,6 +91,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); + JobsRunner::print_deps(&targets, JobType::Build { production }); let results = join_all( targets .iter() @@ -99,6 +105,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); + JobsRunner::print_deps(&targets, JobType::Clean); let results = join_all( targets .iter() @@ -116,6 +123,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); + JobsRunner::print_deps(&targets, JobType::Test { production }); let results = join_all( targets .iter() diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 0ee352d4a..401db57ac 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -31,23 +31,25 @@ mod wasm; mod wrapper; #[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] +//NOTE: The order of targets must match the running-order between them because it's used for +// solving their dependencies-graph using BTreeMap pub enum Target { /// Represents the path `application/apps/indexer` Core, + /// Represents the path `application/platform` + Shared, /// Represents the path `application/apps/rustcore/rs-bindings` Binding, /// Represents the path `application/apps/rustcore/ts-bindings` Wrapper, + /// Represents the path `application/apps/rustcore/wasm-bindings` + Wasm, /// Represents the path `application/client` Client, - /// Represents the path `application/platform` - Shared, /// Represents the path `application/holder` App, /// Represents the path `cli` Cli, - /// Represents the path `application/apps/rustcore/wasm-bindings` - Wasm, /// Represents the path `application/apps/precompiled/updater Updater, } From 362bec716e27560e415a4f09ceaee9e28bf6823a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 31 May 2024 14:41:32 +0200 Subject: [PATCH 116/174] CLI: Rewriting To Solve Dependencies upfront.. Jobs are run sequentially while in development to solve jobs logic at first before delving into concurrency --- cli/src/jobs_runner/job_definition.rs | 2 +- cli/src/jobs_runner/mod.rs | 36 ++++++++++++++++++-- cli/src/main.rs | 48 ++++----------------------- 3 files changed, 40 insertions(+), 46 deletions(-) diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 9671206a3..c870c3ca0 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -15,7 +15,7 @@ impl JobDefinition { format!("{} {}", self.target, self.job_type) } - async fn run(&self) -> Option> { + pub async fn run(&self) -> Option> { let res = match self.job_type { JobType::Lint => self.target.check().await, JobType::Build { production } => self.target.build(production).await, diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 04a57be69..d32254b03 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -28,9 +28,39 @@ pub struct JobsRunner { } impl JobsRunner { - pub fn print_deps(targets: &[Target], main_job: JobType) { + pub async fn run_jobs( + targets: &[Target], + main_job: JobType, + ) -> Vec> { let jobs_tree = jobs_resolver::resolve(targets, main_job); - dbg!(jobs_tree); - todo!() + + // This is needed for assertions while in development only, and it will be removed once the + // concurrent solution is implemented. + let mut finished = BTreeSet::new(); + + let mut results = Vec::new(); + + for (job_def, deps) in jobs_tree { + assert!( + deps.iter().all(|def| finished.contains(def)), + "Jobs deps must be resolved before running it" + ); + + let Some(res) = job_def.run().await else { + if cfg!(debug_assertions) { + panic!( + "Jobs tree should contain only runnable jobs. JobDefinition: {job_def:?}" + ); + } else { + continue; + } + }; + + results.push(res); + + finished.insert(job_def); + } + + results } } diff --git a/cli/src/main.rs b/cli/src/main.rs index 84d505f2c..dd735d779 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -73,14 +73,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - JobsRunner::print_deps(&targets, JobType::Lint); - let results = join_all( - targets - .iter() - .map(|module| module.check()) - .collect::>(), - ) - .await; + let results = JobsRunner::run_jobs(&targets, JobType::Lint).await; (JobType::Lint, results) } Command::Build { @@ -91,28 +84,14 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - JobsRunner::print_deps(&targets, JobType::Build { production }); - let results = join_all( - targets - .iter() - .map(|module| module.build(production)) - .collect::>(), - ) - .await; + let results = JobsRunner::run_jobs(&targets, JobType::Build { production }).await; (JobType::Build { production }, results) } Command::Clean { target, report } => { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - JobsRunner::print_deps(&targets, JobType::Clean); - let results = join_all( - targets - .iter() - .map(|module| module.reset()) - .collect::>(), - ) - .await; + let results = JobsRunner::run_jobs(&targets, JobType::Clean).await; (JobType::Clean, results) } Command::Test { @@ -123,28 +102,13 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - JobsRunner::print_deps(&targets, JobType::Test { production }); - let results = join_all( - targets - .iter() - .map(|module| module.test(production)) - .collect::>(), - ) - .await; - todo!() - //TODO AAZ: This will replaced with the new logic - // (JobType::Test { production }, results) + let results = JobsRunner::run_jobs(&targets, JobType::Test { production }).await; + (JobType::Test { production }, results) } Command::Run { production } => { resolve_dev_tools().await?; report_opt = ReportOptions::None; - let results = join_all( - Target::all() - .iter() - .map(|module| module.build(production)) - .collect::>(), - ) - .await; + let results = JobsRunner::run_jobs(&Target::all(), JobType::Build { production }).await; (JobType::Run { production }, results) } Command::ResetChecksum { production } => { From c9e09ff4bf1b9f3d557ad0dd4e54f65f6771ade4 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 31 May 2024 14:58:29 +0200 Subject: [PATCH 117/174] CLI: Rewriting To Solve Dependencies upfront.. - Remove TODOs from the code after making sure they are involved in the new logic - Small adjustments on the jobs to make them fit in the new system --- cli/src/target/mod.rs | 74 ++++----------------------------------- cli/src/target/wrapper.rs | 4 --- 2 files changed, 7 insertions(+), 71 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 401db57ac..d56948cbd 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -276,11 +276,6 @@ impl Target { debug_assert!(!test_cmds.is_empty()); - // TODO AAZ: Call build outside - // build method calls install - // let build_results = self.build(false).await?; - // results.extend(build_results); - let caption = format!("Test {}", self); let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { spawn( @@ -313,13 +308,7 @@ impl Target { /// Perform Linting Checks on the giving target pub async fn check(&self) -> Result { match self.kind() { - TargetKind::Ts => { - //TODO AAZ: Check needs install on TS - // if let Some(install_result) = self.install(false).await? { - // todo!() - // } - self.ts_lint().await - } + TargetKind::Ts => self.ts_lint().await, TargetKind::Rs => self.clippy().await, } } @@ -377,17 +366,6 @@ impl Target { let checksum = ChecksumRecords::get(JobType::Clean).await?; checksum.remove_hash_if_exist(*self)?; - self.clean().await - //TODO AAZ: - - // let dist_path = self.cwd().join("dist"); - // let remove_log = format!("removing {}", dist_path.display()); - // fstools::rm_folder(&dist_path).await?; - // - // Ok(results) - } - - async fn clean(&self) -> Result { let mut logs = Vec::new(); let path = match self.kind() { TargetKind::Ts => self.cwd().join("node_modules"), @@ -399,6 +377,11 @@ impl Target { fstools::rm_folder(&path).await?; + let dist_path = self.cwd().join("dist"); + let remove_log = format!("removing {}", dist_path.display()); + logs.push(remove_log); + fstools::rm_folder(&dist_path).await?; + let job = format!("Clean {}", self); Ok(SpawnResult::create_for_fs(job, logs)) @@ -409,20 +392,6 @@ impl Target { let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; checksum_rec.register_job(*self)?; - //TODO AAZ: Dependencies should be resolved before running the jobs - // let deps: Vec = self.deps(); - // for module in deps { - // let status = module.build(prod).await.with_context(|| { - // format!( - // "Error while building the dependciy {} for target {}", - // module, self - // ) - // })?; - // results.extend(status); - // if results.iter().any(|res| !res.status.success()) { - // return Ok(results); - // } - // } let path = get_root().join(self.cwd()); let cmd = self.build_cmd(prod).await?; let caption = format!("Build {}", self); @@ -442,41 +411,12 @@ impl Target { // } // - //TODO AAZ: Install Run outside of build - // if let Some(install_result) = self.install(false).await? { - // results.push(install_result); - // } let spawn_opt = SpawnOptions { has_skip_info: true, ..Default::default() }; - let status = spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await?; - if !status.status.success() { - Ok(status) - } else { - //TODO AAZ: This should be solved before running the tasks - // Taken from a discussion on GitHub: - // To build an npm package you would need (in most cases) to be in dev-mode - install dev-dependencies + dependencies. - // But to prepare a package for production, you have to remove dev-dependencies. - // That's not an issue, if npm-package is published in npmjs; but we are coping packages manually in a right destination - // and before copy it, we have to reinstall it to get rid of dev-dependencies. - // if matches!(self.kind(), TargetKind::Ts) && prod { - // let clean_res = self.clean().await?; - // results.push(clean_res); - // if let Some(install_res) = self.install(prod).await? { - // results.push(install_res); - // } - // } - - // TODO AAZ: After Build should be called separately - // let res = self.after_build(prod).await?; - // if let Some(result) = res { - // results.push(result); - // } - - Ok(status) - } + spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await } /// Performs build process without checking the current builds states diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 091e85f79..b98bf7de9 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -40,10 +40,6 @@ const TEST_SPECS: [&str; 14] = [ ]; pub async fn run_test() -> Result { - //TODO AAZ: Build should happen outside of test - // let build_results = Target::Wrapper.build(false).await?; - // results.extend(build_results); - let cwd = Target::Wrapper.cwd(); //TODO AAZ: Append build_spec_path to SpawnResult From 4468322a30bc0053ff8c96328cb97657742396df Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 3 Jun 2024 12:24:14 +0200 Subject: [PATCH 118/174] CLI: Rewriting To Solve Dependencies upfront.. - Rewrite the tracker changing its logic to receive all the jobs at first and set them all to pending, then changing their state to running when run is called - Each job definition will get one progress bar only. - fs tools and tests updates their corresponding progress bar and don't spawn a new one. - The call for starting the jobs and finishing them happens in one place in the code when run is called on the job definition. - Small refactoring on ts lint to remove duplicate call for get root. - Provide new method to get the relative root from the targets and remove the method to get the relative path from the absolute one. - Create run_intern method and call it in run method for job definition so it can be used in the unit tests and to provide a clean way to run the jobs without showing progress bar if that is needed for CI pipeline in the future. --- cli/src/fstools.rs | 56 ++++--- cli/src/jobs_runner/job_definition.rs | 40 ++++- cli/src/jobs_runner/mod.rs | 21 +-- cli/src/location.rs | 6 - cli/src/main.rs | 11 +- cli/src/spawner.rs | 95 +++--------- cli/src/target/app.rs | 9 +- cli/src/target/binding.rs | 12 +- cli/src/target/mod.rs | 82 ++++++---- cli/src/target/shared.rs | 9 +- cli/src/target/wrapper.rs | 21 ++- cli/src/tracker.rs | 215 ++++++++++++++++---------- 12 files changed, 325 insertions(+), 252 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 3cf83afe4..f0f70d7d7 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -6,23 +6,22 @@ use std::fmt::Display; use std::sync::mpsc; use std::{fs, path::PathBuf}; +use crate::jobs_runner::JobDefinition; use crate::tracker::get_tracker; -//TODO AAZ: Tools here shouldn't spawn new trackers. This should be able to send updates to their -// corresponding tracker only. - /// Spawn a job to copy a file, adding the info the report logs pub async fn cp_file( + job_def: JobDefinition, src: PathBuf, dest: PathBuf, report_logs: &mut Vec, ) -> Result<(), Error> { - let tracker = get_tracker().await; - let sequence = tracker.start("copy file").await?; - let msg = format!("copying file: '{}' to '{}'", src.display(), dest.display()); report_logs.push(msg); + let tracker = get_tracker().await; + tracker.msg(job_def, "copying files".into()).await; + fs::copy(&src, &dest).with_context(|| { format!( "Error while copying file '{}' to '{}'", @@ -31,9 +30,9 @@ pub async fn cp_file( ) })?; tracker - .success( - sequence, - &format!("copied: {} to {}", src.display(), dest.display()), + .msg( + job_def, + format!("copied: {} to {}", src.display(), dest.display()), ) .await; Ok(()) @@ -41,19 +40,21 @@ pub async fn cp_file( /// Spawn a job to copy a directory, adding the info the report logs pub async fn cp_folder( + job_def: JobDefinition, src: PathBuf, dest: PathBuf, report_logs: &mut Vec, ) -> Result<(), Error> { - let tracker = get_tracker().await; - let sequence = tracker.start("copy folder").await?; let options = CopyOptions::new(); let (tx, rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(); let path_display = format!("'{}' to '{}'", src.display(), dest.display()); let report_msg = format!("copying directory: {path_display}"); - report_logs.push(report_msg); + report_logs.push(report_msg.clone()); + + let tracker = get_tracker().await; + tracker.msg(job_def, report_msg).await; let _ = tokio::spawn(async move { copy_with_progress(src, dest, &options, |info| { @@ -68,31 +69,30 @@ pub async fn cp_folder( while let Ok(info) = rx.recv() { tracker .msg( - sequence, - &format!( + job_def, + format!( "copied: {} bytes; current: {}", info.copied_bytes, info.file_name ), ) .await; - tracker.progress(sequence, None).await; + tracker.progress(job_def, None).await; } let msg = format!("copied: {path_display}"); - tracker.success(sequence, &msg).await; + tracker.msg(job_def, msg).await; Ok(()) } /// Spawn a job to Copy a collection of files and folders recursively, adding copying info to the /// log records pub async fn cp_many( + job_def: JobDefinition, items: Vec, dest: PathBuf, general_source: impl Display, logs: &mut Vec, ) -> Result<(), Error> { - let tracker = get_tracker().await; - let sequence = tracker.start("copy file and folders").await?; let options = CopyOptions::new(); let (tx, rx) = mpsc::channel(); let path_display = format!("from '{}' to '{}'", general_source, dest.display()); @@ -103,6 +103,8 @@ pub async fn cp_many( .map(|item| format!("Item: '{}' copied to '{}'", item.display(), dest.display())), ); + let tracker = get_tracker().await; + tracker.msg(job_def, "copying files".into()).await; let _ = tokio::spawn(async move { copy_items_with_progress(&items, dest, &options, |info| { if tx.send(info).is_err() { @@ -116,31 +118,35 @@ pub async fn cp_many( while let Ok(info) = rx.recv() { tracker .msg( - sequence, - &format!( + job_def, + format!( "copied: {} bytes; current: {}", info.copied_bytes, info.file_name ), ) .await; - tracker.progress(sequence, None).await; + tracker.progress(job_def, None).await; } let msg = format!("copied files: {path_display}"); - tracker.success(sequence, &msg).await; + tracker.msg(job_def, msg).await; Ok(()) } /// Spawn a job to remove a directory recursively, adding the info the report logs -pub async fn rm_folder(path: &PathBuf) -> Result<(), Error> { +pub async fn rm_folder(job_def: JobDefinition, path: &PathBuf) -> Result<(), Error> { if !path.exists() { return Ok(()); } let tracker = get_tracker().await; - let sequence = tracker.start("remove folder").await?; + tracker + .msg(job_def, format!("removing directory: {}", path.display())) + .await; + fs::remove_dir_all(path)?; + tracker - .success(sequence, &format!("removed: {}", path.display(),)) + .msg(job_def, format!("removed: {}", path.display(),)) .await; Ok(()) } diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index c870c3ca0..4eafccacb 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -1,6 +1,6 @@ -use crate::{job_type::JobType, spawner::SpawnResult, target::Target}; +use crate::{job_type::JobType, spawner::SpawnResult, target::Target, tracker::get_tracker}; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct JobDefinition { pub target: Target, pub job_type: JobType, @@ -15,7 +15,41 @@ impl JobDefinition { format!("{} {}", self.target, self.job_type) } + /// Run the job definition if it has a job, communicating its status with the UI bars pub async fn run(&self) -> Option> { + let tracker = get_tracker().await; + if let Err(err) = tracker.start(*self).await { + return Some(Err(err)); + } + + let res = self.run_intern().await; + + match res.as_ref() { + Some(Ok(res)) => { + if res.status.success() { + if res.skipped.is_some_and(|skipped| skipped) { + tracker.success(*self, "skipped".into()).await; + } else { + tracker.success(*self, String::default()).await; + } + } else { + tracker.fail(*self, "finished with errors".into()).await; + } + } + Some(Err(err)) => { + tracker + .fail(*self, format!("finished with errors. {err}")) + .await + } + None => (), + } + + res + } + + #[inline] + /// Runs the job definition if it has a job + async fn run_intern(&self) -> Option> { let res = match self.job_type { JobType::Lint => self.target.check().await, JobType::Build { production } => self.target.build(production).await, @@ -43,7 +77,7 @@ mod tests { if !target.has_job(&job_type) { let job_def = JobDefinition::new(*target, job_type.clone()); assert!( - job_def.run().await.is_none(), + job_def.run_intern().await.is_none(), "'{}' has no job for '{}' but it returns Some when calling run", target, job_type diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index d32254b03..e6f852081 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -5,15 +5,16 @@ use std::collections::{BTreeMap, BTreeSet}; pub use job_definition::JobDefinition; -use crate::{job_type::JobType, spawner::SpawnResult, target::Target}; +use crate::{job_type::JobType, spawner::SpawnResult, target::Target, tracker::get_tracker}; + +use anyhow::Result; + +type SpawnResultsCollection = Vec>; enum JobPhase { Awaiting(Vec), Running, - //TODO AAZ: Errors on Spawn calls should terminate the execution. Make sure results aren't - // returned if a command fails or for expect reasons Done(SpawnResult), - Skipped, } struct JobState { @@ -28,12 +29,14 @@ pub struct JobsRunner { } impl JobsRunner { - pub async fn run_jobs( - targets: &[Target], - main_job: JobType, - ) -> Vec> { + pub async fn run_jobs(targets: &[Target], main_job: JobType) -> Result { let jobs_tree = jobs_resolver::resolve(targets, main_job); + let tracker = get_tracker().await; + tracker + .start_all(jobs_tree.keys().cloned().collect()) + .await?; + // This is needed for assertions while in development only, and it will be removed once the // concurrent solution is implemented. let mut finished = BTreeSet::new(); @@ -61,6 +64,6 @@ impl JobsRunner { finished.insert(job_def); } - results + Ok(results) } } diff --git a/cli/src/location.rs b/cli/src/location.rs index bd000beb8..5d596ad2b 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -51,9 +51,3 @@ pub fn init_location() -> Result<(), Error> { .expect("Developer Error: init location can't be called more than once"); Ok(()) } - -/// Returns the path relative to the repository root -pub fn to_relative_path(path: &PathBuf) -> &Path { - let root = get_root(); - path.strip_prefix(root).unwrap_or(path) -} diff --git a/cli/src/main.rs b/cli/src/main.rs index dd735d779..a6b593cd4 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -73,7 +73,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Lint).await; + let results = JobsRunner::run_jobs(&targets, JobType::Lint).await?; (JobType::Lint, results) } Command::Build { @@ -84,14 +84,14 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Build { production }).await; + let results = JobsRunner::run_jobs(&targets, JobType::Build { production }).await?; (JobType::Build { production }, results) } Command::Clean { target, report } => { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Clean).await; + let results = JobsRunner::run_jobs(&targets, JobType::Clean).await?; (JobType::Clean, results) } Command::Test { @@ -102,13 +102,14 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Test { production }).await; + let results = JobsRunner::run_jobs(&targets, JobType::Test { production }).await?; (JobType::Test { production }, results) } Command::Run { production } => { resolve_dev_tools().await?; report_opt = ReportOptions::None; - let results = JobsRunner::run_jobs(&Target::all(), JobType::Build { production }).await; + let results = + JobsRunner::run_jobs(&Target::all(), JobType::Build { production }).await?; (JobType::Run { production }, results) } Command::ResetChecksum { production } => { diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 2022800e8..62fbbfe86 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,8 +1,5 @@ // cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); -use crate::{ - location::{get_root, to_relative_path}, - tracker::get_tracker, -}; +use crate::{jobs_runner::JobDefinition, location::get_root, tracker::get_tracker}; use anyhow::{bail, Context}; use core::panic; use futures_lite::{future, FutureExt}; @@ -84,9 +81,9 @@ pub(crate) struct SpawnOptions { /// Spawns and runs a job asynchronously, updating the bar when job infos are available pub async fn spawn( + job_def: JobDefinition, command: String, cwd: Option, - caption: String, environment_vars: impl IntoIterator, opts: Option, ) -> Result { @@ -98,13 +95,6 @@ pub async fn spawn( combined_env_vars.extend(environment_vars); let tracker = get_tracker().await; - let sequence = tracker - .start(&format!( - "{}: {}", - to_relative_path(&cwd).display(), - caption - )) - .await?; let command_result = Command::new(cmd) .current_dir(&cwd) @@ -117,13 +107,7 @@ pub async fn spawn( format!("Error While running the command '{cmd}'\nwith arguments: {parts:?}") }); - let mut child = match command_result { - Ok(child) => child, - Err(err) => { - tracker.fail(sequence, format!("{err:#}").as_str()).await; - return Err(err); - } - }; + let mut child = command_result?; let mut report_lines: Vec = vec![]; let drain_stdout_stderr = { @@ -147,9 +131,9 @@ pub async fn spawn( break; } else { if !opts.suppress_msg { - tracker.msg(sequence, &stdout_line).await; + tracker.msg(job_def, stdout_line.clone()).await; } - tracker.progress(sequence, None).await; + tracker.progress(job_def, None).await; storage_report.push(stdout_line); } } @@ -158,7 +142,7 @@ pub async fn spawn( if stderr_read_bytes == 0 { break; } else { - tracker.progress(sequence, None).await; + tracker.progress(job_def, None).await; if !stderr_line.trim().is_empty() { storage_report.push(stderr_line); } @@ -173,23 +157,11 @@ pub async fn spawn( } }; - let status = match drain_stdout_stderr + let status = drain_stdout_stderr .or(async move { Ok(Some(child.wait().await?)) }) - .await - { - Ok(status) => status, - Err(err) => { - tracker.fail(sequence, &err.to_string()).await; - return Err(err); - } - }; - if let Some(status) = status { - if status.success() { - tracker.success(sequence, "").await; - } else { - tracker.fail(sequence, "finished with errors").await; - } + .await?; + if let Some(status) = status { let skipped = if opts.has_skip_info { Some(false) } else { @@ -199,14 +171,11 @@ pub async fn spawn( Ok(SpawnResult { report: report_lines, status, - job: caption, + job: job_def.job_title(), cmd: command, skipped, }) } else { - tracker - .fail(sequence, "Fail to get exist status of spawned command") - .await; bail!("Fail to get exist status of spawned command") } } @@ -214,9 +183,9 @@ pub async fn spawn( /// Suspend the progress bars and run the giving blocking command using `Stdio::inherit()` /// This is used with commands that doesn't work with `Stdio::piped()` pub async fn spawn_blocking( + job_def: JobDefinition, command: String, cwd: Option, - caption: String, environment_vars: impl IntoIterator, ) -> Result { let cwd = cwd.unwrap_or_else(|| get_root().clone()); @@ -233,32 +202,12 @@ pub async fn spawn_blocking( let tracker = get_tracker().await; - let sequence = tracker - .start(&format!( - "{}: {}", - to_relative_path(&cwd).display(), - caption - )) - .await?; - - let status = match tracker.suspend_and_run(child).await { - Ok(status) => status, - Err(err) => { - tracker.fail(sequence, &err.to_string()).await; - bail!("Error While running the command '{cmd}'\nwith arguments: {parts:?}\ncwd: {}\n Error Info: {err}", cwd.display()); - } - }; - - if status.success() { - tracker.success(sequence, "").await; - } else { - tracker.fail(sequence, "finished with errors").await; - } + let status = tracker.suspend_and_run(child).await?; Ok(SpawnResult { report: Vec::new(), status, - job: caption, + job: job_def.job_title(), cmd: command, skipped: None, }) @@ -266,22 +215,14 @@ pub async fn spawn_blocking( /// This spawns a new task and return immediately showing that the job has been skipped pub async fn spawn_skip( + job_def: JobDefinition, command: String, cwd: Option, - caption: String, ) -> anyhow::Result { let cwd = cwd.unwrap_or_else(|| get_root().clone()); - let tracker = get_tracker().await; - let sequence = tracker - .start(&format!( - "{}: {}", - to_relative_path(&cwd).display(), - caption - )) - .await?; - - tracker.success(sequence, "skipped").await; - - Ok(SpawnResult::create_for_skipped(caption, command)) + Ok(SpawnResult::create_for_skipped( + job_def.job_title(), + command, + )) } diff --git a/cli/src/target/app.rs b/cli/src/target/app.rs index 1ea0c5f17..69f31e713 100644 --- a/cli/src/target/app.rs +++ b/cli/src/target/app.rs @@ -2,12 +2,13 @@ use std::fs; use anyhow::{bail, Context}; -use crate::{fstools, spawner::SpawnResult}; +use crate::{fstools, job_type::JobType, jobs_runner::JobDefinition, spawner::SpawnResult}; use super::{client::get_dist_path, Target}; -pub async fn copy_client_to_app(prod: bool) -> Result { +pub async fn copy_client_to_app(job_def: JobDefinition) -> Result { let mut report_logs = Vec::new(); + let prod = job_def.job_type.is_production().unwrap_or(false); let src = get_dist_path(prod); let dest = Target::App.cwd().join("dist"); if !src.exists() { @@ -25,10 +26,10 @@ pub async fn copy_client_to_app(prod: bool) -> Result anyhow::Result { )) } -pub async fn copy_index_node() -> Result { +pub async fn copy_index_node(job_def: JobDefinition) -> Result { let mut report_logs = Vec::new(); // *** Copy `index.node` from rs to ts bindings dist *** @@ -56,6 +61,7 @@ pub async fn copy_index_node() -> Result { } fstools::cp_file( + job_def, src_file.clone(), ts_dist_native_dir.join("index.node"), &mut report_logs, @@ -70,7 +76,7 @@ pub async fn copy_index_node() -> Result { let dir_tests = Target::Wrapper.cwd().join("src").join("native"); let mod_file = dir_tests.join("index.node"); - fstools::cp_file(src_file, mod_file, &mut report_logs).await?; + fstools::cp_file(job_def, src_file, mod_file, &mut report_logs).await?; Ok(SpawnResult::create_for_fs( "Copying `index.node` from rs to ts bindings".into(), diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index d56948cbd..e7006f7bb 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -13,8 +13,10 @@ use crate::{ dev_tools::DevTool, fstools, job_type::JobType, + jobs_runner::JobDefinition, location::get_root, spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, + tracker::{self, get_tracker}, }; use target_kind::TargetKind; @@ -161,6 +163,12 @@ impl Target { /// Provides the absolute path to the target code pub fn cwd(&self) -> PathBuf { let root = get_root(); + let relative_path = self.relative_cwd(); + + root.join(relative_path) + } + + pub fn relative_cwd(&self) -> PathBuf { let sub_parts = match self { Target::Core => ["application", "apps", "indexer"].iter(), Target::Binding => ["application", "apps", "rustcore", "rs-bindings"].iter(), @@ -173,9 +181,7 @@ impl Target { Target::Updater => ["application", "apps", "precompiled", "updater"].iter(), }; - let sub_path: PathBuf = sub_parts.collect(); - - root.join(sub_path) + sub_parts.collect() } /// Provide the kind of the target between Rust or Type-Script @@ -241,18 +247,28 @@ impl Target { // wrapper (ts-bindings) install in the rs bindings install. // Since rs bindings is a dependency for ts bindings, we don't need to call to install // on ts bindings again. - Target::Binding => install_general(&Target::Wrapper, prod).await, + Target::Binding => { + install_general( + &Target::Wrapper, + prod, + Some(JobDefinition::new( + Target::Binding, + JobType::Install { production: prod }, + )), + ) + .await + } Target::Wrapper => None, // For app we don't need --production - Target::App => install_general(&Target::App, false).await, - rest_targets => install_general(rest_targets, prod).await, + Target::App => install_general(&Target::App, false, None).await, + rest_targets => install_general(rest_targets, prod, None).await, } } /// Run tests for the giving the target pub async fn test(&self, production: bool) -> Option> { match self { - Target::Wrapper => Some(wrapper::run_test().await), + Target::Wrapper => Some(wrapper::run_test(production).await), rest_targets => rest_targets.run_test_general(production).await, } } @@ -276,12 +292,12 @@ impl Target { debug_assert!(!test_cmds.is_empty()); - let caption = format!("Test {}", self); + let job_def = JobDefinition::new(*self, JobType::Test { production }); let spawn_results = join_all(test_cmds.into_iter().map(|cmd| { spawn( + job_def, cmd.command, Some(cmd.cwd), - caption.clone(), iter::empty(), cmd.spawn_opts, ) @@ -316,13 +332,13 @@ impl Target { /// Perform Linting the Building the giving target since linting Type-Script doesn't check for /// compiling errors async fn ts_lint(&self) -> Result { - let path = get_root().join(self.cwd()); - let caption = format!("TS Lint {}", self); + let path = self.cwd(); let yarn_cmd = DevTool::Yarn.path().await.to_string_lossy(); + let job_def = JobDefinition::new(*self, JobType::Lint); let status = spawn( + job_def, format!("{} run lint", yarn_cmd), Some(path.clone()), - caption, iter::empty(), None, ) @@ -331,11 +347,10 @@ impl Target { return Ok(status); } - let caption = format!("Build {}", self); spawn( + job_def, format!("{} run build", yarn_cmd), Some(path), - caption, iter::empty(), None, ) @@ -347,14 +362,14 @@ impl Target { let path = get_root().join(self.cwd()); let cargo_path = DevTool::Cargo.path().await; - let caption = format!("Clippy {}", self); + let job_def = JobDefinition::new(*self, JobType::Lint); spawn( + job_def, format!( "{} clippy --color always --all --all-features -- -D warnings", cargo_path.to_string_lossy() ), Some(path), - caption, iter::empty(), None, ) @@ -363,6 +378,9 @@ impl Target { /// Clean the given target, removing it from the checksum tracker as well. pub async fn reset(&self) -> anyhow::Result { + let job_def = JobDefinition::new(*self, JobType::Clean); + let tracker = get_tracker().await; + let checksum = ChecksumRecords::get(JobType::Clean).await?; checksum.remove_hash_if_exist(*self)?; @@ -375,12 +393,12 @@ impl Target { let remove_log = format!("removing directory {}", path.display()); logs.push(remove_log); - fstools::rm_folder(&path).await?; + fstools::rm_folder(job_def, &path).await?; let dist_path = self.cwd().join("dist"); let remove_log = format!("removing {}", dist_path.display()); logs.push(remove_log); - fstools::rm_folder(&dist_path).await?; + fstools::rm_folder(job_def, &dist_path).await?; let job = format!("Clean {}", self); @@ -394,7 +412,6 @@ impl Target { let path = get_root().join(self.cwd()); let cmd = self.build_cmd(prod).await?; - let caption = format!("Build {}", self); //TODO AAZ: Skipping jobs should be resolved before running the jobs // let mut skip_task = false; @@ -416,18 +433,21 @@ impl Target { ..Default::default() }; - spawn(cmd, Some(path), caption, iter::empty(), Some(spawn_opt)).await + let job_def = JobDefinition::new(*self, JobType::Build { production: prod }); + + spawn(job_def, cmd, Some(path), iter::empty(), Some(spawn_opt)).await } /// Performs build process without checking the current builds states /// Perform any needed copy operation after the build is done pub async fn after_build(&self, prod: bool) -> Option> { + let job_def = JobDefinition::new(*self, JobType::AfterBuild { production: prod }); let res = match self { - Target::Binding => binding::copy_index_node().await, - Target::Wrapper => wrapper::copy_binding_to_app().await, - Target::Shared => shared::copy_platform_to_binding().await, - Target::App => app::copy_client_to_app(prod).await, + Target::Binding => binding::copy_index_node(job_def).await, + Target::Wrapper => wrapper::copy_binding_to_app(job_def).await, + Target::Shared => shared::copy_platform_to_binding(job_def).await, + Target::App => app::copy_client_to_app(job_def).await, _ => return None, }; @@ -436,14 +456,24 @@ impl Target { } /// run install using the general routine for the given target +/// * `target`: job target to perform its after build jobs +/// * `prod`: build for production +/// * `overridden_job_def`: override job definition to update tracker message. This is used when +/// target is used for another install command which happens currently with bindings and wrapper +/// targets async fn install_general( target: &Target, prod: bool, + overridden_job_def: Option, ) -> Option> { let cmd = target.kind().install_cmd(prod).await; + let job_def = overridden_job_def.unwrap_or(JobDefinition::new( + *target, + JobType::Install { production: prod }, + )); + if let Some(cmd) = cmd { - let caption = format!("Install {}", target); - let res = spawn(cmd, Some(target.cwd()), caption, iter::empty(), None).await; + let res = spawn(job_def, cmd, Some(target.cwd()), iter::empty(), None).await; Some(res) } else { None diff --git a/cli/src/target/shared.rs b/cli/src/target/shared.rs index 67c865379..aac8f71c0 100644 --- a/cli/src/target/shared.rs +++ b/cli/src/target/shared.rs @@ -2,11 +2,13 @@ use std::fs; use anyhow::Context; -use crate::{fstools, spawner::SpawnResult}; +use crate::{fstools, jobs_runner::JobDefinition, spawner::SpawnResult}; use super::Target; -pub async fn copy_platform_to_binding() -> Result { +pub async fn copy_platform_to_binding( + job_def: JobDefinition, +) -> Result { let mut report_logs = Vec::new(); report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); @@ -16,7 +18,7 @@ pub async fn copy_platform_to_binding() -> Result { let msg = format!("Removing directory: '{}'", platform_dest.display()); report_logs.push(msg); - fstools::rm_folder(&platform_dest).await?; + fstools::rm_folder(job_def, &platform_dest).await?; tokio::fs::create_dir_all(&platform_dest) .await @@ -40,6 +42,7 @@ pub async fn copy_platform_to_binding() -> Result { .collect(); fstools::cp_many( + job_def, entries_to_copy, platform_dest, source.display(), diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index b98bf7de9..d9ce909e1 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -5,6 +5,8 @@ use std::fs; use crate::{ fstools, + job_type::JobType, + jobs_runner::JobDefinition, spawner::{spawn, spawn_blocking, SpawnResult}, }; @@ -39,7 +41,7 @@ const TEST_SPECS: [&str; 14] = [ "promises", ]; -pub async fn run_test() -> Result { +pub async fn run_test(production: bool) -> Result { let cwd = Target::Wrapper.cwd(); //TODO AAZ: Append build_spec_path to SpawnResult @@ -73,6 +75,8 @@ pub async fn run_test() -> Result { let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); let mut final_result: Option = None; + + let job_def = JobDefinition::new(Target::Binding, JobType::Test { production }); for spec in TEST_SPECS { let caption = format!("Test {}: {}", Target::Wrapper, spec); let spec_file_name = format!("session.{spec}.spec.js"); @@ -82,9 +86,9 @@ pub async fn run_test() -> Result { spec_file_path.to_string_lossy() ); let res = spawn_blocking( + job_def, command, Some(cwd.clone()), - caption.clone(), vec![(String::from("ELECTRON_RUN_AS_NODE"), String::from("1"))], ) .await?; @@ -98,14 +102,14 @@ pub async fn run_test() -> Result { final_result.ok_or_else(|| anyhow!("Wrapper doesn't have test specs")) } -pub async fn copy_binding_to_app() -> Result { +pub async fn copy_binding_to_app(job_def: JobDefinition) -> Result { let mut report_logs = Vec::new(); // *** Copying TS Bindings *** report_logs.push(String::from("Copying ts-bindings to electron...")); let rustcore_dest = Target::App.cwd().join("node_modules").join("rustcore"); - fstools::rm_folder(&rustcore_dest).await?; + fstools::rm_folder(job_def, &rustcore_dest).await?; let msg = format!("Removing directory: '{}'", rustcore_dest.display()); report_logs.push(msg); @@ -130,6 +134,7 @@ pub async fn copy_binding_to_app() -> Result { .collect(); fstools::cp_many( + job_def, ts_entries_to_copy, rustcore_dest.clone(), ts_source.display(), @@ -144,13 +149,13 @@ pub async fn copy_binding_to_app() -> Result { native_dir_path.display() )); - fstools::rm_folder(&native_dir_path).await?; + fstools::rm_folder(job_def, &native_dir_path).await?; // *** Copy Platform rustcore to electron *** report_logs.push(String::from("Copying platform rustcore in to electron...")); let platform_dest = rustcore_dest.join("node_modules").join("platform"); - fstools::rm_folder(&platform_dest).await?; + fstools::rm_folder(job_def, &platform_dest).await?; fs::create_dir_all(&platform_dest).with_context(|| { format!( "Error while creating directory: {}", @@ -175,6 +180,7 @@ pub async fn copy_binding_to_app() -> Result { .collect(); fstools::cp_many( + job_def, platform_entries_to_copy.clone(), platform_dest, platform_src.display(), @@ -186,7 +192,7 @@ pub async fn copy_binding_to_app() -> Result { report_logs.push(String::from("Copying platform in to electron...")); let platform_dest2 = Target::App.cwd().join("node_modules").join("platform"); - fstools::rm_folder(&platform_dest2).await?; + fstools::rm_folder(job_def, &platform_dest2).await?; fs::create_dir_all(&platform_dest2).with_context(|| { format!( "Error while creating directory: {}", @@ -195,6 +201,7 @@ pub async fn copy_binding_to_app() -> Result { })?; fstools::cp_many( + job_def, platform_entries_to_copy, platform_dest2, platform_src.display(), diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 16669ffe6..8cb806e15 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Context, Error}; use console::style; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use std::{ + collections::BTreeMap, process::{Command, ExitStatus}, time::Instant, }; @@ -10,6 +11,8 @@ use tokio::sync::{ oneshot, OnceCell, }; +use crate::jobs_runner::JobDefinition; + const TIME_BAR_WIDTH: usize = 5; #[derive(Clone, Debug)] @@ -34,14 +37,16 @@ impl std::fmt::Display for OperationResult { #[derive(Debug)] /// Represents tasks information that can be sent to and from the tracker pub enum Tick { + /// Change job status from awaiting to started. + Started(JobDefinition, oneshot::Sender<()>), /// Start a job giving the job name and the sender to return the job number. - Started(String, oneshot::Sender), - /// Update the job with the given id providing an optional progress value. - Progress(usize, Option), - /// Send a message to the job with the giving id - Message(usize, String), - /// Sets the job with the given id as finished providing the job result and a message - Finished(usize, OperationResult, String), + StartAll(Vec, oneshot::Sender<()>), + /// Update the job providing an optional progress value. + Progress(JobDefinition, Option), + /// Send a message to the job + Message(JobDefinition, String), + /// Sets the job as finished providing the job result and a message + Finished(JobDefinition, OperationResult, String), #[allow(dead_code)] /// Prints the given text outside the progress bar Print(String), @@ -56,24 +61,36 @@ pub struct Tracker { tx: UnboundedSender, } +enum JobBarPhase { + Pending, + Running(Instant), + Finished((OperationResult, u64)), +} + struct JobBarState { name: String, bar: ProgressBar, - start_time: Instant, - result: Option<(OperationResult, u64)>, + phase: JobBarPhase, } impl JobBarState { - fn start_job(name: String, bar: ProgressBar) -> Self { - let start_time = Instant::now(); - + fn new(name: String, bar: ProgressBar) -> Self { Self { name, bar, - start_time, - result: None, + phase: JobBarPhase::Pending, } } + + fn start(&mut self) { + assert!( + matches!(self.phase, JobBarPhase::Pending), + "Start can be called on pending jobs only" + ); + + let start_time = Instant::now(); + self.phase = JobBarPhase::Running(start_time); + } } pub async fn get_tracker() -> &'static Tracker { @@ -96,71 +113,94 @@ impl Tracker { async move { let mut max_time_len = 0; let max = u64::MAX; - let mut bars: Vec = Vec::new(); + let mut bars: BTreeMap = BTreeMap::new(); let mp = MultiProgress::new(); let start_time = Instant::now(); while let Some(tick) = rx.recv().await { match tick { - Tick::Started(job, tx_response) => { - let bar = mp.add(ProgressBar::new(max)); - bar.set_style(spinner_style.clone()); - let job_bar = JobBarState::start_job(job, bar); - bars.push(job_bar); + Tick::Started(job_def, tx_response) => { + let Some(job) = bars.get_mut(&job_def) else { + unreachable!("Job must exist in progress bar before starting it. Job Info: {job_def:?}") + }; + if matches!(job.phase, JobBarPhase::Pending){ + job.start(); + Self::refresh_all_bars(&mut bars, max_time_len, None); + } + + if let Err(_) = tx_response.send(()) { + let _ = mp.println(format!("Fail to send response while starting the jobs")); + } + } + Tick::StartAll(jobs, tx_response ) => { + for job in jobs.into_iter() { + let bar = mp.add(ProgressBar::new(max)); + bar.set_style(spinner_style.clone()); + let bar_text = format!("{}: {}", job.target.relative_cwd().display(), job.job_title()); + let job_bar = JobBarState::new(bar_text, bar); + bars.insert(job, job_bar); + } + Self::refresh_all_bars(&mut bars, max_time_len, None); - let job_number = bars.len(); - if let Err(e) = tx_response.send(job_number) { - let _ = mp.println(format!("Fail to send response: {e}")); + if let Err(_) = tx_response.send(()) { + let _ = mp.println(format!("Fail to send response while starting the jobs")); } } - Tick::Message(job_number, log) => { - let idx = job_number.checked_sub(1).expect("Job number can't be zero"); - if let Some(job_bar) = bars.get(idx) { - job_bar.bar.set_message(log); + Tick::Message(job_def, log) => { + match bars.get(&job_def) { + Some(job_bar) => job_bar.bar.set_message(log), + None => unreachable!("Job must exist in progress bar before messaging it. Job Info: {job_def:?}"), } + } - Tick::Progress(job_number, pos) => { - let idx = job_number.checked_sub(1).expect("Job number can't be zero"); - if let Some(job_bar) = bars.get(idx) { - if let Some(pos) = pos { - job_bar.bar.set_position(pos); - } else { - job_bar.bar.inc(1); - } + Tick::Progress(job_def, pos) => { + let Some(job_bar) = bars.get(&job_def) else { + unreachable!("Job must exist in progress bar before changing it progress. Job Info: {job_def:?}") + }; + + if let Some(pos) = pos { + job_bar.bar.set_position(pos); + } else { + job_bar.bar.inc(1); } } - Tick::Finished(job_number, result, msg) => { - let idx = job_number.checked_sub(1).expect("Job number can't be zero"); + Tick::Finished(job_def, result, msg) => { let jobs_count_txt = bars.len().to_string(); - if let Some(job_bar) = bars.get_mut(idx) { - // It doesn't make sense to show that a job is done in 0 seconds - let time = job_bar.start_time.elapsed().as_secs().max(1); + let Some(job_bar) = bars.get_mut(&job_def)else { + unreachable!("Job must exist in progress bar before finishing it. Job Info: {job_def:?}") + }; - max_time_len = max_time_len.max(Self::count_digits(time)); + // It doesn't make sense to show that a job is done in 0 seconds + let time = match job_bar.phase { + JobBarPhase::Running(start_time) => start_time.elapsed().as_secs().max(1), + _ => unreachable!("Job must be running when finish is called"), + }; - let seq_width = jobs_count_txt.len(); - let job = job_bar.name.as_str(); - job_bar.bar.set_prefix(format!( - "[{job_number:seq_width$}/{jobs_count_txt}][{result}][{time:max_time_len$}s][{job}].", - )); - job_bar.bar.finish_with_message(msg); - job_bar.result.replace((result, time)); + max_time_len = max_time_len.max(Self::count_digits(time)); - Self::refresh_all_bars(&mut bars, max_time_len, None); - } + let seq_width = jobs_count_txt.len(); + let job = job_bar.name.as_str(); + job_bar.bar.finish_with_message(msg); + job_bar.phase = JobBarPhase::Finished((result, time)); + + Self::refresh_all_bars(&mut bars, max_time_len, None); } Tick::Print(msg) => { let _ = mp.println(msg); } Tick::Shutdown(tx_response) => { - bars.iter_mut().for_each(|job_bar| { - if !job_bar.bar.is_finished() { - let time = job_bar.start_time.elapsed().as_secs().max(1); - job_bar.result.replace((OperationResult::Success, time)); - max_time_len = max_time_len.max(Self::count_digits(time)); + // Finish jobs that are still running + for (_job_def, job_bar) in bars.iter_mut(){ + let time = match job_bar.phase{ + JobBarPhase::Pending => 1 , + JobBarPhase::Running(start_time) => start_time.elapsed().as_secs().max(1), + JobBarPhase::Finished(_) => continue , + }; + + job_bar.phase = JobBarPhase::Finished((OperationResult::Failed, time)); + max_time_len = max_time_len.max(Self::count_digits(time)); - job_bar.bar.finish(); - } - }); + job_bar.bar.finish(); + } // Insert graphic bar for the running duration of each bars let total_time = start_time.elapsed().as_secs().max(1) as usize; @@ -193,21 +233,20 @@ impl Tracker { } fn refresh_all_bars( - bars: &mut Vec, + bars: &mut BTreeMap, max_time_len: usize, total_time: Option, ) { let jobs_count_txt = bars.len().to_string(); - bars.iter_mut().enumerate().for_each(|(idx, job_bar)| { + bars.iter_mut().enumerate().for_each(|(idx, (_job_def ,job_bar))| { let job_number = idx + 1; let seq_width = jobs_count_txt.len(); let job = job_bar.name.as_str(); - let line_prefix = match job_bar.result.as_ref() { - None => { - format!("[{job_number:seq_width$}/{jobs_count_txt}][....][{job}]") - } - Some((res, time)) => { + let line_prefix = match &job_bar.phase { + JobBarPhase::Pending => format!("[{job_number:seq_width$}/{jobs_count_txt}][{}][{job}]", style("wait").bold().blue()), + JobBarPhase::Running(_) => format!("[{job_number:seq_width$}/{jobs_count_txt}][....][{job}]"), + JobBarPhase::Finished((res, time)) => { if let Some(total_time) = total_time { let finish_limit = (*time as usize * TIME_BAR_WIDTH) / total_time; let time_bar: String = (0..TIME_BAR_WIDTH).map(|idx| if idx <= finish_limit {'█'}else {'░'}).collect(); @@ -215,7 +254,7 @@ impl Tracker { }else { format!("[{job_number:seq_width$}/{jobs_count_txt}][{res}][{time:max_time_len$}s][{job}].") } - } + }, }; job_bar.bar.set_prefix(line_prefix); @@ -236,46 +275,54 @@ impl Tracker { count } - /// Start a job giving the job name and the sender to return the job number. - pub async fn start(&self, job: &str) -> Result { + pub async fn start_all(&self, jobs: Vec) -> Result<(), Error> { let (tx_response, rx_response) = oneshot::channel(); self.tx - .send(Tick::Started(job.to_string(), tx_response)) + .send(Tick::StartAll(jobs, tx_response)) .context("Fail to send tick")?; - rx_response.await.context("Fail to receive tick") + rx_response.await.context("Fail to receive tick start all") + } + + /// Change job status from awaiting to started. + pub async fn start(&self, job_def: JobDefinition) -> Result<(), Error> { + let (tx_response, rx_response) = oneshot::channel(); + self.tx + .send(Tick::Started(job_def, tx_response)) + .context("Fail to send tick")?; + rx_response.await.context("Fail to receive tick Start Single") } - /// Update the job with the given id providing an optional progress value. - pub async fn progress(&self, job_number: usize, pos: Option) { - if let Err(e) = self.tx.send(Tick::Progress(job_number, pos)) { + /// Update the job providing an optional progress value. + pub async fn progress(&self, job_def: JobDefinition, pos: Option) { + if let Err(e) = self.tx.send(Tick::Progress(job_def, pos)) { eprintln!("Fail to communicate with tracker: {e}"); } } - /// Send a message to the job with the giving id - pub async fn msg(&self, job_number: usize, log: &str) { - if let Err(e) = self.tx.send(Tick::Message(job_number, log.to_string())) { + /// Send a message to the job + pub async fn msg(&self, job_def: JobDefinition, log: String) { + if let Err(e) = self.tx.send(Tick::Message(job_def, log)) { eprintln!("Fail to communicate with tracker: {e}"); } } - /// Sets the job with the given id as finished providing successful result and a message - pub async fn success(&self, job_number: usize, msg: &str) { + /// Sets the job as finished providing successful result and a message + pub async fn success(&self, job_def: JobDefinition, msg: String) { if let Err(e) = self.tx.send(Tick::Finished( - job_number, + job_def, OperationResult::Success, - msg.to_string(), + msg, )) { eprintln!("Fail to communicate with tracker: {e}"); } } - /// Sets the job with the given id as finished providing failed result and a message - pub async fn fail(&self, job_number: usize, msg: &str) { + /// Sets the job as finished providing failed result and a message + pub async fn fail(&self, job_def: JobDefinition, msg: String) { if let Err(e) = self.tx.send(Tick::Finished( - job_number, + job_def, OperationResult::Failed, - msg.to_string(), + msg, )) { eprintln!("Fail to communicate with tracker: {e}"); } From e6d882cd3b0b18ade8d14c76a0b4a7bd9c98f6d2 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 3 Jun 2024 13:03:40 +0200 Subject: [PATCH 119/174] CLI: Rewriting To Solve Dependencies upfront.. Refactoring: - Remove unused build state - Remove unused varialbes and use statements - Move suppressing warning to structs under development only --- cli/src/build_state.rs | 34 ---------------------------------- cli/src/checksum_records.rs | 3 +++ cli/src/jobs_runner/mod.rs | 3 +++ cli/src/location.rs | 1 - cli/src/main.rs | 5 ----- cli/src/spawner.rs | 12 ++++-------- cli/src/target/app.rs | 2 +- cli/src/target/binding.rs | 7 +------ cli/src/target/mod.rs | 13 +++---------- cli/src/target/wrapper.rs | 5 ++--- cli/src/tracker.rs | 3 --- 11 files changed, 17 insertions(+), 71 deletions(-) delete mode 100644 cli/src/build_state.rs diff --git a/cli/src/build_state.rs b/cli/src/build_state.rs deleted file mode 100644 index e3a78f2d4..000000000 --- a/cli/src/build_state.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::{collections::HashMap, sync::Mutex}; - -use tokio::sync::{oneshot, OnceCell}; - -use crate::{spawner::SpawnResult, target::Target}; - -type BuildResult = Result, anyhow::Error>; - -#[derive(Debug)] -pub enum BuildState { - Running(Vec>), - Finished(BuildResult), -} - -#[derive(Debug)] -//TODO AAZ: Delete the whole struct -pub struct BuildStatesTracker { - pub states_map: Mutex>, -} - -impl BuildStatesTracker { - fn new() -> Self { - let states_map = Mutex::new(HashMap::new()); - Self { states_map } - } - - pub async fn get() -> &'static BuildStatesTracker { - static BUILD_STATES: OnceCell = OnceCell::const_new(); - - BUILD_STATES - .get_or_init(|| async { BuildStatesTracker::new() }) - .await - } -} diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index d07e82da1..788ae759c 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -1,3 +1,6 @@ +//TODO AAZ: Remove this when done prototyping +#![allow(dead_code, unused_imports, unused)] + use std::{ collections::{btree_map, BTreeMap, BTreeSet}, fs::{self, File}, diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index e6f852081..0154f364c 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -1,3 +1,6 @@ +//TODO AAZ: Remove this when done prototyping +#![allow(dead_code, unused_imports, unused)] + mod job_definition; mod jobs_resolver; diff --git a/cli/src/location.rs b/cli/src/location.rs index 5d596ad2b..ff48c1b8d 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,7 +1,6 @@ use anyhow::{bail, Context, Error}; use git2::Repository; -use std::path::Path; use std::{env::current_dir, path::PathBuf}; use tokio::sync::OnceCell; diff --git a/cli/src/main.rs b/cli/src/main.rs index a6b593cd4..0d16b23f1 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,8 +1,4 @@ -//TODO AAZ: Remove this when done prototyping -#![allow(dead_code, unused_imports, unused)] - mod app_runner; -mod build_state; mod checksum_records; mod cli_args; mod dev_environment; @@ -22,7 +18,6 @@ use checksum_records::ChecksumRecords; use clap::Parser; use cli_args::{CargoCli, Command}; use dev_environment::{print_env_info, resolve_dev_tools}; -use futures::future::join_all; use job_type::JobType; use jobs_runner::JobsRunner; use location::init_location; diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 62fbbfe86..35dc4a707 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,4 +1,6 @@ -// cmd.envs(vec![("PATH", "/bin"), ("TERM", "xterm-256color")]); +// TODO AAZ: Remove this when done prototyping +#![allow(dead_code, unused_imports, unused)] + use crate::{jobs_runner::JobDefinition, location::get_root, tracker::get_tracker}; use anyhow::{bail, Context}; use core::panic; @@ -214,13 +216,7 @@ pub async fn spawn_blocking( } /// This spawns a new task and return immediately showing that the job has been skipped -pub async fn spawn_skip( - job_def: JobDefinition, - command: String, - cwd: Option, -) -> anyhow::Result { - let cwd = cwd.unwrap_or_else(|| get_root().clone()); - +pub async fn spawn_skip(job_def: JobDefinition, command: String) -> anyhow::Result { Ok(SpawnResult::create_for_skipped( job_def.job_title(), command, diff --git a/cli/src/target/app.rs b/cli/src/target/app.rs index 69f31e713..15fbe37f4 100644 --- a/cli/src/target/app.rs +++ b/cli/src/target/app.rs @@ -2,7 +2,7 @@ use std::fs; use anyhow::{bail, Context}; -use crate::{fstools, job_type::JobType, jobs_runner::JobDefinition, spawner::SpawnResult}; +use crate::{fstools, jobs_runner::JobDefinition, spawner::SpawnResult}; use super::{client::get_dist_path, Target}; diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index 21d79e0ec..3e6ae7106 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -2,12 +2,7 @@ use std::fs; use anyhow::{bail, Context}; -use crate::{ - fstools, - job_type::{self, JobType}, - jobs_runner::JobDefinition, - spawner::SpawnResult, -}; +use crate::{fstools, jobs_runner::JobDefinition, spawner::SpawnResult}; use super::Target; diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index e7006f7bb..8f7bc0386 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,22 +1,16 @@ -use anyhow::{anyhow, bail, Context}; +use anyhow::bail; use clap::ValueEnum; -use futures::{ - future::{join_all, BoxFuture}, - FutureExt, -}; +use futures::future::join_all; use std::{iter, path::PathBuf, str::FromStr}; -use tokio::sync::oneshot; use crate::{ - build_state::{BuildState, BuildStatesTracker}, checksum_records::ChecksumRecords, dev_tools::DevTool, fstools, job_type::JobType, jobs_runner::JobDefinition, location::get_root, - spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, - tracker::{self, get_tracker}, + spawner::{spawn, SpawnOptions, SpawnResult}, }; use target_kind::TargetKind; @@ -379,7 +373,6 @@ impl Target { /// Clean the given target, removing it from the checksum tracker as well. pub async fn reset(&self) -> anyhow::Result { let job_def = JobDefinition::new(*self, JobType::Clean); - let tracker = get_tracker().await; let checksum = ChecksumRecords::get(JobType::Clean).await?; checksum.remove_hash_if_exist(*self)?; diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index d9ce909e1..75c759ea4 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,4 +1,4 @@ -use std::{iter, path::PathBuf}; +use std::path::PathBuf; use anyhow::{anyhow, Context}; use std::fs; @@ -7,7 +7,7 @@ use crate::{ fstools, job_type::JobType, jobs_runner::JobDefinition, - spawner::{spawn, spawn_blocking, SpawnResult}, + spawner::{spawn_blocking, SpawnResult}, }; use super::Target; @@ -78,7 +78,6 @@ pub async fn run_test(production: bool) -> Result { let job_def = JobDefinition::new(Target::Binding, JobType::Test { production }); for spec in TEST_SPECS { - let caption = format!("Test {}: {}", Target::Wrapper, spec); let spec_file_name = format!("session.{spec}.spec.js"); let spec_file_path = specs_dir_path.join(spec_file_name); let command = format!( diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 8cb806e15..d612e030b 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -164,7 +164,6 @@ impl Tracker { } } Tick::Finished(job_def, result, msg) => { - let jobs_count_txt = bars.len().to_string(); let Some(job_bar) = bars.get_mut(&job_def)else { unreachable!("Job must exist in progress bar before finishing it. Job Info: {job_def:?}") }; @@ -177,8 +176,6 @@ impl Tracker { max_time_len = max_time_len.max(Self::count_digits(time)); - let seq_width = jobs_count_txt.len(); - let job = job_bar.name.as_str(); job_bar.bar.finish_with_message(msg); job_bar.phase = JobBarPhase::Finished((result, time)); From 0f61f2171eecb9e01001514cdfb297b9d8c6ec6e Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 3 Jun 2024 16:56:42 +0200 Subject: [PATCH 120/174] CLI: Rewriting To Solve Dependencies upfront.. - Integrate calculating checksums in the jobs processes for build-related jobs - Handle the special case of the "Clean" Command with the checksums, because it cleans the files for development and production at the same time and the checksums must be deleted from the both of them too. (Code can still be improved if no optimizations can be done for the main build logic here) --- cli/src/checksum_records.rs | 56 ++++++++++++++------ cli/src/job_type.rs | 10 ++++ cli/src/jobs_runner/job_definition.rs | 16 +++--- cli/src/jobs_runner/mod.rs | 42 +++++++++++++-- cli/src/target/mod.rs | 74 +++++++++++++++++---------- 5 files changed, 144 insertions(+), 54 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 788ae759c..3daa3e027 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -21,7 +21,6 @@ const FILE_NAME_PROD: &str = ".build_chksum_prod"; #[derive(Debug)] pub struct ChecksumRecords { items: Mutex, - job_type: JobType, } #[derive(Debug, Default)] @@ -33,41 +32,65 @@ struct ChecksumItems { impl ChecksumRecords { pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { // calculate should be involved when build is called at some point of the job - let calculate_involved = match &job_type { + let (calculate_involved, prod) = match &job_type { JobType::Lint => return Ok(()), - JobType::Build { production: _ } - | JobType::Run { production: _ } - | JobType::Test { production: _ } => true, - JobType::Clean - | JobType::Install { production: _ } - | JobType::AfterBuild { production: _ } => false, + JobType::Build { production } + | JobType::Run { production } + | JobType::Test { production } => (true, *production), + // With clean we need to remove the items from both development and production + JobType::Clean => (false, false), + JobType::Install { production } | JobType::AfterBuild { production } => { + (false, *production) + } }; - let records = Self::get(job_type).await?; + let records = Self::get(prod).await?; if calculate_involved { records.calculate_involved_hashes()?; } + records - .persist_hashes() + .persist_hashes(prod) .context("Error while saving the updated hashes")?; + // Hashes must be removed from production if clean is called because it doesn't + // differentiate between development and production. + if matches!(job_type, JobType::Clean) { + let prod_records = + Self::load(true).context("Error while loading production recoreds")?; + + let dev_items = records + .items + .lock() + .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; + + // With clean job, the involved targets are the ones that has been deleted. + for target in &dev_items.involved_targets { + prod_records.remove_hash_if_exist(*target); + } + + prod_records + .persist_hashes(true) + .context("Error while saving the updated hashes")?; + } + Ok(()) } - pub async fn get(job_type: JobType) -> anyhow::Result<&'static ChecksumRecords> { + pub async fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); CHECKSUM_RECORDS - .get_or_init(|| async { ChecksumRecords::load(job_type) }) + .get_or_init(|| async { ChecksumRecords::load(production) }) .await .as_ref() .map_err(|err| anyhow!("{err}")) } /// Loads the persisted records from checksums file if exist - fn load(job_type: JobType) -> anyhow::Result { - let file_path = Self::get_file_path(job_type.is_production().is_some_and(|prod| prod)); + fn load(production: bool) -> anyhow::Result { + let file_path = Self::get_file_path(production); let items = if file_path.exists() { let file_content = fs::read_to_string(file_path)?; @@ -82,7 +105,6 @@ impl ChecksumRecords { Ok(Self { items: Mutex::new(items), - job_type, }) } @@ -191,8 +213,8 @@ impl ChecksumRecords { Ok(()) } - fn persist_hashes(&self) -> anyhow::Result<()> { - let file_path = Self::get_file_path(self.job_type.is_production().is_some_and(|prod| prod)); + fn persist_hashes(&self, production: bool) -> anyhow::Result<()> { + let file_path = Self::get_file_path(production); let mut file = File::create(file_path)?; let items = self diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index b66b89c5c..ffe50b11e 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -59,6 +59,16 @@ impl JobType { | JobType::AfterBuild { production: _ } => Vec::new(), } } + + /// Returns if the job type is part of the build process (install, build, or after build) + pub fn is_part_of_build(&self) -> bool { + matches!( + self, + JobType::Install { production: _ } + | JobType::Build { production: _ } + | JobType::AfterBuild { production: _ } + ) + } } #[cfg(test)] diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 4eafccacb..afdc286ef 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -16,13 +16,13 @@ impl JobDefinition { } /// Run the job definition if it has a job, communicating its status with the UI bars - pub async fn run(&self) -> Option> { + pub async fn run(&self, skip: bool) -> Option> { let tracker = get_tracker().await; if let Err(err) = tracker.start(*self).await { return Some(Err(err)); } - let res = self.run_intern().await; + let res = self.run_intern(skip).await; match res.as_ref() { Some(Ok(res)) => { @@ -49,12 +49,14 @@ impl JobDefinition { #[inline] /// Runs the job definition if it has a job - async fn run_intern(&self) -> Option> { + async fn run_intern(&self, skip: bool) -> Option> { let res = match self.job_type { JobType::Lint => self.target.check().await, - JobType::Build { production } => self.target.build(production).await, - JobType::Install { production } => return self.target.install(production).await, - JobType::AfterBuild { production } => return self.target.after_build(production).await, + JobType::Build { production } => self.target.build(production, skip).await, + JobType::Install { production } => return self.target.install(production, skip).await, + JobType::AfterBuild { production } => { + return self.target.after_build(production, skip).await + } JobType::Clean => self.target.reset().await, JobType::Test { production } => return self.target.test(production).await, JobType::Run { production: _ } => return None, @@ -77,7 +79,7 @@ mod tests { if !target.has_job(&job_type) { let job_def = JobDefinition::new(*target, job_type.clone()); assert!( - job_def.run_intern().await.is_none(), + job_def.run_intern(false).await.is_none(), "'{}' has no job for '{}' but it returns Some when calling run", target, job_type diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 0154f364c..08e5beb3a 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -8,7 +8,13 @@ use std::collections::{BTreeMap, BTreeSet}; pub use job_definition::JobDefinition; -use crate::{job_type::JobType, spawner::SpawnResult, target::Target, tracker::get_tracker}; +use crate::{ + checksum_records::ChecksumRecords, + job_type::{self, JobType}, + spawner::SpawnResult, + target::Target, + tracker::get_tracker, +}; use anyhow::Result; @@ -40,10 +46,12 @@ impl JobsRunner { .start_all(jobs_tree.keys().cloned().collect()) .await?; - // This is needed for assertions while in development only, and it will be removed once the - // concurrent solution is implemented. + // TODO AAZ: This is needed for assertions while in development only, and it will be removed + // once the concurrent solution is implemented. let mut finished = BTreeSet::new(); + let mut skipped_map = BTreeMap::new(); + let mut results = Vec::new(); for (job_def, deps) in jobs_tree { @@ -52,7 +60,33 @@ impl JobsRunner { "Jobs deps must be resolved before running it" ); - let Some(res) = job_def.run().await else { + let skip = if job_def.job_type.is_part_of_build() { + // Check if target is already registered and checked + if let Some(skip) = skipped_map.get(&job_def.target) { + *skip + } else { + let prod = job_def.job_type.is_production().is_some_and(|prod| prod); + let checksum_rec = ChecksumRecords::get(prod).await?; + checksum_rec.register_job(job_def.target)?; + + if job_def + .target + .deps() + .iter() + .all(|dep| skipped_map.get(dep).is_some_and(|skip| *skip)) + { + let calc_skip = !checksum_rec.check_changed(job_def.target)?; + skipped_map.insert(job_def.target, calc_skip); + calc_skip + } else { + false + } + } + } else { + false + }; + + let Some(res) = job_def.run(skip).await else { if cfg!(debug_assertions) { panic!( "Jobs tree should contain only runnable jobs. JobDefinition: {job_def:?}" diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8f7bc0386..8904c7549 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -10,7 +10,7 @@ use crate::{ job_type::JobType, jobs_runner::JobDefinition, location::get_root, - spawner::{spawn, SpawnOptions, SpawnResult}, + spawner::{spawn, spawn_skip, SpawnOptions, SpawnResult}, }; use target_kind::TargetKind; @@ -235,7 +235,21 @@ impl Target { } /// Installs the needed module to perform the development task - pub async fn install(&self, prod: bool) -> Option> { + pub async fn install( + &self, + prod: bool, + skip: bool, + ) -> Option> { + if skip { + return Some( + spawn_skip( + JobDefinition::new(*self, JobType::Install { production: prod }), + format!("Install command for {self}"), + ) + .await, + ); + } + match self { // We must install ts binding tools before running rs bindings, therefore we call // wrapper (ts-bindings) install in the rs bindings install. @@ -253,8 +267,19 @@ impl Target { .await } Target::Wrapper => None, - // For app we don't need --production - Target::App => install_general(&Target::App, false, None).await, + // For app we don't need --production, but we still needed to be defined to comunicate + // with UI bars + Target::App => { + install_general( + &Target::App, + false, + Some(JobDefinition::new( + *self, + JobType::Install { production: prod }, + )), + ) + .await + } rest_targets => install_general(rest_targets, prod, None).await, } } @@ -374,7 +399,9 @@ impl Target { pub async fn reset(&self) -> anyhow::Result { let job_def = JobDefinition::new(*self, JobType::Clean); - let checksum = ChecksumRecords::get(JobType::Clean).await?; + // Clean doesn't differentiate between development and production, and both of them will be + // cleaned from the files when the data are persisted. + let checksum = ChecksumRecords::get(false).await?; checksum.remove_hash_if_exist(*self)?; let mut logs = Vec::new(); @@ -399,28 +426,10 @@ impl Target { } /// Runs build considering the currently running builds and already finished ones as well. - pub async fn build(&self, prod: bool) -> Result { - let checksum_rec = ChecksumRecords::get(JobType::Build { production: prod }).await?; - checksum_rec.register_job(*self)?; - + pub async fn build(&self, prod: bool, skip: bool) -> Result { let path = get_root().join(self.cwd()); let cmd = self.build_cmd(prod).await?; - //TODO AAZ: Skipping jobs should be resolved before running the jobs - // let mut skip_task = false; - // - // let all_skipped = results.iter().all(|r| { - // r.skipped.unwrap_or({ - // // Tasks with no skip info are irrelevant - // true - // }) - // }); - // - // if all_skipped { - // skip_task = !checksum_rec.check_changed(*self)?; - // } - // - let spawn_opt = SpawnOptions { has_skip_info: true, ..Default::default() @@ -428,14 +437,27 @@ impl Target { let job_def = JobDefinition::new(*self, JobType::Build { production: prod }); - spawn(job_def, cmd, Some(path), iter::empty(), Some(spawn_opt)).await + if skip { + spawn_skip(job_def, cmd).await + } else { + spawn(job_def, cmd, Some(path), iter::empty(), Some(spawn_opt)).await + } } /// Performs build process without checking the current builds states /// Perform any needed copy operation after the build is done - pub async fn after_build(&self, prod: bool) -> Option> { + pub async fn after_build( + &self, + prod: bool, + skip: bool, + ) -> Option> { let job_def = JobDefinition::new(*self, JobType::AfterBuild { production: prod }); + + if skip { + return Some(spawn_skip(job_def, "Multiple file system commands".into()).await); + } + let res = match self { Target::Binding => binding::copy_index_node(job_def).await, Target::Wrapper => wrapper::copy_binding_to_app(job_def).await, From 086f557b88a7db5b06dcf495c9b5834008d7d167 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 4 Jun 2024 10:37:02 +0200 Subject: [PATCH 121/174] CLI: Rewriting To Solve Dependencies upfront.. Re-apply building wrapper specs before running their tests if their directory doesn't exist --- cli/src/target/wrapper.rs | 47 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 75c759ea4..8fbc3fa0f 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{iter, path::PathBuf}; use anyhow::{anyhow, Context}; use std::fs; @@ -7,7 +7,7 @@ use crate::{ fstools, job_type::JobType, jobs_runner::JobDefinition, - spawner::{spawn_blocking, SpawnResult}, + spawner::{spawn, spawn_blocking, SpawnResult}, }; use super::Target; @@ -42,27 +42,29 @@ const TEST_SPECS: [&str; 14] = [ ]; pub async fn run_test(production: bool) -> Result { + let job_def = JobDefinition::new(Target::Binding, JobType::Test { production }); + let mut final_result: Option = None; + let cwd = Target::Wrapper.cwd(); - //TODO AAZ: Append build_spec_path to SpawnResult - // let build_spec_path = cwd.join("spec"); - // //TODO: This check exists in rake implementation but it need to be improved. - // // The check should cover if the test themselves or the code under the tests has been changed. - // if !build_spec_path.join("build").exists() { - // let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); - // let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); - // - // let spec_res = spawn( - // build_spec_cmd, - // Some(build_spec_path), - // "Build Specs".into(), - // iter::empty(), - // None, - // ) - // .await?; - // - // results.push(spec_res); - // } + let build_spec_path = cwd.join("spec"); + //TODO: This check exists in rake implementation but it need to be improved. + // The check should cover if the test themselves or the code under the tests has been changed. + if !build_spec_path.join("build").exists() { + let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); + let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); + + let spec_res = spawn( + job_def, + build_spec_cmd, + Some(build_spec_path), + iter::empty(), + None, + ) + .await?; + + final_result = Some(spec_res); + } let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); let electron_path = electron_path.to_string_lossy(); @@ -74,9 +76,6 @@ pub async fn run_test(production: bool) -> Result { let specs_dir_path: PathBuf = ["spec", "build", "spec"].iter().collect(); - let mut final_result: Option = None; - - let job_def = JobDefinition::new(Target::Binding, JobType::Test { production }); for spec in TEST_SPECS { let spec_file_name = format!("session.{spec}.spec.js"); let spec_file_path = specs_dir_path.join(spec_file_name); From 261076438c1eacfc9c3483b11ed6d515d0963a98 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 4 Jun 2024 13:19:18 +0200 Subject: [PATCH 122/174] CLI: Rewriting To Solve Dependencies upfront.. Install TS always in development mode and reinstall it in production mode after building the app before copying the files --- cli/src/job_type.rs | 6 +- cli/src/jobs_runner/job_definition.rs | 4 +- cli/src/target/mod.rs | 91 ++++++++++++++++----------- 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index ffe50b11e..0835ee6e6 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -44,9 +44,9 @@ impl JobType { match self { JobType::Lint => vec![JobType::Install { production: false }], JobType::Build { production } => vec![ - JobType::Install { - production: *production, - }, + // Install run always in development at first then it should get reinstalled with + // production after build command is ran. + JobType::Install { production: false }, JobType::AfterBuild { production: *production, }, diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index afdc286ef..d69853d71 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -53,7 +53,9 @@ impl JobDefinition { let res = match self.job_type { JobType::Lint => self.target.check().await, JobType::Build { production } => self.target.build(production, skip).await, - JobType::Install { production } => return self.target.install(production, skip).await, + JobType::Install { production } => { + return self.target.install(production, skip, None).await + } JobType::AfterBuild { production } => { return self.target.after_build(production, skip).await } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8904c7549..007916c29 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -235,10 +235,16 @@ impl Target { } /// Installs the needed module to perform the development task + /// + /// * `prod`: run install in production + /// * `skip`: skip the task + /// * `overridden_job_type`: override job type to communicate with tracker when install is ran + /// from within another task pub async fn install( &self, prod: bool, skip: bool, + overridden_job_type: Option, ) -> Option> { if skip { return Some( @@ -250,37 +256,20 @@ impl Target { ); } + let job_type = overridden_job_type.unwrap_or(JobType::Install { production: prod }); + match self { // We must install ts binding tools before running rs bindings, therefore we call // wrapper (ts-bindings) install in the rs bindings install. // Since rs bindings is a dependency for ts bindings, we don't need to call to install // on ts bindings again. Target::Binding => { - install_general( - &Target::Wrapper, - prod, - Some(JobDefinition::new( - Target::Binding, - JobType::Install { production: prod }, - )), - ) - .await + install_general(Target::Wrapper, prod, job_type, Some(Target::Binding)).await } Target::Wrapper => None, - // For app we don't need --production, but we still needed to be defined to comunicate - // with UI bars - Target::App => { - install_general( - &Target::App, - false, - Some(JobDefinition::new( - *self, - JobType::Install { production: prod }, - )), - ) - .await - } - rest_targets => install_general(rest_targets, prod, None).await, + // For app we don't need --production + Target::App => install_general(Target::App, false, job_type, None).await, + rest_targets => install_general(*rest_targets, prod, job_type, None).await, } } @@ -452,13 +441,40 @@ impl Target { prod: bool, skip: bool, ) -> Option> { - let job_def = JobDefinition::new(*self, JobType::AfterBuild { production: prod }); + let job_type = JobType::AfterBuild { production: prod }; + let job_def = JobDefinition::new(*self, job_type); if skip { return Some(spawn_skip(job_def, "Multiple file system commands".into()).await); } - let res = match self { + // Taken from a discussion on GitHub: + // To build an npm package you would need (in most cases) to be in dev-mode - install dev-dependencies + dependencies, + // therefore we always install in development mode at first. + // But to prepare a package for production, you have to remove dev-dependencies. + // That's not an issue, if npm-package is published in npmjs; but we are coping packages manually in a right destination + // and before copy it, we have to reinstall it to get rid of dev-dependencies. + let reinstall_res = if prod && matches!(self.kind(), TargetKind::Ts) { + let node_path = self.cwd().join("node_modules"); + let remove_log = format!("removing directory {}", node_path.display()); + + if let Err(err) = fstools::rm_folder(job_def, &node_path).await { + return Some(Err(err)); + } + + match self.install(true, false, Some(job_type)).await { + Some(Ok(mut spawn_res)) => { + spawn_res.report.insert(0, remove_log); + Some(spawn_res) + } + Some(Err(err)) => return Some(Err(err)), + None => None, + } + } else { + None + }; + + let after_res = match self { Target::Binding => binding::copy_index_node(job_def).await, Target::Wrapper => wrapper::copy_binding_to_app(job_def).await, Target::Shared => shared::copy_platform_to_binding(job_def).await, @@ -466,26 +482,31 @@ impl Target { _ => return None, }; - Some(res) + match (after_res, reinstall_res) { + (res, None) => Some(res), + (Err(err), _) => Some(Err(err)), + (Ok(after_res), Some(mut install_res)) => { + install_res.append(after_res); + Some(Ok(install_res)) + } + } } } /// run install using the general routine for the given target /// * `target`: job target to perform its after build jobs /// * `prod`: build for production -/// * `overridden_job_def`: override job definition to update tracker message. This is used when -/// target is used for another install command which happens currently with bindings and wrapper -/// targets +/// * `job_type`: job type to communicate with `tracker` +/// * `overridden_target`: override target to communicate with `tracker` when install is called +/// from within another task. async fn install_general( - target: &Target, + target: Target, prod: bool, - overridden_job_def: Option, + job_type: JobType, + overridden_target: Option, ) -> Option> { let cmd = target.kind().install_cmd(prod).await; - let job_def = overridden_job_def.unwrap_or(JobDefinition::new( - *target, - JobType::Install { production: prod }, - )); + let job_def = JobDefinition::new(overridden_target.unwrap_or(target), job_type); if let Some(cmd) = cmd { let res = spawn(job_def, cmd, Some(target.cwd()), iter::empty(), None).await; From 0b0ac896cf54bd21ea88f7bf16077c8ccd5e6783 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 4 Jun 2024 13:29:08 +0200 Subject: [PATCH 123/174] CLI: Rewriting To Solve Dependencies upfront.. Skip build-related jobs if any of their prequels has failed --- cli/src/jobs_runner/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 08e5beb3a..1a9dc4371 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -51,6 +51,7 @@ impl JobsRunner { let mut finished = BTreeSet::new(); let mut skipped_map = BTreeMap::new(); + let mut failed_jobs = Vec::new(); let mut results = Vec::new(); @@ -61,8 +62,12 @@ impl JobsRunner { ); let skip = if job_def.job_type.is_part_of_build() { + // Skip if any prequel job of this target has failed + if failed_jobs.contains(&job_def.target) { + true + } // Check if target is already registered and checked - if let Some(skip) = skipped_map.get(&job_def.target) { + else if let Some(skip) = skipped_map.get(&job_def.target) { *skip } else { let prod = job_def.job_type.is_production().is_some_and(|prod| prod); @@ -96,6 +101,10 @@ impl JobsRunner { } }; + if res.is_err() { + failed_jobs.push(job_def.target); + } + results.push(res); finished.insert(job_def); From 850269897ba2cd018eecccbbddf2cdd223e395ba Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 4 Jun 2024 14:00:48 +0200 Subject: [PATCH 124/174] CLI: Rewriting To Solve Dependencies upfront.. Provide new options on dot print function to print all the jobs and their relations --- cli/src/cli_args.rs | 6 +++++- cli/src/jobs_runner/mod.rs | 2 +- cli/src/main.rs | 8 ++++++-- cli/src/print_dot.rs | 34 ++++++++++++++++++++++++++++++++-- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/cli/src/cli_args.rs b/cli/src/cli_args.rs index a4b5e7501..a640c41ba 100644 --- a/cli/src/cli_args.rs +++ b/cli/src/cli_args.rs @@ -29,7 +29,11 @@ pub enum Command { Environment(EnvironmentCommand), /// Prints an overview of targets dependencies in print-dot format for `Graphviz` #[clap(visible_alias = "dot")] - PrintDot, + PrintDot { + /// Show all jobs and their relations + #[arg(short, long, default_value_t = false)] + all_jobs: bool, + }, /// Runs linting & clippy for all or the specified targets Lint { /// Target to lint, by default whole application will be linted diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 1a9dc4371..549bd6f2b 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -2,7 +2,7 @@ #![allow(dead_code, unused_imports, unused)] mod job_definition; -mod jobs_resolver; +pub mod jobs_resolver; use std::collections::{BTreeMap, BTreeSet}; diff --git a/cli/src/main.rs b/cli/src/main.rs index 0d16b23f1..7d66819c6 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -60,8 +60,12 @@ async fn main() -> Result<(), Error> { return Ok(()); } }, - Command::PrintDot => { - print_dot::print_dependencies(); + Command::PrintDot { all_jobs } => { + if all_jobs { + print_dot::print_dependencies_jobs(); + } else { + print_dot::print_dependencies_targets(); + } return Ok(()); } Command::Lint { target, report } => { diff --git a/cli/src/print_dot.rs b/cli/src/print_dot.rs index 20dd92de4..2fc0f5ea8 100644 --- a/cli/src/print_dot.rs +++ b/cli/src/print_dot.rs @@ -1,7 +1,11 @@ -use crate::target::Target; +use crate::{ + job_type::JobType, + jobs_runner::{jobs_resolver, JobDefinition}, + target::Target, +}; /// Prints an overview of targets dependencies in print-dot format for `Graphviz` -pub fn print_dependencies() { +pub fn print_dependencies_targets() { println!("digraph dependencies {{"); for target in Target::all() { @@ -12,3 +16,29 @@ pub fn print_dependencies() { println!("}}"); } + +/// Prints an overview of jobs dependencies in print-dot format for `Graphviz` +pub fn print_dependencies_jobs() { + let deps_tree = jobs_resolver::resolve(Target::all(), JobType::Build { production: false }); + println!("digraph dependencies {{"); + + for (job, deps) in deps_tree { + let job_txt = job_to_dot_string(&job); + for dep in deps { + println!(r#" "{job_txt}" -> "{}""#, job_to_dot_string(&dep)); + } + } + + println!("}}"); +} + +fn job_to_dot_string(job_def: &JobDefinition) -> String { + let job_type = match job_def.job_type { + JobType::Install { production: _ } => "Install", + JobType::Build { production: _ } => "Build", + JobType::AfterBuild { production: _ } => "After Build (Copy & Reinstall)", + _ => unreachable!("Only build-related jobs are included in dot print"), + }; + + format!("{}: {job_type}", job_def.target) +} From e5dfe680ebffc5f841e680457681a9403bbf5a03 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 4 Jun 2024 14:18:14 +0200 Subject: [PATCH 125/174] Build CLI: Adjust App Dependencies - Remove redundant dependency between App and Shared since Wrapper and Client has it. - Add Core and Updater to app dependencies - Adjust run command build app target only to skip building the CLI tool because it's not part of the app - Print dots provides now all apps dependencies and their graph --- cli/src/jobs_runner/jobs_resolver.rs | 2 ++ cli/src/main.rs | 2 +- cli/src/target/mod.rs | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 19c3c1939..333f675d6 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -166,11 +166,13 @@ mod tests { #[test] fn flatten_app_target() { let expected = BTreeSet::from([ + Target::Core, Target::Shared, Target::Binding, Target::Wrapper, Target::Client, Target::Wasm, + Target::Updater, Target::App, ]); assert_eq!(flatten_targets_for_build(&[Target::App]), expected); diff --git a/cli/src/main.rs b/cli/src/main.rs index 7d66819c6..00ada4916 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -108,7 +108,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = ReportOptions::None; let results = - JobsRunner::run_jobs(&Target::all(), JobType::Build { production }).await?; + JobsRunner::run_jobs(&[Target::App], JobType::Build { production }).await?; (JobType::Run { production }, results) } Command::ResetChecksum { production } => { diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 007916c29..812a5d949 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -42,12 +42,12 @@ pub enum Target { Wasm, /// Represents the path `application/client` Client, + /// Represents the path `application/apps/precompiled/updater + Updater, /// Represents the path `application/holder` App, /// Represents the path `cli` Cli, - /// Represents the path `application/apps/precompiled/updater - Updater, } pub struct TestCommand { @@ -197,7 +197,12 @@ impl Target { Target::Binding => vec![Target::Shared], Target::Wrapper => vec![Target::Binding, Target::Shared], Target::Client => vec![Target::Shared, Target::Wasm], - Target::App => vec![Target::Shared, Target::Wrapper, Target::Client], + Target::App => vec![ + Target::Wrapper, + Target::Client, + Target::Core, + Target::Updater, + ], } } From 55fbb3abe6daed343c2e88b316a6de20c8fd07b6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 5 Jun 2024 17:43:42 +0200 Subject: [PATCH 126/174] Build CLI: Run jobs concurrently Jobs will be spawned in their own tasks returning their results via channels --- cli/src/jobs_runner/mod.rs | 194 ++++++++++++++++++++++--------------- cli/src/main.rs | 12 +-- 2 files changed, 122 insertions(+), 84 deletions(-) diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 549bd6f2b..6930bdb99 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -1,18 +1,13 @@ -//TODO AAZ: Remove this when done prototyping -#![allow(dead_code, unused_imports, unused)] - mod job_definition; pub mod jobs_resolver; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; pub use job_definition::JobDefinition; +use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use crate::{ - checksum_records::ChecksumRecords, - job_type::{self, JobType}, - spawner::SpawnResult, - target::Target, + checksum_records::ChecksumRecords, job_type::JobType, spawner::SpawnResult, target::Target, tracker::get_tracker, }; @@ -20,96 +15,141 @@ use anyhow::Result; type SpawnResultsCollection = Vec>; +#[derive(Debug, Clone)] enum JobPhase { - Awaiting(Vec), + Awaiting(Vec), Running, - Done(SpawnResult), + Done, } -struct JobState { - phase: JobPhase, - job_number: usize, -} +pub async fn run(targets: &[Target], main_job: JobType) -> Result { + let jobs_tree = jobs_resolver::resolve(targets, main_job); -pub struct JobsRunner { - // BTreeMap keeps the jobs in logical ordering - jobs: BTreeMap, - resolved_targets: Vec, -} + let tracker = get_tracker().await; + tracker + .start_all(jobs_tree.keys().cloned().collect()) + .await?; -impl JobsRunner { - pub async fn run_jobs(targets: &[Target], main_job: JobType) -> Result { - let jobs_tree = jobs_resolver::resolve(targets, main_job); + let mut jobs_status: BTreeMap = jobs_tree + .into_iter() + .map(|(job, deps)| (job, JobPhase::Awaiting(deps))) + .collect(); - let tracker = get_tracker().await; - tracker - .start_all(jobs_tree.keys().cloned().collect()) - .await?; + let (tx, mut rx) = unbounded_channel::<(JobDefinition, Result)>(); - // TODO AAZ: This is needed for assertions while in development only, and it will be removed - // once the concurrent solution is implemented. - let mut finished = BTreeSet::new(); + let mut skipped_map = BTreeMap::new(); + let mut failed_jobs = Vec::new(); - let mut skipped_map = BTreeMap::new(); - let mut failed_jobs = Vec::new(); + // Spawn free job at first + spawn_jobs(tx.clone(), &mut jobs_status, &mut skipped_map, &failed_jobs).await?; - let mut results = Vec::new(); + let mut results = Vec::new(); - for (job_def, deps) in jobs_tree { - assert!( - deps.iter().all(|def| finished.contains(def)), - "Jobs deps must be resolved before running it" - ); + while let Some((job_def, result)) = rx.recv().await { + jobs_status + .entry(job_def) + .and_modify(|phase| *phase = JobPhase::Done); - let skip = if job_def.job_type.is_part_of_build() { - // Skip if any prequel job of this target has failed - if failed_jobs.contains(&job_def.target) { - true - } - // Check if target is already registered and checked - else if let Some(skip) = skipped_map.get(&job_def.target) { - *skip - } else { - let prod = job_def.job_type.is_production().is_some_and(|prod| prod); - let checksum_rec = ChecksumRecords::get(prod).await?; - checksum_rec.register_job(job_def.target)?; - - if job_def - .target - .deps() - .iter() - .all(|dep| skipped_map.get(dep).is_some_and(|skip| *skip)) - { - let calc_skip = !checksum_rec.check_changed(job_def.target)?; - skipped_map.insert(job_def.target, calc_skip); - calc_skip - } else { - false - } - } - } else { - false - }; + if result.is_err() { + failed_jobs.push(job_def.target); + } - let Some(res) = job_def.run(skip).await else { - if cfg!(debug_assertions) { - panic!( - "Jobs tree should contain only runnable jobs. JobDefinition: {job_def:?}" - ); - } else { + results.push(result); + + let mut all_done = true; + + // Remove finished job from waiting lists to the awaiting jobs. + // And check if all jobs are done at same time. + for (_, mut phase) in jobs_status.iter_mut() { + let deps = match &mut phase { + JobPhase::Awaiting(deps) => { + all_done = false; + deps + } + JobPhase::Running => { + all_done = false; continue; } + JobPhase::Done => continue, }; - if res.is_err() { - failed_jobs.push(job_def.target); + if let Some(dep_idx) = deps.iter().position(|j| *j == job_def) { + let _ = deps.swap_remove(dep_idx); } + } - results.push(res); + if all_done { + return Ok(results); + } + + spawn_jobs(tx.clone(), &mut jobs_status, &mut skipped_map, &failed_jobs).await?; + } - finished.insert(job_def); + Ok(results) +} + +/// Iterate over jobs states maps and spawn the jobs that aren't waiting for any jobs to be done +async fn spawn_jobs( + sender: UnboundedSender<(JobDefinition, Result)>, + jobs_status: &mut BTreeMap, + skipped_map: &mut BTreeMap, + failed_jobs: &[Target], +) -> Result<()> { + for (job_def, phase) in jobs_status.iter_mut() { + let JobPhase::Awaiting(deps) = phase else { + continue; + }; + + if !deps.is_empty() { + continue; } - Ok(results) + let skip = if job_def.job_type.is_part_of_build() { + // Skip if any prequel job of this target has failed + if failed_jobs.contains(&job_def.target) { + true + } + // Check if target is already registered and checked + else if let Some(skip) = skipped_map.get(&job_def.target) { + *skip + } else { + let prod = job_def.job_type.is_production().is_some_and(|prod| prod); + let checksum_rec = ChecksumRecords::get(prod).await?; + checksum_rec.register_job(job_def.target)?; + + if job_def + .target + .deps() + .iter() + .all(|dep| skipped_map.get(dep).is_some_and(|skip| *skip)) + { + let calc_skip = !checksum_rec.check_changed(job_def.target)?; + skipped_map.insert(job_def.target, calc_skip); + calc_skip + } else { + false + } + } + } else { + false + }; + + let sender = sender.clone(); + let job_def = *job_def; + tokio::spawn(async move { + let result = job_def.run(skip).await; + + let result = match result { + Some(res) => res, + None => panic!("Spawned jobs already resolved and must have return value."), + }; + + if let Err(_) = sender.send((job_def, result)) { + eprintln!("Job results can't be sent to receiver"); + }; + }); + + *phase = JobPhase::Running; } + Ok(()) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 00ada4916..ca6b4582d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -19,7 +19,6 @@ use clap::Parser; use cli_args::{CargoCli, Command}; use dev_environment::{print_env_info, resolve_dev_tools}; use job_type::JobType; -use jobs_runner::JobsRunner; use location::init_location; use spawner::SpawnResult; use std::{ @@ -72,7 +71,7 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Lint).await?; + let results = jobs_runner::run(&targets, JobType::Lint).await?; (JobType::Lint, results) } Command::Build { @@ -83,14 +82,14 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Build { production }).await?; + let results = jobs_runner::run(&targets, JobType::Build { production }).await?; (JobType::Build { production }, results) } Command::Clean { target, report } => { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Clean).await?; + let results = jobs_runner::run(&targets, JobType::Clean).await?; (JobType::Clean, results) } Command::Test { @@ -101,14 +100,13 @@ async fn main() -> Result<(), Error> { resolve_dev_tools().await?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); - let results = JobsRunner::run_jobs(&targets, JobType::Test { production }).await?; + let results = jobs_runner::run(&targets, JobType::Test { production }).await?; (JobType::Test { production }, results) } Command::Run { production } => { resolve_dev_tools().await?; report_opt = ReportOptions::None; - let results = - JobsRunner::run_jobs(&[Target::App], JobType::Build { production }).await?; + let results = jobs_runner::run(&[Target::App], JobType::Build { production }).await?; (JobType::Run { production }, results) } Command::ResetChecksum { production } => { From 91d75f343990a0ee34ef274108918e6b0b3e9444 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 07:38:56 +0200 Subject: [PATCH 127/174] Build CLI: Remove TODOs & Clippy fixes --- cli/src/checksum_records.rs | 2 +- cli/src/jobs_runner/mod.rs | 2 +- cli/src/spawner.rs | 4 ---- cli/src/target/mod.rs | 2 +- cli/src/tracker.rs | 8 ++++---- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 3daa3e027..1f825901c 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -67,7 +67,7 @@ impl ChecksumRecords { // With clean job, the involved targets are the ones that has been deleted. for target in &dev_items.involved_targets { - prod_records.remove_hash_if_exist(*target); + prod_records.remove_hash_if_exist(*target)?; } prod_records diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 6930bdb99..edcbe0637 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -144,7 +144,7 @@ async fn spawn_jobs( None => panic!("Spawned jobs already resolved and must have return value."), }; - if let Err(_) = sender.send((job_def, result)) { + if sender.send((job_def, result)).is_err() { eprintln!("Job results can't be sent to receiver"); }; }); diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 35dc4a707..922d36220 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,6 +1,3 @@ -// TODO AAZ: Remove this when done prototyping -#![allow(dead_code, unused_imports, unused)] - use crate::{jobs_runner::JobDefinition, location::get_root, tracker::get_tracker}; use anyhow::{bail, Context}; use core::panic; @@ -15,7 +12,6 @@ use tokio::{ }; #[derive(Clone, Debug)] -//TODO AAZ: Move this to its own file pub struct SpawnResult { pub report: Vec, pub status: ExitStatus, diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 812a5d949..8fa91d687 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -324,7 +324,7 @@ impl Target { Err(err) => return Some(Err(err)), }; - while let Some(next_result) = spawn_results.next() { + for next_result in spawn_results { match next_result { Ok(next_res) => result.append(next_res), Err(err) => return Some(Err(err)), diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index d612e030b..539026b59 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -127,8 +127,8 @@ impl Tracker { Self::refresh_all_bars(&mut bars, max_time_len, None); } - if let Err(_) = tx_response.send(()) { - let _ = mp.println(format!("Fail to send response while starting the jobs")); + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response while starting the jobs"); } } Tick::StartAll(jobs, tx_response ) => { @@ -141,8 +141,8 @@ impl Tracker { } Self::refresh_all_bars(&mut bars, max_time_len, None); - if let Err(_) = tx_response.send(()) { - let _ = mp.println(format!("Fail to send response while starting the jobs")); + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response while starting the jobs"); } } Tick::Message(job_def, log) => { From 22364a4bf1678a104baf00b1d63886ecbe399be9 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 07:42:09 +0200 Subject: [PATCH 128/174] Build CLI: Remove redundant async move block - Async move block doesn't serve any benefits in the spawner async run function --- cli/src/tracker.rs | 226 +++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 111 deletions(-) diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 539026b59..326312806 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -43,7 +43,7 @@ pub enum Tick { StartAll(Vec, oneshot::Sender<()>), /// Update the job providing an optional progress value. Progress(JobDefinition, Option), - /// Send a message to the job + /// Send a message to the job Message(JobDefinition, String), /// Sets the job as finished providing the job result and a message Finished(JobDefinition, OperationResult, String), @@ -110,122 +110,126 @@ impl Tracker { let spinner_style = ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}")? .tick_chars("▂▃▅▆▇▆▅▃▂ "); - async move { - let mut max_time_len = 0; - let max = u64::MAX; - let mut bars: BTreeMap = BTreeMap::new(); - let mp = MultiProgress::new(); - let start_time = Instant::now(); - while let Some(tick) = rx.recv().await { - match tick { - Tick::Started(job_def, tx_response) => { - let Some(job) = bars.get_mut(&job_def) else { - unreachable!("Job must exist in progress bar before starting it. Job Info: {job_def:?}") - }; - if matches!(job.phase, JobBarPhase::Pending){ - job.start(); - Self::refresh_all_bars(&mut bars, max_time_len, None); - } - - if tx_response.send(()).is_err() { - let _ = mp.println("Fail to send response while starting the jobs"); - } + let mut max_time_len = 0; + let max = u64::MAX; + let mut bars: BTreeMap = BTreeMap::new(); + let mp = MultiProgress::new(); + let start_time = Instant::now(); + while let Some(tick) = rx.recv().await { + match tick { + Tick::Started(job_def, tx_response) => { + let Some(job) = bars.get_mut(&job_def) else { + unreachable!("Job must exist in progress bar before starting it. Job Info: {job_def:?}") + }; + if matches!(job.phase, JobBarPhase::Pending) { + job.start(); + Self::refresh_all_bars(&mut bars, max_time_len, None); } - Tick::StartAll(jobs, tx_response ) => { - for job in jobs.into_iter() { - let bar = mp.add(ProgressBar::new(max)); - bar.set_style(spinner_style.clone()); - let bar_text = format!("{}: {}", job.target.relative_cwd().display(), job.job_title()); - let job_bar = JobBarState::new(bar_text, bar); - bars.insert(job, job_bar); - } - Self::refresh_all_bars(&mut bars, max_time_len, None); - if tx_response.send(()).is_err() { - let _ = mp.println("Fail to send response while starting the jobs"); - } + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response while starting the jobs"); } - Tick::Message(job_def, log) => { - match bars.get(&job_def) { - Some(job_bar) => job_bar.bar.set_message(log), - None => unreachable!("Job must exist in progress bar before messaging it. Job Info: {job_def:?}"), - } - + } + Tick::StartAll(jobs, tx_response) => { + for job in jobs.into_iter() { + let bar = mp.add(ProgressBar::new(max)); + bar.set_style(spinner_style.clone()); + let bar_text = format!( + "{}: {}", + job.target.relative_cwd().display(), + job.job_title() + ); + let job_bar = JobBarState::new(bar_text, bar); + bars.insert(job, job_bar); } - Tick::Progress(job_def, pos) => { - let Some(job_bar) = bars.get(&job_def) else { - unreachable!("Job must exist in progress bar before changing it progress. Job Info: {job_def:?}") - }; - if let Some(pos) = pos { - job_bar.bar.set_position(pos); - } else { - job_bar.bar.inc(1); - } + Self::refresh_all_bars(&mut bars, max_time_len, None); + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response while starting the jobs"); + } + } + Tick::Message(job_def, log) => match bars.get(&job_def) { + Some(job_bar) => job_bar.bar.set_message(log), + None => unreachable!( + "Job must exist in progress bar before messaging it. Job Info: {job_def:?}" + ), + }, + Tick::Progress(job_def, pos) => { + let Some(job_bar) = bars.get(&job_def) else { + unreachable!("Job must exist in progress bar before changing it progress. Job Info: {job_def:?}") + }; + + if let Some(pos) = pos { + job_bar.bar.set_position(pos); + } else { + job_bar.bar.inc(1); } - Tick::Finished(job_def, result, msg) => { - let Some(job_bar) = bars.get_mut(&job_def)else { - unreachable!("Job must exist in progress bar before finishing it. Job Info: {job_def:?}") - }; + } + Tick::Finished(job_def, result, msg) => { + let Some(job_bar) = bars.get_mut(&job_def) else { + unreachable!("Job must exist in progress bar before finishing it. Job Info: {job_def:?}") + }; + + // It doesn't make sense to show that a job is done in 0 seconds + let time = match job_bar.phase { + JobBarPhase::Running(start_time) => start_time.elapsed().as_secs().max(1), + _ => unreachable!("Job must be running when finish is called"), + }; - // It doesn't make sense to show that a job is done in 0 seconds + max_time_len = max_time_len.max(Self::count_digits(time)); + + job_bar.bar.finish_with_message(msg); + job_bar.phase = JobBarPhase::Finished((result, time)); + + Self::refresh_all_bars(&mut bars, max_time_len, None); + } + Tick::Print(msg) => { + let _ = mp.println(msg); + } + Tick::Shutdown(tx_response) => { + // Finish jobs that are still running + for (_job_def, job_bar) in bars.iter_mut() { let time = match job_bar.phase { - JobBarPhase::Running(start_time) => start_time.elapsed().as_secs().max(1), - _ => unreachable!("Job must be running when finish is called"), + JobBarPhase::Pending => 1, + JobBarPhase::Running(start_time) => { + start_time.elapsed().as_secs().max(1) + } + JobBarPhase::Finished(_) => continue, }; + job_bar.phase = JobBarPhase::Finished((OperationResult::Failed, time)); max_time_len = max_time_len.max(Self::count_digits(time)); - job_bar.bar.finish_with_message(msg); - job_bar.phase = JobBarPhase::Finished((result, time)); - - Self::refresh_all_bars(&mut bars, max_time_len, None); + job_bar.bar.finish(); } - Tick::Print(msg) => { - let _ = mp.println(msg); + + // Insert graphic bar for the running duration of each bars + let total_time = start_time.elapsed().as_secs().max(1) as usize; + Self::refresh_all_bars(&mut bars, max_time_len, Some(total_time)); + + // Insert total time bar + let total_bar = mp.add(ProgressBar::new((bars.len() + 1) as u64)); + total_bar.set_style(spinner_style.clone()); + total_bar.set_prefix(format!("[total] done all in {total_time}s.")); + total_bar.finish(); + + bars.clear(); + // let _ = mp.clear(); + if tx_response.send(()).is_err() { + let _ = mp.println("Fail to send response"); } - Tick::Shutdown(tx_response) => { - // Finish jobs that are still running - for (_job_def, job_bar) in bars.iter_mut(){ - let time = match job_bar.phase{ - JobBarPhase::Pending => 1 , - JobBarPhase::Running(start_time) => start_time.elapsed().as_secs().max(1), - JobBarPhase::Finished(_) => continue , - }; - - job_bar.phase = JobBarPhase::Finished((OperationResult::Failed, time)); - max_time_len = max_time_len.max(Self::count_digits(time)); - - job_bar.bar.finish(); - } - - // Insert graphic bar for the running duration of each bars - let total_time = start_time.elapsed().as_secs().max(1) as usize; - Self::refresh_all_bars(&mut bars, max_time_len, Some(total_time)); - - // Insert total time bar - let total_bar = mp.add(ProgressBar::new((bars.len() + 1) as u64)); - total_bar.set_style(spinner_style.clone()); - total_bar.set_prefix(format!("[total] done all in {total_time}s.")); - total_bar.finish(); - - bars.clear(); - // let _ = mp.clear(); - if tx_response.send(()).is_err() { - let _ = mp.println("Fail to send response"); - } - break; + break; + } + Tick::SuspendAndRun(mut command, tx_response) => { + let status = mp + .suspend(|| command.status()) + .context("Error while executing command"); + if tx_response.send(status).is_err() { + let _ = mp.println("Fail to send response"); } - Tick::SuspendAndRun(mut command, tx_response ) => { - let status = mp.suspend(|| {command.status()}).context("Error while executing command"); - if tx_response.send(status).is_err() { - let _ = mp.println("Fail to send response"); - } - }, } } } - .await; Ok(()) } @@ -286,7 +290,9 @@ impl Tracker { self.tx .send(Tick::Started(job_def, tx_response)) .context("Fail to send tick")?; - rx_response.await.context("Fail to receive tick Start Single") + rx_response + .await + .context("Fail to receive tick Start Single") } /// Update the job providing an optional progress value. @@ -296,7 +302,7 @@ impl Tracker { } } - /// Send a message to the job + /// Send a message to the job pub async fn msg(&self, job_def: JobDefinition, log: String) { if let Err(e) = self.tx.send(Tick::Message(job_def, log)) { eprintln!("Fail to communicate with tracker: {e}"); @@ -305,22 +311,20 @@ impl Tracker { /// Sets the job as finished providing successful result and a message pub async fn success(&self, job_def: JobDefinition, msg: String) { - if let Err(e) = self.tx.send(Tick::Finished( - job_def, - OperationResult::Success, - msg, - )) { + if let Err(e) = self + .tx + .send(Tick::Finished(job_def, OperationResult::Success, msg)) + { eprintln!("Fail to communicate with tracker: {e}"); } } /// Sets the job as finished providing failed result and a message pub async fn fail(&self, job_def: JobDefinition, msg: String) { - if let Err(e) = self.tx.send(Tick::Finished( - job_def, - OperationResult::Failed, - msg, - )) { + if let Err(e) = self + .tx + .send(Tick::Finished(job_def, OperationResult::Failed, msg)) + { eprintln!("Fail to communicate with tracker: {e}"); } } From 0abd9b72c18ae204d0d90a3dca38924d8ae32dcb Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 08:18:22 +0200 Subject: [PATCH 129/174] Build CLI Cleaning up: Visibilities & Tracker print - Set Visibilities to private on functions that don't need to be public - Use print method on tracker instead of printing to stderr - Remove forgotten TODO --- cli/src/checksum_records.rs | 3 --- cli/src/jobs_runner/mod.rs | 7 ++++++- cli/src/location.rs | 2 +- cli/src/tracker.rs | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 1f825901c..8541a0208 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -1,6 +1,3 @@ -//TODO AAZ: Remove this when done prototyping -#![allow(dead_code, unused_imports, unused)] - use std::{ collections::{btree_map, BTreeMap, BTreeSet}, fs::{self, File}, diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index edcbe0637..69ba8a3e3 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -145,7 +145,12 @@ async fn spawn_jobs( }; if sender.send((job_def, result)).is_err() { - eprintln!("Job results can't be sent to receiver"); + let tracker = get_tracker().await; + tracker + .print(format!( + "Error: Job results can't be sent to receiver. Job: {job_def:?}" + )) + .await; }; }); diff --git a/cli/src/location.rs b/cli/src/location.rs index ff48c1b8d..33ad475e9 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -13,7 +13,7 @@ pub struct Location { } impl Location { - pub fn new() -> Result { + fn new() -> Result { let current_dir = current_dir()?; let repo = Repository::discover(current_dir).context("Fail to find chipmunk root directory")?; diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 326312806..54d52f2fb 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -100,13 +100,13 @@ pub async fn get_tracker() -> &'static Tracker { } impl Tracker { - pub fn new() -> Self { + fn new() -> Self { let (tx, rx): (UnboundedSender, UnboundedReceiver) = unbounded_channel(); tokio::spawn(Tracker::run(rx)); Self { tx } } - pub async fn run(mut rx: UnboundedReceiver) -> Result<(), Error> { + async fn run(mut rx: UnboundedReceiver) -> Result<(), Error> { let spinner_style = ProgressStyle::with_template("{spinner} {prefix:.bold.dim} {wide_msg}")? .tick_chars("▂▃▅▆▇▆▅▃▂ "); @@ -339,7 +339,7 @@ impl Tracker { } /// Prints the given text outside the progress bar - pub async fn _print(&self, msg: String) { + pub async fn print(&self, msg: String) { if let Err(e) = self .tx .send(Tick::Print(msg)) From ce5b895dce8ec5396c3bf2113a01d76c7b8cc271 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 08:19:24 +0200 Subject: [PATCH 130/174] Build CLI: Add missing documentations & Rename --- cli/dir_checksum/src/hash_digest.rs | 2 ++ cli/src/checksum_records.rs | 3 +++ cli/src/dev_tools.rs | 1 + cli/src/job_type.rs | 2 ++ cli/src/jobs_runner/job_definition.rs | 5 +++++ cli/src/jobs_runner/mod.rs | 2 +- cli/src/target/mod.rs | 4 +++- cli/src/tracker.rs | 4 +++- 8 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cli/dir_checksum/src/hash_digest.rs b/cli/dir_checksum/src/hash_digest.rs index 95cf54817..8e375e032 100644 --- a/cli/dir_checksum/src/hash_digest.rs +++ b/cli/dir_checksum/src/hash_digest.rs @@ -51,10 +51,12 @@ impl<'a> From<&'a HashDigest> for &'a [u8; OUT_LEN] { } impl HashDigest { + /// Returns a reference to the hash as bytes. pub fn as_bytes(&self) -> &[u8; OUT_LEN] { self.hash.as_bytes() } + /// Convert the hash to bytes consuming itself. pub fn to_bytes(self) -> [u8; OUT_LEN] { self.hash.into() } diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 8541a0208..ad8078a0e 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -27,6 +27,8 @@ struct ChecksumItems { } impl ChecksumRecords { + /// Update checksum records for involved jobs depending on the job type. + /// It will calculate new checksums if build tasks were involved. pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { // calculate should be involved when build is called at some point of the job let (calculate_involved, prod) = match &job_type { @@ -75,6 +77,7 @@ impl ChecksumRecords { Ok(()) } + /// Returns a reference to checksums records manager singleton pub async fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index d1db69387..ca0d73149 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -31,6 +31,7 @@ impl Display for DevTool { } impl DevTool { + /// Returns all needed tools to chipmunk development pub fn all() -> &'static [DevTool] { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 0835ee6e6..b5bb0d1ed 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -73,6 +73,8 @@ impl JobType { #[cfg(test)] impl JobType { + /// Returns all existing targets with production set to false for the types with + /// production infos pub fn all() -> &'static [JobType] { if cfg!(debug_assertions) { // This check to remember to add the newly added enums to this function diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index d69853d71..08f93714f 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -1,6 +1,10 @@ use crate::{job_type::JobType, spawner::SpawnResult, target::Target, tracker::get_tracker}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +/// Represents a development job definition +/// +/// * `target`: Job Target (Shared, Client...) +/// * `job_type`: Job Type (Build, Test...) pub struct JobDefinition { pub target: Target, pub job_type: JobType, @@ -11,6 +15,7 @@ impl JobDefinition { Self { target, job_type } } + /// Provide formatted job title with target and job type infos pub fn job_title(&self) -> String { format!("{} {}", self.target, self.job_type) } diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 69ba8a3e3..edf06ab47 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -27,7 +27,7 @@ pub async fn run(targets: &[Target], main_job: JobType) -> Result = jobs_tree diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8fa91d687..184a75c1f 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -154,7 +154,7 @@ impl Target { .as_slice() } - /// Provides the absolute path to the target code + /// Provides the absolute path to the target pub fn cwd(&self) -> PathBuf { let root = get_root(); let relative_path = self.relative_cwd(); @@ -162,6 +162,7 @@ impl Target { root.join(relative_path) } + /// Provides the relative path to the target starting from chipmunk root directory pub fn relative_cwd(&self) -> PathBuf { let sub_parts = match self { Target::Core => ["application", "apps", "indexer"].iter(), @@ -206,6 +207,7 @@ impl Target { } } + /// Returns if the current target has a job to the given job type pub fn has_job(&self, job_type: &JobType) -> bool { match job_type { JobType::Lint | JobType::Clean | JobType::Build { production: _ } => true, diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 54d52f2fb..d4122e251 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -276,7 +276,9 @@ impl Tracker { count } - pub async fn start_all(&self, jobs: Vec) -> Result<(), Error> { + /// Registers all the given jobs setting their status to awaiting. + /// This function should be called once on the start of running the tasks + pub async fn register_all(&self, jobs: Vec) -> Result<(), Error> { let (tx_response, rx_response) = oneshot::channel(); self.tx .send(Tick::StartAll(jobs, tx_response)) From f06e10d0fab240b01fa006ea0239080aebbfc339 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 10:56:23 +0200 Subject: [PATCH 131/174] Build CLI: Adjustments for using Yarn 4 after merge - Remove copying files from platform to bindings. - Remove copying files from bindings to client. - Remove unused functions and files. --- cli/src/fstools.rs | 51 ------------ cli/src/jobs_runner/jobs_resolver.rs | 9 --- cli/src/target/mod.rs | 8 +- cli/src/target/shared.rs | 57 ------------- cli/src/target/wrapper.rs | 117 +-------------------------- 5 files changed, 2 insertions(+), 240 deletions(-) delete mode 100644 cli/src/target/shared.rs diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index f0f70d7d7..ebfc322ec 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -1,8 +1,6 @@ extern crate fs_extra; use anyhow::{Context, Error}; -use fs_extra::copy_items_with_progress; use fs_extra::dir::{copy_with_progress, CopyOptions, TransitProcess, TransitProcessResult}; -use std::fmt::Display; use std::sync::mpsc; use std::{fs, path::PathBuf}; @@ -84,55 +82,6 @@ pub async fn cp_folder( Ok(()) } -/// Spawn a job to Copy a collection of files and folders recursively, adding copying info to the -/// log records -pub async fn cp_many( - job_def: JobDefinition, - items: Vec, - dest: PathBuf, - general_source: impl Display, - logs: &mut Vec, -) -> Result<(), Error> { - let options = CopyOptions::new(); - let (tx, rx) = mpsc::channel(); - let path_display = format!("from '{}' to '{}'", general_source, dest.display()); - - logs.extend( - items - .iter() - .map(|item| format!("Item: '{}' copied to '{}'", item.display(), dest.display())), - ); - - let tracker = get_tracker().await; - tracker.msg(job_def, "copying files".into()).await; - let _ = tokio::spawn(async move { - copy_items_with_progress(&items, dest, &options, |info| { - if tx.send(info).is_err() { - eprintln!("Fail to send copying progress"); - } - TransitProcessResult::ContinueOrAbort - }) - }) - .await - .with_context(|| format!("Error while copying: {path_display}"))?; - while let Ok(info) = rx.recv() { - tracker - .msg( - job_def, - format!( - "copied: {} bytes; current: {}", - info.copied_bytes, info.file_name - ), - ) - .await; - tracker.progress(job_def, None).await; - } - - let msg = format!("copied files: {path_display}"); - tracker.msg(job_def, msg).await; - Ok(()) -} - /// Spawn a job to remove a directory recursively, adding the info the report logs pub async fn rm_folder(job_def: JobDefinition, path: &PathBuf) -> Result<(), Error> { if !path.exists() { diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 333f675d6..994151048 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -245,19 +245,11 @@ mod tests { JobType::Install { production }, )], ), - ( - JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), - vec![ - JobDefinition::new(Target::Shared, JobType::Install { production }), - JobDefinition::new(Target::Shared, JobType::Build { production }), - ], - ), ( JobDefinition::new(Target::Binding, JobType::Install { production }), vec![ JobDefinition::new(Target::Shared, JobType::Install { production }), JobDefinition::new(Target::Shared, JobType::Build { production }), - JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), ], ), ( @@ -265,7 +257,6 @@ mod tests { vec![ JobDefinition::new(Target::Shared, JobType::Install { production }), JobDefinition::new(Target::Shared, JobType::Build { production }), - JobDefinition::new(Target::Shared, JobType::AfterBuild { production }), JobDefinition::new(Target::Binding, JobType::Install { production }), ], ), diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 184a75c1f..25969983d 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -20,7 +20,6 @@ mod binding; mod cli; mod client; mod core; -mod shared; mod target_kind; mod updater; mod wasm; @@ -217,10 +216,7 @@ impl Target { Target::Binding | Target::Client | Target::Shared | Target::App ) } - JobType::AfterBuild { production: _ } => matches!( - self, - Target::Shared | Target::Binding | Target::Wrapper | Target::App - ), + JobType::AfterBuild { production: _ } => matches!(self, Target::Binding | Target::App), JobType::Test { production: _ } => matches!( self, Target::Wrapper | Target::Core | Target::Cli | Target::Wasm @@ -483,8 +479,6 @@ impl Target { let after_res = match self { Target::Binding => binding::copy_index_node(job_def).await, - Target::Wrapper => wrapper::copy_binding_to_app(job_def).await, - Target::Shared => shared::copy_platform_to_binding(job_def).await, Target::App => app::copy_client_to_app(job_def).await, _ => return None, }; diff --git a/cli/src/target/shared.rs b/cli/src/target/shared.rs deleted file mode 100644 index aac8f71c0..000000000 --- a/cli/src/target/shared.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::fs; - -use anyhow::Context; - -use crate::{fstools, jobs_runner::JobDefinition, spawner::SpawnResult}; - -use super::Target; - -pub async fn copy_platform_to_binding( - job_def: JobDefinition, -) -> Result { - let mut report_logs = Vec::new(); - - report_logs.push(String::from("Start Job: Copying Platform to Bindings...")); - - let platform_dest = Target::Wrapper.cwd().join("node_modules").join("platform"); - - let msg = format!("Removing directory: '{}'", platform_dest.display()); - report_logs.push(msg); - - fstools::rm_folder(job_def, &platform_dest).await?; - - tokio::fs::create_dir_all(&platform_dest) - .await - .with_context(|| format!("Error while creating directory {}", platform_dest.display()))?; - - let source = Target::Shared.cwd(); - - // This part to get all the needed files and folders to copy - let entries_to_copy: Vec<_> = fs::read_dir(&source) - .with_context(|| { - format!( - "Error while reading directory content: {}", - source.display() - ) - })? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name() - .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) - }) - .collect(); - - fstools::cp_many( - job_def, - entries_to_copy, - platform_dest, - source.display(), - &mut report_logs, - ) - .await?; - - Ok(SpawnResult::create_for_fs( - "Copying Platform to Bindings".into(), - report_logs, - )) -} diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 8fbc3fa0f..c6f700a8c 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,10 +1,8 @@ use std::{iter, path::PathBuf}; -use anyhow::{anyhow, Context}; -use std::fs; +use anyhow::anyhow; use crate::{ - fstools, job_type::JobType, jobs_runner::JobDefinition, spawner::{spawn, spawn_blocking, SpawnResult}, @@ -99,116 +97,3 @@ pub async fn run_test(production: bool) -> Result { final_result.ok_or_else(|| anyhow!("Wrapper doesn't have test specs")) } - -pub async fn copy_binding_to_app(job_def: JobDefinition) -> Result { - let mut report_logs = Vec::new(); - - // *** Copying TS Bindings *** - report_logs.push(String::from("Copying ts-bindings to electron...")); - let rustcore_dest = Target::App.cwd().join("node_modules").join("rustcore"); - - fstools::rm_folder(job_def, &rustcore_dest).await?; - - let msg = format!("Removing directory: '{}'", rustcore_dest.display()); - report_logs.push(msg); - - fs::create_dir_all(&rustcore_dest) - .with_context(|| format!("Error while creating directory {}", rustcore_dest.display()))?; - - // This part to get all the needed files and folders to copy - let ts_source = Target::Wrapper.cwd(); - let with_context = fs::read_dir(&ts_source).with_context(|| { - format!( - "Error while reading directory content: {}", - ts_source.display() - ) - }); - let ts_entries_to_copy: Vec<_> = with_context? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name() - .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) - }) - .collect(); - - fstools::cp_many( - job_def, - ts_entries_to_copy, - rustcore_dest.clone(), - ts_source.display(), - &mut report_logs, - ) - .await?; - - // *** Remove native folder *** - let native_dir_path = rustcore_dest.join("native"); - report_logs.push(format!( - "Removing the directory '{}'", - native_dir_path.display() - )); - - fstools::rm_folder(job_def, &native_dir_path).await?; - - // *** Copy Platform rustcore to electron *** - report_logs.push(String::from("Copying platform rustcore in to electron...")); - let platform_dest = rustcore_dest.join("node_modules").join("platform"); - - fstools::rm_folder(job_def, &platform_dest).await?; - fs::create_dir_all(&platform_dest).with_context(|| { - format!( - "Error while creating directory: {}", - platform_dest.display() - ) - })?; - - let platform_src = Target::Shared.cwd(); - - let platform_entries_to_copy: Vec<_> = fs::read_dir(&platform_src) - .with_context(|| { - format!( - "Error while reading directory content: {}", - platform_src.display() - ) - })? - .filter_map(|entry_res| entry_res.ok().map(|entry| entry.path())) - .filter(|path| { - path.file_name() - .is_some_and(|file_name| !file_name.to_string_lossy().starts_with("node_modules")) - }) - .collect(); - - fstools::cp_many( - job_def, - platform_entries_to_copy.clone(), - platform_dest, - platform_src.display(), - &mut report_logs, - ) - .await?; - - // *** Copy Platform to electron *** - report_logs.push(String::from("Copying platform in to electron...")); - let platform_dest2 = Target::App.cwd().join("node_modules").join("platform"); - - fstools::rm_folder(job_def, &platform_dest2).await?; - fs::create_dir_all(&platform_dest2).with_context(|| { - format!( - "Error while creating directory: {}", - platform_dest2.display() - ) - })?; - - fstools::cp_many( - job_def, - platform_entries_to_copy, - platform_dest2, - platform_src.display(), - &mut report_logs, - ) - .await?; - - Ok(SpawnResult::create_for_fs( - "Copy TS Bindings and Platform to Electron".into(), - report_logs, - )) -} From 7b8c7e06013580d1a111d79297436691e2d1c595 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 11:24:22 +0200 Subject: [PATCH 132/174] Build CLI: Run missing Yarn install command for Wasm Wasm Target have the kind RS but it needs to run `yarn install'. --- cli/src/target/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 25969983d..dfbe6a3d4 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -213,7 +213,7 @@ impl Target { JobType::Install { production: _ } => { matches!( self, - Target::Binding | Target::Client | Target::Shared | Target::App + Target::Binding | Target::Client | Target::Shared | Target::App | Target::Wasm ) } JobType::AfterBuild { production: _ } => matches!(self, Target::Binding | Target::App), @@ -506,7 +506,12 @@ async fn install_general( job_type: JobType, overridden_target: Option, ) -> Option> { - let cmd = target.kind().install_cmd(prod).await; + let cmd = match target { + // Wasm needs `yarn install` command despite having the kind `TargetKind::Rs` + Target::Wasm => TargetKind::Ts.install_cmd(prod).await, + t => t.kind().install_cmd(prod).await, + }; + let job_def = JobDefinition::new(overridden_target.unwrap_or(target), job_type); if let Some(cmd) = cmd { From b14819cb9b1f208e27b8245f09f9b7cf43127299 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 13:06:50 +0200 Subject: [PATCH 133/174] Build CLI: Add missing paths to clean command Targets `Wasm` and `Wrapper` have extra paths that need to be cleaned --- cli/src/target/mod.rs | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index dfbe6a3d4..9812865f1 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -2,6 +2,7 @@ use anyhow::bail; use clap::ValueEnum; use futures::future::join_all; use std::{iter, path::PathBuf, str::FromStr}; +use tokio::fs; use crate::{ checksum_records::ChecksumRecords, @@ -397,20 +398,35 @@ impl Target { checksum.remove_hash_if_exist(*self)?; let mut logs = Vec::new(); + let mut paths_to_remove = vec![self.cwd().join("dist")]; let path = match self.kind() { TargetKind::Ts => self.cwd().join("node_modules"), TargetKind::Rs => self.cwd().join("target"), }; + paths_to_remove.push(path); - let remove_log = format!("removing directory {}", path.display()); - logs.push(remove_log); + match self { + Target::Wasm => { + paths_to_remove.push(self.cwd().join("pkg")); + paths_to_remove.push(self.cwd().join("test_output")); + } + Target::Wrapper => { + paths_to_remove.push(self.cwd().join("spec").join("build")); + let index_node_path = self.cwd().join("src").join("native").join("index.node"); + if index_node_path.exists() { + logs.push(format!("removing file: {}", index_node_path.display())); + fs::remove_file(index_node_path).await?; + } + } + _ => {} + } - fstools::rm_folder(job_def, &path).await?; + for path in paths_to_remove.into_iter().filter(|p| p.exists()) { + let remove_log = format!("removing directory {}", path.display()); + logs.push(remove_log); - let dist_path = self.cwd().join("dist"); - let remove_log = format!("removing {}", dist_path.display()); - logs.push(remove_log); - fstools::rm_folder(job_def, &dist_path).await?; + fstools::rm_folder(job_def, &path).await?; + } let job = format!("Clean {}", self); From 8943884d503cadb59f8ab3d41da68aa722d606fa Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 6 Jun 2024 15:18:57 +0200 Subject: [PATCH 134/174] Build CLI: Solve Build TS targets with Linting - TS Targets must be built when linting is running on it because linting doesn't cover type checks - This led to problem that all involved targets to build TS ones must be built in the correct order with respect to concurrency. - Therefore build has been added to Lint job type and excluded manually from Rust jobs that aren't involved in building TS targets - Build Spawn has been removed from ts_lint() method since we are calling build command on it. - Lint is involved now in checksums calculations because build is involved in their pipeline too. --- cli/src/checksum_records.rs | 3 ++- cli/src/job_type.rs | 5 +++-- cli/src/jobs_runner/jobs_resolver.rs | 31 ++++++++++++++++++++++++++-- cli/src/target/mod.rs | 14 +------------ 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index ad8078a0e..b1639e21a 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -32,7 +32,8 @@ impl ChecksumRecords { pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { // calculate should be involved when build is called at some point of the job let (calculate_involved, prod) = match &job_type { - JobType::Lint => return Ok(()), + // Linting build targets for TS targets and their dependencies + JobType::Lint => (true, false), JobType::Build { production } | JobType::Run { production } | JobType::Test { production } => (true, *production), diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index b5bb0d1ed..f7308abc4 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -4,11 +4,11 @@ use std::fmt::Display; //NOTE: The order of job types must match the runnig-order of them because it's used by // solving their dependencies-graph using BTreeMap pub enum JobType { - Lint, Clean, Install { production: bool }, Build { production: bool }, AfterBuild { production: bool }, + Lint, Test { production: bool }, Run { production: bool }, } @@ -42,7 +42,8 @@ impl JobType { /// Returns job types that are involved with this job and should run with it. pub fn get_involved_jobs(&self) -> Vec { match self { - JobType::Lint => vec![JobType::Install { production: false }], + // Linting TS needs to building too to check for type errors + JobType::Lint => vec![JobType::Build { production: false }], JobType::Build { production } => vec![ // Install run always in development at first then it should get reinstalled with // production after build command is ran. diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 994151048..548318fe9 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -23,7 +23,10 @@ pub fn resolve( let mut jobs_tree: BTreeMap> = BTreeMap::new(); for target in involved_targets { - for job in involved_jobs.iter().filter(|j| target.has_job(j)) { + for job in involved_jobs + .iter() + .filter(|j| is_job_involved(&target, j, &main_job)) + { // Start with dependencies from other targets (Applies for Build & Install jobs only) // Install jobs are involved here too because copying the files in the after build // process could delete the current files. @@ -101,6 +104,25 @@ fn flatten_targets_for_build(targets: &[Target]) -> BTreeSet { resolved_targets } +/// Check if job involved depending if the target has a job for the current job type + Additional +/// filter based on the main job type (Currently used because TS linting require all build steps) +/// +/// * `target`: Job Target +/// * `current_job`: Current job type to check if it has job for the given target +/// * `main_job`: Main job type, which is used for the additional filter +fn is_job_involved(target: &Target, current_job: &JobType, main_job: &JobType) -> bool { + let additional_filter = match (main_job, target) { + // For linting TS targets we need to build all their dependencies targets that have impact + // on building TS. Therefore we can exclude Core and CLI and Updater only. + (JobType::Lint, Target::Core | Target::Cli | Target::Updater) => { + matches!(current_job, JobType::Lint) + } + _ => true, + }; + + additional_filter && target.has_job(current_job) +} + #[cfg(test)] mod tests { use super::*; @@ -114,7 +136,12 @@ mod tests { #[test] fn flatten_lint_job() { let production = false; - let expected_lint = BTreeSet::from([JobType::Lint, JobType::Install { production }]); + let expected_lint = BTreeSet::from([ + JobType::Lint, + JobType::Install { production }, + JobType::Build { production }, + JobType::AfterBuild { production }, + ]); assert_eq!(flatten_jobs(JobType::Lint), expected_lint); } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 9812865f1..a7511d3f3 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -347,21 +347,9 @@ impl Target { let path = self.cwd(); let yarn_cmd = DevTool::Yarn.path().await.to_string_lossy(); let job_def = JobDefinition::new(*self, JobType::Lint); - let status = spawn( - job_def, - format!("{} run lint", yarn_cmd), - Some(path.clone()), - iter::empty(), - None, - ) - .await?; - if !status.status.success() { - return Ok(status); - } - spawn( job_def, - format!("{} run build", yarn_cmd), + format!("{} run lint", yarn_cmd), Some(path), iter::empty(), None, From 3a9171f173b5c764e114f4469b75849ed643a9f6 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 7 Jun 2024 10:54:20 +0200 Subject: [PATCH 135/174] Build CLI: Define commands and their args explicitly Processes Commands will be explicitly defined instead delivering them as strings that will be spliced in the spawner again because this led to problems when the path for the command has a white space in it (Like C:\Program Files... in Windows) --- cli/src/spawner.rs | 29 +++++++------ cli/src/target/binding.rs | 17 +++++--- cli/src/target/cli.rs | 25 +++++++---- cli/src/target/core.rs | 25 +++++++---- cli/src/target/mod.rs | 82 ++++++++++++++++++++++------------- cli/src/target/target_kind.rs | 46 +++++++++++++------- cli/src/target/updater.rs | 16 +++++-- cli/src/target/wasm.rs | 39 ++++++++++++----- cli/src/target/wrapper.rs | 17 +++++--- 9 files changed, 191 insertions(+), 105 deletions(-) diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 922d36220..8155fe6ca 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -1,4 +1,6 @@ -use crate::{jobs_runner::JobDefinition, location::get_root, tracker::get_tracker}; +use crate::{ + jobs_runner::JobDefinition, location::get_root, target::ProcessCommand, tracker::get_tracker, +}; use anyhow::{bail, Context}; use core::panic; use futures_lite::{future, FutureExt}; @@ -80,29 +82,30 @@ pub(crate) struct SpawnOptions { /// Spawns and runs a job asynchronously, updating the bar when job infos are available pub async fn spawn( job_def: JobDefinition, - command: String, + command: ProcessCommand, cwd: Option, environment_vars: impl IntoIterator, opts: Option, ) -> Result { let opts = opts.unwrap_or_default(); let cwd = cwd.unwrap_or_else(|| get_root().clone()); - let mut parts = command.split(' ').collect::>(); - let cmd = parts.remove(0); let mut combined_env_vars = vec![(String::from("TERM"), String::from("xterm-256color"))]; combined_env_vars.extend(environment_vars); let tracker = get_tracker().await; - let command_result = Command::new(cmd) + let command_result = Command::new(&command.cmd) .current_dir(&cwd) - .args(&parts) + .args(&command.args) .envs(combined_env_vars) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .with_context(|| { - format!("Error While running the command '{cmd}'\nwith arguments: {parts:?}") + format!( + "Error While running the command '{}'\nwith arguments: {:?}", + command.cmd, command.args + ) }); let mut child = command_result?; @@ -170,7 +173,7 @@ pub async fn spawn( report: report_lines, status, job: job_def.job_title(), - cmd: command, + cmd: command.to_string(), skipped, }) } else { @@ -182,20 +185,18 @@ pub async fn spawn( /// This is used with commands that doesn't work with `Stdio::piped()` pub async fn spawn_blocking( job_def: JobDefinition, - command: String, + command: ProcessCommand, cwd: Option, environment_vars: impl IntoIterator, ) -> Result { let cwd = cwd.unwrap_or_else(|| get_root().clone()); - let mut parts = command.split(' ').collect::>(); - let cmd = parts.remove(0); let mut combined_env_vars = vec![(String::from("TERM"), String::from("xterm-256color"))]; combined_env_vars.extend(environment_vars); - let mut child = std::process::Command::new(cmd); + let mut child = std::process::Command::new(&command.cmd); child.current_dir(&cwd); - child.args(&parts); + child.args(&command.args); child.envs(combined_env_vars); let tracker = get_tracker().await; @@ -206,7 +207,7 @@ pub async fn spawn_blocking( report: Vec::new(), status, job: job_def.job_title(), - cmd: command, + cmd: command.to_string(), skipped: None, }) } diff --git a/cli/src/target/binding.rs b/cli/src/target/binding.rs index 3e6ae7106..4933887e5 100644 --- a/cli/src/target/binding.rs +++ b/cli/src/target/binding.rs @@ -4,9 +4,9 @@ use anyhow::{bail, Context}; use crate::{fstools, jobs_runner::JobDefinition, spawner::SpawnResult}; -use super::Target; +use super::{ProcessCommand, Target}; -pub fn get_build_cmd(prod: bool) -> anyhow::Result { +pub fn get_build_cmd(prod: bool) -> anyhow::Result { let mut path = Target::Wrapper.cwd(); path.push("node_modules"); path.push(".bin"); @@ -20,11 +20,14 @@ pub fn get_build_cmd(prod: bool) -> anyhow::Result { path.push("electron-build-env"); } - Ok(format!( - "{} nj-cli build{}", - path.to_string_lossy(), - //TODO: Ruby code build always in release mode - if prod { " --release" } else { "" } + let mut args = vec![String::from("nj-cli"), String::from("build")]; + + if prod { + args.push("--release".into()); + } + Ok(ProcessCommand::new( + path.to_string_lossy().to_string(), + args, )) } diff --git a/cli/src/target/cli.rs b/cli/src/target/cli.rs index 401b894c0..50b889668 100644 --- a/cli/src/target/cli.rs +++ b/cli/src/target/cli.rs @@ -1,14 +1,21 @@ -use crate::{dev_tools::DevTool, target::Target}; +use crate::{ + dev_tools::DevTool, + target::{ProcessCommand, Target}, +}; -use super::TestCommand; +use super::TestSpawnCommand; -pub async fn get_test_cmds(production: bool) -> Vec { +pub async fn get_test_cmds(production: bool) -> Vec { let cargo_path = DevTool::Cargo.path().await; - let cmd = format!( - "{} +stable test{} --color always", - cargo_path.to_string_lossy(), - if production { " -r" } else { "" } - ); - vec![TestCommand::new(cmd, Target::Cli.cwd(), None)] + let mut args = vec![String::from("+stable"), String::from("test")]; + if production { + args.push("-r".into()); + } + args.push("--color".into()); + args.push("always".into()); + + let cmd = ProcessCommand::new(cargo_path.to_string_lossy().to_string(), args); + + vec![TestSpawnCommand::new(cmd, Target::Cli.cwd(), None)] } diff --git a/cli/src/target/core.rs b/cli/src/target/core.rs index 31f7f165f..261b5bf13 100644 --- a/cli/src/target/core.rs +++ b/cli/src/target/core.rs @@ -1,14 +1,21 @@ -use crate::{dev_tools::DevTool, target::Target}; +use crate::{ + dev_tools::DevTool, + target::{ProcessCommand, Target}, +}; -use super::TestCommand; +use super::TestSpawnCommand; -pub async fn get_test_cmds(production: bool) -> Vec { +pub async fn get_test_cmds(production: bool) -> Vec { let cargo_path = DevTool::Cargo.path().await; - let cmd = format!( - "{} +stable test{} --color always", - cargo_path.to_string_lossy(), - if production { " -r" } else { "" } - ); - vec![TestCommand::new(cmd, Target::Core.cwd(), None)] + let mut args = vec![String::from("+stable"), String::from("test")]; + if production { + args.push("-r".into()); + } + args.push("--color".into()); + args.push("always".into()); + + let cmd = ProcessCommand::new(cargo_path.to_string_lossy().to_string(), args); + + vec![TestSpawnCommand::new(cmd, Target::Core.cwd(), None)] } diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index a7511d3f3..49e6eab51 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -1,7 +1,7 @@ use anyhow::bail; use clap::ValueEnum; use futures::future::join_all; -use std::{iter, path::PathBuf, str::FromStr}; +use std::{fmt::Display, iter, path::PathBuf, str::FromStr}; use tokio::fs; use crate::{ @@ -50,14 +50,35 @@ pub enum Target { Cli, } -pub struct TestCommand { - command: String, +#[derive(Debug, Clone)] +/// Represents a command to run with `process::Command` and its arguments +pub struct ProcessCommand { + pub cmd: String, + pub args: Vec, +} + +impl ProcessCommand { + pub fn new(cmd: String, args: Vec) -> Self { + Self { cmd, args } + } +} + +impl Display for ProcessCommand { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {}", self.cmd, self.args.join(" ")) + } +} + +#[derive(Debug, Clone)] +/// Represents command and spawns infos to run with `Target::Test` +pub struct TestSpawnCommand { + command: ProcessCommand, cwd: PathBuf, spawn_opts: Option, } -impl TestCommand { - fn new(command: String, cwd: PathBuf, spawn_opts: Option) -> Self { +impl TestSpawnCommand { + fn new(command: ProcessCommand, cwd: PathBuf, spawn_opts: Option) -> Self { Self { command, cwd, @@ -227,7 +248,7 @@ impl Target { } /// Provide the command that should be used in to build the target - pub async fn build_cmd(&self, prod: bool) -> anyhow::Result { + pub async fn build_cmd(&self, prod: bool) -> anyhow::Result { let build_cmd = match self { Target::Binding => binding::get_build_cmd(prod)?, Target::Wasm => wasm::get_build_cmd(prod).await, @@ -286,7 +307,7 @@ impl Target { } /// Provides the test commands for the given target if available - async fn test_cmds(&self, production: bool) -> Option> { + async fn test_cmds(&self, production: bool) -> Option> { match self { Target::Core => Some(core::get_test_cmds(production).await), Target::Cli => Some(cli::get_test_cmds(production).await), @@ -345,35 +366,38 @@ impl Target { /// compiling errors async fn ts_lint(&self) -> Result { let path = self.cwd(); - let yarn_cmd = DevTool::Yarn.path().await.to_string_lossy(); let job_def = JobDefinition::new(*self, JobType::Lint); - spawn( - job_def, - format!("{} run lint", yarn_cmd), - Some(path), - iter::empty(), - None, - ) - .await + + let yarn_cmd = DevTool::Yarn.path().await; + let command = ProcessCommand::new( + yarn_cmd.to_string_lossy().to_string(), + vec![String::from("run"), String::from("lint")], + ); + spawn(job_def, command, Some(path), iter::empty(), None).await } /// Runs Clippy for the given rust target async fn clippy(&self) -> Result { let path = get_root().join(self.cwd()); - let cargo_path = DevTool::Cargo.path().await; let job_def = JobDefinition::new(*self, JobType::Lint); - spawn( - job_def, - format!( - "{} clippy --color always --all --all-features -- -D warnings", - cargo_path.to_string_lossy() - ), - Some(path), - iter::empty(), - None, - ) - .await + + let cargo_path = DevTool::Cargo.path().await; + let command = ProcessCommand::new( + cargo_path.to_string_lossy().to_string(), + vec![ + String::from("clippy"), + String::from("--color"), + String::from("always"), + String::from("--all"), + String::from("--all-features"), + String::from("--"), + String::from("-D"), + String::from("warnings"), + ], + ); + + spawn(job_def, command, Some(path), iter::empty(), None).await } /// Clean the given target, removing it from the checksum tracker as well. @@ -434,7 +458,7 @@ impl Target { let job_def = JobDefinition::new(*self, JobType::Build { production: prod }); if skip { - spawn_skip(job_def, cmd).await + spawn_skip(job_def, cmd.to_string()).await } else { spawn(job_def, cmd, Some(path), iter::empty(), Some(spawn_opt)).await } diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index cfc6b2e71..7c019ee71 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -1,5 +1,7 @@ use crate::dev_tools::DevTool; +use super::ProcessCommand; + #[derive(Debug, Clone)] pub enum TargetKind { /// TypeScript @@ -10,35 +12,47 @@ pub enum TargetKind { impl TargetKind { /// Provide the general build command for each target type - pub async fn build_cmd(&self, prod: bool) -> String { + pub async fn build_cmd(&self, prod: bool) -> ProcessCommand { match self { TargetKind::Ts => { let yarn_path = DevTool::Yarn.path().await; - format!( - "{} run {}", - yarn_path.to_string_lossy(), - if prod { "prod" } else { "build" } - ) + let mut args = vec![String::from("run")]; + if prod { + args.push("prod".into()); + } else { + args.push("build".into()); + } + + ProcessCommand::new(yarn_path.to_string_lossy().to_string(), args) } TargetKind::Rs => { let cargo_path = DevTool::Cargo.path().await; - format!( - "{} build --color always{}", - cargo_path.to_string_lossy(), - if prod { " --release" } else { "" } - ) + let mut args = vec![ + String::from("build"), + String::from("--color"), + String::from("always"), + ]; + if prod { + args.push("--release".into()); + } + + ProcessCommand::new(cargo_path.to_string_lossy().to_string(), args) } } } /// Provide the general install command for each target type - pub async fn install_cmd(&self, prod: bool) -> Option { + pub async fn install_cmd(&self, prod: bool) -> Option { match self { TargetKind::Ts => { let yarn_path = DevTool::Yarn.path().await; - Some(format!( - "{} install{}", - yarn_path.to_string_lossy(), - if prod { " --production" } else { "" } + let mut args = vec![String::from("install")]; + if prod { + args.push("--production".into()); + } + + Some(ProcessCommand::new( + yarn_path.to_string_lossy().to_string(), + args, )) } TargetKind::Rs => None, diff --git a/cli/src/target/updater.rs b/cli/src/target/updater.rs index 334606d3d..b2256ce7a 100644 --- a/cli/src/target/updater.rs +++ b/cli/src/target/updater.rs @@ -1,10 +1,18 @@ use crate::dev_tools::DevTool; -pub async fn get_build_cmd() -> String { +use super::ProcessCommand; + +pub async fn get_build_cmd() -> ProcessCommand { let cargo_path = DevTool::Cargo.path().await; - format!( - "{} +stable build --color always --release", - cargo_path.to_string_lossy() + ProcessCommand::new( + cargo_path.to_string_lossy().to_string(), + vec![ + String::from("+stable"), + String::from("build"), + String::from("--color"), + String::from("always"), + String::from("--release"), + ], ) } diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index 95691566f..c72d8ade2 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -1,34 +1,49 @@ use crate::{dev_tools::DevTool, spawner::SpawnOptions, target::Target}; -use super::TestCommand; +use super::{ProcessCommand, TestSpawnCommand}; -pub async fn get_build_cmd(prod: bool) -> String { +pub async fn get_build_cmd(prod: bool) -> ProcessCommand { let wasm_pack_path = DevTool::WasmPack.path().await; let env = if prod { "--release" } else { "--dev" }; - format!( - "{} build {env} --target bundler --color always", - wasm_pack_path.to_string_lossy() + ProcessCommand::new( + wasm_pack_path.to_string_lossy().to_string(), + vec![ + String::from("build"), + String::from(env), + String::from("--target"), + String::from("bundler"), + String::from("--color"), + String::from("always"), + ], ) } -pub async fn get_test_cmds() -> Vec { +pub async fn get_test_cmds() -> Vec { let cwd = Target::Wasm.cwd(); let npm_path = DevTool::Npm.path().await; let wasm_pack_path = DevTool::WasmPack.path().await; vec![ - TestCommand::new( - format!( - "{} test --node --color always", - wasm_pack_path.to_string_lossy() + TestSpawnCommand::new( + ProcessCommand::new( + wasm_pack_path.to_string_lossy().to_string(), + vec![ + String::from("test"), + String::from("--node"), + String::from("--color"), + String::from("always"), + ], ), cwd.clone(), None, ), - TestCommand::new( - format!("{} run test", npm_path.to_string_lossy()), + TestSpawnCommand::new( + ProcessCommand::new( + npm_path.to_string_lossy().to_string(), + vec![String::from("run"), String::from("test")], + ), cwd.join("spec"), Some(SpawnOptions { suppress_msg: true, diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index c6f700a8c..24bd1d6d0 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -8,7 +8,7 @@ use crate::{ spawner::{spawn, spawn_blocking, SpawnResult}, }; -use super::Target; +use super::{ProcessCommand, Target}; const TEST_SPECS: [&str; 14] = [ // TODO: @@ -50,7 +50,11 @@ pub async fn run_test(production: bool) -> Result { // The check should cover if the test themselves or the code under the tests has been changed. if !build_spec_path.join("build").exists() { let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); - let build_spec_cmd = format!("{} -p tsconfig.json", test_builder_path.to_string_lossy()); + + let build_spec_cmd = ProcessCommand::new( + test_builder_path.to_string_lossy().to_string(), + vec![String::from("-p"), String::from("tsconfig.json")], + ); let spec_res = spawn( job_def, @@ -77,9 +81,12 @@ pub async fn run_test(production: bool) -> Result { for spec in TEST_SPECS { let spec_file_name = format!("session.{spec}.spec.js"); let spec_file_path = specs_dir_path.join(spec_file_name); - let command = format!( - "{electron_path} {jasmine_path} {}", - spec_file_path.to_string_lossy() + let command = ProcessCommand::new( + electron_path.to_string(), + vec![ + jasmine_path.to_string(), + spec_file_path.to_string_lossy().to_string(), + ], ); let res = spawn_blocking( job_def, From 90ddfded21c2bb1e0e9f1f44c840bdd16cc46098 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 14 Jun 2024 12:06:24 +0200 Subject: [PATCH 136/174] Build CLI: More Comments & Documentations --- cli/src/jobs_runner/jobs_resolver.rs | 5 +++++ cli/src/jobs_runner/mod.rs | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 548318fe9..de7300aa9 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -4,6 +4,8 @@ use crate::{job_type::JobType, target::Target}; use super::JobDefinition; +/// Resolve tasks dependcies for the given targets and job, +/// returning dependcies map for the tasks pub fn resolve( targets: &[Target], main_job: JobType, @@ -68,6 +70,7 @@ pub fn resolve( jobs_tree } +/// Returns all involved job types according to the given job type. fn flatten_jobs(main_job: JobType) -> BTreeSet { fn flatten_rec(job: JobType, involved_jobs: &mut BTreeSet) { if !involved_jobs.insert(job) { @@ -85,6 +88,7 @@ fn flatten_jobs(main_job: JobType) -> BTreeSet { jobs } +/// Returns all involved targets for the given target for build tasks fn flatten_targets_for_build(targets: &[Target]) -> BTreeSet { fn flatten_rec(target: Target, involved_targets: &mut BTreeSet) { if !involved_targets.insert(target) { @@ -114,6 +118,7 @@ fn is_job_involved(target: &Target, current_job: &JobType, main_job: &JobType) - let additional_filter = match (main_job, target) { // For linting TS targets we need to build all their dependencies targets that have impact // on building TS. Therefore we can exclude Core and CLI and Updater only. + // Rust Linting doesn't need build and need to be excluded in the additional filter. (JobType::Lint, Target::Core | Target::Cli | Target::Updater) => { matches!(current_job, JobType::Lint) } diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index edf06ab47..acef29c6c 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -16,12 +16,18 @@ use anyhow::Result; type SpawnResultsCollection = Vec>; #[derive(Debug, Clone)] +/// Represents the current state of the task. enum JobPhase { + /// Job is waiting to the jobs in the list to finish Awaiting(Vec), + /// Job is running currently Running, + /// Job is finished Done, } +/// Runs all the needed tasks for the given targets and the main jobs asynchronously, +/// returning a list of the tasks results pub async fn run(targets: &[Target], main_job: JobType) -> Result { let jobs_tree = jobs_resolver::resolve(targets, main_job); @@ -46,6 +52,7 @@ pub async fn run(targets: &[Target], main_job: JobType) -> Result Result Date: Mon, 1 Jul 2024 13:48:06 +0200 Subject: [PATCH 137/174] Build CLI: Replace tokio::OnceCell with OnceLock - Use `OnceLock` from the standard library after it got stabilized and remove all the no needed any more async/await everywhere in the app. --- cli/src/app_runner.rs | 2 +- cli/src/checksum_records.rs | 14 ++--- cli/src/dev_environment.rs | 8 +-- cli/src/dev_tools.rs | 82 +++++++++++++-------------- cli/src/fstools.rs | 6 +- cli/src/jobs_runner/job_definition.rs | 2 +- cli/src/jobs_runner/mod.rs | 6 +- cli/src/location.rs | 6 +- cli/src/main.rs | 18 +++--- cli/src/spawner.rs | 4 +- cli/src/target/cli.rs | 4 +- cli/src/target/core.rs | 4 +- cli/src/target/mod.rs | 30 +++++----- cli/src/target/target_kind.rs | 10 ++-- cli/src/target/updater.rs | 4 +- cli/src/target/wasm.rs | 10 ++-- cli/src/tracker.rs | 9 +-- 17 files changed, 105 insertions(+), 114 deletions(-) diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index 73cb39cdb..7d6407a03 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -7,7 +7,7 @@ use crate::{dev_tools::DevTool, target::Target}; pub async fn run_app() -> io::Result { let electron_path = Target::App.cwd(); - let yarn_path = DevTool::Yarn.path().await; + let yarn_path = DevTool::Yarn.path(); let electron_arg = if cfg!(windows) { "electron-win" diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index b1639e21a..3f417eb2d 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -3,12 +3,11 @@ use std::{ fs::{self, File}, io::Write, path::PathBuf, - sync::Mutex, + sync::{Mutex, OnceLock}, }; use anyhow::{anyhow, Context}; use dir_checksum::{calc_combined_checksum, HashDigest}; -use tokio::sync::OnceCell; use crate::{job_type::JobType, location::get_root, target::Target}; @@ -29,7 +28,7 @@ struct ChecksumItems { impl ChecksumRecords { /// Update checksum records for involved jobs depending on the job type. /// It will calculate new checksums if build tasks were involved. - pub async fn update_and_save(job_type: JobType) -> anyhow::Result<()> { + pub fn update_and_save(job_type: JobType) -> anyhow::Result<()> { // calculate should be involved when build is called at some point of the job let (calculate_involved, prod) = match &job_type { // Linting build targets for TS targets and their dependencies @@ -44,7 +43,7 @@ impl ChecksumRecords { } }; - let records = Self::get(prod).await?; + let records = Self::get(prod)?; if calculate_involved { records.calculate_involved_hashes()?; @@ -79,12 +78,11 @@ impl ChecksumRecords { } /// Returns a reference to checksums records manager singleton - pub async fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { - static CHECKSUM_RECORDS: OnceCell> = OnceCell::const_new(); + pub fn get(production: bool) -> anyhow::Result<&'static ChecksumRecords> { + static CHECKSUM_RECORDS: OnceLock> = OnceLock::new(); CHECKSUM_RECORDS - .get_or_init(|| async { ChecksumRecords::load(production) }) - .await + .get_or_init(|| ChecksumRecords::load(production)) .as_ref() .map_err(|err| anyhow!("{err}")) } diff --git a/cli/src/dev_environment.rs b/cli/src/dev_environment.rs index dbf0c18e3..23f9ebc8b 100644 --- a/cli/src/dev_environment.rs +++ b/cli/src/dev_environment.rs @@ -5,10 +5,10 @@ use anyhow::bail; use crate::dev_tools::DevTool; /// Resolve the paths for all development tool returning an Error if any of them can't be resolved -pub async fn resolve_dev_tools() -> anyhow::Result<()> { +pub fn resolve_dev_tools() -> anyhow::Result<()> { let mut errors = None; for tool in DevTool::all() { - let Err(err) = tool.resolve().await else { + let Err(err) = tool.resolve() else { continue; }; @@ -44,10 +44,10 @@ pub async fn resolve_dev_tools() -> anyhow::Result<()> { /// Prints the information of the needed tools for the development if available, otherwise prints /// error information to `stderr` -pub async fn print_env_info() { +pub fn print_env_info() { for tool in DevTool::all() { println!("{tool} Info:"); - match tool.resolve().await { + match tool.resolve() { Ok(cmd) => { if let Err(err) = Command::new(cmd).arg(tool.version_args()).status() { eprintln!("Error while retrieving dependency's information: {err}"); diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index ca0d73149..79e7cb015 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -1,7 +1,6 @@ use anyhow::{anyhow, Result}; -use std::{fmt::Display, path::PathBuf}; +use std::{fmt::Display, path::PathBuf, sync::OnceLock}; -use tokio::sync::OnceCell; use which::{which_all_global, which_global}; #[derive(Debug, Clone, Copy)] @@ -77,84 +76,79 @@ impl DevTool { } /// Resolve the path of the tool if exists. Returning an Error when not possible - pub async fn resolve(&self) -> &'static Result { + pub fn resolve(&self) -> &'static Result { match self { - DevTool::Node => resolve_node().await, - DevTool::Npm => resolve_npm().await, - DevTool::Yarn => resolve_yarn().await, - DevTool::RustUp => resolve_rustup().await, - DevTool::Cargo => resolve_cargo().await, - DevTool::WasmPack => resolve_wasm_pack().await, - DevTool::NjCli => resolve_nj_cli().await, + DevTool::Node => resolve_node(), + DevTool::Npm => resolve_npm(), + DevTool::Yarn => resolve_yarn(), + DevTool::RustUp => resolve_rustup(), + DevTool::Cargo => resolve_cargo(), + DevTool::WasmPack => resolve_wasm_pack(), + DevTool::NjCli => resolve_nj_cli(), } } /// Get the path of the resolved tool. Panics if the tool can't be resolved - pub async fn path(&self) -> &'static PathBuf { + pub fn path(&self) -> &'static PathBuf { self.resolve() - .await .as_ref() .expect("Developer Error: Cmd has already been resolved") } } -async fn resolve_node() -> &'static Result { - static NODE: OnceCell> = OnceCell::const_new(); +fn resolve_node() -> &'static Result { + static NODE: OnceLock> = OnceLock::new(); - NODE.get_or_init(|| async { find_cmd("node") }).await + NODE.get_or_init(|| find_cmd("node")) } fn find_cmd(cmd: &str) -> Result { which_global(cmd).map_err(|err| anyhow!("Command `{cmd}` couldn't be resolved. Err: {err}")) } -async fn resolve_npm() -> &'static Result { - static NPM: OnceCell> = OnceCell::const_new(); +fn resolve_npm() -> &'static Result { + static NPM: OnceLock> = OnceLock::new(); - NPM.get_or_init(|| async { find_cmd("npm") }).await + NPM.get_or_init(|| find_cmd("npm")) } -async fn resolve_yarn() -> &'static Result { - static YARN: OnceCell> = OnceCell::const_new(); +fn resolve_yarn() -> &'static Result { + static YARN: OnceLock> = OnceLock::new(); - YARN.get_or_init(|| async { find_cmd("yarn") }).await + YARN.get_or_init(|| find_cmd("yarn")) } -async fn resolve_rustup() -> &'static Result { - static RUSTUP: OnceCell> = OnceCell::const_new(); +fn resolve_rustup() -> &'static Result { + static RUSTUP: OnceLock> = OnceLock::new(); - RUSTUP.get_or_init(|| async { find_cmd("rustup") }).await + RUSTUP.get_or_init(|| find_cmd("rustup")) } -async fn resolve_cargo() -> &'static Result { - static CARGO: OnceCell> = OnceCell::const_new(); +fn resolve_cargo() -> &'static Result { + static CARGO: OnceLock> = OnceLock::new(); if cfg!(windows) { // Rust adds its toolchain to PATH in windows which must be filtered out - CARGO - .get_or_init(|| async { - let mut paths = which_all_global("cargo")?; - - paths - .find(|p| p.components().any(|c| c.as_os_str() == ".cargo")) - .ok_or_else(|| anyhow!("The command 'cargo' can't be found")) - }) - .await + CARGO.get_or_init(|| { + let mut paths = which_all_global("cargo")?; + + paths + .find(|p| p.components().any(|c| c.as_os_str() == ".cargo")) + .ok_or_else(|| anyhow!("The command 'cargo' can't be found")) + }) } else { - CARGO.get_or_init(|| async { find_cmd("cargo") }).await + CARGO.get_or_init(|| find_cmd("cargo")) } } -async fn resolve_wasm_pack() -> &'static Result { - static WASM_PACK: OnceCell> = OnceCell::const_new(); +fn resolve_wasm_pack() -> &'static Result { + static WASM_PACK: OnceLock> = OnceLock::new(); - WASM_PACK - .get_or_init(|| async { find_cmd("wasm-pack") }) - .await + WASM_PACK.get_or_init(|| find_cmd("wasm-pack")) } -async fn resolve_nj_cli() -> &'static Result { - static NJ_CLI: OnceCell> = OnceCell::const_new(); +fn resolve_nj_cli() -> &'static Result { + static NJ_CLI: OnceLock> = OnceLock::new(); - NJ_CLI.get_or_init(|| async { find_cmd("nj-cli") }).await + NJ_CLI.get_or_init(|| find_cmd("nj-cli")) } diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index ebfc322ec..3fac08ae4 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -17,7 +17,7 @@ pub async fn cp_file( let msg = format!("copying file: '{}' to '{}'", src.display(), dest.display()); report_logs.push(msg); - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker.msg(job_def, "copying files".into()).await; fs::copy(&src, &dest).with_context(|| { @@ -51,7 +51,7 @@ pub async fn cp_folder( let report_msg = format!("copying directory: {path_display}"); report_logs.push(report_msg.clone()); - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker.msg(job_def, report_msg).await; let _ = tokio::spawn(async move { @@ -87,7 +87,7 @@ pub async fn rm_folder(job_def: JobDefinition, path: &PathBuf) -> Result<(), Err if !path.exists() { return Ok(()); } - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker .msg(job_def, format!("removing directory: {}", path.display())) .await; diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 08f93714f..d3c83aa2d 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -22,7 +22,7 @@ impl JobDefinition { /// Run the job definition if it has a job, communicating its status with the UI bars pub async fn run(&self, skip: bool) -> Option> { - let tracker = get_tracker().await; + let tracker = get_tracker(); if let Err(err) = tracker.start(*self).await { return Some(Err(err)); } diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index acef29c6c..9c41dd201 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -31,7 +31,7 @@ enum JobPhase { pub async fn run(targets: &[Target], main_job: JobType) -> Result { let jobs_tree = jobs_resolver::resolve(targets, main_job); - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker .register_all(jobs_tree.keys().cloned().collect()) .await?; @@ -123,7 +123,7 @@ async fn spawn_jobs( } else { // Calculate target checksums and compare it the persisted one let prod = job_def.job_type.is_production().is_some_and(|prod| prod); - let checksum_rec = ChecksumRecords::get(prod).await?; + let checksum_rec = ChecksumRecords::get(prod)?; checksum_rec.register_job(job_def.target)?; // Check if all dependent jobs are skipped, then do the checksum calculations @@ -156,7 +156,7 @@ async fn spawn_jobs( }; if sender.send((job_def, result)).is_err() { - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker .print(format!( "Error: Job results can't be sent to receiver. Job: {job_def:?}" diff --git a/cli/src/location.rs b/cli/src/location.rs index 33ad475e9..a18ba7509 100644 --- a/cli/src/location.rs +++ b/cli/src/location.rs @@ -1,11 +1,9 @@ use anyhow::{bail, Context, Error}; use git2::Repository; -use std::{env::current_dir, path::PathBuf}; +use std::{env::current_dir, path::PathBuf, sync::OnceLock}; -use tokio::sync::OnceCell; - -pub static LOCATION: OnceCell = OnceCell::const_new(); +pub static LOCATION: OnceLock = OnceLock::new(); #[derive(Clone, Debug)] pub struct Location { diff --git a/cli/src/main.rs b/cli/src/main.rs index ca6b4582d..fe243dddf 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -50,12 +50,12 @@ async fn main() -> Result<(), Error> { let (job_type, results) = match command { Command::Environment(sub_command) => match sub_command { EnvironmentCommand::Check => { - resolve_dev_tools().await?; + resolve_dev_tools()?; println!("All needed tools for development are installed"); return Ok(()); } EnvironmentCommand::Print => { - print_env_info().await; + print_env_info(); return Ok(()); } }, @@ -68,7 +68,7 @@ async fn main() -> Result<(), Error> { return Ok(()); } Command::Lint { target, report } => { - resolve_dev_tools().await?; + resolve_dev_tools()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = jobs_runner::run(&targets, JobType::Lint).await?; @@ -79,14 +79,14 @@ async fn main() -> Result<(), Error> { production, report, } => { - resolve_dev_tools().await?; + resolve_dev_tools()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = jobs_runner::run(&targets, JobType::Build { production }).await?; (JobType::Build { production }, results) } Command::Clean { target, report } => { - resolve_dev_tools().await?; + resolve_dev_tools()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = jobs_runner::run(&targets, JobType::Clean).await?; @@ -97,14 +97,14 @@ async fn main() -> Result<(), Error> { production, report, } => { - resolve_dev_tools().await?; + resolve_dev_tools()?; report_opt = get_report_option(report)?; let targets = get_targets_or_default(target); let results = jobs_runner::run(&targets, JobType::Test { production }).await?; (JobType::Test { production }, results) } Command::Run { production } => { - resolve_dev_tools().await?; + resolve_dev_tools()?; report_opt = ReportOptions::None; let results = jobs_runner::run(&[Target::App], JobType::Build { production }).await?; (JobType::Run { production }, results) @@ -130,7 +130,7 @@ async fn main() -> Result<(), Error> { }; // Shutdown and show results & report - let tracker = get_tracker().await; + let tracker = get_tracker(); tracker.shutdown().await?; let mut success: bool = true; for (idx, res) in results.iter().enumerate() { @@ -174,7 +174,7 @@ async fn main() -> Result<(), Error> { bail!("Some task were failed") }; - ChecksumRecords::update_and_save(job_type).await?; + ChecksumRecords::update_and_save(job_type)?; if matches!(job_type, JobType::Run { production: _ }) { println!("Starting chipmunk..."); diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 8155fe6ca..8c923cebb 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -92,7 +92,7 @@ pub async fn spawn( let mut combined_env_vars = vec![(String::from("TERM"), String::from("xterm-256color"))]; combined_env_vars.extend(environment_vars); - let tracker = get_tracker().await; + let tracker = get_tracker(); let command_result = Command::new(&command.cmd) .current_dir(&cwd) @@ -199,7 +199,7 @@ pub async fn spawn_blocking( child.args(&command.args); child.envs(combined_env_vars); - let tracker = get_tracker().await; + let tracker = get_tracker(); let status = tracker.suspend_and_run(child).await?; diff --git a/cli/src/target/cli.rs b/cli/src/target/cli.rs index 50b889668..afa730fe8 100644 --- a/cli/src/target/cli.rs +++ b/cli/src/target/cli.rs @@ -5,8 +5,8 @@ use crate::{ use super::TestSpawnCommand; -pub async fn get_test_cmds(production: bool) -> Vec { - let cargo_path = DevTool::Cargo.path().await; +pub fn get_test_cmds(production: bool) -> Vec { + let cargo_path = DevTool::Cargo.path(); let mut args = vec![String::from("+stable"), String::from("test")]; if production { diff --git a/cli/src/target/core.rs b/cli/src/target/core.rs index 261b5bf13..19794febf 100644 --- a/cli/src/target/core.rs +++ b/cli/src/target/core.rs @@ -5,8 +5,8 @@ use crate::{ use super::TestSpawnCommand; -pub async fn get_test_cmds(production: bool) -> Vec { - let cargo_path = DevTool::Cargo.path().await; +pub fn get_test_cmds(production: bool) -> Vec { + let cargo_path = DevTool::Cargo.path(); let mut args = vec![String::from("+stable"), String::from("test")]; if production { diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 49e6eab51..8bc314724 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -248,12 +248,12 @@ impl Target { } /// Provide the command that should be used in to build the target - pub async fn build_cmd(&self, prod: bool) -> anyhow::Result { + pub fn build_cmd(&self, prod: bool) -> anyhow::Result { let build_cmd = match self { Target::Binding => binding::get_build_cmd(prod)?, - Target::Wasm => wasm::get_build_cmd(prod).await, - Target::Updater => updater::get_build_cmd().await, - rest_targets => rest_targets.kind().build_cmd(prod).await, + Target::Wasm => wasm::get_build_cmd(prod), + Target::Updater => updater::get_build_cmd(), + rest_targets => rest_targets.kind().build_cmd(prod), }; Ok(build_cmd) @@ -307,11 +307,11 @@ impl Target { } /// Provides the test commands for the given target if available - async fn test_cmds(&self, production: bool) -> Option> { + fn test_cmds(&self, production: bool) -> Option> { match self { - Target::Core => Some(core::get_test_cmds(production).await), - Target::Cli => Some(cli::get_test_cmds(production).await), - Target::Wasm => Some(wasm::get_test_cmds().await), + Target::Core => Some(core::get_test_cmds(production)), + Target::Cli => Some(cli::get_test_cmds(production)), + Target::Wasm => Some(wasm::get_test_cmds()), _ => None, } } @@ -321,7 +321,7 @@ impl Target { &self, production: bool, ) -> Option> { - let test_cmds = self.test_cmds(production).await?; + let test_cmds = self.test_cmds(production)?; debug_assert!(!test_cmds.is_empty()); @@ -368,7 +368,7 @@ impl Target { let path = self.cwd(); let job_def = JobDefinition::new(*self, JobType::Lint); - let yarn_cmd = DevTool::Yarn.path().await; + let yarn_cmd = DevTool::Yarn.path(); let command = ProcessCommand::new( yarn_cmd.to_string_lossy().to_string(), vec![String::from("run"), String::from("lint")], @@ -382,7 +382,7 @@ impl Target { let job_def = JobDefinition::new(*self, JobType::Lint); - let cargo_path = DevTool::Cargo.path().await; + let cargo_path = DevTool::Cargo.path(); let command = ProcessCommand::new( cargo_path.to_string_lossy().to_string(), vec![ @@ -406,7 +406,7 @@ impl Target { // Clean doesn't differentiate between development and production, and both of them will be // cleaned from the files when the data are persisted. - let checksum = ChecksumRecords::get(false).await?; + let checksum = ChecksumRecords::get(false)?; checksum.remove_hash_if_exist(*self)?; let mut logs = Vec::new(); @@ -448,7 +448,7 @@ impl Target { /// Runs build considering the currently running builds and already finished ones as well. pub async fn build(&self, prod: bool, skip: bool) -> Result { let path = get_root().join(self.cwd()); - let cmd = self.build_cmd(prod).await?; + let cmd = self.build_cmd(prod)?; let spawn_opt = SpawnOptions { has_skip_info: true, @@ -536,8 +536,8 @@ async fn install_general( ) -> Option> { let cmd = match target { // Wasm needs `yarn install` command despite having the kind `TargetKind::Rs` - Target::Wasm => TargetKind::Ts.install_cmd(prod).await, - t => t.kind().install_cmd(prod).await, + Target::Wasm => TargetKind::Ts.install_cmd(prod), + t => t.kind().install_cmd(prod), }; let job_def = JobDefinition::new(overridden_target.unwrap_or(target), job_type); diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index 7c019ee71..2a368de5a 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -12,10 +12,10 @@ pub enum TargetKind { impl TargetKind { /// Provide the general build command for each target type - pub async fn build_cmd(&self, prod: bool) -> ProcessCommand { + pub fn build_cmd(&self, prod: bool) -> ProcessCommand { match self { TargetKind::Ts => { - let yarn_path = DevTool::Yarn.path().await; + let yarn_path = DevTool::Yarn.path(); let mut args = vec![String::from("run")]; if prod { args.push("prod".into()); @@ -26,7 +26,7 @@ impl TargetKind { ProcessCommand::new(yarn_path.to_string_lossy().to_string(), args) } TargetKind::Rs => { - let cargo_path = DevTool::Cargo.path().await; + let cargo_path = DevTool::Cargo.path(); let mut args = vec![ String::from("build"), String::from("--color"), @@ -41,10 +41,10 @@ impl TargetKind { } } /// Provide the general install command for each target type - pub async fn install_cmd(&self, prod: bool) -> Option { + pub fn install_cmd(&self, prod: bool) -> Option { match self { TargetKind::Ts => { - let yarn_path = DevTool::Yarn.path().await; + let yarn_path = DevTool::Yarn.path(); let mut args = vec![String::from("install")]; if prod { args.push("--production".into()); diff --git a/cli/src/target/updater.rs b/cli/src/target/updater.rs index b2256ce7a..82f86acd9 100644 --- a/cli/src/target/updater.rs +++ b/cli/src/target/updater.rs @@ -2,8 +2,8 @@ use crate::dev_tools::DevTool; use super::ProcessCommand; -pub async fn get_build_cmd() -> ProcessCommand { - let cargo_path = DevTool::Cargo.path().await; +pub fn get_build_cmd() -> ProcessCommand { + let cargo_path = DevTool::Cargo.path(); ProcessCommand::new( cargo_path.to_string_lossy().to_string(), diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index c72d8ade2..c6ce3b709 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -2,8 +2,8 @@ use crate::{dev_tools::DevTool, spawner::SpawnOptions, target::Target}; use super::{ProcessCommand, TestSpawnCommand}; -pub async fn get_build_cmd(prod: bool) -> ProcessCommand { - let wasm_pack_path = DevTool::WasmPack.path().await; +pub fn get_build_cmd(prod: bool) -> ProcessCommand { + let wasm_pack_path = DevTool::WasmPack.path(); let env = if prod { "--release" } else { "--dev" }; ProcessCommand::new( @@ -19,11 +19,11 @@ pub async fn get_build_cmd(prod: bool) -> ProcessCommand { ) } -pub async fn get_test_cmds() -> Vec { +pub fn get_test_cmds() -> Vec { let cwd = Target::Wasm.cwd(); - let npm_path = DevTool::Npm.path().await; - let wasm_pack_path = DevTool::WasmPack.path().await; + let npm_path = DevTool::Npm.path(); + let wasm_pack_path = DevTool::WasmPack.path(); vec![ TestSpawnCommand::new( diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index d4122e251..968085abe 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -4,11 +4,12 @@ use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use std::{ collections::BTreeMap, process::{Command, ExitStatus}, + sync::OnceLock, time::Instant, }; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, - oneshot, OnceCell, + oneshot, }; use crate::jobs_runner::JobDefinition; @@ -93,10 +94,10 @@ impl JobBarState { } } -pub async fn get_tracker() -> &'static Tracker { - static TRACKER: OnceCell = OnceCell::const_new(); +pub fn get_tracker() -> &'static Tracker { + static TRACKER: OnceLock = OnceLock::new(); - TRACKER.get_or_init(|| async { Tracker::new() }).await + TRACKER.get_or_init(Tracker::new) } impl Tracker { From 9fc9c320ab154eab6c643911a0507eeb3ffe61aa Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 11:20:54 +0200 Subject: [PATCH 138/174] Build CLI: Clean Integration tests & Fix clean Wasm - Python scripts for integration tests for clean function. - The script calls clean function and check for all the paths that supposed to be deleted. - Add missing directory to cleaning WASM, which has be uncovered in the new integration tests --- cli/integration_tests/.gitignore | 1 + cli/integration_tests/clean.py | 77 ++++++++++++++++++++++++++++++++ cli/integration_tests/utls.py | 26 +++++++++++ cli/src/target/mod.rs | 1 + 4 files changed, 105 insertions(+) create mode 100644 cli/integration_tests/.gitignore create mode 100644 cli/integration_tests/clean.py create mode 100644 cli/integration_tests/utls.py diff --git a/cli/integration_tests/.gitignore b/cli/integration_tests/.gitignore new file mode 100644 index 000000000..bee8a64b7 --- /dev/null +++ b/cli/integration_tests/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/cli/integration_tests/clean.py b/cli/integration_tests/clean.py new file mode 100644 index 000000000..295d0ccca --- /dev/null +++ b/cli/integration_tests/clean.py @@ -0,0 +1,77 @@ +import subprocess +from pathlib import Path +from utls import get_root + +# This will be needed for debugging only +# from pprint import pprint + +CLEAN_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "clean", + # We need to set targets explicitly because we need to clean everything expect the build CLI tools + # binaries to avoid failing on Windows when CLI tool tries to remove it's own binary. + "core", + "shared", + "binding", + "wrapper", + "wasm", + "client", + "updater", + "app", +] + +# These paths must not exist after clean build is done. +# The paths are relative starting from `chipmunk_root/application` +PATHS_TO_CHECK = [ + # Core + "apps/indexer/target", + # Shared + "platform/dist", + "platform/node_modules", + # Binding + "apps/rustcore/rs-bindings/dist", + # These sub paths must not exist after clean build is done. + "apps/rustcore/rs-bindings/target", + # Wrapper + "apps/rustcore/ts-bindings/dist", + "apps/rustcore/ts-bindings/node_modules", + "apps/rustcore/ts-bindings/spec/build", + "apps/rustcore/ts-bindings/src/native/index.node", + # Wasm + "apps/rustcore/wasm-bindings/pkg", + "apps/rustcore/wasm-bindings/node_modules", + "apps/rustcore/wasm-bindings/test_output", + # Client + "client/dist", + "client/node_modules", + # Updater + "apps/precompiled/updater/target", + # App + "holder/dist", + "holder/node_modules", +] + + +def get_removed_paths() -> list[Path]: + """Provides the paths for the directories that must be removed after running the clean command""" + root_dir = get_root() + application_dir = root_dir.joinpath("application") + return [application_dir.joinpath(sub_dir) for sub_dir in PATHS_TO_CHECK] + + +def run_clean_command(): + print("Running clean command...") + subprocess.run(CLEAN_COMMAND, check=True) + for path in get_removed_paths(): + if path.exists(): + raise AssertionError(f"Path exists after clean. Path: {path}") + + print("*** Check for Clean Command Succeeded ***") + + +if __name__ == "__main__": + run_clean_command() diff --git a/cli/integration_tests/utls.py b/cli/integration_tests/utls.py new file mode 100644 index 000000000..d7de50c7d --- /dev/null +++ b/cli/integration_tests/utls.py @@ -0,0 +1,26 @@ +from pathlib import Path + + +def get_root() -> Path: + """Get and validate the root directory of chipmunk repository + + Raises: + IOError: If validation of root directory fails + + Returns: + Root directory of chipmunk + """ + + # We are using the utls file with the assumption that is path is `root/cli/integration_tests` + root_dir = Path(__file__).parent.parent.parent + + # Root Dir checks. This checks depends on the current chipmunk directories' structure + sub_dirs = ( + root_dir.joinpath(dir_name) + for dir_name in ["application", "developing", "cli", "scripts"] + ) + + if any(not path.exists() for path in sub_dirs): + raise SystemError(f"Root directory verification fail. Root Dir: {root_dir}") + + return root_dir diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 8bc314724..0e728b020 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -421,6 +421,7 @@ impl Target { Target::Wasm => { paths_to_remove.push(self.cwd().join("pkg")); paths_to_remove.push(self.cwd().join("test_output")); + paths_to_remove.push(self.cwd().join("node_modules")); } Target::Wrapper => { paths_to_remove.push(self.cwd().join("spec").join("build")); From dbbbf19974d5fc2df915015370d1496d9027fbcd Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 11:36:01 +0200 Subject: [PATCH 139/174] Build CLI: Integration tests for Test command --- cli/integration_tests/test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 cli/integration_tests/test.py diff --git a/cli/integration_tests/test.py b/cli/integration_tests/test.py new file mode 100644 index 000000000..7abd2af7c --- /dev/null +++ b/cli/integration_tests/test.py @@ -0,0 +1,20 @@ +import subprocess + +TEST_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "test", +] + + +def run_test_command(): + print("Running test command...") + subprocess.run(TEST_COMMAND, check=True) + print("*** Check for Test Command Succeeded ***") + + +if __name__ == "__main__": + run_test_command() From d53566612544e444de7aadffd4230cb002baf408 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 13:17:50 +0200 Subject: [PATCH 140/174] Build CLI: Integration tests for Build command - Tests call build for app target and checks for build paths in all other targets if they exist - Tests compare last modification date for checksum file in development build with the same file before build if exists --- cli/integration_tests/build.py | 102 +++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 cli/integration_tests/build.py diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py new file mode 100644 index 000000000..b5ff14cb8 --- /dev/null +++ b/cli/integration_tests/build.py @@ -0,0 +1,102 @@ +import subprocess +from pathlib import Path +from utls import get_root +from datetime import datetime + +BUILD_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "build", + # Provide app target only and it should pull all other targets expect for build CLI, which isn't + # possible to build on Windows because it's not allowed to replace a binary while it's running. + "app", +] + +# These paths must exist after build build is done. +# The paths are relative starting from `chipmunk_root/application` +APP_PATHS_TO_CHECK = [ + # Core + "apps/indexer/target", + # Shared + "platform/dist", + "platform/node_modules", + # Binding + "apps/rustcore/rs-bindings/dist", + # These sub paths must not exist after clean build is done. + "apps/rustcore/rs-bindings/target", + # Wrapper + "apps/rustcore/ts-bindings/dist", + "apps/rustcore/ts-bindings/node_modules", + "apps/rustcore/ts-bindings/src/native/index.node", + # Wasm + "apps/rustcore/wasm-bindings/pkg", + "apps/rustcore/wasm-bindings/node_modules", + # Client + "client/dist", + "client/node_modules", + # Updater + "apps/precompiled/updater/target", + # App + "holder/dist", + "holder/node_modules", +] + +# The name of the file where the checksum are saved for development build +CHECKSUM_FILE_NAME = ".build_chksum_dev" + + +def get_build_paths(root_dir: Path) -> list[Path]: + """Provides the paths for the directories that must be created after running the build command""" + application_dir = root_dir.joinpath("application") + build_paths = [application_dir.joinpath(sub_dir) for sub_dir in APP_PATHS_TO_CHECK] + return build_paths + + +def run_build_command(): + print("Running build command...") + root_dir = get_root() + # The path for the file where build checksums are saved. + # This file must be written on every call of build. + checksum_path = root_dir.joinpath(CHECKSUM_FILE_NAME) + + # Get last modification date for checksum file if exists otherwise get minimal date + checksum_modified_before = ( + datetime.fromtimestamp(checksum_path.stat().st_mtime) + if checksum_path.exists() + else datetime.min + ) + subprocess.run(BUILD_COMMAND, check=True) + + print("Checking created directories...") + for path in get_build_paths(root_dir): + if not path.exists(): + raise AssertionError(f"Path doesn't exist after build. Path: {path}") + + print("Checking created directories Succeeded") + + # Checksum Records file Checks: File must exist after build, + # and it must have a more resent modification date compared to before build + print("Checking Checksum Records file...") + + assert ( + checksum_path.exists() + ), f"Checksum record file doesn't exist after build. File Path: {checksum_path}" + + checksum_modified_after = datetime.fromtimestamp(checksum_path.stat().st_mtime) + + assert ( + checksum_modified_after > checksum_modified_before + ), f"Checksum file modification date after build isn't greater as before build.\ + Before: {checksum_modified_before} , After: {checksum_modified_after}" + + print("Checking Checksum Records file Succeeded") + + # All Done + print("*** Check for Build Command Succeeded ***") + + +if __name__ == "__main__": + run_build_command() From 9cdb416d8eb3970adf97f5ff77e4c31b5203b016 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 15:29:22 +0200 Subject: [PATCH 141/174] fBuild CLI: Integration tests for Checksum Checks - Tests to insure if files change in a target directory then the target and all other targets depending on it will be rebuilt. - Tests creates a file in platform and checks that platform and all other targets until app(electron) targets are rebuilt --- cli/integration_tests/build.py | 120 ++++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 10 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index b5ff14cb8..2132f75d7 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -2,7 +2,9 @@ from pathlib import Path from utls import get_root from datetime import datetime +from typing import Dict +# Build command to be used in all build tests BUILD_COMMAND = [ "cargo", "run", @@ -15,9 +17,9 @@ "app", ] -# These paths must exist after build build is done. +# These paths must exist after build command has been finished. # The paths are relative starting from `chipmunk_root/application` -APP_PATHS_TO_CHECK = [ +APP_PATHS_FOR_BUILD_CHECK = [ # Core "apps/indexer/target", # Shared @@ -25,7 +27,6 @@ "platform/node_modules", # Binding "apps/rustcore/rs-bindings/dist", - # These sub paths must not exist after clean build is done. "apps/rustcore/rs-bindings/target", # Wrapper "apps/rustcore/ts-bindings/dist", @@ -51,11 +52,14 @@ def get_build_paths(root_dir: Path) -> list[Path]: """Provides the paths for the directories that must be created after running the build command""" application_dir = root_dir.joinpath("application") - build_paths = [application_dir.joinpath(sub_dir) for sub_dir in APP_PATHS_TO_CHECK] + build_paths = [ + application_dir.joinpath(sub_dir) for sub_dir in APP_PATHS_FOR_BUILD_CHECK + ] return build_paths -def run_build_command(): +def _build_general_check(): + """Runs Build command for app targets and checks if all build directories + checksum file are created""" print("Running build command...") root_dir = get_root() # The path for the file where build checksums are saved. @@ -70,6 +74,7 @@ def run_build_command(): ) subprocess.run(BUILD_COMMAND, check=True) + # Check if all the targets directory exists after running build command. print("Checking created directories...") for path in get_build_paths(root_dir): if not path.exists(): @@ -79,7 +84,7 @@ def run_build_command(): # Checksum Records file Checks: File must exist after build, # and it must have a more resent modification date compared to before build - print("Checking Checksum Records file...") + print("Checking Checksum Records file changed...") assert ( checksum_path.exists() @@ -91,12 +96,107 @@ def run_build_command(): checksum_modified_after > checksum_modified_before ), f"Checksum file modification date after build isn't greater as before build.\ Before: {checksum_modified_before} , After: {checksum_modified_after}" + print("Checking if Checksum Records file changed Succeeded") - print("Checking Checksum Records file Succeeded") - # All Done - print("*** Check for Build Command Succeeded ***") +############ CHECKSUM RECORDS TESTS ########### + +# Name of the temporary file which will be used for testing if file changes will lead to rebuild +# the direct target and all other targets that depend on here. +# This file should be created then deleted after the test is done. +TEMP_FILE_NAME = "tmp_for_cli_test.txt" + +# Content of temporary file for testing checksum checks. +# It should provide a clear message for the users to delete the file if the file somehow didn't +# get deleted after the test has finished. +TEMP_FILE_CONTENT = """This file is created to test the build CLI tool only and it should be deleted after each test. +Please delete this file manually if it still exists after build CLI tests are done, +and please consider opening an issue if you can reproduce this behavior""" + +# Paths of files and directory in platform target and all other targets depending on it. +# The modification date for this files will be read before the checksum test starts, then it will be compared after +# the build command has finished to insure that the those targets have been rebuilt. +APP_PATHS_FOR_CHECKSUM_CHECK = [ + # Shared + "platform/dist/lib.js", + # Binding + "apps/rustcore/rs-bindings/dist/index.node", + # Wrapper + "apps/rustcore/ts-bindings/dist/index.js", + # Client + "client/dist", + # App + "holder/dist/app.js", +] + + +def _build_checksum_check(): + """!!!This function must run directly after a full build!!! + It creates a dummy file in platform directory and checks that all dependencies (Binding, Wrapper, Client, App) + has been newly built + """ + root_dir = get_root() + application_dir = root_dir.joinpath("application") + + # Get and validate checksum file + chksum_file = root_dir.joinpath(CHECKSUM_FILE_NAME) + assert ( + chksum_file.exists() + ), f"Checksum File must exist before running checksum tests. File Path: {chksum_file}" + + # Save modification date for build paths before start to compare them later. + modifi_before_start: Dict[Path, datetime] = {} + for sub_path in APP_PATHS_FOR_CHECKSUM_CHECK: + sub_path = application_dir.joinpath(sub_path) + assert ( + sub_path.exists() + ), f"Build Path must exist before checksum tests starts. Path {sub_path}" + modifi_date = datetime.fromtimestamp(sub_path.stat().st_mtime) + modifi_before_start[sub_path] = modifi_date + + # Define temporary file path in platform directory to insure it will be rebuilt + # with all other targets depending on it. + temp_file_path = application_dir.joinpath(f"platform/{TEMP_FILE_NAME}") + assert ( + not temp_file_path.exists() + ), f"Temporary file can't exist before checksum test start. File Path: {temp_file_path}" + + try: + # Create temporary file in platform directory + with open(temp_file_path, "w") as f: + f.write(TEMP_FILE_CONTENT) + + # Run build command + subprocess.run(BUILD_COMMAND, check=True) + + # Compare modification date for involved targets + for path, modifi_before in modifi_before_start.items(): + modifi_after = datetime.fromtimestamp(path.stat().st_mtime) + assert ( + modifi_after > modifi_before + ), f"Target modified date after must be more recent than before.\n\ + Target Path: {path}.\n\ + Before: {modifi_before}, After: {modifi_after}" + finally: + # Insure temporary file is deleted + if temp_file_path.exists(): + temp_file_path.unlink() + + +################ RUN ALL TESTS ###################### + + +def run_build_tests(): + print("Running General Checks for Build Command...") + _build_general_check() + print("*** General Check for Build Command Succeeded ***") + + print("---------------------------------------------------------------") + + print("Running Checksum Checks for Build Command...") + _build_checksum_check() + print("*** Checksum Check for Build Command Succeeded ***") if __name__ == "__main__": - run_build_command() + run_build_tests() From 9a96d91e1074f322549acd84ebf6094ed5d5f95a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 15:38:25 +0200 Subject: [PATCH 142/174] Build CLI: Integration tests for Lint command --- cli/integration_tests/lint.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 cli/integration_tests/lint.py diff --git a/cli/integration_tests/lint.py b/cli/integration_tests/lint.py new file mode 100644 index 000000000..c54f96af0 --- /dev/null +++ b/cli/integration_tests/lint.py @@ -0,0 +1,20 @@ +import subprocess + +LINT_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "lint", +] + + +def run_lint_command(): + print("Running Lint command...") + subprocess.run(LINT_COMMAND, check=True) + print("*** Check for Lint Command Succeeded ***") + + +if __name__ == "__main__": + run_lint_command() From 7e4b7ba4cb48ccd3fc4d22abfde46a0ef1d2d4e8 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 16:08:43 +0200 Subject: [PATCH 143/174] Build CLI Tests: Environment & Print_dot & Shell completion --- cli/integration_tests/environment.py | 37 ++++++++++++++++++++++++++++ cli/integration_tests/print_dot.py | 35 ++++++++++++++++++++++++++ cli/integration_tests/shell_compl.py | 26 +++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 cli/integration_tests/environment.py create mode 100644 cli/integration_tests/print_dot.py create mode 100644 cli/integration_tests/shell_compl.py diff --git a/cli/integration_tests/environment.py b/cli/integration_tests/environment.py new file mode 100644 index 000000000..ff42223a6 --- /dev/null +++ b/cli/integration_tests/environment.py @@ -0,0 +1,37 @@ +import subprocess + +ENVIRONMENT_CHECK_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "environment", + "check", +] + +ENVIRONMENT_PRINT_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "environment", + "print", +] + + +def run_environment_commands(): + """Runs environment command to check the installed development tools for chipmunk, + and the command to print infos about those tools""" + print("Running Environment Check command...") + subprocess.run(ENVIRONMENT_CHECK_COMMAND, check=True) + print("*** Environment Check Command Succeeded ***") + + print("Running Environment Print command...") + subprocess.run(ENVIRONMENT_PRINT_COMMAND, check=True) + print("*** Environment Print Command Succeeded ***") + + +if __name__ == "__main__": + run_environment_commands() diff --git a/cli/integration_tests/print_dot.py b/cli/integration_tests/print_dot.py new file mode 100644 index 000000000..043547944 --- /dev/null +++ b/cli/integration_tests/print_dot.py @@ -0,0 +1,35 @@ +import subprocess + +PRINT_DOT_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "print-dot", +] + +PRINT_DOT_ALL_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "print-dot", + "-a", +] + + +def run_print_dot_commands(): + """Runs print dot commands for both targets and tasks""" + print("Running General Print Dot command...") + subprocess.run(PRINT_DOT_COMMAND, check=True) + print("*** General Print Dot command Succeeded ***") + + print("Running All Print Dot command...") + subprocess.run(PRINT_DOT_ALL_COMMAND, check=True) + print("*** All Print Dot command Succeeded ***") + + +if __name__ == "__main__": + run_print_dot_commands() diff --git a/cli/integration_tests/shell_compl.py b/cli/integration_tests/shell_compl.py new file mode 100644 index 000000000..19ba20ebd --- /dev/null +++ b/cli/integration_tests/shell_compl.py @@ -0,0 +1,26 @@ +import subprocess + +PRINT_COMPLETION_COMMAND = [ + "cargo", + "run", + "-r", + "--", + "chipmunk", + "shell-completion", +] + +SHELLS = ["bash", "elvish", "fish", "powershell", "zsh"] + + +def run_print_dot_commands(): + """Runs commands to generate shell completion on all the available shells""" + for shell in SHELLS: + print(f"Runnig Shell Completion command for {shell}") + shell_command = PRINT_COMPLETION_COMMAND.copy() + shell_command.append(shell) + subprocess.run(shell_command, check=True) + print(f"*** Shell Completion Command for {shell} Succeeded ***") + + +if __name__ == "__main__": + run_print_dot_commands() From 6f7c806198fc49059a83bca2f04f12206fdd2af2 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 16:33:57 +0200 Subject: [PATCH 144/174] Build CLI: Integration tests for reset checksum - Add new test for build tests to insure that checksum file get deleted when reset checksum command is called - Provide documentation for run build tests functions --- cli/integration_tests/build.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index 2132f75d7..68a7e5aae 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -183,10 +183,35 @@ def _build_checksum_check(): temp_file_path.unlink() +############## RESET CHECKSUM COMMAND ################ + +RESET_CHECKSUM_COMMAND = ["cargo", "run", "-r", "--", "chipmunk", "reset-checksum"] + + +def _run_reset_checksum_command(): + checksum_path = get_root().joinpath(CHECKSUM_FILE_NAME) + assert ( + checksum_path.exists() + ), f"Checksum file must exist before running 'reset-checksum' command. File Path: {checksum_path}" + + subprocess.run(RESET_CHECKSUM_COMMAND, check=True) + + assert ( + not checksum_path.exists() + ), f"Checksum file must not exist after running 'reset-checksum' command. File Path: {checksum_path}" + + pass + + ################ RUN ALL TESTS ###################### def run_build_tests(): + """Runs many tests for build command: + - General Test: General test for build command and the created directories and checksum files. + - Checksum Test: Test how changing files in some target will affect the build on it and other targets depending on it. + - Reset Checksum Test: Make sure checksum file will deleted when the command is called. + """ print("Running General Checks for Build Command...") _build_general_check() print("*** General Check for Build Command Succeeded ***") @@ -197,6 +222,12 @@ def run_build_tests(): _build_checksum_check() print("*** Checksum Check for Build Command Succeeded ***") + print("---------------------------------------------------------------") + + print("Running Reset Checksum Command...") + _run_reset_checksum_command() + print("*** Test Running Reset Checksum Command Succeeded ***") + if __name__ == "__main__": run_build_tests() From 6ec35054ee01061fb218ea0725df8b47d142f0b4 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 16:48:00 +0200 Subject: [PATCH 145/174] Build CLI: documentation for Integration tests Add more documentation for python modules and remove not needed code comments --- cli/integration_tests/build.py | 5 +++++ cli/integration_tests/clean.py | 9 +++++---- cli/integration_tests/environment.py | 4 ++++ cli/integration_tests/lint.py | 5 +++++ cli/integration_tests/print_dot.py | 4 ++++ cli/integration_tests/shell_compl.py | 4 ++++ cli/integration_tests/test.py | 5 +++++ cli/integration_tests/utls.py | 4 ++++ 8 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index 68a7e5aae..db115b981 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -1,3 +1,8 @@ +""" +Provides methods to test the Build command and checksum implementation to watch source code changes +in Chipmunk Build CLI Tool +""" + import subprocess from pathlib import Path from utls import get_root diff --git a/cli/integration_tests/clean.py b/cli/integration_tests/clean.py index 295d0ccca..acfd2b8f6 100644 --- a/cli/integration_tests/clean.py +++ b/cli/integration_tests/clean.py @@ -1,10 +1,11 @@ +""" +Provides methods to test the Clean command in Chipmunk Build CLI Tool +""" + import subprocess from pathlib import Path from utls import get_root -# This will be needed for debugging only -# from pprint import pprint - CLEAN_COMMAND = [ "cargo", "run", @@ -34,7 +35,6 @@ "platform/node_modules", # Binding "apps/rustcore/rs-bindings/dist", - # These sub paths must not exist after clean build is done. "apps/rustcore/rs-bindings/target", # Wrapper "apps/rustcore/ts-bindings/dist", @@ -64,6 +64,7 @@ def get_removed_paths() -> list[Path]: def run_clean_command(): + """Runs Clean Commands on all targets and insure that all build directories are delted.""" print("Running clean command...") subprocess.run(CLEAN_COMMAND, check=True) for path in get_removed_paths(): diff --git a/cli/integration_tests/environment.py b/cli/integration_tests/environment.py index ff42223a6..9bfa41a4e 100644 --- a/cli/integration_tests/environment.py +++ b/cli/integration_tests/environment.py @@ -1,3 +1,7 @@ +""" +Provides methods to test the Environment commands in Chipmunk Build CLI Tool +""" + import subprocess ENVIRONMENT_CHECK_COMMAND = [ diff --git a/cli/integration_tests/lint.py b/cli/integration_tests/lint.py index c54f96af0..594b432a8 100644 --- a/cli/integration_tests/lint.py +++ b/cli/integration_tests/lint.py @@ -1,3 +1,7 @@ +""" +Provides methods to test the Lint command in Chipmunk Build CLI Tool +""" + import subprocess LINT_COMMAND = [ @@ -11,6 +15,7 @@ def run_lint_command(): + """Runs lint command for all targets. This test will fail on linting errors as well.""" print("Running Lint command...") subprocess.run(LINT_COMMAND, check=True) print("*** Check for Lint Command Succeeded ***") diff --git a/cli/integration_tests/print_dot.py b/cli/integration_tests/print_dot.py index 043547944..d449bc563 100644 --- a/cli/integration_tests/print_dot.py +++ b/cli/integration_tests/print_dot.py @@ -1,3 +1,7 @@ +""" +Provides methods to test the Print-dot commands in Chipmunk Build CLI Tool +""" + import subprocess PRINT_DOT_COMMAND = [ diff --git a/cli/integration_tests/shell_compl.py b/cli/integration_tests/shell_compl.py index 19ba20ebd..d8422f675 100644 --- a/cli/integration_tests/shell_compl.py +++ b/cli/integration_tests/shell_compl.py @@ -1,3 +1,7 @@ +""" +Provides methods to test the Shell Completion command for varity of shells in Chipmunk Build CLI Tool +""" + import subprocess PRINT_COMPLETION_COMMAND = [ diff --git a/cli/integration_tests/test.py b/cli/integration_tests/test.py index 7abd2af7c..47bb9c312 100644 --- a/cli/integration_tests/test.py +++ b/cli/integration_tests/test.py @@ -1,3 +1,7 @@ +""" +Provides methods to test the Test Command in Chipmunk Build CLI Tool +""" + import subprocess TEST_COMMAND = [ @@ -11,6 +15,7 @@ def run_test_command(): + """Runs test command on all targets. This test will fail on test errors of chipmunk targets as well.""" print("Running test command...") subprocess.run(TEST_COMMAND, check=True) print("*** Check for Test Command Succeeded ***") diff --git a/cli/integration_tests/utls.py b/cli/integration_tests/utls.py index d7de50c7d..787ed496c 100644 --- a/cli/integration_tests/utls.py +++ b/cli/integration_tests/utls.py @@ -1,3 +1,7 @@ +""" +Utilities function to be used among the modules to test Chipmunk Build CLI Tool +""" + from pathlib import Path From dd29c78e472d9f4c90c18b3f79cbc375ddb22234 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Mon, 22 Jul 2024 16:49:33 +0200 Subject: [PATCH 146/174] Build CLI: Clippy Fix --- cli/dir_checksum/src/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/dir_checksum/src/input.rs b/cli/dir_checksum/src/input.rs index 49d1c3c16..292cdf60f 100644 --- a/cli/dir_checksum/src/input.rs +++ b/cli/dir_checksum/src/input.rs @@ -54,7 +54,7 @@ fn maybe_memmap_file(file: &File) -> Result, HashError> { let map = if !metadata.is_file() { // Not a real file. None - } else if file_size > isize::max_value() as u64 { + } else if file_size > isize::MAX as u64 { // Too long to safely map. // https://github.com/danburkert/memmap-rs/issues/69 None From 66fdecad45a23bf4a39d3ee0241c2b5a51afdeab Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Jul 2024 12:38:36 +0200 Subject: [PATCH 147/174] Build CLI Integ Tests: Refactoring & Improvements - Commands will be printed to the console with bold before running it - Provide function to print colored text to the stdout and apply it for the main statement before and after running each test --- cli/integration_tests/build.py | 21 ++++++------- cli/integration_tests/clean.py | 9 +++--- cli/integration_tests/environment.py | 14 ++++----- cli/integration_tests/lint.py | 8 ++--- cli/integration_tests/print_dot.py | 14 ++++----- cli/integration_tests/shell_compl.py | 8 ++--- cli/integration_tests/test.py | 8 ++--- cli/integration_tests/utls.py | 46 ++++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 42 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index db115b981..b94764654 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -3,9 +3,8 @@ in Chipmunk Build CLI Tool """ -import subprocess from pathlib import Path -from utls import get_root +from utls import get_root, run_command, print_green_bold, print_blue_bold from datetime import datetime from typing import Dict @@ -77,7 +76,7 @@ def _build_general_check(): if checksum_path.exists() else datetime.min ) - subprocess.run(BUILD_COMMAND, check=True) + run_command(BUILD_COMMAND) # Check if all the targets directory exists after running build command. print("Checking created directories...") @@ -172,7 +171,7 @@ def _build_checksum_check(): f.write(TEMP_FILE_CONTENT) # Run build command - subprocess.run(BUILD_COMMAND, check=True) + run_command(BUILD_COMMAND) # Compare modification date for involved targets for path, modifi_before in modifi_before_start.items(): @@ -199,7 +198,7 @@ def _run_reset_checksum_command(): checksum_path.exists() ), f"Checksum file must exist before running 'reset-checksum' command. File Path: {checksum_path}" - subprocess.run(RESET_CHECKSUM_COMMAND, check=True) + run_command(RESET_CHECKSUM_COMMAND) assert ( not checksum_path.exists() @@ -217,21 +216,21 @@ def run_build_tests(): - Checksum Test: Test how changing files in some target will affect the build on it and other targets depending on it. - Reset Checksum Test: Make sure checksum file will deleted when the command is called. """ - print("Running General Checks for Build Command...") + print_blue_bold("Running General Checks for Build Command...") _build_general_check() - print("*** General Check for Build Command Succeeded ***") + print_green_bold("*** General Check for Build Command Succeeded ***") print("---------------------------------------------------------------") - print("Running Checksum Checks for Build Command...") + print_blue_bold("Running Checksum Checks for Build Command...") _build_checksum_check() - print("*** Checksum Check for Build Command Succeeded ***") + print_green_bold("*** Checksum Check for Build Command Succeeded ***") print("---------------------------------------------------------------") - print("Running Reset Checksum Command...") + print_blue_bold("Running Reset Checksum Command...") _run_reset_checksum_command() - print("*** Test Running Reset Checksum Command Succeeded ***") + print_green_bold("*** Test Running Reset Checksum Command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/clean.py b/cli/integration_tests/clean.py index acfd2b8f6..3114bcc19 100644 --- a/cli/integration_tests/clean.py +++ b/cli/integration_tests/clean.py @@ -2,9 +2,8 @@ Provides methods to test the Clean command in Chipmunk Build CLI Tool """ -import subprocess from pathlib import Path -from utls import get_root +from utls import run_command, print_blue_bold, print_green_bold, get_root CLEAN_COMMAND = [ "cargo", @@ -65,13 +64,13 @@ def get_removed_paths() -> list[Path]: def run_clean_command(): """Runs Clean Commands on all targets and insure that all build directories are delted.""" - print("Running clean command...") - subprocess.run(CLEAN_COMMAND, check=True) + print_blue_bold("Running clean command...") + run_command(CLEAN_COMMAND) for path in get_removed_paths(): if path.exists(): raise AssertionError(f"Path exists after clean. Path: {path}") - print("*** Check for Clean Command Succeeded ***") + print_green_bold("*** Check for Clean Command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/environment.py b/cli/integration_tests/environment.py index 9bfa41a4e..82a1b7d98 100644 --- a/cli/integration_tests/environment.py +++ b/cli/integration_tests/environment.py @@ -2,7 +2,7 @@ Provides methods to test the Environment commands in Chipmunk Build CLI Tool """ -import subprocess +from utls import run_command, print_blue_bold, print_green_bold ENVIRONMENT_CHECK_COMMAND = [ "cargo", @@ -28,13 +28,13 @@ def run_environment_commands(): """Runs environment command to check the installed development tools for chipmunk, and the command to print infos about those tools""" - print("Running Environment Check command...") - subprocess.run(ENVIRONMENT_CHECK_COMMAND, check=True) - print("*** Environment Check Command Succeeded ***") + print_blue_bold("Running Environment Check command...") + run_command(ENVIRONMENT_CHECK_COMMAND) + print_green_bold("*** Environment Check Command Succeeded ***") - print("Running Environment Print command...") - subprocess.run(ENVIRONMENT_PRINT_COMMAND, check=True) - print("*** Environment Print Command Succeeded ***") + print_blue_bold("Running Environment Print command...") + run_command(ENVIRONMENT_PRINT_COMMAND) + print_green_bold("*** Environment Print Command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/lint.py b/cli/integration_tests/lint.py index 594b432a8..5e476a620 100644 --- a/cli/integration_tests/lint.py +++ b/cli/integration_tests/lint.py @@ -2,7 +2,7 @@ Provides methods to test the Lint command in Chipmunk Build CLI Tool """ -import subprocess +from utls import run_command, print_green_bold, print_blue_bold LINT_COMMAND = [ "cargo", @@ -16,9 +16,9 @@ def run_lint_command(): """Runs lint command for all targets. This test will fail on linting errors as well.""" - print("Running Lint command...") - subprocess.run(LINT_COMMAND, check=True) - print("*** Check for Lint Command Succeeded ***") + print_blue_bold("Running Lint command...") + run_command(LINT_COMMAND) + print_green_bold("*** Check for Lint Command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/print_dot.py b/cli/integration_tests/print_dot.py index d449bc563..43287ca4f 100644 --- a/cli/integration_tests/print_dot.py +++ b/cli/integration_tests/print_dot.py @@ -2,7 +2,7 @@ Provides methods to test the Print-dot commands in Chipmunk Build CLI Tool """ -import subprocess +from utls import run_command, print_green_bold, print_blue_bold PRINT_DOT_COMMAND = [ "cargo", @@ -26,13 +26,13 @@ def run_print_dot_commands(): """Runs print dot commands for both targets and tasks""" - print("Running General Print Dot command...") - subprocess.run(PRINT_DOT_COMMAND, check=True) - print("*** General Print Dot command Succeeded ***") + print_blue_bold("Running General Print Dot command...") + run_command(PRINT_DOT_COMMAND) + print_green_bold("*** General Print Dot command Succeeded ***") - print("Running All Print Dot command...") - subprocess.run(PRINT_DOT_ALL_COMMAND, check=True) - print("*** All Print Dot command Succeeded ***") + print_blue_bold("Running All Print Dot command...") + run_command(PRINT_DOT_ALL_COMMAND) + print_green_bold("*** All Print Dot command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/shell_compl.py b/cli/integration_tests/shell_compl.py index d8422f675..2ee73f1ae 100644 --- a/cli/integration_tests/shell_compl.py +++ b/cli/integration_tests/shell_compl.py @@ -2,7 +2,7 @@ Provides methods to test the Shell Completion command for varity of shells in Chipmunk Build CLI Tool """ -import subprocess +from utls import run_command, print_blue_bold, print_green_bold PRINT_COMPLETION_COMMAND = [ "cargo", @@ -19,11 +19,11 @@ def run_print_dot_commands(): """Runs commands to generate shell completion on all the available shells""" for shell in SHELLS: - print(f"Runnig Shell Completion command for {shell}") + print_blue_bold(f"Runnig Shell Completion command for {shell}") shell_command = PRINT_COMPLETION_COMMAND.copy() shell_command.append(shell) - subprocess.run(shell_command, check=True) - print(f"*** Shell Completion Command for {shell} Succeeded ***") + run_command(shell_command) + print_green_bold(f"*** Shell Completion Command for {shell} Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/test.py b/cli/integration_tests/test.py index 47bb9c312..3e3a6565b 100644 --- a/cli/integration_tests/test.py +++ b/cli/integration_tests/test.py @@ -2,7 +2,7 @@ Provides methods to test the Test Command in Chipmunk Build CLI Tool """ -import subprocess +from utls import run_command, print_blue_bold, print_green_bold TEST_COMMAND = [ "cargo", @@ -16,9 +16,9 @@ def run_test_command(): """Runs test command on all targets. This test will fail on test errors of chipmunk targets as well.""" - print("Running test command...") - subprocess.run(TEST_COMMAND, check=True) - print("*** Check for Test Command Succeeded ***") + print_blue_bold("Running test command...") + run_command(TEST_COMMAND) + print_green_bold("*** Check for Test Command Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/utls.py b/cli/integration_tests/utls.py index 787ed496c..0d157e4e6 100644 --- a/cli/integration_tests/utls.py +++ b/cli/integration_tests/utls.py @@ -3,6 +3,7 @@ """ from pathlib import Path +import subprocess def get_root() -> Path: @@ -28,3 +29,48 @@ def get_root() -> Path: raise SystemError(f"Root directory verification fail. Root Dir: {root_dir}") return root_dir + + +def run_command(command_args: list[str]): + """Runs the commands and its arguments after printing the command to stdout + + Args: + command_args: The command and its arguments in one string list + """ + command_txt = " ".join(command_args) + print_bold(f"Command: {command_txt}") + subprocess.run(command_args, check=True) + + +class bcolors: + """Color codes representation in ANSI""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def print_bold(text: str): + """Prints the given text to stdout with bold attribute""" + print(f"{bcolors.BOLD}{text}{bcolors.ENDC}") + + +def print_blue_bold(text: str): + """Prints the given text to stdout with bold and blue attribute""" + print(f"{bcolors.BOLD}{bcolors.OKBLUE}{text}{bcolors.ENDC}{bcolors.ENDC}") + + +def print_green_bold(text: str): + """Prints the given text to stdout with bold and green attribute""" + print(f"{bcolors.BOLD}{bcolors.OKGREEN}{text}{bcolors.ENDC}{bcolors.ENDC}") + + +def print_red_bold(text: str): + """Prints the given text to stdout with bold and red attribute""" + print(f"{bcolors.BOLD}{bcolors.FAIL}{text}{bcolors.ENDC}{bcolors.ENDC}") From bec1a75582c45266fcd22a10615dcd06ab000652 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Jul 2024 13:08:43 +0200 Subject: [PATCH 148/174] Build CLI Integration Tests: Run all tests Provide module to run all the integration tests in sequence while printing information about the running tests especially in failing case --- cli/integration_tests/run_all.py | 88 +++++++++++++++++++ cli/integration_tests/shell_compl.py | 4 +- .../{test.py => test_cmd.py} | 0 cli/integration_tests/utls.py | 5 ++ 4 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 cli/integration_tests/run_all.py rename cli/integration_tests/{test.py => test_cmd.py} (100%) diff --git a/cli/integration_tests/run_all.py b/cli/integration_tests/run_all.py new file mode 100644 index 000000000..4492f632f --- /dev/null +++ b/cli/integration_tests/run_all.py @@ -0,0 +1,88 @@ +""" +Provide method to run the tests for all the commands provided by Chipmunk Build CLI Tool +""" + +from utls import print_red_bold, print_blue_bold, print_green_bold, print_cyan +from build import run_build_tests +from clean import run_clean_command +from environment import run_environment_commands +from lint import run_lint_command +from print_dot import run_print_dot_commands +from shell_compl import run_shell_completion_commands +from test_cmd import run_test_command + + +def print_err(cmd_name: str): + print_red_bold(f"Error while running tests for {cmd_name} commands") + + +def print_separtor(): + print_cyan("------------------------------------------------------------------") + + +def run_all(): + """Run the tests for all commands provided by Chipmunk Build CLI Tool""" + print_blue_bold("Running tests for all commands of Chipmunk Build CLI Tool") + + ### Environment ### + try: + run_environment_commands() + except Exception: + print_err("Environment") + raise + print_separtor() + + ### Lint ### + try: + run_lint_command() + except Exception: + print_err("Lint") + raise + print_separtor() + + ### Clean ### + try: + run_clean_command() + except Exception: + print_err("Clean") + raise + print_separtor() + + ### Build ### + try: + run_build_tests() + except Exception: + print_err("Build") + raise + print_separtor() + + ### Test ### + try: + run_test_command() + except Exception: + print_err("Test") + raise + print_separtor() + + ### Print Dots ### + try: + run_print_dot_commands() + except Exception: + print_err("Print Dots") + raise + print_separtor() + + ### Shell Completion ### + try: + run_shell_completion_commands() + except Exception: + print_err("Shell Completion") + raise + + print_green_bold( + "******** Tests for all commands of Chipmunk Build CLI Tool succeeded ********" + ) + + +if __name__ == "__main__": + run_all() diff --git a/cli/integration_tests/shell_compl.py b/cli/integration_tests/shell_compl.py index 2ee73f1ae..fbb581fab 100644 --- a/cli/integration_tests/shell_compl.py +++ b/cli/integration_tests/shell_compl.py @@ -16,7 +16,7 @@ SHELLS = ["bash", "elvish", "fish", "powershell", "zsh"] -def run_print_dot_commands(): +def run_shell_completion_commands(): """Runs commands to generate shell completion on all the available shells""" for shell in SHELLS: print_blue_bold(f"Runnig Shell Completion command for {shell}") @@ -27,4 +27,4 @@ def run_print_dot_commands(): if __name__ == "__main__": - run_print_dot_commands() + run_shell_completion_commands() diff --git a/cli/integration_tests/test.py b/cli/integration_tests/test_cmd.py similarity index 100% rename from cli/integration_tests/test.py rename to cli/integration_tests/test_cmd.py diff --git a/cli/integration_tests/utls.py b/cli/integration_tests/utls.py index 0d157e4e6..a0380a808 100644 --- a/cli/integration_tests/utls.py +++ b/cli/integration_tests/utls.py @@ -74,3 +74,8 @@ def print_green_bold(text: str): def print_red_bold(text: str): """Prints the given text to stdout with bold and red attribute""" print(f"{bcolors.BOLD}{bcolors.FAIL}{text}{bcolors.ENDC}{bcolors.ENDC}") + + +def print_cyan(text: str): + """Prints the given text to stdout with cyan color attribute""" + print(f"{bcolors.OKCYAN}{text}{bcolors.ENDC}") From f42fed4b7b3c6bbecdaeb25fe1ef38ecda19ba58 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Jul 2024 14:00:58 +0200 Subject: [PATCH 149/174] Clippy Fixes for Windows. Clippy fails currently on windows machine and have been quickly fixed in this PR instead of creating a new PR and having another merge from master --- application/apps/indexer/sources/src/command/process.rs | 3 --- application/apps/indexer/sources/src/serial/serialport.rs | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/application/apps/indexer/sources/src/command/process.rs b/application/apps/indexer/sources/src/command/process.rs index 9ba6a102f..5edf14246 100644 --- a/application/apps/indexer/sources/src/command/process.rs +++ b/application/apps/indexer/sources/src/command/process.rs @@ -14,9 +14,6 @@ use tokio::{ use tokio_stream::StreamExt; use tokio_util::codec::{self, FramedRead, LinesCodec}; -#[cfg(windows)] -use std::os::windows::process::CommandExt; - lazy_static! { static ref GROUP_RE: Regex = Regex::new(r#"".*?""#).expect("Regex must compile (fail with GROUP_RE)"); diff --git a/application/apps/indexer/sources/src/serial/serialport.rs b/application/apps/indexer/sources/src/serial/serialport.rs index 38a51d481..8c82830fb 100644 --- a/application/apps/indexer/sources/src/serial/serialport.rs +++ b/application/apps/indexer/sources/src/serial/serialport.rs @@ -103,6 +103,8 @@ impl SerialSource { .stop_bits(stop_bits(&config.stop_bits)) .open_native_async() { + // We get warning on windows because `port` doesn't need to be mutated there + #[cfg_attr(windows, allow(unused_mut))] Ok(mut port) => { #[cfg(unix)] if let Err(err) = port.set_exclusive(config.exclusive) { From 307f0aa05cc571ee6d687fedbc0a4e4f56529ff2 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Jul 2024 15:37:11 +0200 Subject: [PATCH 150/174] Build CLI Integ test: Improve checksum test - Add checks for targets that must not be involved in the build to insure that they will be stay unchanged - Fix typos --- cli/integration_tests/build.py | 45 ++++++++++++++++++++++++++++------ cli/src/job_type.rs | 4 +-- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index b94764654..e41a1df8c 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -120,7 +120,7 @@ def _build_general_check(): # Paths of files and directory in platform target and all other targets depending on it. # The modification date for this files will be read before the checksum test starts, then it will be compared after # the build command has finished to insure that the those targets have been rebuilt. -APP_PATHS_FOR_CHECKSUM_CHECK = [ +INVOLVED_PATHS_CHECKSUM_CHECK = [ # Shared "platform/dist/lib.js", # Binding @@ -133,6 +133,18 @@ def _build_general_check(): "holder/dist/app.js", ] +# Paths of files and directory that must be not changed after running the build with the changes in platform +# The modification date for this files will be read before the checksum test starts, then it will be compared after +# the build command has finished to insure that the those targets have not been rebuilt. +PATHS_NON_INVOLVED_CHECKSUM_CHECK = [ + # Core + "apps/indexer/target", + # Wasm + "apps/rustcore/wasm-bindings/pkg", + # Updater + "apps/precompiled/updater/target", +] + def _build_checksum_check(): """!!!This function must run directly after a full build!!! @@ -148,15 +160,25 @@ def _build_checksum_check(): chksum_file.exists() ), f"Checksum File must exist before running checksum tests. File Path: {chksum_file}" - # Save modification date for build paths before start to compare them later. - modifi_before_start: Dict[Path, datetime] = {} - for sub_path in APP_PATHS_FOR_CHECKSUM_CHECK: + # Save modification date for build paths that must change before start to compare them later. + modifi_involved_before_start: Dict[Path, datetime] = {} + for sub_path in INVOLVED_PATHS_CHECKSUM_CHECK: + sub_path = application_dir.joinpath(sub_path) + assert ( + sub_path.exists() + ), f"Build Path must exist before checksum tests starts. Path {sub_path}" + modifi_date = datetime.fromtimestamp(sub_path.stat().st_mtime) + modifi_involved_before_start[sub_path] = modifi_date + + # Save modification date for build paths that must sta before start to compare them later. + modifi_non_involved_before_start: Dict[Path, datetime] = {} + for sub_path in PATHS_NON_INVOLVED_CHECKSUM_CHECK: sub_path = application_dir.joinpath(sub_path) assert ( sub_path.exists() ), f"Build Path must exist before checksum tests starts. Path {sub_path}" modifi_date = datetime.fromtimestamp(sub_path.stat().st_mtime) - modifi_before_start[sub_path] = modifi_date + modifi_non_involved_before_start[sub_path] = modifi_date # Define temporary file path in platform directory to insure it will be rebuilt # with all other targets depending on it. @@ -174,11 +196,20 @@ def _build_checksum_check(): run_command(BUILD_COMMAND) # Compare modification date for involved targets - for path, modifi_before in modifi_before_start.items(): + for path, modifi_before in modifi_involved_before_start.items(): modifi_after = datetime.fromtimestamp(path.stat().st_mtime) assert ( modifi_after > modifi_before - ), f"Target modified date after must be more recent than before.\n\ + ), f"Involved target modification date after must be more recent than before.\n\ + Target Path: {path}.\n\ + Before: {modifi_before}, After: {modifi_after}" + + # Compare modification date for not involved targets + for path, modifi_before in modifi_involved_before_start.items(): + modifi_after = datetime.fromtimestamp(path.stat().st_mtime) + assert ( + modifi_after != modifi_before + ), f"Not involved target modification date must not be changed.\n\ Target Path: {path}.\n\ Before: {modifi_before}, After: {modifi_after}" finally: diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index f7308abc4..b17823052 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -1,7 +1,7 @@ use std::fmt::Display; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -//NOTE: The order of job types must match the runnig-order of them because it's used by +//NOTE: The order of job types must match the running-order of them because it's used by // solving their dependencies-graph using BTreeMap pub enum JobType { Clean, @@ -20,7 +20,7 @@ impl Display for JobType { JobType::Clean => write!(f, "Clean"), JobType::Build { production: _ } => write!(f, "Build"), JobType::Install { production: _ } => write!(f, "Install"), - JobType::AfterBuild { production: _ } => write!(f, "After Bulid"), + JobType::AfterBuild { production: _ } => write!(f, "After Build"), JobType::Test { production: _ } => write!(f, "Test"), JobType::Run { production: _ } => write!(f, "Run"), } From d8f38b7eaaf107d36c97c2d1b5d0019d47e53111 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 23 Jul 2024 17:42:53 +0200 Subject: [PATCH 151/174] Build CLI Integ test: Fix files modification time - Fix Windows file system errors (creation time can be bigger than modification time) by returning the bigger between them and checking for any of the targets to be more recent after build instead of testing all of them like in Unix because of the unreliability of Windows file system - On Unix we return the last time the meta data of a file has been changed --- cli/integration_tests/build.py | 63 +++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index e41a1df8c..67c3b7afc 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -7,6 +7,7 @@ from utls import get_root, run_command, print_green_bold, print_blue_bold from datetime import datetime from typing import Dict +import platform # Build command to be used in all build tests BUILD_COMMAND = [ @@ -72,7 +73,7 @@ def _build_general_check(): # Get last modification date for checksum file if exists otherwise get minimal date checksum_modified_before = ( - datetime.fromtimestamp(checksum_path.stat().st_mtime) + get_last_modification_date(checksum_path) if checksum_path.exists() else datetime.min ) @@ -94,7 +95,7 @@ def _build_general_check(): checksum_path.exists() ), f"Checksum record file doesn't exist after build. File Path: {checksum_path}" - checksum_modified_after = datetime.fromtimestamp(checksum_path.stat().st_mtime) + checksum_modified_after = get_last_modification_date(checksum_path) assert ( checksum_modified_after > checksum_modified_before @@ -146,6 +147,28 @@ def _build_general_check(): ] +def get_last_modification_date(path: Path) -> datetime: + """Gets the last modification date for the given path on different platforms + On Unix it will return the last time the meta data of the file has been changed + On Windows it will return the more recent between creating and last modification times + """ + # Get file stats + stats = path.stat() + + # On Unix, return the last time any of the file meta data has changed + if platform.system() != "Windows": + return datetime.fromtimestamp(stats.st_ctime) + + # On Windows, return the greater of st_mtime and st_birthtime because time information on Windows can be misleading, + # like it's possible to get a creation time that is more recent than the modification time. + else: + most_recent_time = max( + stats.st_mtime, + stats.st_birthtime, + ) + return datetime.fromtimestamp(most_recent_time) + + def _build_checksum_check(): """!!!This function must run directly after a full build!!! It creates a dummy file in platform directory and checks that all dependencies (Binding, Wrapper, Client, App) @@ -167,7 +190,7 @@ def _build_checksum_check(): assert ( sub_path.exists() ), f"Build Path must exist before checksum tests starts. Path {sub_path}" - modifi_date = datetime.fromtimestamp(sub_path.stat().st_mtime) + modifi_date = get_last_modification_date(sub_path) modifi_involved_before_start[sub_path] = modifi_date # Save modification date for build paths that must sta before start to compare them later. @@ -177,7 +200,7 @@ def _build_checksum_check(): assert ( sub_path.exists() ), f"Build Path must exist before checksum tests starts. Path {sub_path}" - modifi_date = datetime.fromtimestamp(sub_path.stat().st_mtime) + modifi_date = get_last_modification_date(sub_path) modifi_non_involved_before_start[sub_path] = modifi_date # Define temporary file path in platform directory to insure it will be rebuilt @@ -195,18 +218,34 @@ def _build_checksum_check(): # Run build command run_command(BUILD_COMMAND) - # Compare modification date for involved targets - for path, modifi_before in modifi_involved_before_start.items(): - modifi_after = datetime.fromtimestamp(path.stat().st_mtime) + # Compare modification date for involved targets on different platforms + if platform.system() != "Windows": + # On Unix systems we compare all the involved targets to insure the checksum solution is + # working and to test the build dependencies logic + for path, modifi_before in modifi_involved_before_start.items(): + modifi_after = get_last_modification_date(path) + assert ( + modifi_after > modifi_before + ), f"Involved target modification date after must be more recent than before.\n\ + Target Path: {path}.\n\ + Before: {modifi_before}, After: {modifi_after}" + else: + # On Windows it's enough that only one of the involved target has more recent date + # because the file system here isn't reliable in delivering the current time of the + # latest change on a file or directory + date_changed = False + for path, modifi_before in modifi_involved_before_start.items(): + modifi_after = get_last_modification_date(path) + date_changed = modifi_after > modifi_before + if date_changed: + break assert ( - modifi_after > modifi_before - ), f"Involved target modification date after must be more recent than before.\n\ - Target Path: {path}.\n\ - Before: {modifi_before}, After: {modifi_after}" + date_changed + ), "None of the involved targets' modification date is more recent than before build" # Compare modification date for not involved targets for path, modifi_before in modifi_involved_before_start.items(): - modifi_after = datetime.fromtimestamp(path.stat().st_mtime) + modifi_after = get_last_modification_date(path) assert ( modifi_after != modifi_before ), f"Not involved target modification date must not be changed.\n\ From df8f58888beebb7a137ac32bcd14a033d26c786a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Jul 2024 10:56:58 +0200 Subject: [PATCH 152/174] Build CLI Integ test: Fix checksum test - Use paths that shouldn't be modified and fix the check from not equal to equal --- cli/integration_tests/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index 67c3b7afc..ef5fe8b6f 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -244,10 +244,10 @@ def _build_checksum_check(): ), "None of the involved targets' modification date is more recent than before build" # Compare modification date for not involved targets - for path, modifi_before in modifi_involved_before_start.items(): + for path, modifi_before in modifi_non_involved_before_start.items(): modifi_after = get_last_modification_date(path) assert ( - modifi_after != modifi_before + modifi_after == modifi_before ), f"Not involved target modification date must not be changed.\n\ Target Path: {path}.\n\ Before: {modifi_before}, After: {modifi_after}" From 7e3bf460451d01fff32736ef135e486d3550d719 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Jul 2024 11:32:29 +0200 Subject: [PATCH 153/174] Build CLI: Fix running wrapper tests on Windows - Path for test runner on Windows must be resolved with which crate because node executables get the extension `.cmd` on Windows sometimes --- cli/src/target/wrapper.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 24bd1d6d0..92c4a6149 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -1,6 +1,6 @@ use std::{iter, path::PathBuf}; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use crate::{ job_type::JobType, @@ -49,10 +49,18 @@ pub async fn run_test(production: bool) -> Result { //TODO: This check exists in rake implementation but it need to be improved. // The check should cover if the test themselves or the code under the tests has been changed. if !build_spec_path.join("build").exists() { - let test_builder_path = cwd.join("node_modules").join(".bin").join("tsc"); + // Finding tsc path on differnet platforms + let mut test_runner_path = cwd.join("node_modules").join(".bin"); + if cfg!(windows) { + let tsc_path = which::which_in("tsc", Some(&test_runner_path), &test_runner_path) + .context("Error while resolving tsc bin path on Windows")?; + test_runner_path = tsc_path; + } else { + test_runner_path.push("tsc"); + } let build_spec_cmd = ProcessCommand::new( - test_builder_path.to_string_lossy().to_string(), + test_runner_path.to_string_lossy().to_string(), vec![String::from("-p"), String::from("tsconfig.json")], ); @@ -68,7 +76,14 @@ pub async fn run_test(production: bool) -> Result { final_result = Some(spec_res); } - let electron_path: PathBuf = [".", "node_modules", ".bin", "electron"].iter().collect(); + let mut electron_path: PathBuf = cwd.join("node_modules").join(".bin"); + if cfg!(windows) { + electron_path = which::which_in("electron", Some(&electron_path), &electron_path) + .context("Error while resolving electron bin path on Windows")?; + } else { + electron_path.push("electron"); + } + // "electron"].iter().collect(); let electron_path = electron_path.to_string_lossy(); let jasmine_path: PathBuf = [".", "node_modules", "jasmine", "bin", "jasmine.js"] From 0f710fa0c345f1fbef7f5660299fb85eaaa5fa21 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Jul 2024 14:33:30 +0200 Subject: [PATCH 154/174] Build CLI: Add integration tests to README - Add details about running the integration tests on each change in the build CLI tool and request to add new tests with new features - Fix broken link for chipmunk main contribution page --- cli/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cli/README.md b/cli/README.md index 509515e6c..64233821a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -108,4 +108,20 @@ Next, copy the chipmunk-completion.bash file to your bash completion directory ( ## Contributing -See our [contribution](contribution.md) guide for details + +Contributions in any part of Chipmunk are very welcome! + +After making any changes to this build CLI tool, please run the integration tests to ensure that all the provided commands in this tool are still working as expected. Additionally, consider adding new tests when introducing new features and commands. + +To run all the tests, execute the Python file `chipmunk/cli/integration_tests/run_all.py` from within the `chipmunk/cli` directory. For example: + +```bash +# Move to cli directory +cd cli +# Run python file +python ./integration_tests/run_all.py +``` +Please be aware that these tests will run on your local copy of Chipmunk. This process will rebuild the project and run all linting and tests on the entire solution. + +For more details, please see our [contribution guid](../contribution.md) + From d5bbbdf5159ce11436f45cc10a0dd07eed7dc450 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Jul 2024 15:36:16 +0200 Subject: [PATCH 155/174] Build CLI: Fix bindings tests on Windows - Add another command for testing on windows with forward slashes - Provide the matching command for each platform in the Build CLI Tool --- application/apps/rustcore/wasm-bindings/package.json | 3 ++- cli/src/target/wasm.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/application/apps/rustcore/wasm-bindings/package.json b/application/apps/rustcore/wasm-bindings/package.json index fcfb879dc..2c6eb90ae 100644 --- a/application/apps/rustcore/wasm-bindings/package.json +++ b/application/apps/rustcore/wasm-bindings/package.json @@ -5,7 +5,8 @@ "scripts": { "build": "node_modules/.bin/webpack", "start": "node_modules/.bin/webpack-dev-server", - "test": "node_modules/.bin/karma start" + "test": "node_modules/.bin/karma start", + "test_win": "node_modules\\.bin\\karma start" }, "author": "", "license": "MIT", diff --git a/cli/src/target/wasm.rs b/cli/src/target/wasm.rs index c6ce3b709..d3e1738f5 100644 --- a/cli/src/target/wasm.rs +++ b/cli/src/target/wasm.rs @@ -25,6 +25,8 @@ pub fn get_test_cmds() -> Vec { let npm_path = DevTool::Npm.path(); let wasm_pack_path = DevTool::WasmPack.path(); + let npm_test_command = if cfg!(windows) { "test_win" } else { "test" }; + vec![ TestSpawnCommand::new( ProcessCommand::new( @@ -42,7 +44,7 @@ pub fn get_test_cmds() -> Vec { TestSpawnCommand::new( ProcessCommand::new( npm_path.to_string_lossy().to_string(), - vec![String::from("run"), String::from("test")], + vec![String::from("run"), String::from(npm_test_command)], ), cwd.join("spec"), Some(SpawnOptions { From c9f9897ad46677f27b7e63ac715cb4bca24a934f Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 24 Jul 2024 17:08:49 +0200 Subject: [PATCH 156/174] Build CLI Integ Tests: Comments & Cleaning Up - Refactoring and Providing more comments to the scripts - Fixing typos --- cli/integration_tests/build.py | 74 +++++++++++++++++----------- cli/integration_tests/clean.py | 18 ++++--- cli/integration_tests/print_dot.py | 6 ++- cli/integration_tests/run_all.py | 32 ++++++------ cli/integration_tests/shell_compl.py | 5 +- cli/integration_tests/utls.py | 4 +- 6 files changed, 81 insertions(+), 58 deletions(-) diff --git a/cli/integration_tests/build.py b/cli/integration_tests/build.py index ef5fe8b6f..1623222d6 100644 --- a/cli/integration_tests/build.py +++ b/cli/integration_tests/build.py @@ -1,6 +1,6 @@ """ Provides methods to test the Build command and checksum implementation to watch source code changes -in Chipmunk Build CLI Tool +in Chipmunk Build CLI Tool. """ from pathlib import Path @@ -9,7 +9,40 @@ from typing import Dict import platform -# Build command to be used in all build tests +####################################################################### +######################## RUN ALL TESTS ############################ +####################################################################### + + +def run_build_tests(): + """Runs many tests for build command: + - General Test: General test for build command and the created directories and checksum files. + - Checksum Test: Test how changing files in some target will affect the build on it and other targets depending on it. + - Reset Checksum Test: Make sure checksum file will deleted when the command is called. + """ + print_blue_bold("Running General Checks for Build Command...") + _build_general_check() + print_green_bold("*** General Check for Build Command Succeeded ***") + + print("---------------------------------------------------------------") + + print_blue_bold("Running Checksum Checks for Build Command...") + _build_checksum_check() + print_green_bold("*** Checksum Check for Build Command Succeeded ***") + + print("---------------------------------------------------------------") + + print_blue_bold("Running Reset Checksum Command...") + _run_reset_checksum_command() + print_green_bold("*** Test Running Reset Checksum Command Succeeded ***") + + +####################################################################### +################ GENERAL TEST FOR BUILD COMMAND ################### +####################################################################### + +# Build command to be used in general build test +# This command should build all the targets expect the Build CLI Tool BUILD_COMMAND = [ "cargo", "run", @@ -104,7 +137,9 @@ def _build_general_check(): print("Checking if Checksum Records file changed Succeeded") -############ CHECKSUM RECORDS TESTS ########### +####################################################################### +#################### CHECKSUM RECORDS TESTS ####################### +####################################################################### # Name of the temporary file which will be used for testing if file changes will lead to rebuild # the direct target and all other targets that depend on here. @@ -257,12 +292,17 @@ def _build_checksum_check(): temp_file_path.unlink() -############## RESET CHECKSUM COMMAND ################ +####################################################################### +#################### RESET CHECKSUM COMMAND ####################### +####################################################################### RESET_CHECKSUM_COMMAND = ["cargo", "run", "-r", "--", "chipmunk", "reset-checksum"] def _run_reset_checksum_command(): + """This test will run reset-checksum command in development mode, then it insure that + the checksum file has been deleted""" + checksum_path = get_root().joinpath(CHECKSUM_FILE_NAME) assert ( checksum_path.exists() @@ -277,31 +317,5 @@ def _run_reset_checksum_command(): pass -################ RUN ALL TESTS ###################### - - -def run_build_tests(): - """Runs many tests for build command: - - General Test: General test for build command and the created directories and checksum files. - - Checksum Test: Test how changing files in some target will affect the build on it and other targets depending on it. - - Reset Checksum Test: Make sure checksum file will deleted when the command is called. - """ - print_blue_bold("Running General Checks for Build Command...") - _build_general_check() - print_green_bold("*** General Check for Build Command Succeeded ***") - - print("---------------------------------------------------------------") - - print_blue_bold("Running Checksum Checks for Build Command...") - _build_checksum_check() - print_green_bold("*** Checksum Check for Build Command Succeeded ***") - - print("---------------------------------------------------------------") - - print_blue_bold("Running Reset Checksum Command...") - _run_reset_checksum_command() - print_green_bold("*** Test Running Reset Checksum Command Succeeded ***") - - if __name__ == "__main__": run_build_tests() diff --git a/cli/integration_tests/clean.py b/cli/integration_tests/clean.py index 3114bcc19..80b8d1b6a 100644 --- a/cli/integration_tests/clean.py +++ b/cli/integration_tests/clean.py @@ -1,5 +1,7 @@ """ Provides methods to test the Clean command in Chipmunk Build CLI Tool +Clean command will be invoked here for all targets but the CLI tool itself, +then it checks that all the paths that must be removed don't exist anymore. """ from pathlib import Path @@ -55,15 +57,8 @@ ] -def get_removed_paths() -> list[Path]: - """Provides the paths for the directories that must be removed after running the clean command""" - root_dir = get_root() - application_dir = root_dir.joinpath("application") - return [application_dir.joinpath(sub_dir) for sub_dir in PATHS_TO_CHECK] - - def run_clean_command(): - """Runs Clean Commands on all targets and insure that all build directories are delted.""" + """Runs Clean Commands on all targets and insure that all build directories are deleted.""" print_blue_bold("Running clean command...") run_command(CLEAN_COMMAND) for path in get_removed_paths(): @@ -73,5 +68,12 @@ def run_clean_command(): print_green_bold("*** Check for Clean Command Succeeded ***") +def get_removed_paths() -> list[Path]: + """Provides the paths for the directories that must be removed after running the clean command""" + root_dir = get_root() + application_dir = root_dir.joinpath("application") + return [application_dir.joinpath(sub_dir) for sub_dir in PATHS_TO_CHECK] + + if __name__ == "__main__": run_clean_command() diff --git a/cli/integration_tests/print_dot.py b/cli/integration_tests/print_dot.py index 43287ca4f..01c3224ed 100644 --- a/cli/integration_tests/print_dot.py +++ b/cli/integration_tests/print_dot.py @@ -4,6 +4,7 @@ from utls import run_command, print_green_bold, print_blue_bold +# Command and Args for the general print dot command PRINT_DOT_COMMAND = [ "cargo", "run", @@ -13,6 +14,7 @@ "print-dot", ] +# Command and Args for the print dot command with `--all` flag PRINT_DOT_ALL_COMMAND = [ "cargo", "run", @@ -30,9 +32,9 @@ def run_print_dot_commands(): run_command(PRINT_DOT_COMMAND) print_green_bold("*** General Print Dot command Succeeded ***") - print_blue_bold("Running All Print Dot command...") + print_blue_bold("Running Print Dot command with flag `--all`...") run_command(PRINT_DOT_ALL_COMMAND) - print_green_bold("*** All Print Dot command Succeeded ***") + print_green_bold("*** Print Dot command with flag `--all` Succeeded ***") if __name__ == "__main__": diff --git a/cli/integration_tests/run_all.py b/cli/integration_tests/run_all.py index 4492f632f..4bec7fad0 100644 --- a/cli/integration_tests/run_all.py +++ b/cli/integration_tests/run_all.py @@ -1,5 +1,7 @@ """ Provide method to run the tests for all the commands provided by Chipmunk Build CLI Tool +All the tests build and run the current build CLI implementation in release mode, therefore +it must be invoked from withing `Chipmunk/cli` directory """ from utls import print_red_bold, print_blue_bold, print_green_bold, print_cyan @@ -12,14 +14,6 @@ from test_cmd import run_test_command -def print_err(cmd_name: str): - print_red_bold(f"Error while running tests for {cmd_name} commands") - - -def print_separtor(): - print_cyan("------------------------------------------------------------------") - - def run_all(): """Run the tests for all commands provided by Chipmunk Build CLI Tool""" print_blue_bold("Running tests for all commands of Chipmunk Build CLI Tool") @@ -30,7 +24,7 @@ def run_all(): except Exception: print_err("Environment") raise - print_separtor() + print_separator() ### Lint ### try: @@ -38,7 +32,7 @@ def run_all(): except Exception: print_err("Lint") raise - print_separtor() + print_separator() ### Clean ### try: @@ -46,7 +40,7 @@ def run_all(): except Exception: print_err("Clean") raise - print_separtor() + print_separator() ### Build ### try: @@ -54,7 +48,7 @@ def run_all(): except Exception: print_err("Build") raise - print_separtor() + print_separator() ### Test ### try: @@ -62,7 +56,7 @@ def run_all(): except Exception: print_err("Test") raise - print_separtor() + print_separator() ### Print Dots ### try: @@ -70,7 +64,7 @@ def run_all(): except Exception: print_err("Print Dots") raise - print_separtor() + print_separator() ### Shell Completion ### try: @@ -84,5 +78,15 @@ def run_all(): ) +def print_err(cmd_name: str): + """Prints a formatted error with the main command name""" + print_red_bold(f"Error while running tests for {cmd_name} commands") + + +def print_separator(): + """Prints a colored separator between main commands""" + print_cyan("------------------------------------------------------------------") + + if __name__ == "__main__": run_all() diff --git a/cli/integration_tests/shell_compl.py b/cli/integration_tests/shell_compl.py index fbb581fab..f2f214239 100644 --- a/cli/integration_tests/shell_compl.py +++ b/cli/integration_tests/shell_compl.py @@ -1,5 +1,5 @@ """ -Provides methods to test the Shell Completion command for varity of shells in Chipmunk Build CLI Tool +Provides methods to test the Shell Completion command for variety of shells in Chipmunk Build CLI Tool """ from utls import run_command, print_blue_bold, print_green_bold @@ -13,13 +13,14 @@ "shell-completion", ] +# These are all the supported shells for completion SHELLS = ["bash", "elvish", "fish", "powershell", "zsh"] def run_shell_completion_commands(): """Runs commands to generate shell completion on all the available shells""" for shell in SHELLS: - print_blue_bold(f"Runnig Shell Completion command for {shell}") + print_blue_bold(f"Running Shell Completion command for {shell}") shell_command = PRINT_COMPLETION_COMMAND.copy() shell_command.append(shell) run_command(shell_command) diff --git a/cli/integration_tests/utls.py b/cli/integration_tests/utls.py index a0380a808..ed71e2f5b 100644 --- a/cli/integration_tests/utls.py +++ b/cli/integration_tests/utls.py @@ -10,7 +10,7 @@ def get_root() -> Path: """Get and validate the root directory of chipmunk repository Raises: - IOError: If validation of root directory fails + SystemError: If validation of root directory fails Returns: Root directory of chipmunk @@ -32,7 +32,7 @@ def get_root() -> Path: def run_command(command_args: list[str]): - """Runs the commands and its arguments after printing the command to stdout + """Runs the commands and its arguments after printing it to stdout Args: command_args: The command and its arguments in one string list From b94bddcf1f46cd6af1628f24dbf732c64656840e Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 26 Jul 2024 09:43:37 +0200 Subject: [PATCH 157/174] Build CLI: Clippy Fixes with docs indentation --- cli/dir_checksum/src/lib.rs | 2 +- cli/src/target/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/dir_checksum/src/lib.rs b/cli/dir_checksum/src/lib.rs index ca4e7d7fb..e995ff367 100644 --- a/cli/dir_checksum/src/lib.rs +++ b/cli/dir_checksum/src/lib.rs @@ -49,7 +49,7 @@ where /// returning its result /// /// * `calc_fn`: Function that will be called inside the function with the directory path and -/// the created hasher +/// the created hasher fn run_intern(dir_path: P, calc_fn: F) -> Result where F: Fn(&Path, &mut blake3::Hasher) -> Result + Sync + Send, diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 0e728b020..2c64b934c 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -264,7 +264,7 @@ impl Target { /// * `prod`: run install in production /// * `skip`: skip the task /// * `overridden_job_type`: override job type to communicate with tracker when install is ran - /// from within another task + /// from within another task pub async fn install( &self, prod: bool, @@ -528,7 +528,7 @@ impl Target { /// * `prod`: build for production /// * `job_type`: job type to communicate with `tracker` /// * `overridden_target`: override target to communicate with `tracker` when install is called -/// from within another task. +/// from within another task. async fn install_general( target: Target, prod: bool, From 27c5707d87c30931cc2d1b0b3828150cb4ce7f68 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Jul 2024 11:42:37 +0200 Subject: [PATCH 158/174] Build CLI: Basic integrations in CI system - Use the CLI tool with GitHub Actions for Pull Requests along side with the current ruby actions for comparison - Install latest rust manually instead of the deprecated GitHub Action - Use Cargo cache action for speed up installing the build CLI tool - Dependencies are installed manually for now but this can improved --- .github/workflows/pullrequest_check.yml | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/pullrequest_check.yml b/.github/workflows/pullrequest_check.yml index 833bb9154..a382e1413 100644 --- a/.github/workflows/pullrequest_check.yml +++ b/.github/workflows/pullrequest_check.yml @@ -3,6 +3,56 @@ name: Checks on: [pull_request] jobs: + all_lint: + name: Run all lints using CLI tool + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Latest Rust + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup default ${{ env.RUST_CHANNEL }} + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.3 + - name: Install wasm-pack + run: cargo install wasm-pack + - name: Install nj-cli + run: cargo install nj-cli + - name: Install Build CLI tool + run: cargo install --path=cli + - name: libudev-dev + run: sudo apt-get install -y libudev-dev + - name: enable corepack for yarnpkg upgrade + run: corepack enable + - name: Run Lints + run: cargo chipmunk lint -r + all_test: + name: Run all tests using CLI tool + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Latest Rust + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup default ${{ env.RUST_CHANNEL }} + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.3 + - name: Install wasm-pack + run: cargo install wasm-pack + - name: Install nj-cli + run: cargo install nj-cli + - name: Install Build CLI tool + run: cargo install --path=cli + - name: libudev-dev + run: sudo apt-get install -y libudev-dev + - name: enable corepack for yarnpkg upgrade + run: | + npm install tslib + corepack enable + - name: Run Tests + run: cargo chipmunk test -r ts_lint: runs-on: ubuntu-latest steps: From c391944d228374cf264129f1a13d259f7babb49a Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Jul 2024 16:05:59 +0200 Subject: [PATCH 159/174] Build CLI: Fix install in production issue - Install command will always run in develop mode initially and then the command will rerun in production mode if it's active after the build is done. - This problem was solved by setting the production flag on false always for install job definition which lead to problem by calculating the checksums because production flag on install was always false leading to loading the checksums on development always even on production builds - The issue is solved by assigning the production flag in the internal command since we are calling it anyway with the production flag in the reinstall process --- cli/src/job_type.rs | 6 +++--- cli/src/jobs_runner/job_definition.rs | 8 +++++++- cli/src/target/mod.rs | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index b17823052..19ced08bf 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -45,9 +45,9 @@ impl JobType { // Linting TS needs to building too to check for type errors JobType::Lint => vec![JobType::Build { production: false }], JobType::Build { production } => vec![ - // Install run always in development at first then it should get reinstalled with - // production after build command is ran. - JobType::Install { production: false }, + JobType::Install { + production: *production, + }, JobType::AfterBuild { production: *production, }, diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index d3c83aa2d..d1e6e996b 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -58,8 +58,14 @@ impl JobDefinition { let res = match self.job_type { JobType::Lint => self.target.check().await, JobType::Build { production } => self.target.build(production, skip).await, + // Install run always in development at first then it should get reinstalled with + // production after build command is ran. + // We must deliver the correct jobtype though for the communication with tracker. JobType::Install { production } => { - return self.target.install(production, skip, None).await + return self + .target + .install(false, skip, Some(JobType::Install { production })) + .await; } JobType::AfterBuild { production } => { return self.target.after_build(production, skip).await diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 2c64b934c..4b8446b09 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -264,7 +264,7 @@ impl Target { /// * `prod`: run install in production /// * `skip`: skip the task /// * `overridden_job_type`: override job type to communicate with tracker when install is ran - /// from within another task + /// from within another task or when the production flag must be ignored pub async fn install( &self, prod: bool, From 3bb57c3d760ccd32a3d09af1c9050933d004deb5 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 30 Jul 2024 16:28:02 +0200 Subject: [PATCH 160/174] Build CLI: Remove checksum of opposite build - Production and development use the same artifacts which will lead to false positives when the artifacts are modified via another build but the checksum of source files still the same. - To solve this problem we will reset the checksums of the opposite build production type when build is involved in the current process --- cli/src/checksum_records.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 3f417eb2d..2061f033d 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -47,6 +47,23 @@ impl ChecksumRecords { if calculate_involved { records.calculate_involved_hashes()?; + + // Production and development use the same artifacts which will lead to false + // positives when the artifacts are modified via another build but the checksum of + // source files still the same. + // To solve this problem we will reset the checksums of the opposite build production + // type when build is involved in the current process + let outdated_record_prod = !prod; + Self::remove_records_file(outdated_record_prod).with_context(|| { + format!( + "Error while remove the outdated {} build checksum records", + if outdated_record_prod { + "production" + } else { + "development" + } + ) + })?; } records From cc7814cba2c13c420b35acc2c2674cfcfd88ae9c Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 31 Jul 2024 09:55:51 +0200 Subject: [PATCH 161/174] Build CLI: Update dependencies --- cli/Cargo.lock | 559 ++++++++++++++++++------------------------------- 1 file changed, 209 insertions(+), 350 deletions(-) diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 0be655565..699374ab1 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -28,63 +28,64 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" [[package]] name = "arrayvec" @@ -94,15 +95,15 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -115,35 +116,29 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "rayon", + "rayon-core", ] [[package]] name = "bstr" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", "serde", @@ -151,9 +146,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cargo-chipmunk" @@ -176,9 +171,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", @@ -192,9 +187,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.8" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" dependencies = [ "clap_builder", "clap_derive", @@ -202,9 +197,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" dependencies = [ "anstream", "anstyle", @@ -214,18 +209,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.2" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79504325bf38b10165b02e89b4347300f855f273c4cb30c4a3209e6583275e" +checksum = "c6ae69fbb0833c6fcd5a8d4b8609f108c7ad95fc11e248d853ff2c42a90df26a" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" dependencies = [ "heck", "proc-macro2", @@ -235,27 +230,27 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -285,9 +280,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "dir_checksum" @@ -304,9 +299,9 @@ dependencies = [ [[package]] name = "either" -version = "1.11.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encode_unicode" @@ -316,12 +311,12 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -356,9 +351,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -371,9 +366,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -381,15 +376,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -398,9 +393,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -419,9 +414,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -430,21 +425,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -460,17 +455,17 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "git2" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" dependencies = [ - "bitflags 2.4.2", + "bitflags", "libc", "libgit2-sys", "log", @@ -494,15 +489,15 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "home" @@ -510,7 +505,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -541,9 +536,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", @@ -554,33 +549,39 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libgit2-sys" @@ -612,9 +613,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.15" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" dependencies = [ "cc", "libc", @@ -624,15 +625,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -640,15 +641,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -661,32 +662,23 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ "hermit-abi", "libc", + "wasi", + "windows-sys", ] [[package]] @@ -697,9 +689,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.2" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] @@ -712,9 +704,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.100" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae94056a791d0e1217d18b6cbdccb02c61e3054fc69893607f4067e3bb0b1fd1" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", @@ -730,9 +722,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -740,15 +732,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -759,9 +751,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -777,24 +769,24 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -858,18 +850,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -878,9 +870,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "remove_dir_all" @@ -893,9 +885,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" @@ -903,11 +895,11 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.2", + "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -927,18 +919,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -947,9 +939,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -965,31 +957,31 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.39" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -1008,18 +1000,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.55" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3de26b0965292219b4287ff031fcba86837900fe9cd2b34ea8ad893c0953d2" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.55" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268026685b2be38d7103e9e507c938a1fcb3d7e6eb15e87870b617bf37b6d581" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", @@ -1028,9 +1020,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -1043,28 +1035,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -1094,15 +1085,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -1111,9 +1102,9 @@ dependencies = [ [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "vcpkg" @@ -1123,9 +1114,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "waker-fn" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" [[package]] name = "walkdir" @@ -1145,9 +1136,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "which" -version = "6.0.1" +version = "6.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8211e4f58a2b2805adfbefbc07bab82958fc91e3836339b1ab7ae32465dce0d7" +checksum = "3d9c5ed668ee1f17edb3b627225343d210006a90bb1e3745ce1f30b1fb115075" dependencies = [ "either", "home", @@ -1173,11 +1164,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -1186,210 +1177,78 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winsafe" From d7ee117566e37173ba0dab48c2a253a26b772470 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Fri, 2 Aug 2024 11:33:21 +0200 Subject: [PATCH 162/174] Build CLI: Move created CI jobs to their own action Move the new CI checks using the CLI tool to their own GitHub action that will run manually only since those jobs are still work on progress and not needed to run along side with the current CI checks using rake commands --- .github/workflows/build_cli_checks.yml | 59 +++++++++++++++++++++++++ .github/workflows/pullrequest_check.yml | 50 --------------------- 2 files changed, 59 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/build_cli_checks.yml diff --git a/.github/workflows/build_cli_checks.yml b/.github/workflows/build_cli_checks.yml new file mode 100644 index 000000000..3804f7dd2 --- /dev/null +++ b/.github/workflows/build_cli_checks.yml @@ -0,0 +1,59 @@ +# This action runs the checks for pull requests using the build CLI Tool. +name: Build CLI Tool Checks + +# Using the tool in GitHub Actions stills work in progress and will be invoked manually +# until it's ready to replace the current rake workflows. +on: + workflow_dispatch: + +jobs: + all_lint: + name: Run all lints using CLI tool + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Latest Rust + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup default ${{ env.RUST_CHANNEL }} + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.3 + - name: Install wasm-pack + run: cargo install wasm-pack + - name: Install nj-cli + run: cargo install nj-cli + - name: Install Build CLI tool + run: cargo install --path=cli + - name: libudev-dev + run: sudo apt-get install -y libudev-dev + - name: enable corepack for yarnpkg upgrade + run: corepack enable + - name: Run Lints + run: cargo chipmunk lint -r + all_test: + name: Run all tests using CLI tool + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Latest Rust + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup default ${{ env.RUST_CHANNEL }} + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.3 + - name: Install wasm-pack + run: cargo install wasm-pack + - name: Install nj-cli + run: cargo install nj-cli + - name: Install Build CLI tool + run: cargo install --path=cli + - name: libudev-dev + run: sudo apt-get install -y libudev-dev + - name: enable corepack for yarnpkg upgrade + run: | + npm install tslib + corepack enable + - name: Run Tests + run: cargo chipmunk test -r diff --git a/.github/workflows/pullrequest_check.yml b/.github/workflows/pullrequest_check.yml index a382e1413..833bb9154 100644 --- a/.github/workflows/pullrequest_check.yml +++ b/.github/workflows/pullrequest_check.yml @@ -3,56 +3,6 @@ name: Checks on: [pull_request] jobs: - all_lint: - name: Run all lints using CLI tool - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Latest Rust - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup default ${{ env.RUST_CHANNEL }} - - name: Rust Cache - uses: Swatinem/rust-cache@v2.7.3 - - name: Install wasm-pack - run: cargo install wasm-pack - - name: Install nj-cli - run: cargo install nj-cli - - name: Install Build CLI tool - run: cargo install --path=cli - - name: libudev-dev - run: sudo apt-get install -y libudev-dev - - name: enable corepack for yarnpkg upgrade - run: corepack enable - - name: Run Lints - run: cargo chipmunk lint -r - all_test: - name: Run all tests using CLI tool - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Latest Rust - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup default ${{ env.RUST_CHANNEL }} - - name: Rust Cache - uses: Swatinem/rust-cache@v2.7.3 - - name: Install wasm-pack - run: cargo install wasm-pack - - name: Install nj-cli - run: cargo install nj-cli - - name: Install Build CLI tool - run: cargo install --path=cli - - name: libudev-dev - run: sudo apt-get install -y libudev-dev - - name: enable corepack for yarnpkg upgrade - run: | - npm install tslib - corepack enable - - name: Run Tests - run: cargo chipmunk test -r ts_lint: runs-on: ubuntu-latest steps: From d5ebd9a62c5e471b2a225e24ca37a4bf566ab3ad Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 09:54:55 +0200 Subject: [PATCH 163/174] Build CLI: Enum instead of boolean for checksum * Use Enum instead of boolean for the checksum comparison of the given target to improve code readability and avoid potential bugs * Fix Typo & Improve documentations --- cli/src/checksum_records.rs | 27 ++++++++++++++++++---- cli/src/jobs_runner/mod.rs | 46 ++++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 2061f033d..65681a217 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -25,6 +25,14 @@ struct ChecksumItems { involved_targets: BTreeSet, } +/// Represents the comparison's result between the saved Checksum and the calculate one for the +/// build target +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ChecksumCompareResult { + Same, + Changed, +} + impl ChecksumRecords { /// Update checksum records for involved jobs depending on the job type. /// It will calculate new checksums if build tasks were involved. @@ -74,7 +82,7 @@ impl ChecksumRecords { // differentiate between development and production. if matches!(job_type, JobType::Clean) { let prod_records = - Self::load(true).context("Error while loading production recoreds")?; + Self::load(true).context("Error while loading production records")?; let dev_items = records .items @@ -173,8 +181,11 @@ impl ChecksumRecords { } /// Calculate the current checksum for the given target and compare it to the saved one. + /// + /// # Panics + /// /// This method panics if the provided target isn't registered - pub fn check_changed(&self, target: Target) -> anyhow::Result { + pub fn compare_checksum(&self, target: Target) -> anyhow::Result { let items = self .items .lock() @@ -183,12 +194,20 @@ impl ChecksumRecords { assert!(items.involved_targets.contains(&target)); let saved_hash = match items.map.get(&target) { Some(hash) => hash, - None => return Ok(true), + // If there is no existing checksum to compare with, then the checksums state has + // changed. + None => return Ok(ChecksumCompareResult::Changed), }; let current_hash = Self::calc_hash_for_target(&target)?; - Ok(current_hash != *saved_hash) + let comparison = if current_hash == *saved_hash { + ChecksumCompareResult::Same + } else { + ChecksumCompareResult::Changed + }; + + Ok(comparison) } fn calc_hash_for_target(target: &Target) -> anyhow::Result { diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 9c41dd201..1f29e7d6c 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -7,7 +7,10 @@ pub use job_definition::JobDefinition; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use crate::{ - checksum_records::ChecksumRecords, job_type::JobType, spawner::SpawnResult, target::Target, + checksum_records::{ChecksumCompareResult, ChecksumRecords}, + job_type::JobType, + spawner::SpawnResult, + target::Target, tracker::get_tracker, }; @@ -43,11 +46,17 @@ pub async fn run(targets: &[Target], main_job: JobType) -> Result)>(); - let mut skipped_map = BTreeMap::new(); + let mut checksum_compare_map = BTreeMap::new(); let mut failed_jobs = Vec::new(); // Spawn free job at first - spawn_jobs(tx.clone(), &mut jobs_status, &mut skipped_map, &failed_jobs).await?; + spawn_jobs( + tx.clone(), + &mut jobs_status, + &mut checksum_compare_map, + &failed_jobs, + ) + .await?; let mut results = Vec::new(); @@ -90,7 +99,13 @@ pub async fn run(targets: &[Target], main_job: JobType) -> Result Result)>, jobs_status: &mut BTreeMap, - skipped_map: &mut BTreeMap, + checksum_compare_map: &mut BTreeMap, failed_jobs: &[Target], ) -> Result<()> { for (job_def, phase) in jobs_status.iter_mut() { @@ -118,8 +133,8 @@ async fn spawn_jobs( true } // Check if target is already registered and checked - else if let Some(skip) = skipped_map.get(&job_def.target) { - *skip + else if let Some(&chksm_compare) = checksum_compare_map.get(&job_def.target) { + chksm_compare == ChecksumCompareResult::Same } else { // Calculate target checksums and compare it the persisted one let prod = job_def.job_type.is_production().is_some_and(|prod| prod); @@ -127,15 +142,14 @@ async fn spawn_jobs( checksum_rec.register_job(job_def.target)?; // Check if all dependent jobs are skipped, then do the checksum calculations - if job_def - .target - .deps() - .iter() - .all(|dep| skipped_map.get(dep).is_some_and(|skip| *skip)) - { - let calc_skip = !checksum_rec.check_changed(job_def.target)?; - skipped_map.insert(job_def.target, calc_skip); - calc_skip + if job_def.target.deps().iter().all(|dep| { + checksum_compare_map + .get(dep) + .is_some_and(|&chksm| chksm == ChecksumCompareResult::Same) + }) { + let chksm_compare = checksum_rec.compare_checksum(job_def.target)?; + checksum_compare_map.insert(job_def.target, chksm_compare); + chksm_compare == ChecksumCompareResult::Same } else { false } From d7b53be421bae1329f0efa767ab9f3b6187492d4 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 11:59:56 +0200 Subject: [PATCH 164/174] Build CLI: Remove build steps from tests & refactoring * Remove the unnecessary build steps from rust targets for running tests * Remove the build steps from TS targets that don't have tests and aren't dependencies of other TS targets that have tests. * Add assertion for that case if we add tests to those targets in the future. * Refactor the function to filter out the not included jobs by moving the additional filter to a closure that will run after the standard filter which allowed us to add the assertion mentioned above --- cli/src/job_type.rs | 1 + cli/src/jobs_runner/jobs_resolver.rs | 86 ++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 19ced08bf..7d074bdfd 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -52,6 +52,7 @@ impl JobType { production: *production, }, ], + // Only TS and WASM Tests need to build before running the tests JobType::Run { production } | JobType::Test { production } => vec![JobType::Build { production: *production, }], diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index de7300aa9..2de92f9eb 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -4,8 +4,8 @@ use crate::{job_type::JobType, target::Target}; use super::JobDefinition; -/// Resolve tasks dependcies for the given targets and job, -/// returning dependcies map for the tasks +/// Resolve tasks dependencies for the given targets and job, +/// returning dependencies map for the tasks pub fn resolve( targets: &[Target], main_job: JobType, @@ -109,23 +109,70 @@ fn flatten_targets_for_build(targets: &[Target]) -> BTreeSet { } /// Check if job involved depending if the target has a job for the current job type + Additional -/// filter based on the main job type (Currently used because TS linting require all build steps) +/// filter based on the main job type. +/// The additional filter is currently used because linting and running tests on TS and WASM targets +/// require all build steps to be done on them and their dependencies. /// /// * `target`: Job Target /// * `current_job`: Current job type to check if it has job for the given target /// * `main_job`: Main job type, which is used for the additional filter fn is_job_involved(target: &Target, current_job: &JobType, main_job: &JobType) -> bool { - let additional_filter = match (main_job, target) { - // For linting TS targets we need to build all their dependencies targets that have impact - // on building TS. Therefore we can exclude Core and CLI and Updater only. - // Rust Linting doesn't need build and need to be excluded in the additional filter. - (JobType::Lint, Target::Core | Target::Cli | Target::Updater) => { - matches!(current_job, JobType::Lint) + // This filter handle the special cases of adding build steps for TS and WASM lints and tests + // and remove those jobs from the not involved targets + let additional_filter = || { + match main_job { + // Linting for TS and WASM targets inquire that those targets are built + JobType::Lint => match target { + // Linting for Rust targets doesn't need any build and must be excluded in the additional filter. + Target::Core | Target::Cli | Target::Updater => { + matches!(current_job, JobType::Lint) + } + // TS and Bindings targets need to be built with all their dependencies to perform the + // needed type checks on TypeScript + Target::Shared + | Target::Binding + | Target::Wrapper + | Target::Wasm + | Target::Client + | Target::App => true, + }, + + // Tests for TS and WASM targets inquire that those targets are built + JobType::Test { production: _ } => match target { + // Running tests for rust jobs doesn't inquire running build on them. + Target::Core | Target::Cli | Target::Updater => { + matches!(current_job, JobType::Test { production: _ }) + } + + // Only TS and WASM Bindings have tests that inquire running build on them and their dependencies + // before running the actual tests. + Target::Wrapper | Target::Wasm => true, + + // Shared and Bindings don't have tests but they should be built for Wrapper and Wasm + // tests + Target::Shared | Target::Binding => { + assert!( + !matches!(current_job, JobType::Test { production: _ }), + "Shared and Bindings targets don't have test jobs currently" + ); + true + } + + // Client and App doesn't have tests and they are not dependencies other TS and WASM + // targets. + Target::Client | Target::App => { + assert!( + !matches!(current_job, JobType::Test { production: _ }), + "Client and App targets don't have test jobs currently" + ); + false + } + }, + _ => true, } - _ => true, }; - additional_filter && target.has_job(current_job) + target.has_job(current_job) && additional_filter() } #[cfg(test)] @@ -242,19 +289,10 @@ mod tests { #[test] fn resolve_test_core() { let production = false; - let expected = BTreeMap::from([ - ( - JobDefinition::new(Target::Core, JobType::Build { production }), - vec![], - ), - ( - JobDefinition::new(Target::Core, JobType::Test { production }), - vec![JobDefinition::new( - Target::Core, - JobType::Build { production: false }, - )], - ), - ]); + let expected = BTreeMap::from([( + JobDefinition::new(Target::Core, JobType::Test { production }), + vec![], + )]); assert_eq!( expected, From fde21f29c7424628cfe901f1da30295a3d46fca5 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 14:08:14 +0200 Subject: [PATCH 165/174] Build CLI: Handle match branches explicitly * Replace `matches!()` macro and catch-all place holder in match statements for targets and job types with explicit enum types to enforce handling new types to targets and job enums. * The build system has grown to be complicated, therefore it's useful to get assistant from the compiler when new targets or job types are added --- cli/src/job_type.rs | 13 +++--- cli/src/jobs_runner/jobs_resolver.rs | 43 ++++++++++++-------- cli/src/print_dot.rs | 7 +++- cli/src/target/mod.rs | 60 +++++++++++++++++++++------- 4 files changed, 87 insertions(+), 36 deletions(-) diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 7d074bdfd..15c74c3db 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -64,12 +64,15 @@ impl JobType { /// Returns if the job type is part of the build process (install, build, or after build) pub fn is_part_of_build(&self) -> bool { - matches!( - self, + match self { JobType::Install { production: _ } - | JobType::Build { production: _ } - | JobType::AfterBuild { production: _ } - ) + | JobType::Build { production: _ } + | JobType::AfterBuild { production: _ } => true, + JobType::Clean + | JobType::Lint + | JobType::Test { production: _ } + | JobType::Run { production: _ } => false, + } } } diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 2de92f9eb..6c079af3b 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -30,24 +30,31 @@ pub fn resolve( .filter(|j| is_job_involved(&target, j, &main_job)) { // Start with dependencies from other targets (Applies for Build & Install jobs only) - // Install jobs are involved here too because copying the files in the after build - // process could delete the current files. - let mut dep_jobs = if matches!( - job, - JobType::Build { production: _ } | JobType::Install { production: _ } - ) { - let deps = flatten_targets_for_build(target.deps().as_slice()); + let mut dep_jobs = match job { + // Install jobs are involved here too because copying the files in the after build + // process could delete the current files. + JobType::Build { production: _ } | JobType::Install { production: _ } => { + let deps = flatten_targets_for_build(target.deps().as_slice()); + + // Jobs of the dependencies are already included in the jobs tree because we + // are iterating through targets and jobs in the matching order of their + // dependencies relations. + jobs_tree + .keys() + .filter(|job_def| deps.contains(&job_def.target)) + .cloned() + .collect() + } - jobs_tree - .keys() - .filter(|job_def| deps.contains(&job_def.target)) - .cloned() - .collect() - } else { - Vec::new() + // Other job types doesn't have dependencies + JobType::Clean + | JobType::AfterBuild { production: _ } + | JobType::Lint + | JobType::Test { production: _ } + | JobType::Run { production: _ } => Vec::new(), }; - // Add dependencies from the same target + // Add dependencies jobs from the same target // NOTE: This relays on that JobType enums are listed in the current order dep_jobs.extend( jobs_tree @@ -168,7 +175,11 @@ fn is_job_involved(target: &Target, current_job: &JobType, main_job: &JobType) - false } }, - _ => true, + JobType::Clean + | JobType::Install { production: _ } + | JobType::Build { production: _ } + | JobType::AfterBuild { production: _ } + | JobType::Run { production: _ } => true, } }; diff --git a/cli/src/print_dot.rs b/cli/src/print_dot.rs index 2fc0f5ea8..4d81ce282 100644 --- a/cli/src/print_dot.rs +++ b/cli/src/print_dot.rs @@ -37,7 +37,12 @@ fn job_to_dot_string(job_def: &JobDefinition) -> String { JobType::Install { production: _ } => "Install", JobType::Build { production: _ } => "Build", JobType::AfterBuild { production: _ } => "After Build (Copy & Reinstall)", - _ => unreachable!("Only build-related jobs are included in dot print"), + JobType::Clean + | JobType::Lint + | JobType::Test { production: _ } + | JobType::Run { production: _ } => { + unreachable!("Only build-related jobs are included in dot print") + } }; format!("{}: {job_type}", job_def.target) diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 4b8446b09..f19744b90 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -232,17 +232,32 @@ impl Target { pub fn has_job(&self, job_type: &JobType) -> bool { match job_type { JobType::Lint | JobType::Clean | JobType::Build { production: _ } => true, - JobType::Install { production: _ } => { - matches!( - self, - Target::Binding | Target::Client | Target::Shared | Target::App | Target::Wasm - ) - } - JobType::AfterBuild { production: _ } => matches!(self, Target::Binding | Target::App), - JobType::Test { production: _ } => matches!( - self, - Target::Wrapper | Target::Core | Target::Cli | Target::Wasm - ), + + JobType::Install { production: _ } => match self { + Target::Binding | Target::Client | Target::Shared | Target::App | Target::Wasm => { + true + } + Target::Core | Target::Wrapper | Target::Updater | Target::Cli => false, + }, + + JobType::AfterBuild { production: _ } => match self { + Target::Binding | Target::App => true, + Target::Core + | Target::Shared + | Target::Wrapper + | Target::Wasm + | Target::Client + | Target::Updater + | Target::Cli => false, + }, + JobType::Test { production: _ } => match self { + Target::Wrapper | Target::Core | Target::Cli | Target::Wasm => true, + Target::Shared + | Target::Binding + | Target::Client + | Target::Updater + | Target::App => false, + }, JobType::Run { production: _ } => false, } } @@ -312,7 +327,12 @@ impl Target { Target::Core => Some(core::get_test_cmds(production)), Target::Cli => Some(cli::get_test_cmds(production)), Target::Wasm => Some(wasm::get_test_cmds()), - _ => None, + Target::Shared + | Target::Binding + | Target::Wrapper + | Target::Client + | Target::Updater + | Target::App => None, } } @@ -431,7 +451,13 @@ impl Target { fs::remove_file(index_node_path).await?; } } - _ => {} + Target::Core + | Target::Shared + | Target::Binding + | Target::Client + | Target::Updater + | Target::App + | Target::Cli => {} } for path in paths_to_remove.into_iter().filter(|p| p.exists()) { @@ -509,7 +535,13 @@ impl Target { let after_res = match self { Target::Binding => binding::copy_index_node(job_def).await, Target::App => app::copy_client_to_app(job_def).await, - _ => return None, + Target::Core + | Target::Shared + | Target::Wrapper + | Target::Wasm + | Target::Client + | Target::Updater + | Target::Cli => return None, }; match (after_res, reinstall_res) { From e332dfec69c559c7865ce0e4da15d9df39dca128 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 14:58:58 +0200 Subject: [PATCH 166/174] Build CLI: Remove unused async on functions * Remove async from the declarations of functions that don't have await inside of them --- cli/src/fstools.rs | 46 +++++++++++---------------- cli/src/jobs_runner/job_definition.rs | 12 +++---- cli/src/jobs_runner/mod.rs | 16 ++++------ cli/src/spawner.rs | 6 ++-- cli/src/target/app.rs | 2 +- cli/src/target/binding.rs | 7 ++-- cli/src/target/mod.rs | 6 ++-- cli/src/tracker.rs | 10 +++--- 8 files changed, 44 insertions(+), 61 deletions(-) diff --git a/cli/src/fstools.rs b/cli/src/fstools.rs index 3fac08ae4..0e43d4f33 100644 --- a/cli/src/fstools.rs +++ b/cli/src/fstools.rs @@ -8,7 +8,7 @@ use crate::jobs_runner::JobDefinition; use crate::tracker::get_tracker; /// Spawn a job to copy a file, adding the info the report logs -pub async fn cp_file( +pub fn cp_file( job_def: JobDefinition, src: PathBuf, dest: PathBuf, @@ -18,7 +18,7 @@ pub async fn cp_file( report_logs.push(msg); let tracker = get_tracker(); - tracker.msg(job_def, "copying files".into()).await; + tracker.msg(job_def, "copying files".into()); fs::copy(&src, &dest).with_context(|| { format!( @@ -27,12 +27,10 @@ pub async fn cp_file( dest.display() ) })?; - tracker - .msg( - job_def, - format!("copied: {} to {}", src.display(), dest.display()), - ) - .await; + tracker.msg( + job_def, + format!("copied: {} to {}", src.display(), dest.display()), + ); Ok(()) } @@ -52,7 +50,7 @@ pub async fn cp_folder( report_logs.push(report_msg.clone()); let tracker = get_tracker(); - tracker.msg(job_def, report_msg).await; + tracker.msg(job_def, report_msg); let _ = tokio::spawn(async move { copy_with_progress(src, dest, &options, |info| { @@ -65,37 +63,31 @@ pub async fn cp_folder( .await .with_context(|| format!("Error while copying directory: {path_display}"))?; while let Ok(info) = rx.recv() { - tracker - .msg( - job_def, - format!( - "copied: {} bytes; current: {}", - info.copied_bytes, info.file_name - ), - ) - .await; - tracker.progress(job_def, None).await; + tracker.msg( + job_def, + format!( + "copied: {} bytes; current: {}", + info.copied_bytes, info.file_name + ), + ); + tracker.progress(job_def, None); } let msg = format!("copied: {path_display}"); - tracker.msg(job_def, msg).await; + tracker.msg(job_def, msg); Ok(()) } /// Spawn a job to remove a directory recursively, adding the info the report logs -pub async fn rm_folder(job_def: JobDefinition, path: &PathBuf) -> Result<(), Error> { +pub fn rm_folder(job_def: JobDefinition, path: &PathBuf) -> Result<(), Error> { if !path.exists() { return Ok(()); } let tracker = get_tracker(); - tracker - .msg(job_def, format!("removing directory: {}", path.display())) - .await; + tracker.msg(job_def, format!("removing directory: {}", path.display())); fs::remove_dir_all(path)?; - tracker - .msg(job_def, format!("removed: {}", path.display(),)) - .await; + tracker.msg(job_def, format!("removed: {}", path.display(),)); Ok(()) } diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index d1e6e996b..8afeeddd0 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -33,19 +33,15 @@ impl JobDefinition { Some(Ok(res)) => { if res.status.success() { if res.skipped.is_some_and(|skipped| skipped) { - tracker.success(*self, "skipped".into()).await; + tracker.success(*self, "skipped".into()); } else { - tracker.success(*self, String::default()).await; + tracker.success(*self, String::default()); } } else { - tracker.fail(*self, "finished with errors".into()).await; + tracker.fail(*self, "finished with errors".into()); } } - Some(Err(err)) => { - tracker - .fail(*self, format!("finished with errors. {err}")) - .await - } + Some(Err(err)) => tracker.fail(*self, format!("finished with errors. {err}")), None => (), } diff --git a/cli/src/jobs_runner/mod.rs b/cli/src/jobs_runner/mod.rs index 1f29e7d6c..b1527b1f1 100644 --- a/cli/src/jobs_runner/mod.rs +++ b/cli/src/jobs_runner/mod.rs @@ -55,8 +55,7 @@ pub async fn run(targets: &[Target], main_job: JobType) -> Result Result)>, jobs_status: &mut BTreeMap, checksum_compare_map: &mut BTreeMap, @@ -171,11 +169,9 @@ async fn spawn_jobs( if sender.send((job_def, result)).is_err() { let tracker = get_tracker(); - tracker - .print(format!( - "Error: Job results can't be sent to receiver. Job: {job_def:?}" - )) - .await; + tracker.print(format!( + "Error: Job results can't be sent to receiver. Job: {job_def:?}" + )); }; }); diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index 8c923cebb..e92757113 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -132,9 +132,9 @@ pub async fn spawn( break; } else { if !opts.suppress_msg { - tracker.msg(job_def, stdout_line.clone()).await; + tracker.msg(job_def, stdout_line.clone()); } - tracker.progress(job_def, None).await; + tracker.progress(job_def, None); storage_report.push(stdout_line); } } @@ -143,7 +143,7 @@ pub async fn spawn( if stderr_read_bytes == 0 { break; } else { - tracker.progress(job_def, None).await; + tracker.progress(job_def, None); if !stderr_line.trim().is_empty() { storage_report.push(stderr_line); } diff --git a/cli/src/target/app.rs b/cli/src/target/app.rs index 15fbe37f4..94e087149 100644 --- a/cli/src/target/app.rs +++ b/cli/src/target/app.rs @@ -26,7 +26,7 @@ pub async fn copy_client_to_app(job_def: JobDefinition) -> Result anyhow::Result { )) } -pub async fn copy_index_node(job_def: JobDefinition) -> Result { +pub fn copy_index_node(job_def: JobDefinition) -> Result { let mut report_logs = Vec::new(); // *** Copy `index.node` from rs to ts bindings dist *** @@ -63,8 +63,7 @@ pub async fn copy_index_node(job_def: JobDefinition) -> Result Result binding::copy_index_node(job_def).await, + Target::Binding => binding::copy_index_node(job_def), Target::App => app::copy_client_to_app(job_def).await, Target::Core | Target::Shared diff --git a/cli/src/tracker.rs b/cli/src/tracker.rs index 968085abe..f1c6677dc 100644 --- a/cli/src/tracker.rs +++ b/cli/src/tracker.rs @@ -299,21 +299,21 @@ impl Tracker { } /// Update the job providing an optional progress value. - pub async fn progress(&self, job_def: JobDefinition, pos: Option) { + pub fn progress(&self, job_def: JobDefinition, pos: Option) { if let Err(e) = self.tx.send(Tick::Progress(job_def, pos)) { eprintln!("Fail to communicate with tracker: {e}"); } } /// Send a message to the job - pub async fn msg(&self, job_def: JobDefinition, log: String) { + pub fn msg(&self, job_def: JobDefinition, log: String) { if let Err(e) = self.tx.send(Tick::Message(job_def, log)) { eprintln!("Fail to communicate with tracker: {e}"); } } /// Sets the job as finished providing successful result and a message - pub async fn success(&self, job_def: JobDefinition, msg: String) { + pub fn success(&self, job_def: JobDefinition, msg: String) { if let Err(e) = self .tx .send(Tick::Finished(job_def, OperationResult::Success, msg)) @@ -323,7 +323,7 @@ impl Tracker { } /// Sets the job as finished providing failed result and a message - pub async fn fail(&self, job_def: JobDefinition, msg: String) { + pub fn fail(&self, job_def: JobDefinition, msg: String) { if let Err(e) = self .tx .send(Tick::Finished(job_def, OperationResult::Failed, msg)) @@ -342,7 +342,7 @@ impl Tracker { } /// Prints the given text outside the progress bar - pub async fn print(&self, msg: String) { + pub fn print(&self, msg: String) { if let Err(e) = self .tx .send(Tick::Print(msg)) From b0e70992eb7839d52544082da1face63dd65a25f Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 15:07:25 +0200 Subject: [PATCH 167/174] Build CLI: Value instead of reference for small enums * Use the value itself instead of a reference for target enums because their size is only 1 Kilobyte, which make it more efficient to copy it instead of copying a reference (usize) to it. * This applies to JobDefinition and DevTool enums as well --- cli/src/checksum_records.rs | 6 +++--- cli/src/dev_tools.rs | 8 ++++---- cli/src/jobs_runner/job_definition.rs | 18 +++++++++--------- cli/src/jobs_runner/jobs_resolver.rs | 4 ++-- cli/src/target/mod.rs | 14 +++++++------- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 65681a217..330b46179 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -199,7 +199,7 @@ impl ChecksumRecords { None => return Ok(ChecksumCompareResult::Changed), }; - let current_hash = Self::calc_hash_for_target(&target)?; + let current_hash = Self::calc_hash_for_target(target)?; let comparison = if current_hash == *saved_hash { ChecksumCompareResult::Same @@ -210,7 +210,7 @@ impl ChecksumRecords { Ok(comparison) } - fn calc_hash_for_target(target: &Target) -> anyhow::Result { + fn calc_hash_for_target(target: Target) -> anyhow::Result { let path = target.cwd(); calc_combined_checksum(path).with_context(|| { format!("Error while calculating the current hash for target: {target}",) @@ -238,7 +238,7 @@ impl ChecksumRecords { .map_err(|err| anyhow!("Error while acquiring items jobs mutex: Error {err}"))?; for target in items.involved_targets.clone() { - let hash = Self::calc_hash_for_target(&target)?; + let hash = Self::calc_hash_for_target(target)?; match items.map.entry(target) { btree_map::Entry::Occupied(mut o) => *o.get_mut() = hash, btree_map::Entry::Vacant(e) => _ = e.insert(hash), diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index 79e7cb015..dd4e909e0 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -58,7 +58,7 @@ impl DevTool { } /// Provide the suggested way to install the tool - pub fn install_hint(&self) -> Option<&'static str> { + pub fn install_hint(self) -> Option<&'static str> { match self { DevTool::Node | DevTool::Npm | DevTool::RustUp | DevTool::Cargo => None, DevTool::Yarn => Some("npm install --global yarn"), @@ -68,7 +68,7 @@ impl DevTool { } /// Provide the command line argument to get the version of the installed tool - pub fn version_args(&self) -> &'static str { + pub fn version_args(self) -> &'static str { match self { DevTool::Node | DevTool::Npm | DevTool::Yarn => "-v", DevTool::RustUp | DevTool::Cargo | DevTool::WasmPack | DevTool::NjCli => "-V", @@ -76,7 +76,7 @@ impl DevTool { } /// Resolve the path of the tool if exists. Returning an Error when not possible - pub fn resolve(&self) -> &'static Result { + pub fn resolve(self) -> &'static Result { match self { DevTool::Node => resolve_node(), DevTool::Npm => resolve_npm(), @@ -89,7 +89,7 @@ impl DevTool { } /// Get the path of the resolved tool. Panics if the tool can't be resolved - pub fn path(&self) -> &'static PathBuf { + pub fn path(self) -> &'static PathBuf { self.resolve() .as_ref() .expect("Developer Error: Cmd has already been resolved") diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 8afeeddd0..3cb37d8ed 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -16,14 +16,14 @@ impl JobDefinition { } /// Provide formatted job title with target and job type infos - pub fn job_title(&self) -> String { + pub fn job_title(self) -> String { format!("{} {}", self.target, self.job_type) } /// Run the job definition if it has a job, communicating its status with the UI bars - pub async fn run(&self, skip: bool) -> Option> { + pub async fn run(self, skip: bool) -> Option> { let tracker = get_tracker(); - if let Err(err) = tracker.start(*self).await { + if let Err(err) = tracker.start(self).await { return Some(Err(err)); } @@ -33,15 +33,15 @@ impl JobDefinition { Some(Ok(res)) => { if res.status.success() { if res.skipped.is_some_and(|skipped| skipped) { - tracker.success(*self, "skipped".into()); + tracker.success(self, "skipped".into()); } else { - tracker.success(*self, String::default()); + tracker.success(self, String::default()); } } else { - tracker.fail(*self, "finished with errors".into()); + tracker.fail(self, "finished with errors".into()); } } - Some(Err(err)) => tracker.fail(*self, format!("finished with errors. {err}")), + Some(Err(err)) => tracker.fail(self, format!("finished with errors. {err}")), None => (), } @@ -50,7 +50,7 @@ impl JobDefinition { #[inline] /// Runs the job definition if it has a job - async fn run_intern(&self, skip: bool) -> Option> { + async fn run_intern(self, skip: bool) -> Option> { let res = match self.job_type { JobType::Lint => self.target.check().await, JobType::Build { production } => self.target.build(production, skip).await, @@ -85,7 +85,7 @@ mod tests { async fn target_has_job() { for target in Target::all() { for job_type in JobType::all() { - if !target.has_job(&job_type) { + if !target.has_job(*job_type) { let job_def = JobDefinition::new(*target, job_type.clone()); assert!( job_def.run_intern(false).await.is_none(), diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index 6c079af3b..ba3d3bd16 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -27,7 +27,7 @@ pub fn resolve( for target in involved_targets { for job in involved_jobs .iter() - .filter(|j| is_job_involved(&target, j, &main_job)) + .filter(|&&j| is_job_involved(target, j, &main_job)) { // Start with dependencies from other targets (Applies for Build & Install jobs only) let mut dep_jobs = match job { @@ -123,7 +123,7 @@ fn flatten_targets_for_build(targets: &[Target]) -> BTreeSet { /// * `target`: Job Target /// * `current_job`: Current job type to check if it has job for the given target /// * `main_job`: Main job type, which is used for the additional filter -fn is_job_involved(target: &Target, current_job: &JobType, main_job: &JobType) -> bool { +fn is_job_involved(target: Target, current_job: JobType, main_job: &JobType) -> bool { // This filter handle the special cases of adding build steps for TS and WASM lints and tests // and remove those jobs from the not involved targets let additional_filter = || { diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 560514821..0aaf3aad4 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -176,7 +176,7 @@ impl Target { } /// Provides the absolute path to the target - pub fn cwd(&self) -> PathBuf { + pub fn cwd(self) -> PathBuf { let root = get_root(); let relative_path = self.relative_cwd(); @@ -184,7 +184,7 @@ impl Target { } /// Provides the relative path to the target starting from chipmunk root directory - pub fn relative_cwd(&self) -> PathBuf { + pub fn relative_cwd(self) -> PathBuf { let sub_parts = match self { Target::Core => ["application", "apps", "indexer"].iter(), Target::Binding => ["application", "apps", "rustcore", "rs-bindings"].iter(), @@ -201,7 +201,7 @@ impl Target { } /// Provide the kind of the target between Rust or Type-Script - pub fn kind(&self) -> TargetKind { + pub fn kind(self) -> TargetKind { match self { Target::Binding | Target::Core | Target::Cli | Target::Wasm | Target::Updater => { TargetKind::Rs @@ -211,7 +211,7 @@ impl Target { } /// Provides the target which this target depend on - pub fn deps(&self) -> Vec { + pub fn deps(self) -> Vec { match self { Target::Core | Target::Cli | Target::Shared | Target::Wasm | Target::Updater => { Vec::new() @@ -229,7 +229,7 @@ impl Target { } /// Returns if the current target has a job to the given job type - pub fn has_job(&self, job_type: &JobType) -> bool { + pub fn has_job(self, job_type: JobType) -> bool { match job_type { JobType::Lint | JobType::Clean | JobType::Build { production: _ } => true, @@ -263,7 +263,7 @@ impl Target { } /// Provide the command that should be used in to build the target - pub fn build_cmd(&self, prod: bool) -> anyhow::Result { + pub fn build_cmd(self, prod: bool) -> anyhow::Result { let build_cmd = match self { Target::Binding => binding::get_build_cmd(prod)?, Target::Wasm => wasm::get_build_cmd(prod), @@ -322,7 +322,7 @@ impl Target { } /// Provides the test commands for the given target if available - fn test_cmds(&self, production: bool) -> Option> { + fn test_cmds(self, production: bool) -> Option> { match self { Target::Core => Some(core::get_test_cmds(production)), Target::Cli => Some(cli::get_test_cmds(production)), From 199be8a5b68a6cbfb493c02cb37fe65bb0873523 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Tue, 6 Aug 2024 15:31:59 +0200 Subject: [PATCH 168/174] Build CLI: Comments for Match Arms & Clippy Fixes * Add comments in code to emphasize that explicit match arms with targets and job types should be used for keep the compiler assistance when adding new targets or job types. * Use reference for a string instead of moving the whole string since it's not consumed inside the function. --- cli/src/checksum_records.rs | 4 ++-- cli/src/job_type.rs | 5 ++++- cli/src/target/mod.rs | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 330b46179..9e996d644 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -118,7 +118,7 @@ impl ChecksumRecords { let items = if file_path.exists() { let file_content = fs::read_to_string(file_path)?; - let map = Self::parse_hashes(file_content)?; + let map = Self::parse_hashes(&file_content)?; ChecksumItems { map, involved_targets: BTreeSet::new(), @@ -157,7 +157,7 @@ impl ChecksumRecords { Ok(()) } - fn parse_hashes(text: String) -> anyhow::Result> { + fn parse_hashes(text: &str) -> anyhow::Result> { let mut hashes = BTreeMap::new(); for (target, hash) in text.lines().filter_map(|line| line.split_once(':')) { diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 15c74c3db..923f5784a 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -1,8 +1,11 @@ use std::fmt::Display; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -//NOTE: The order of job types must match the running-order of them because it's used by +// * NOTE: The order of job types must match the running-order of them because it's used by // solving their dependencies-graph using BTreeMap +// +// * NOTE: We provide all job types in match arms without using wild-card matching nor +// `matches!()` macro to keep the compiler assistance when adding new job types. pub enum JobType { Clean, Install { production: bool }, diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 0aaf3aad4..51f33b397 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -27,8 +27,11 @@ mod wasm; mod wrapper; #[derive(Debug, ValueEnum, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] -//NOTE: The order of targets must match the running-order between them because it's used for +// * NOTE: The order of targets must match the running-order between them because it's used for // solving their dependencies-graph using BTreeMap +// +// * NOTE: We provide all targets with match arms without using wild-card matching nor +// `matches!()` macro to keep the compiler assistance when adding new targets. pub enum Target { /// Represents the path `application/apps/indexer` Core, From 8b3a84f54b31c8e156d8ae81fc3a6001ec13f585 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 7 Aug 2024 09:26:18 +0200 Subject: [PATCH 169/174] Build CLI: Rename dev_tools functions names --- cli/src/dev_environment.rs | 4 ++-- cli/src/dev_tools.rs | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cli/src/dev_environment.rs b/cli/src/dev_environment.rs index 23f9ebc8b..244ae3807 100644 --- a/cli/src/dev_environment.rs +++ b/cli/src/dev_environment.rs @@ -8,7 +8,7 @@ use crate::dev_tools::DevTool; pub fn resolve_dev_tools() -> anyhow::Result<()> { let mut errors = None; for tool in DevTool::all() { - let Err(err) = tool.resolve() else { + let Err(err) = tool.resolve_path() else { continue; }; @@ -47,7 +47,7 @@ pub fn resolve_dev_tools() -> anyhow::Result<()> { pub fn print_env_info() { for tool in DevTool::all() { println!("{tool} Info:"); - match tool.resolve() { + match tool.resolve_path() { Ok(cmd) => { if let Err(err) = Command::new(cmd).arg(tool.version_args()).status() { eprintln!("Error while retrieving dependency's information: {err}"); diff --git a/cli/src/dev_tools.rs b/cli/src/dev_tools.rs index dd4e909e0..5bc8e3f83 100644 --- a/cli/src/dev_tools.rs +++ b/cli/src/dev_tools.rs @@ -76,27 +76,27 @@ impl DevTool { } /// Resolve the path of the tool if exists. Returning an Error when not possible - pub fn resolve(self) -> &'static Result { + pub fn resolve_path(self) -> &'static Result { match self { - DevTool::Node => resolve_node(), - DevTool::Npm => resolve_npm(), - DevTool::Yarn => resolve_yarn(), - DevTool::RustUp => resolve_rustup(), - DevTool::Cargo => resolve_cargo(), - DevTool::WasmPack => resolve_wasm_pack(), - DevTool::NjCli => resolve_nj_cli(), + DevTool::Node => resolve_node_path(), + DevTool::Npm => resolve_npm_path(), + DevTool::Yarn => resolve_yarn_path(), + DevTool::RustUp => resolve_rustup_path(), + DevTool::Cargo => resolve_cargo_path(), + DevTool::WasmPack => resolve_wasm_pack_path(), + DevTool::NjCli => resolve_nj_cli_path(), } } /// Get the path of the resolved tool. Panics if the tool can't be resolved pub fn path(self) -> &'static PathBuf { - self.resolve() + self.resolve_path() .as_ref() .expect("Developer Error: Cmd has already been resolved") } } -fn resolve_node() -> &'static Result { +fn resolve_node_path() -> &'static Result { static NODE: OnceLock> = OnceLock::new(); NODE.get_or_init(|| find_cmd("node")) @@ -106,25 +106,25 @@ fn find_cmd(cmd: &str) -> Result { which_global(cmd).map_err(|err| anyhow!("Command `{cmd}` couldn't be resolved. Err: {err}")) } -fn resolve_npm() -> &'static Result { +fn resolve_npm_path() -> &'static Result { static NPM: OnceLock> = OnceLock::new(); NPM.get_or_init(|| find_cmd("npm")) } -fn resolve_yarn() -> &'static Result { +fn resolve_yarn_path() -> &'static Result { static YARN: OnceLock> = OnceLock::new(); YARN.get_or_init(|| find_cmd("yarn")) } -fn resolve_rustup() -> &'static Result { +fn resolve_rustup_path() -> &'static Result { static RUSTUP: OnceLock> = OnceLock::new(); RUSTUP.get_or_init(|| find_cmd("rustup")) } -fn resolve_cargo() -> &'static Result { +fn resolve_cargo_path() -> &'static Result { static CARGO: OnceLock> = OnceLock::new(); if cfg!(windows) { @@ -141,13 +141,13 @@ fn resolve_cargo() -> &'static Result { } } -fn resolve_wasm_pack() -> &'static Result { +fn resolve_wasm_pack_path() -> &'static Result { static WASM_PACK: OnceLock> = OnceLock::new(); WASM_PACK.get_or_init(|| find_cmd("wasm-pack")) } -fn resolve_nj_cli() -> &'static Result { +fn resolve_nj_cli_path() -> &'static Result { static NJ_CLI: OnceLock> = OnceLock::new(); NJ_CLI.get_or_init(|| find_cmd("nj-cli")) From b03df95feaebb04ed212ca7fcbf33e3206097526 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 7 Aug 2024 09:54:55 +0200 Subject: [PATCH 170/174] Build CLI: Dir Check: Integration tests fixes * Remove redundant test * Use assigning to _ to directly drop file after creating it --- cli/dir_checksum/tests/integration_tests.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/cli/dir_checksum/tests/integration_tests.rs b/cli/dir_checksum/tests/integration_tests.rs index 6511b16ba..8d78192a3 100644 --- a/cli/dir_checksum/tests/integration_tests.rs +++ b/cli/dir_checksum/tests/integration_tests.rs @@ -16,21 +16,6 @@ fn create_tmp_dir_with_file(dir_name: &'static str) -> anyhow::Result<(TempDir, Ok((tmp_dir, file_path)) } -#[test] -fn hash_combinations_file_identical() -> anyhow::Result<()> { - let (tmp_dir, _) = create_tmp_dir_with_file("comb_file_identical")?; - - let original_hash = calc_combined_checksum(tmp_dir.path())?; - - assert_eq!( - original_hash, - calc_combined_checksum(tmp_dir.path())?, - "Hashes of the same directory must be identical" - ); - - Ok(()) -} - #[test] fn hash_combinations_add_then_remove_file() -> anyhow::Result<()> { let (tmp_dir, _) = create_tmp_dir_with_file("comb_add_remove_file")?; @@ -132,8 +117,7 @@ fn hash_individual_many_files() -> anyhow::Result<()> { // Create empty file let empty_file_path = tmp_dir.path().join("empty.txt"); - let empty_file = File::create(&empty_file_path)?; - drop(empty_file); + let _ = File::create(&empty_file_path)?; let items = calc_individual_checksum(tmp_dir.path())?; From edf4893b0368188fd334f85aa6500b920e9327fa Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 7 Aug 2024 10:41:37 +0200 Subject: [PATCH 171/174] Build CLI: Add missing documentation --- cli/src/checksum_records.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/src/checksum_records.rs b/cli/src/checksum_records.rs index 9e996d644..c1e343dec 100644 --- a/cli/src/checksum_records.rs +++ b/cli/src/checksum_records.rs @@ -15,6 +15,11 @@ const FILE_NAME_DEV: &str = ".build_chksum_dev"; const FILE_NAME_PROD: &str = ".build_chksum_prod"; #[derive(Debug)] +/// Manages and compares the file states for the targets between current and previous builds. +/// It calculates the checksums of the files for each targets and saves them to a file after +/// each build, and for the next build it'll calculate the checksum again and compare it with +/// the saved one. +/// It also manages loading and clearing the saved checksum records as well. pub struct ChecksumRecords { items: Mutex, } From 1851c6f03b1ab9e78f13a9f0f62188ab7d1f6c09 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 7 Aug 2024 11:01:19 +0200 Subject: [PATCH 172/174] Build CLI: Function yarn command & Code comments * Use one function to create process commands with yarn as cmd and the given arguments and use it in code when needed * Remove TODO comment from ts testing file and improve the relating comment for it in `wrapper.rs file --- .../rustcore/ts-bindings/spec/session.jobs.spec.ts | 1 - cli/src/target/mod.rs | 13 ++++++++----- cli/src/target/target_kind.rs | 13 +++++-------- cli/src/target/wrapper.rs | 12 ++++++------ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts index d1b61562b..15e98d96a 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.jobs.spec.ts @@ -133,7 +133,6 @@ describe('Jobs', function () { it(config.regular.list[4], function () { return runner(config.regular, 4, async (logger, done, collector) => { const jobs = collector(await Jobs.create()) as Jobs; - //TODO: If we comment out these lines, build CLI with Stdio::piped() works. const profiles = await jobs.getShellProfiles(); expect(profiles.length > 0).toBe(true); finish(jobs, done); diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index 51f33b397..f2848d7f3 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -391,11 +391,7 @@ impl Target { let path = self.cwd(); let job_def = JobDefinition::new(*self, JobType::Lint); - let yarn_cmd = DevTool::Yarn.path(); - let command = ProcessCommand::new( - yarn_cmd.to_string_lossy().to_string(), - vec![String::from("run"), String::from("lint")], - ); + let command = yarn_command(vec![String::from("run"), String::from("lint")]); spawn(job_def, command, Some(path), iter::empty(), None).await } @@ -585,3 +581,10 @@ async fn install_general( None } } + +/// Proivdes a process command with yarn as [`ProcessCommand::cmd`] and the given arguments +/// as [`ProcessCommand::args`] +fn yarn_command(args: Vec) -> ProcessCommand { + let yarn_path = DevTool::Yarn.path(); + ProcessCommand::new(yarn_path.to_string_lossy().to_string(), args) +} diff --git a/cli/src/target/target_kind.rs b/cli/src/target/target_kind.rs index 2a368de5a..fbbfcf9ec 100644 --- a/cli/src/target/target_kind.rs +++ b/cli/src/target/target_kind.rs @@ -1,6 +1,6 @@ use crate::dev_tools::DevTool; -use super::ProcessCommand; +use super::{yarn_command, ProcessCommand}; #[derive(Debug, Clone)] pub enum TargetKind { @@ -15,7 +15,6 @@ impl TargetKind { pub fn build_cmd(&self, prod: bool) -> ProcessCommand { match self { TargetKind::Ts => { - let yarn_path = DevTool::Yarn.path(); let mut args = vec![String::from("run")]; if prod { args.push("prod".into()); @@ -23,7 +22,7 @@ impl TargetKind { args.push("build".into()); } - ProcessCommand::new(yarn_path.to_string_lossy().to_string(), args) + yarn_command(args) } TargetKind::Rs => { let cargo_path = DevTool::Cargo.path(); @@ -44,16 +43,14 @@ impl TargetKind { pub fn install_cmd(&self, prod: bool) -> Option { match self { TargetKind::Ts => { - let yarn_path = DevTool::Yarn.path(); let mut args = vec![String::from("install")]; if prod { args.push("--production".into()); } - Some(ProcessCommand::new( - yarn_path.to_string_lossy().to_string(), - args, - )) + let command = yarn_command(args); + + Some(command) } TargetKind::Rs => None, } diff --git a/cli/src/target/wrapper.rs b/cli/src/target/wrapper.rs index 92c4a6149..6d7b8b322 100644 --- a/cli/src/target/wrapper.rs +++ b/cli/src/target/wrapper.rs @@ -16,13 +16,13 @@ const TEST_SPECS: [&str; 14] = [ // out-of-memory error in electron app, even if only this job was running (by // commenting out the other specs). // - // The error happens while executing line 137 from the file `session.jobs.spec.ts` when - // we spawn the command using Stdio::piped() in the spawn command (line 74 in file - // `spawner.rs`). Either Commenting out the line from `session.jobs.spec.ts` file or - // using Stdio::inherit() in `spawner.rs` prevent this error from happening. + // This error happens while executing function `jobs.getShellProfiles()` in file `session.jobs.spec.ts` + // which will call rust function `get_valid_profiles()` in `indexer/session/src/unbound/commands/shells.rs` + // using the crate `envvars` which panics with piped shells. // - // The current work-around to blocking run the all the test commands sequentially using inherit - // Stdio::inherit suspending the progress bars until all tests are done. + // The current work-around is blocking the progress bars temporally and running the tests + // sequentially using `Stdio::inherit` to keep using the main shell, printing the results + // of the test directly to standard out, then the progress bars will be shown again. "jobs", "search", "values", From 8f46ee2691ae07e44c4ce3c24f5c367d6ca91730 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Wed, 7 Aug 2024 16:38:02 +0200 Subject: [PATCH 173/174] Build CLI: Enable kill on drop for async commands Tokio Commands has the option to kill the underline process when the command is dropped. This will provide more grateful cancellation and shut down functionality --- cli/src/app_runner.rs | 1 + cli/src/spawner.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/cli/src/app_runner.rs b/cli/src/app_runner.rs index 7d6407a03..8e766a752 100644 --- a/cli/src/app_runner.rs +++ b/cli/src/app_runner.rs @@ -18,6 +18,7 @@ pub async fn run_app() -> io::Result { Command::new(yarn_path) .current_dir(electron_path) .args(["run", electron_arg]) + .kill_on_drop(true) .status() .await } diff --git a/cli/src/spawner.rs b/cli/src/spawner.rs index e92757113..7a9c3ccf9 100644 --- a/cli/src/spawner.rs +++ b/cli/src/spawner.rs @@ -100,6 +100,7 @@ pub async fn spawn( .envs(combined_env_vars) .stdout(Stdio::piped()) .stderr(Stdio::piped()) + .kill_on_drop(true) .spawn() .with_context(|| { format!( From aeb2a2c882ff6ddbe7a7f6929d93664efb08d938 Mon Sep 17 00:00:00 2001 From: Ammar Abou Zor Date: Thu, 8 Aug 2024 15:19:12 +0200 Subject: [PATCH 174/174] Build CLI Refactoring: Use {..} in Match patterns Use { .. } for wild card matching for JobType where production value should be ignored. --- cli/src/job_type.rs | 33 +++++++++++---------------- cli/src/jobs_runner/job_definition.rs | 2 +- cli/src/jobs_runner/jobs_resolver.rs | 26 ++++++++++----------- cli/src/main.rs | 2 +- cli/src/print_dot.rs | 11 ++++----- cli/src/target/mod.rs | 10 ++++---- 6 files changed, 37 insertions(+), 47 deletions(-) diff --git a/cli/src/job_type.rs b/cli/src/job_type.rs index 923f5784a..34bc586d3 100644 --- a/cli/src/job_type.rs +++ b/cli/src/job_type.rs @@ -21,11 +21,11 @@ impl Display for JobType { match self { JobType::Lint => write!(f, "Lint"), JobType::Clean => write!(f, "Clean"), - JobType::Build { production: _ } => write!(f, "Build"), - JobType::Install { production: _ } => write!(f, "Install"), - JobType::AfterBuild { production: _ } => write!(f, "After Build"), - JobType::Test { production: _ } => write!(f, "Test"), - JobType::Run { production: _ } => write!(f, "Run"), + JobType::Build { .. } => write!(f, "Build"), + JobType::Install { .. } => write!(f, "Install"), + JobType::AfterBuild { .. } => write!(f, "After Build"), + JobType::Test { .. } => write!(f, "Test"), + JobType::Run { .. } => write!(f, "Run"), } } } @@ -59,22 +59,15 @@ impl JobType { JobType::Run { production } | JobType::Test { production } => vec![JobType::Build { production: *production, }], - JobType::Clean - | JobType::Install { production: _ } - | JobType::AfterBuild { production: _ } => Vec::new(), + JobType::Clean | JobType::Install { .. } | JobType::AfterBuild { .. } => Vec::new(), } } /// Returns if the job type is part of the build process (install, build, or after build) pub fn is_part_of_build(&self) -> bool { match self { - JobType::Install { production: _ } - | JobType::Build { production: _ } - | JobType::AfterBuild { production: _ } => true, - JobType::Clean - | JobType::Lint - | JobType::Test { production: _ } - | JobType::Run { production: _ } => false, + JobType::Install { .. } | JobType::Build { .. } | JobType::AfterBuild { .. } => true, + JobType::Clean | JobType::Lint | JobType::Test { .. } | JobType::Run { .. } => false, } } } @@ -89,11 +82,11 @@ impl JobType { match JobType::Lint { JobType::Lint => (), JobType::Clean => (), - JobType::Build { production: _ } => (), - JobType::Install { production: _ } => (), - JobType::AfterBuild { production: _ } => (), - JobType::Test { production: _ } => (), - JobType::Run { production: _ } => (), + JobType::Build { .. } => (), + JobType::Install { .. } => (), + JobType::AfterBuild { .. } => (), + JobType::Test { .. } => (), + JobType::Run { .. } => (), }; } diff --git a/cli/src/jobs_runner/job_definition.rs b/cli/src/jobs_runner/job_definition.rs index 3cb37d8ed..170c4b05e 100644 --- a/cli/src/jobs_runner/job_definition.rs +++ b/cli/src/jobs_runner/job_definition.rs @@ -68,7 +68,7 @@ impl JobDefinition { } JobType::Clean => self.target.reset().await, JobType::Test { production } => return self.target.test(production).await, - JobType::Run { production: _ } => return None, + JobType::Run { .. } => return None, }; Some(res) diff --git a/cli/src/jobs_runner/jobs_resolver.rs b/cli/src/jobs_runner/jobs_resolver.rs index ba3d3bd16..8abeb1cc1 100644 --- a/cli/src/jobs_runner/jobs_resolver.rs +++ b/cli/src/jobs_runner/jobs_resolver.rs @@ -14,7 +14,7 @@ pub fn resolve( let has_build_deps = involved_jobs .iter() - .any(|job| matches!(job, JobType::Build { production: _ })); + .any(|job| matches!(job, JobType::Build { .. })); let involved_targets = if has_build_deps { flatten_targets_for_build(targets) @@ -33,7 +33,7 @@ pub fn resolve( let mut dep_jobs = match job { // Install jobs are involved here too because copying the files in the after build // process could delete the current files. - JobType::Build { production: _ } | JobType::Install { production: _ } => { + JobType::Build { .. } | JobType::Install { .. } => { let deps = flatten_targets_for_build(target.deps().as_slice()); // Jobs of the dependencies are already included in the jobs tree because we @@ -48,10 +48,10 @@ pub fn resolve( // Other job types doesn't have dependencies JobType::Clean - | JobType::AfterBuild { production: _ } + | JobType::AfterBuild { .. } | JobType::Lint - | JobType::Test { production: _ } - | JobType::Run { production: _ } => Vec::new(), + | JobType::Test { .. } + | JobType::Run { .. } => Vec::new(), }; // Add dependencies jobs from the same target @@ -145,10 +145,10 @@ fn is_job_involved(target: Target, current_job: JobType, main_job: &JobType) -> }, // Tests for TS and WASM targets inquire that those targets are built - JobType::Test { production: _ } => match target { + JobType::Test { .. } => match target { // Running tests for rust jobs doesn't inquire running build on them. Target::Core | Target::Cli | Target::Updater => { - matches!(current_job, JobType::Test { production: _ }) + matches!(current_job, JobType::Test { .. }) } // Only TS and WASM Bindings have tests that inquire running build on them and their dependencies @@ -159,7 +159,7 @@ fn is_job_involved(target: Target, current_job: JobType, main_job: &JobType) -> // tests Target::Shared | Target::Binding => { assert!( - !matches!(current_job, JobType::Test { production: _ }), + !matches!(current_job, JobType::Test { .. }), "Shared and Bindings targets don't have test jobs currently" ); true @@ -169,17 +169,17 @@ fn is_job_involved(target: Target, current_job: JobType, main_job: &JobType) -> // targets. Target::Client | Target::App => { assert!( - !matches!(current_job, JobType::Test { production: _ }), + !matches!(current_job, JobType::Test { .. }), "Client and App targets don't have test jobs currently" ); false } }, JobType::Clean - | JobType::Install { production: _ } - | JobType::Build { production: _ } - | JobType::AfterBuild { production: _ } - | JobType::Run { production: _ } => true, + | JobType::Install { .. } + | JobType::Build { .. } + | JobType::AfterBuild { .. } + | JobType::Run { .. } => true, } }; diff --git a/cli/src/main.rs b/cli/src/main.rs index fe243dddf..84b0fcb61 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -176,7 +176,7 @@ async fn main() -> Result<(), Error> { ChecksumRecords::update_and_save(job_type)?; - if matches!(job_type, JobType::Run { production: _ }) { + if matches!(job_type, JobType::Run { .. }) { println!("Starting chipmunk..."); let status = app_runner::run_app().await?; if !status.success() { diff --git a/cli/src/print_dot.rs b/cli/src/print_dot.rs index 4d81ce282..ef8c4ab57 100644 --- a/cli/src/print_dot.rs +++ b/cli/src/print_dot.rs @@ -34,13 +34,10 @@ pub fn print_dependencies_jobs() { fn job_to_dot_string(job_def: &JobDefinition) -> String { let job_type = match job_def.job_type { - JobType::Install { production: _ } => "Install", - JobType::Build { production: _ } => "Build", - JobType::AfterBuild { production: _ } => "After Build (Copy & Reinstall)", - JobType::Clean - | JobType::Lint - | JobType::Test { production: _ } - | JobType::Run { production: _ } => { + JobType::Install { .. } => "Install", + JobType::Build { .. } => "Build", + JobType::AfterBuild { .. } => "After Build (Copy & Reinstall)", + JobType::Clean | JobType::Lint | JobType::Test { .. } | JobType::Run { .. } => { unreachable!("Only build-related jobs are included in dot print") } }; diff --git a/cli/src/target/mod.rs b/cli/src/target/mod.rs index f2848d7f3..bb2c7355d 100644 --- a/cli/src/target/mod.rs +++ b/cli/src/target/mod.rs @@ -234,16 +234,16 @@ impl Target { /// Returns if the current target has a job to the given job type pub fn has_job(self, job_type: JobType) -> bool { match job_type { - JobType::Lint | JobType::Clean | JobType::Build { production: _ } => true, + JobType::Lint | JobType::Clean | JobType::Build { .. } => true, - JobType::Install { production: _ } => match self { + JobType::Install { .. } => match self { Target::Binding | Target::Client | Target::Shared | Target::App | Target::Wasm => { true } Target::Core | Target::Wrapper | Target::Updater | Target::Cli => false, }, - JobType::AfterBuild { production: _ } => match self { + JobType::AfterBuild { .. } => match self { Target::Binding | Target::App => true, Target::Core | Target::Shared @@ -253,7 +253,7 @@ impl Target { | Target::Updater | Target::Cli => false, }, - JobType::Test { production: _ } => match self { + JobType::Test { .. } => match self { Target::Wrapper | Target::Core | Target::Cli | Target::Wasm => true, Target::Shared | Target::Binding @@ -261,7 +261,7 @@ impl Target { | Target::Updater | Target::App => false, }, - JobType::Run { production: _ } => false, + JobType::Run { .. } => false, } }