refactor: data-driven backend project templates (#3894)

Part of https://dfinity.atlassian.net/browse/SDK-1805

Author: ericswanson-dfinity

Cargo.lock

@@ -35,6 +35,22 @@ teardown() {
   assert_command_fail dfx new 'a:b'
 }
 
+@test "dfx new --help shows possible backend template names" {
+  assert_command dfx new --help
+  assert_match "\[possible values.*motoko.*\]"
+  assert_match "\[possible values.*rust.*\]"
+  assert_match "\[possible values.*kybra.*\]"
+  assert_match "\[possible values.*azle.*\]"
+}
+
+@test "dfx new --type <bad type> shows possible values" {
+  assert_command_fail dfx new --type bad_type
+  assert_match "\[possible values.*motoko.*\]"
+  assert_match "\[possible values.*rust.*\]"
+  assert_match "\[possible values.*kybra.*\]"
+  assert_match "\[possible values.*azle.*\]"
+}
+
 @test "dfx new readmes contain appropriate links" {
   assert_command dfx new --type rust e2e_rust --no-frontend
   assert_command grep "https://docs.rs/ic-cdk" e2e_rust/README.md

e2e/tests-dfx/new.bash

@@ -26,6 +26,7 @@ humantime-serde = "1.1.1"
 ic-agent.workspace = true
 ic-utils.workspace = true
 ic-identity-hsm.workspace = true
+itertools.workspace = true
 k256 = { version = "0.11.4", features = ["pem"] }
 keyring.workspace = true
 lazy_static.workspace = true

src/dfx-core/Cargo.toml

@@ -1,3 +1,4 @@
 pub mod cache;
 pub mod directories;
 pub mod model;
+pub mod project_templates;

src/dfx-core/src/config/mod.rs

@@ -0,0 +1,91 @@
+use itertools::Itertools;
+use std::collections::BTreeMap;
+use std::io;
+use std::sync::OnceLock;
+
+type GetArchiveFn = fn() -> Result<tar::Archive<flate2::read::GzDecoder<&'static [u8]>>, io::Error>;
+
+#[derive(Debug, Clone)]
+pub enum ResourceLocation {
+    /// The template's assets are compiled into the dfx binary
+    Bundled { get_archive_fn: GetArchiveFn },
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum Category {
+    Backend,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub struct ProjectTemplateName(pub String);
+
+#[derive(Debug, Clone)]
+pub struct ProjectTemplate {
+    /// The name of the template as specified on the command line,
+    /// for example `--type rust`
+    pub name: ProjectTemplateName,
+
+    /// The name used for display and sorting
+    pub display: String,
+
+    /// How to obtain the template's files
+    pub resource_location: ResourceLocation,
+
+    /// Used to determine which CLI group (`--type`, `--backend`, `--frontend`)
+    /// as well as for interactive selection
+    pub category: Category,
+
+    /// If true, run `cargo update` after creating the project
+    pub update_cargo_lockfile: bool,
+
+    /// If true, patch in the any_js template files
+    pub has_js: bool,
+
+    /// The sort order is fixed rather than settable in properties:
+    /// - motoko=0
+    /// - rust=1
+    /// - everything else=2 (and then by display name)
+    pub sort_order: u32,
+}
+
+type ProjectTemplates = BTreeMap<ProjectTemplateName, ProjectTemplate>;
+
+static PROJECT_TEMPLATES: OnceLock<ProjectTemplates> = OnceLock::new();
+
+pub fn populate(builtin_templates: Vec<ProjectTemplate>) {
+    let templates = builtin_templates
+        .iter()
+        .map(|t| (t.name.clone(), t.clone()))
+        .collect();
+
+    PROJECT_TEMPLATES.set(templates).unwrap();
+}
+
+pub fn get_project_template(name: &ProjectTemplateName) -> ProjectTemplate {
+    PROJECT_TEMPLATES.get().unwrap().get(name).cloned().unwrap()
+}
+
+pub fn get_sorted_templates(category: Category) -> Vec<ProjectTemplate> {
+    PROJECT_TEMPLATES
+        .get()
+        .unwrap()
+        .values()
+        .filter(|t| t.category == category)
+        .cloned()
+        .sorted_by(|a, b| {
+            a.sort_order
+                .cmp(&b.sort_order)
+                .then_with(|| a.display.cmp(&b.display))
+        })
+        .collect()
+}
+
+pub fn project_template_cli_names(category: Category) -> Vec<String> {
+    PROJECT_TEMPLATES
+        .get()
+        .unwrap()
+        .values()
+        .filter(|t| t.category == category)
+        .map(|t| t.name.0.clone())
+        .collect()
+}

src/dfx-core/src/config/project_templates.rs

@@ -7,8 +7,13 @@ use crate::lib::program;
 use crate::util::assets;
 use crate::util::clap::parsers::project_name_parser;
 use anyhow::{anyhow, bail, ensure, Context};
+use clap::builder::PossibleValuesParser;
 use clap::{Parser, ValueEnum};
 use console::{style, Style};
+use dfx_core::config::project_templates::{
+    get_project_template, get_sorted_templates, project_template_cli_names, Category,
+    ProjectTemplateName, ResourceLocation,
+};
 use dfx_core::json::{load_json_file, save_json_file};
 use dialoguer::theme::ColorfulTheme;
 use dialoguer::{FuzzySelect, MultiSelect};
@@ -38,6 +43,8 @@ const AGENT_JS_DEFAULT_INSTALL_DIST_TAG: &str = "latest";
 // check.
 const CHECK_VERSION_TIMEOUT: Duration = Duration::from_secs(2);
 
+const BACKEND_MOTOKO: &str = "motoko";
+
 /// Creates a new project.
 #[derive(Parser)]
 pub struct NewOpts {
@@ -46,8 +53,8 @@ pub struct NewOpts {
     project_name: String,
 
     /// Choose the type of canister in the starter project.
-    #[arg(long, value_enum)]
-    r#type: Option<BackendType>,
+    #[arg(long, value_parser=backend_project_template_name_parser())]
+    r#type: Option<String>,
 
     /// Provides a preview the directories and files to be created without adding them to the file system.
     #[arg(long)]
@@ -70,24 +77,8 @@ pub struct NewOpts {
     extras: Vec<Extra>,
 }
 
-#[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
-enum BackendType {
-    Motoko,
-    Rust,
-    Azle,
-    Kybra,
-}
-
-impl Display for BackendType {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Motoko => "Motoko",
-            Self::Rust => "Rust",
-            Self::Azle => "TypeScript (Azle)",
-            Self::Kybra => "Python (Kybra)",
-        }
-        .fmt(f)
-    }
+fn backend_project_template_name_parser() -> PossibleValuesParser {
+    PossibleValuesParser::new(project_template_cli_names(Category::Backend))
 }
 
 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
@@ -466,19 +457,17 @@ fn get_agent_js_version_from_npm(dist_tag: &str) -> DfxResult<String> {
 }
 
 pub fn exec(env: &dyn Environment, mut opts: NewOpts) -> DfxResult {
-    use BackendType::*;
     let log = env.get_logger();
     let dry_run = opts.dry_run;
 
-    let r#type = if let Some(r#type) = opts.r#type {
-        r#type
+    let backend_template_name = if let Some(r#type) = opts.r#type {
+        ProjectTemplateName(r#type)
     } else if opts.frontend.is_none() && opts.extras.is_empty() && io::stdout().is_terminal() {
         opts = get_opts_interactively(opts)?;
-        opts.r#type.unwrap()
+        ProjectTemplateName(opts.r#type.unwrap())
     } else {
-        Motoko
+        ProjectTemplateName(BACKEND_MOTOKO.to_string())
     };
-
     let project_name = Path::new(opts.project_name.as_str());
     if project_name.exists() {
         bail!("Cannot create a new project because the directory already exists.");
@@ -560,7 +549,9 @@ pub fn exec(env: &dyn Environment, mut opts: NewOpts) -> DfxResult {
         opts.frontend.unwrap_or(FrontendType::Vanilla)
     };
 
-    if r#type == Azle || frontend.has_js() {
+    let backend = get_project_template(&backend_template_name);
+
+    if backend.has_js || frontend.has_js() {
         write_files_from_entries(
             log,
             &mut assets::new_project_js_files().context("Failed to get JS config archive.")?,
@@ -570,20 +561,19 @@ pub fn exec(env: &dyn Environment, mut opts: NewOpts) -> DfxResult {
         )?;
     }
 
-    // Default to start with motoko
-    let mut new_project_files = match r#type {
-        Rust => assets::new_project_rust_files().context("Failed to get rust archive.")?,
-        Motoko => assets::new_project_motoko_files().context("Failed to get motoko archive.")?,
-        Azle => assets::new_project_azle_files().context("Failed to get azle archive.")?,
-        Kybra => assets::new_project_kybra_files().context("Failed to get kybra archive.")?,
+    match backend.resource_location {
+        ResourceLocation::Bundled { get_archive_fn } => {
+            let mut new_project_files = get_archive_fn()
+                .with_context(|| format!("Failed to get {} archive.", backend.name.0))?;
+            write_files_from_entries(
+                log,
+                &mut new_project_files,
+                project_name,
+                dry_run,
+                &variables,
+            )?;
+        }
     };
-    write_files_from_entries(
-        log,
-        &mut new_project_files,
-        project_name,
-        dry_run,
-        &variables,
-    )?;
 
     if opts.extras.contains(&Extra::InternetIdentity) {
         write_files_from_entries(
@@ -645,7 +635,7 @@ pub fn exec(env: &dyn Environment, mut opts: NewOpts) -> DfxResult {
             init_git(log, project_name)?;
         }
 
-        if r#type == Rust {
+        if backend.update_cargo_lockfile {
             // dfx build will use --locked, so update the lockfile beforehand
             const MSG: &str = "You will need to run it yourself (or a similar command like `cargo vendor`), because `dfx build` will use the --locked flag with Cargo.";
             if let Ok(code) = Command::new("cargo")
@@ -683,17 +673,21 @@ pub fn exec(env: &dyn Environment, mut opts: NewOpts) -> DfxResult {
 }
 
 fn get_opts_interactively(opts: NewOpts) -> DfxResult<NewOpts> {
-    use BackendType::*;
     use Extra::*;
     use FrontendType::*;
     let theme = ColorfulTheme::default();
-    let backends_list = [Motoko, Rust, Azle, Kybra];
+    let backend_templates = get_sorted_templates(Category::Backend);
+    let backends_list = backend_templates
+        .iter()
+        .map(|t| t.display.clone())
+        .collect::<Vec<_>>();
+
     let backend = FuzzySelect::with_theme(&theme)
         .items(&backends_list)
         .default(0)
         .with_prompt("Select a backend language:")
         .interact()?;
-    let backend = backends_list[backend];
+    let backend = &backend_templates[backend];
     let frontends_list = [SvelteKit, React, Vue, Vanilla, SimpleAssets, None];
     let frontend = FuzzySelect::with_theme(&theme)
         .items(&frontends_list)
@@ -715,7 +709,7 @@ fn get_opts_interactively(opts: NewOpts) -> DfxResult<NewOpts> {
     let opts = NewOpts {
         extras,
         frontend: Some(frontend),
-        r#type: Some(backend),
+        r#type: Some(backend.name.0.clone()),
         ..opts
     };
     Ok(opts)

src/dfx/src/commands/new.rs

@@ -1,2 +1,3 @@
 pub mod import;
 pub mod network_mappings;
+pub mod templates;

src/dfx/src/lib/project/mod.rs

@@ -0,0 +1,56 @@
+use crate::util::assets;
+use dfx_core::config::project_templates::{
+    Category, ProjectTemplate, ProjectTemplateName, ResourceLocation,
+};
+
+pub fn builtin_templates() -> Vec<ProjectTemplate> {
+    let motoko = ProjectTemplate {
+        name: ProjectTemplateName("motoko".to_string()),
+        display: "Motoko".to_string(),
+        resource_location: ResourceLocation::Bundled {
+            get_archive_fn: assets::new_project_motoko_files,
+        },
+        category: Category::Backend,
+        sort_order: 0,
+        update_cargo_lockfile: false,
+        has_js: false,
+    };
+
+    let rust = ProjectTemplate {
+        name: ProjectTemplateName("rust".to_string()),
+        display: "Rust".to_string(),
+        resource_location: ResourceLocation::Bundled {
+            get_archive_fn: assets::new_project_rust_files,
+        },
+        category: Category::Backend,
+        sort_order: 1,
+        update_cargo_lockfile: true,
+        has_js: false,
+    };
+
+    let azle = ProjectTemplate {
+        name: ProjectTemplateName("azle".to_string()),
+        display: "Typescript (Azle)".to_string(),
+        resource_location: ResourceLocation::Bundled {
+            get_archive_fn: assets::new_project_azle_files,
+        },
+        category: Category::Backend,
+        sort_order: 2,
+        update_cargo_lockfile: false,
+        has_js: true,
+    };
+
+    let kybra = ProjectTemplate {
+        name: ProjectTemplateName("kybra".to_string()),
+        display: "Python (Kybra)".to_string(),
+        resource_location: ResourceLocation::Bundled {
+            get_archive_fn: assets::new_project_kybra_files,
+        },
+        category: Category::Backend,
+        sort_order: 2,
+        update_cargo_lockfile: false,
+        has_js: false,
+    };
+
+    vec![motoko, rust, azle, kybra]
+}