Skip to content

Commit

Permalink
Plugins Manager: Changes in types & loading.
Browse files Browse the repository at this point in the history 
* Different types for valid and invalid plugins instead of using the
  same time with various states.
* Plugin manager provides different APIs for valid and invalid plugins
  alongside with new function to provide the plugins directories only
  and ones to get one plugin info based on the provided plugin directory
* Change loading internal function to use one function for both plugin
  types, reducing duplicated code.
* Extend proptests and type generation code.

* Provide more APIs for get plugins in both states with full info and
  with paths only + APIs to get infos to one plugin based on the giving
  plugin directory.
* Rename active plugins to installed plugins.
* Adjustments in protocol, bindings and tests in ts-bindings

* Adjustments for API calls in platform, client and holder.

* Plugins Manager: Added unit tests
  • Loading branch information
@AmmarAbouZor
AmmarAbouZor committed Feb 14, 2025
1 parent dee830f commit c02e8a6
Show file tree
Hide file tree
Showing 42 changed files with 1,291 additions and 433 deletions.
6 changes: 3 additions & 3 deletions application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};

use stypes::{PluginType, RenderOptions, SemanticVersion, ValidPluginInfo};
use stypes::{PluginInfo, PluginType, RenderOptions, SemanticVersion};
use wasmtime::component::Component;

use crate::{
Expand Down Expand Up @@ -35,7 +35,7 @@ pub enum PlugVerByteSource {

impl PluginsByteSource {
/// Loads the plugin and extract the needed plugin info if valid.
pub async fn get_info(plugin_path: PathBuf) -> Result<ValidPluginInfo, PluginError> {
pub async fn get_info(plugin_path: PathBuf) -> Result<PluginInfo, PluginError> {
let (component, version) = Self::load(&plugin_path).await?;

let plug_info = match version {
Expand All @@ -52,7 +52,7 @@ impl PluginsByteSource {
}
};

let plugin_info = ValidPluginInfo {
let plugin_info = PluginInfo {
wasm_file_path: plugin_path,
api_version: version,
plugin_version: plug_info.version,
Expand Down
6 changes: 3 additions & 3 deletions application/apps/indexer/plugins_host/src/parser_shared/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::path::{Path, PathBuf};

use stypes::{SemanticVersion, ValidPluginInfo};
use stypes::{PluginInfo, SemanticVersion};
use wasmtime::component::Component;

use crate::{
Expand Down Expand Up @@ -37,7 +37,7 @@ enum PlugVerParser {

impl PluginsParser {
/// Loads the plugin and extract the needed plugin info if valid.
pub async fn get_info(plugin_path: PathBuf) -> Result<ValidPluginInfo, PluginError> {
pub async fn get_info(plugin_path: PathBuf) -> Result<PluginInfo, PluginError> {
let (component, version) = Self::load(&plugin_path).await?;

let plug_info = match version {
Expand All @@ -54,7 +54,7 @@ impl PluginsParser {
}
};

let plugin_info = ValidPluginInfo {
let plugin_info = PluginInfo {
wasm_file_path: plugin_path,
api_version: version,
plugin_version: plug_info.version,
Expand Down
269 changes: 117 additions & 152 deletions application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,127 +6,180 @@ use std::{
path::{Path, PathBuf},
};

use stypes::{InvalidPluginInfo, PluginMetadata};
use stypes::{InvalidPluginEntity, PluginMetadata};

use crate::{
plugins_manager::PluginState, plugins_shared::plugin_errors::PluginError, PluginHostInitError,
PluginType, PluginsByteSource, PluginsParser,
plugins_shared::plugin_errors::PluginError, PluginHostInitError, PluginType, PluginsByteSource,
PluginsParser,
};

use super::{InitError, PluginEntity};

/// Loads all the plugins from the plugin directory
pub async fn load_plugins() -> Result<Vec<PluginEntity>, InitError> {
///
/// # Returns:
///
/// List of valid plugins alongside with other list for invalid ones.
pub async fn load_all_plugins() -> Result<(Vec<PluginEntity>, Vec<InvalidPluginEntity>), InitError>
{
let plugins_dir = paths::plugins_dir()?;
if !plugins_dir.exists() {
log::trace!("Plugins directory doesn't exist. Creating it...");
fs::create_dir_all(plugins_dir)?;
}

let mut plugins = load_all_parsers().await?;
let (mut valid_plugins, mut invalid_plugs) = load_plugins(PluginType::Parser).await?;

let bytesources = load_all_bytesources().await?;
let (valid_sources, invalid_soruces) = load_plugins(PluginType::ByteSource).await?;

plugins.extend(bytesources);
valid_plugins.extend(valid_sources);
invalid_plugs.extend(invalid_soruces);

Ok(plugins)
Ok((valid_plugins, invalid_plugs))
}

/// Loads all parser plugins from their directory.
async fn load_all_parsers() -> Result<Vec<PluginEntity>, InitError> {
let mut parsers = Vec::new();
/// Loads all plugins from the main directory of the provided plugin type.
///
/// # Returns:
///
/// List of valid plugins alongside with other list for invalid ones.
async fn load_plugins(
plug_type: PluginType,
) -> Result<(Vec<PluginEntity>, Vec<InvalidPluginEntity>), InitError> {
let mut valid_plugins = Vec::new();
let mut invalid_plugins = Vec::new();

let plugins_dir = match plug_type {
PluginType::Parser => paths::parser_dir()?,
PluginType::ByteSource => paths::bytesource_dir()?,
};

let parsers_dir = paths::parser_dir()?;
if !parsers_dir.exists() {
log::trace!("Parsers directory doesn't exist. Creating it ...");
fs::create_dir_all(&parsers_dir)?;
if !plugins_dir.exists() {
log::trace!("{plug_type} directory doesn't exist. Creating it...");
fs::create_dir_all(plugins_dir)?;

return Ok(parsers);
return Ok((valid_plugins, invalid_plugins));
}

for dir in get_dirs(&parsers_dir)? {
let parser = load_parser(dir).await?;
parsers.push(parser);
for dir in get_dirs(&plugins_dir)? {
match load_plugin(dir, plug_type).await? {
PluginEntityState::Valid(plugin) => valid_plugins.push(plugin),
PluginEntityState::Invalid(invalid) => invalid_plugins.push(invalid),
}
}

Ok(parsers)
Ok((valid_plugins, invalid_plugins))
}

/// Retrieves all directory form the given directory path
fn get_dirs(dir_path: &PathBuf) -> Result<impl Iterator<Item = PathBuf>, io::Error> {
let dirs = fs::read_dir(dir_path)?
.filter_map(|entry| entry.ok().map(|e| e.path()))
.filter(|path| path.is_dir());

Ok(dirs)
/// Represents the various states of a plugin entity.
/// This type is used internally in this module only.
enum PluginEntityState {
Valid(PluginEntity),
Invalid(InvalidPluginEntity),
}

/// Loads parser infos and metadata from the provided parser directory.
async fn load_parser(dir: PathBuf) -> Result<PluginEntity, InitError> {
let (wasm_file, metadata_file) = match validate_plugin_files(&dir)? {
/// Loads plugin infos and metadata from the provided plugin directory.
///
/// * `plug_dir`: Plugin directory
/// * `plug_type`: Plugin Type
///
/// # Returns:
///
/// Plugins infos and metadata when valid, or error infos when invalid.
async fn load_plugin(
plug_dir: PathBuf,
plug_type: PluginType,
) -> Result<PluginEntityState, InitError> {
let (wasm_file, metadata_file) = match validate_plugin_files(&plug_dir)? {
PluginValidationState::Valid {
wasm_path: wasm,
metadata,
} => (wasm, metadata),
PluginValidationState::Invalid { err_msg } => {
let invalid_entity = PluginEntity {
dir_path: dir,
plugin_type: PluginType::Parser,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
let invalid_entity = InvalidPluginEntity {
dir_path: plug_dir,
plugin_type: plug_type,
error_msgs: vec![err_msg],
};

return Ok(invalid_entity);
return Ok(PluginEntityState::Invalid(invalid_entity));
}
};

let plugin_info = match PluginsParser::get_info(wasm_file).await {
let plug_info_res = match plug_type {
PluginType::Parser => PluginsParser::get_info(wasm_file).await,
PluginType::ByteSource => PluginsByteSource::get_info(wasm_file).await,
};

let plug_info = match plug_info_res {
Ok(info) => info,
// Stop the whole loading on engine errors
Err(PluginError::HostInitError(PluginHostInitError::EngineError(err))) => {
return Err(err.into())
}
Err(err) => {
let err_msg = format!("Loading plugin binary fail. Error: {err}");
let invalid = PluginEntity {
dir_path: dir,
plugin_type: PluginType::Parser,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
let invalid = InvalidPluginEntity {
dir_path: plug_dir,
plugin_type: plug_type,
error_msgs: vec![err_msg],
};

return Ok(invalid);
return Ok(PluginEntityState::Invalid(invalid));
}
};

let plugin_metadata = match metadata_file {
Some(file) => {
let metadata = match parse_metadata(&file) {
Ok(metadata) => metadata,
Err(err_msg) => {
let invalid_entity = PluginEntity {
dir_path: dir,
plugin_type: PluginType::Parser,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
};
return Ok(invalid_entity);
let mut warn_msgs = Vec::new();
let plug_metadata = match metadata_file {
Some(file) => match parse_metadata(&file) {
Ok(metadata) => metadata,
Err(err_msg) => {
warn_msgs.push(format!(
"Parsing metadata file failed with error: {err_msg}"
));
let dir_name = plug_dir
.file_name()
.and_then(|p| p.to_str())
.unwrap_or("Unkown");

PluginMetadata {
name: dir_name.into(),
description: None,
}
};

Some(metadata)
}
},
None => {
warn_msgs.push(String::from("Metadata file not found"));
let dir_name = plug_dir
.file_name()
.and_then(|p| p.to_str())
.unwrap_or("Unkown");

PluginMetadata {
name: dir_name.into(),
description: None,
}
}
None => None,
};

let valid_plugin = PluginEntity {
dir_path: dir,
plugin_type: PluginType::Parser,
state: PluginState::Active(Box::new(plugin_info)),
metadata: plugin_metadata,
dir_path: plug_dir,
plugin_type: plug_type,
info: plug_info,
metadata: plug_metadata,
warn_msgs,
};

Ok(valid_plugin)
Ok(PluginEntityState::Valid(valid_plugin))
}

/// Retrieves all directory form the given directory path
fn get_dirs(dir_path: &PathBuf) -> Result<impl Iterator<Item = PathBuf>, io::Error> {
let dirs = fs::read_dir(dir_path)?
.filter_map(|entry| entry.ok().map(|e| e.path()))
.filter(|path| path.is_dir());

Ok(dirs)
}

/// Represents Plugins State and the corresponding infos.
Expand Down Expand Up @@ -230,91 +283,3 @@ fn parse_metadata(file: &PathBuf) -> Result<PluginMetadata, String> {

toml::from_str(&content).map_err(|err| format!("Parsing metadata file fail. Error {err:#?}"))
}

/// Loads all byte-source plugins from their main directory.
async fn load_all_bytesources() -> Result<Vec<PluginEntity>, InitError> {
let mut bytesources = Vec::new();

let bytesource_dir = paths::bytesource_dir()?;
if !bytesource_dir.exists() {
log::trace!("Bytesources directory doesn't exist. Creating it ...");
fs::create_dir_all(&bytesource_dir)?;

return Ok(bytesources);
}

for dir in get_dirs(&bytesource_dir)? {
let source = load_bytesource(dir).await?;
bytesources.push(source);
}

Ok(bytesources)
}

/// Loads byte-source infos and metadata from the provided parser directory.
async fn load_bytesource(dir: PathBuf) -> Result<PluginEntity, InitError> {
let (wasm_file, metadata_file) = match validate_plugin_files(&dir)? {
PluginValidationState::Valid {
wasm_path: wasm,
metadata,
} => (wasm, metadata),
PluginValidationState::Invalid { err_msg } => {
let invalid_entity = PluginEntity {
dir_path: dir,
plugin_type: PluginType::ByteSource,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
};

return Ok(invalid_entity);
}
};

let plugin_info = match PluginsByteSource::get_info(wasm_file).await {
Ok(info) => info,
// Stop the whole loading on engine errors
Err(PluginError::HostInitError(PluginHostInitError::EngineError(err))) => {
return Err(err.into())
}
Err(err) => {
let err_msg = format!("Loading plugin binary fail. Error: {err}");
let invalid = PluginEntity {
dir_path: dir,
plugin_type: PluginType::ByteSource,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
};

return Ok(invalid);
}
};

let plugin_metadata = match metadata_file {
Some(file) => {
let metadata = match parse_metadata(&file) {
Ok(metadata) => metadata,
Err(err_msg) => {
let invalid_entity = PluginEntity {
dir_path: dir,
plugin_type: PluginType::ByteSource,
state: PluginState::Invalid(Box::new(InvalidPluginInfo::new(err_msg))),
metadata: None,
};
return Ok(invalid_entity);
}
};

Some(metadata)
}
None => None,
};

let valid_plugin = PluginEntity {
dir_path: dir,
plugin_type: PluginType::ByteSource,
state: PluginState::Active(Box::new(plugin_info)),
metadata: plugin_metadata,
};

Ok(valid_plugin)
}
Loading

0 comments on commit c02e8a6

Please sign in to comment.