fixed local-repo rerun defect

local_repo/roles/parse_and_download/library/parallel_tasks.py

@@ -40,6 +40,7 @@
     get_subgroup_dict
 )
 import json
+
 
 def determine_function(task, repo_store_path, csv_file_path, user_data, version_variables):
     """
@@ -67,7 +68,11 @@ def determine_function(task, repo_store_path, csv_file_path, user_data, version_
 
          #version_variables = {"k8s_version": "1.28.1"}
         task_type = task.get("type")
+
         status_file = f'{csv_file_path}/status.csv'
+        if not os.path.exists(status_file) or os.stat(status_file).st_size == 0:
+            with open(status_file, 'w') as file:
+                file.write('name,type,status\n')
 
         if task_type == "manifest":
             return process_manifest, [task, repo_store_path, status_file]
@@ -111,6 +116,7 @@ def generate_pretty_table(task_results, total_duration, overall_status):
     table.add_row(["Total Duration", total_duration,""])
     table.add_row(["Overall Status", overall_status,""])
     return table.get_string()
+
 
 def main():
     """
@@ -197,7 +203,8 @@ def main():
         #slogger.info(f"version var= {version_variables['version']}")
         slogger.info(f"cluster os is= {cluster_os_type}")
         first_entry_dict = dict([list(version_variables.items())[0]])
-
+
+
         # Execute tasks in parallel
         overall_status, task_results = execute_parallel(tasks, determine_function, nthreads, repo_store_path, csv_file_path, log_dir, user_data, first_entry_dict, slogger, timeout)
 
@@ -225,7 +232,6 @@ def main():
         # Generate and log the pretty table
         table_output = generate_pretty_table(task_results, total_duration, overall_status)
         log_table_output(table_output, log_file)
-
         # Update the result dictionary with task results
         result["total_duration"] = total_duration
         result["task_results"] = task_results

local_repo/roles/parse_and_download/library/prepare_tasklist.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
- 
+
 #!/usr/bin/python
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.standard_logger import setup_standard_logger
@@ -20,6 +20,7 @@
     get_software_names,
     check_csv_existence,
     get_failed_software,
+    get_csv_software,
     process_software,
     load_json,
     load_yaml,
@@ -34,16 +35,16 @@
 import json
 import os
 import re
- 
+
 def main():
     """
     The main function that prepares package lists and processes software.
- 
+
     This function initializes the module arguments and sets up the logger.
     It loads the user data from a JSON file and the repository configuration
     data from a YAML file. It retrieves the cluster OS type and version, and
     the list of software names. It also sets up the version variables.
- 
+
     The function then prepares the package lists and processes the software.
     For each software, it checks if it needs a fresh installation. If not,
     it skips the software. For each software, it retrieves the JSON and CSV
@@ -52,91 +53,111 @@ def main():
     and skips it if necessary. It processes the software and stores the tasks
     in the software dictionary. Finally, it transforms the package dictionary
     and parses the repository URLs.
- 
+
     Parameters:
     None
- 
+
     Returns:
     None
- 
+
     Raises:
     Exception: If an error occurs during the processing.
- 
+
     """
     module_args = dict(
         csv_file_path=dict(type="str", required=False, default="/tmp/status_results_table.csv"),
         user_json_file=dict(type="str", required=False, default=""),
         local_repo_config_path=dict(type="str", required=False, default=""),
         log_dir=dict(type="str", required=False, default="/tmp/thread_logs"),
     )
- 
+
     module = AnsibleModule(argument_spec=module_args)
- 
+
     # versions = module.params['versions']
     log_dir = module.params["log_dir"]
     user_json_file = module.params['user_json_file']
     local_repo_config_path = module.params['local_repo_config_path']
     csv_file_path = module.params['csv_file_path']
- 
+
     logger = setup_standard_logger(log_dir)
     start_time = datetime.now().strftime("%I:%M:%S %p")
     logger.info(f"Start execution time: {start_time}")
- 
+
     try:
         user_data = load_json(user_json_file)
         repo_config_data = load_yaml(local_repo_config_path)
- 
+
         cluster_os_type = user_data['cluster_os_type']
         cluster_os_version = user_data['cluster_os_version']
-
+
+        csv_file_path = os.path.join(csv_file_path, "software.csv")
+
         software_list = get_software_names(user_json_file)
-
+        logger.info(f"software_list from software_config: {software_list}")
+
+        fresh_installation = 1 if not check_csv_existence(csv_file_path) else 0
+        logger.info(f"Fresh install: {fresh_installation}")
+
+        csv_softwares = []
+        if not fresh_installation:
+            csv_softwares = get_csv_software(csv_file_path)
+            logger.info(f"software from software.csv: {csv_softwares}")
+        new_software = [software for software in software_list if software not in csv_softwares]
+        logger.info(f"new software list : {new_software}")
+        logger.info(f"Final software_list: {software_list}")
         # Build a dictionary of software_name -> subgroup_list if exists
-       
+
         subgroup_dict , software_names = get_subgroup_dict(user_data)
- 
+
         version_variables = set_version_variables(user_data, software_names, cluster_os_version)
         software_dict = {}
-       
+
         logger.info("Preparing package lists...")
-
+        #csv_file_path = os.path.join(csv_file_path, "software.csv")
         for software in software_list:
             logger.info(f"Processing software: {software}")
-            fresh_installation = 1 if not check_csv_existence(software, csv_file_path) else 0
-
+            logger.info(f"csv_file_path software: {csv_file_path}")
+            #fresh_installation = 1 if not check_csv_existence(software, csv_file_path) else 0
+
             json_path = get_json_file_path(software, cluster_os_type, cluster_os_version, user_json_file)
             csv_path = get_csv_file_path(software, log_dir)
             logger.info(f"csv_path: {csv_path}")
-           
+
             if not json_path:
                 logger.warning(f"Skipping {software}: JSON path does not exist.")
                 continue
-
-
+            if software in new_software:
+                fresh_installation = 1
+
             logger.info(f"{software}: JSON Path: {json_path}, CSV Path: {csv_path}, Fresh Install: {fresh_installation}")
             logger.info(f"Subgroup Data: {subgroup_dict.get(software, None)}")
             logger.info(f"Whole Subgroup Data: {subgroup_dict}")
- 
+
             failed_softwares = get_failed_software(csv_file_path)
+            logger.info(f"failed_softwares : {failed_softwares}")
             if not fresh_installation and software not in failed_softwares:
                 continue
             # Ensure we pass None for subgroup_list when not present
             logger.info(f"json_path: {json_path}")
             logger.info(f"csv_path: {csv_path}")
-            tasks = process_software(software, fresh_installation, json_path, csv_path, subgroup_dict.get(software, None))
+            tasks ,failed_packages = process_software(software, fresh_installation, json_path, csv_path, subgroup_dict.get(software, None))
             logger.info(f"tasks: {tasks}")
-
+            logger.info(f"failed_packages for software:{software} are {failed_packages}")
+            failed_packages = get_failed_software(csv_path)
+            logger.info(f"failed_packages: {failed_packages}")
+
             software_dict[software] = tasks
- 
+
         software_dict=transform_package_dict(software_dict)
         local_config = parse_repo_urls(local_repo_config_path , version_variables)
- 
+
         module.exit_json(changed=False, software_dict=software_dict  , local_config=local_config)
         logger.info(f"Package processing completed: {software_dict}")
- 
+
     except Exception as e:
         logger.error(f"Error occurred: {str(e)}")
         module.fail_json(msg=str(e))
- 
+
 if __name__ == "__main__":
     main()
+

local_repo/roles/parse_and_download/module_utils/parse_and_download.py

@@ -77,24 +77,50 @@ def execute_command(cmd_string,logger,type_json=False):
         # Log function end
         logger.info("#" * 30 + f" {execute_command.__name__} end " + "#" * 30)  # End of function
 
+
 def write_status_to_file(status_file_path, package_name, package_type, status, logger):
     """
-    Writes the status of a package to the specified status file.
-
+    Writes the status of a package to the specified status file. If the package already exists,
+    its status is updated. Otherwise, the package is appended.
+
     Args:
         status_file_path (str): Path to the status file.
         package_name (str): Name of the package.
         package_type (str): Type of the package.
         status (str): Status of the package (e.g., "Success", "Failed").
         logger (Logger): Logger instance for logging.
- 
+
     Returns:
         None
     """
     logger.info("#" * 30 + f" {write_status_to_file.__name__} start " + "#" * 30)  # Start of function
+
     try:
-        with open(status_file_path, "a") as f:
-            f.write(f"{package_name},{package_type},{status}\n")
+        # Check if the file exists and read its contents
+        if os.path.exists(status_file_path):
+            with open(status_file_path, "r") as f:
+                lines = f.readlines()
+
+            # Check if the package name already exists and replace the corresponding line
+            updated = False
+            with open(status_file_path, "w") as f:
+                for line in lines:
+                    if line.startswith(f"{package_name},"):
+                        # Replace the existing line with the new status
+                        f.write(f"{package_name},{package_type},{status}\n")
+                        updated = True
+                    else:
+                        f.write(line)
+
+                # If the package was not found, append the new line
+                if not updated:
+                    f.write(f"{package_name},{package_type},{status}\n")
+        else:
+            # If the file doesn't exist, create it and write the header and the new line
+            with open(status_file_path, "w") as f:
+                f.write("name,type,status\n")
+                f.write(f"{package_name},{package_type},{status}\n")
+
         logger.info(f"Status written to {status_file_path} for {package_name}.")
     except Exception as e:
         logger.error(f"Failed to write to status file: {status_file_path}. Error: {str(e)}")