param_update.py

#!/usr/bin/env python
#
# Copyright (c) 2016-2017 Fraunhofer Institute for Manufacturing Engineering and Automation (IPA)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from autocomplete import AutoComplete
from operator import itemgetter
from sets import Set
import re
import sys
import os.path
import os
import subprocess
import rospkg
import fnmatch
import yaml
import readline


def print_params(input_list):
    if len(input_list) == 0:
        print "No parameters were found - no parameters will be converted.\n"
        return
    print "The following parameters are converted to snake/underscore case:\n"

    description = [('Old parameters:', 'New parameters:'), (' ', ' ')]
    input_list = description + input_list

    for params in input_list:
        print('{:<40} {:<40}'.format(*params))
    print "\n"


def camel_to_snake_line(input_str):
    output_str = re.sub('([ |,])([dbn])([A-Z])+', r'\1\3', input_str)     # Delete prefix d/b/n
    output_str = re.sub('([a-zA-Z])([A-Z][a-z]+)', r'\1_\2', output_str)     #
    output_str = re.sub('([a-z0-9])([A-Z])', r'\1_\2', output_str).lower()
    return output_str


def camel_to_snake_word(input_str):
    output_str = re.sub('^([dbn])([A-Z])+', r'\2', input_str)                       # Delete prefix d/b/n
    output_str = re.sub('([a-zA-Z])([A-Z][a-z]+|[0-9]+)', r'\1_\2', output_str)     #
    output_str = re.sub('([a-z0-9])([A-Z])', r'\1_\2', output_str).lower()
    output_str = re.sub('([a-z]+)([0-9]+)', r'\1_\2', output_str)
    return output_str


def to_upper(match):
    return match.group(1).upper()


def convert_values(values):
    # convert strings
    if 'str' in values[1]:
        values[2] = values[2].replace('"', '')
        values[2] = '"' + values[2] + '"'
        values[1] = 'string'

    # convert bools
    if 'bool' in values[1]:
        values[2] = values[2].lower()


def extend_values(values):
    for i in range(len(values), 5):
        values.append('')


def extract_values_common(values):
    tmp_values = values[:]
    if "string" in tmp_values[1]:
        tmp_values[1] = "std::string"

    ret_line = tmp_values[1] + " " + tmp_values[0] + ";"
    return ret_line


def extract_values_ros(values):
    tmp_values = values[:]
    if "string" in tmp_values[1]:
        tmp_values[1] = "std::string"

    ret_line = "np_.param(\"" + tmp_values[0] + "\", component_config_." + \
        tmp_values[0] + ", (" + tmp_values[1] + ")" + tmp_values[2] + ");"
    return ret_line


def extract_values_cfg(values):
    tmp_values = values[:]
    line_end = ", " + tmp_values[3] + ", " + tmp_values[4]

    line_end = convert_values_cfg(tmp_values, line_end)

    ret_line = "gen.add(\"" + tmp_values[0] + "\", " + tmp_values[1] + \
        "_t, 0, \"Autogenerated parameter based on other files.\", " + tmp_values[2] + line_end + ")"

    return ret_line


def extract_values_yaml(values):
    ret_line = "# The " + values[0] + " [" + values[1] + "]\n" + values[0] + ": " + values[2] + "\n"
    return ret_line


def update_params_ros(line, values):
    tmp_values = values[:]

    if "string" in tmp_values[1]:
        tmp_values[1] = "std::string"

    line_update = line

    if '.setParam(' in line:
        regex_search = r'(\.setParam\(")([a-z_]+)", ?(true|false|[0-9+\-*/.]+|"[^"\']*")\);'
        regex_replace = r'\1\2", ' + tmp_values[2] + r');'
        line_update = re.sub(regex_search, regex_replace, line)

    elif '.param(' in line:
        regex_search = r'(\.param\(")([a-z_A-Z]+)(",[^,]+,) ?\(([a-zA-Z:]+)\)([a-zA-Z]+|[0-9+\-*/.]+|"[^"\'\)]*")\);'
        regex_replace = r'\1\2\3 (' + tmp_values[1] + r')' + tmp_values[2] + r');'
        line_update = re.sub(regex_search, regex_replace, line)

    elif 'ROS_WARN(' in line:
        regex_search = r'ROS_WARN\("([^\[\]"]+)\[([^\]]+)\]([^\[\]"]+)"\);'
        regex_replace = r'ROS_WARN("\g<1>' + tmp_values[2] + r'\g<3>");'
        line_update = re.sub(regex_search, regex_replace, line)

    return line_update


def update_params_cfg(line, values):
    tmp_values = values[:]
    line_end = r'\6'

    line_end = convert_values_cfg(tmp_values, line_end)

    regex_search = r'gen.add\("([a-z_A-Z]+)", ?([a-z]+)_t, ?([0-9]), ?"([^"\']+)", ?([0-9\.]*|[a-zA-Z]*|"[^"\']*")(,? ?[0-9\.]*,? ?[0-9\.]*)\)'
    regex_replace = r'gen.add("\1", ' + tmp_values[1] + r'_t, \3, "\4", ' + tmp_values[2] + line_end + ')'

    line_update = re.sub(regex_search, regex_replace, line)

    return line_update


def convert_params(file_paths, in_zip_names):
    print '\nParameter names will be changed now.\n'
    for current_file_id, current_file in enumerate(file_paths):
        occurrences = 0

        # delete tmp file if exists
        if os.path.exists(current_file + '.tmp'):
            os.system('rm ' + current_file + '.tmp')

        # if file exists then open it
        if os.path.exists(current_file):
            with open(current_file) as in_file, open(current_file + '.tmp', 'w') as out_file:

                # loop through lines of file
                for line in in_file:
                    for old_var, new_var in in_zip_names:
                        old_camel_var = re.sub('_([a-z0-9])', to_upper, old_var)
                        if re.search(r'[^a-zA-Z_0-9]'+old_var+r'[^a-zA-Z_0-9]', line) is not None and old_var is not new_var:
                        # if old_var in line and old_var != new_var:
                            # if line contains an old variable name -> replace it
                            line = line.replace(old_var, new_var)
                            occurrences += 1
                        if re.search(r'[^a-zA-Z_0-9]'+old_camel_var+r'[^a-zA-Z_0-9]', line) is not None:
                        # elif old_camel_var in line:
                            # if line contains an old camel case variable name -> replace it
                            line = line.replace(old_camel_var, new_var)
                            occurrences += 1
                    out_file.write(line)

                in_file.close()
                out_file.close()
                old_file_mode = os.popen("stat --format '%a' " + current_file).read().replace("\n", "")
                os.system('mv ' + current_file + '.tmp' + ' ' + current_file)
                os.system('chmod ' + old_file_mode + ' ' + current_file)

                print "Converted " + str(occurrences) + " parameter occurrences in file \'../" + file_name + file_types[current_file_id] + "\'"
        else:
            print "\nError: File \'" + current_file + "\' does not exist."


def convert_values_cfg(in_values, line_end):
    # first character to upper for bools
    if 'bool' in in_values[1] and len(in_values[2]) > 0:
        in_values[2] = in_values[2][0].upper() + in_values[2][1:]
        line_end = ''
    elif 'str' in in_values[1]:
        in_values[1] = 'str'
        line_end = ''
    elif 'double' in in_values[1] or 'int' in in_values[1]:     # standard in_values
        if len(in_values[3]) <= 0:
            in_values[3] = '0'
        if len(in_values[4]) <= 0:
            in_values[4] = '100'
        line_end = ", " + in_values[3] + ", " + in_values[4]
    return line_end


def find_file(in_root_path, in_file_name):
    matches = []
    for root, dirnames, filenames in os.walk(in_root_path):
        for filename in fnmatch.filter(filenames, in_file_name):
            matches.append(os.path.join(root, filename))
    if len(matches) == 1:
        return matches[0]
    else:
        print "Can't find file with name '" + in_file_name + "'"
        return ''


#####  Ordering of values: [name, type, value, range_lower, range_upper]  #####

# Constants:
# specify static id's for file types
FILE_COMMON = 0
FILE_ROS = 1
FILE_CFG = 2
FILE_YAML = 3

# own package name
THIS_PKG_NAME = 'ipa_code_refactoring'

# Important global variables
rospack = rospkg.RosPack()

# Load config file data
# check for code refactoring pkg
try:
    this_cfg_path = rospack.get_path(THIS_PKG_NAME) + '/cfg/param_update.yaml'
except Exception, e:
    print "Can't find ROS Package with name '" + THIS_PKG_NAME + "'"
    sys.exit()

# read config-yaml file
this_cfg = {}
if os.path.exists(this_cfg_path):
    with open(this_cfg_path, 'r') as stream:
        try:
            this_cfg = yaml.load(stream)
        except Exception as e:
            print(e)
else:
    print "Can't find CFG file with name '" + this_cfg_path + '' + "' in package '" + THIS_PKG_NAME + "'"

# file types
file_types = [this_cfg.get('file_common', '_common.cpp'),
              this_cfg.get('file_ros', '_ros.cpp'),
              this_cfg.get('file_cfg', '.cfg'),
              this_cfg.get('file_yaml', '.yaml')]

## Path to yaml folder
# check for yaml path pkg
cfg_yaml_pkg = this_cfg.get('path_yaml_pkg',
                             'wrong')
try:
    cfg_yaml_pkg = rospack.get_path(cfg_yaml_pkg) + '/'
except Exception, e:
    print "Can't find ROS Package with name '" + cfg_yaml_pkg + "'"
    sys.exit()
cfg_yaml_path = cfg_yaml_pkg + this_cfg.get('path_yaml_folder', 'config/components')

editor = this_cfg.get('editor', 'subl')

# add autocompletion of ros packages
try:
    autocomplete = AutoComplete(rospack.list())
except Exception, e:
    print "Unable to get ROS Package List."
    sys.exit()

readline.set_completer_delims(' \t\n;')
readline.set_completer(autocomplete.complete)
readline.parse_and_bind('tab: complete')

# get package and file name
print "\n1)  Specify the name of the ROS package for parameter updates:"
input_ros_pkg = raw_input("ROS Package:  ")

# find path to ros package
try:
    pkg_path = rospack.get_path(input_ros_pkg)
except Exception, e:
    print "Can't find ROS Package with name '" + input_ros_pkg + "'"
    sys.exit()

print "\n2)  Specify the name of the node if it differs from the ROS package name (optional):"
input_node_name = raw_input("Node name:  ")
print

if len(input_node_name) > 0:
    file_name = input_node_name
else:
    file_name = input_ros_pkg


# find cpp, cfg and yaml files in pkg_path
file_paths = ['', '', '', '']

file_paths[FILE_COMMON] = find_file(pkg_path, file_name + file_types[FILE_COMMON])
if len(file_paths[FILE_COMMON]) <= 0:
    file_paths[FILE_COMMON] = find_file(pkg_path, file_name + '.cpp')
    file_types[FILE_COMMON] = '.cpp'

file_paths[FILE_ROS] = find_file(pkg_path, file_name + file_types[FILE_ROS])
file_paths[FILE_CFG] = find_file(pkg_path, file_name + file_types[FILE_CFG])
file_paths[FILE_YAML] = find_file(cfg_yaml_path, file_name + file_types[FILE_YAML]).replace('_node', '')

# list of methods for updating parameter values
update_params = [update_params_ros, update_params_cfg]
# list of methods for constructing lines for missing parameters
extract_values = [extract_values_common, extract_values_ros, extract_values_cfg, extract_values_yaml]

old_names = []
new_names = []
zip_names = zip(old_names, new_names)

### Regular Expressions ###

# Regexes for finding parameters
# regex for start of variable/parameter declarations
regex_start = re.compile(r'^class.*_config$')
# regex for end of variable/parameter declarations
regex_end = re.compile(r'^};$')
# regex for finding camel-case parameters in common file
regex_params_common_camel = re.compile(r'[ |,]([a-zA-Z0-9]+)[,|;]')
# regex for finding snake-case parameters in common file
regex_params_common_snake = re.compile(r'^[ ]*[a-zA-Z:]+ ([a-zA-Z_0-9]+[a-zA-Z0-9]+);')
# regex for finding any parameters in common file
regex_params_common = re.compile(r'[a-zA-Z:]+ ([^; ]+);')
# regex for finding parameters in ros.cpp file
regex_params_ros = re.compile(r'.param\("([a-zA-Z_0-9]+[a-z0-9]+)"')
# regex for finding parameters in .cfg file
regex_params_cfg = re.compile(r'gen.add\("([a-zA-Z_0-9]+[a-z0-9]+)"')
# regex for finding parameters in .yaml file
regex_params_yaml = re.compile(r'^([a-zA-Z_0-9]+[a-z0-9]+):')

# Regexes for finding values of parameters
# regex for finding param values in common file
regex_values_common = re.compile(r'^[ ]*([a-z:A-Z]+) ([a-z_0-9]+[a-z0-9]+);$')
# regex for finding param values in ros file
regex_values_ros = re.compile(r'\.param\("([a-z_A-Z0-9]+)",[^,]+, ?\(([a-z:]+)\)"?([^")]*)"?\)')
# regex for finding param values in cfg file
regex_values_cfg = re.compile(
    r'gen.add\("([a-z_A-Z0-9]+)", ?([a-z]+)_t, ?[0-9], ?"[^"]+", ?([0-9\.]*|[a-zA-Z]*|"[^"\']*"),? ?([\-0-9\.]+)?,? ?([\-0-9\.]+)?\)')
# regex for finding param values in yaml file
regex_values_yaml = re.compile(r'^([a-z_A-Z0-9]+): ?()([0-9.]+|[a-zA-Z]+|"[^"\']*")')

regex_values = [regex_values_common, regex_values_ros, regex_values_cfg, regex_values_yaml]
regex_params = [regex_params_common, regex_params_ros, regex_params_cfg, regex_params_yaml]



### Find all parameters and their corresponding values in all files with new names ###
found_params_all = [Set([]), Set([]), Set([]), Set([])]
found_params_raw = [Set([]), Set([]), Set([]), Set([])]
extracted_values = [[], [], [], []]
print "\n"

for current_file_id, current_file in enumerate(file_paths):
    if os.path.exists(current_file):
        with open(current_file, 'r+') as f:
            old = f.readlines()                 # Pull the file contents to a list
            f.seek(0)                           # Jump to start, so we overwrite instead of appending

            config_reached = False

            # loop through lines of file
            # add all parameters found to the list of its' corresponding id
            # in found_params
            for line in old:
                if current_file_id == FILE_COMMON:
                    # Find start of config section (if exists)
                    if regex_start.search(line) is not None:
                        config_reached = True

                    # Find end config section
                    if (regex_end.search(line) is not None) and (config_reached):
                        config_reached = False

                if current_file_id > FILE_COMMON or config_reached:
                    params_in_line = regex_params[current_file_id].findall(line)
                    if len(params_in_line) > 0:
                        # Save found raw parameters
                        found_params_raw[current_file_id].update(params_in_line)
                        # Convert all params
                        for i, param in enumerate(params_in_line):
                            params_in_line[i] = camel_to_snake_word(param)
                        # Save found converted parameters
                        found_params_all[current_file_id].update(params_in_line)
                        # Save values for found parameters
                        found_param_values = regex_values[current_file_id].findall(line)
                        if len(found_param_values) > 0:
                            extracted_values[current_file_id].append(list(found_param_values[0]))
                            # Turn around type and name for common
                            if current_file_id == 0:
                                extracted_values[current_file_id][-1] = extracted_values[current_file_id][-1][::-1]
                            # Convert name of parameter
                            extracted_values[
                                current_file_id][-1][0] = camel_to_snake_word(extracted_values[current_file_id][-1][0])

            print "Found " + str(len(found_params_all[current_file_id])) + " parameters in file \'../" + file_name + file_types[current_file_id] + "\'"


# Update parameters that were not found in common

# Build old/new param lists
old_names = []
new_names = []

for current_file_id, found_params_file in enumerate(found_params_all):
    found_params_raw[current_file_id] -= found_params_file
    for raw_param in found_params_raw[current_file_id]:
        if raw_param not in old_names:
            old_names.append(raw_param)
            new_names.append(camel_to_snake_word(raw_param))

zip_names = zip(old_names, new_names)

if len(old_names) > 0 and len(new_names) > 0:
    print '\n\nWrong parameter names have been found, converting files now.\n'
    print_params(zip_names)
    # Convert all variables in files common.cpp, ros.cpp, .cfg and .yaml if they exist
    convert_params(file_paths, zip_names)
else:
    print '\n\nNo parameters with wrong names were found, none will be converted.\n'


# Merge all values in one list
set_of_values = Set()
extracted_values_all = []
iteration_order = [2, 1, 3, 0]  # -> cfg -> ros.cpp -> yaml -> common

for value_file_id in iteration_order:
    for values in extracted_values[value_file_id]:
        if values[0] not in set_of_values:
            # extend values so that it contains at least empty values for all five entries
            extend_values(values)
            # convert values
            convert_values(values)
            # Add all values to list
            extracted_values_all.append(values)
            # Add only the param name to list for checking if it's already there
            set_of_values.add(values[0])
        else:
            for saved_values in extracted_values_all:
                if saved_values[0] == values[0]:
                    for i, value in enumerate(values):
                        if value != '' and saved_values[i] == '':
                            saved_values[i] = value

extracted_values = extracted_values_all


# Change param names/values manually
old_names = []
new_names = []
print "\nDo you want to change the parameter names/values? [y/n]\n"

user_input = raw_input("Choice: ")

if user_input == 'y' or user_input == 'Y':
    print "Type in the new name/value for the parameter, skip by pressing Enter:\n"
    for i, param_values in enumerate(extracted_values):
        user_input = raw_input(param_values[0] + ' (' + param_values[1] + ') = ').replace(' ', '')
        user_input = camel_to_snake_word(user_input)
        if len(user_input) > 0:
            new_names.append(user_input)
            old_names.append(param_values[0])
            extracted_values[i][0] = user_input
        user_input = raw_input('Value [' + param_values[2] + '] = ')
        if len(user_input) > 0:
            extracted_values[i][2] = user_input
        print

# Change parameters
if len(old_names) > 0 and len(new_names) > 0:
    zip_names = zip(old_names, new_names)
    convert_params(file_paths, zip_names)


# Update values of parameters in files
print '\nParameter values will be updated now.\n'
for current_file_id, current_file in enumerate(file_paths[1:3]):
    # delete tmp file if exists
    if os.path.exists(current_file + '.tmp'):
        os.system('rm ' + current_file + '.tmp')

    if os.path.exists(current_file):
        with open(current_file) as in_file, open(current_file + '.tmp', 'w') as out_file:
            # old = f.readlines()                 # Pull the file contents to a list
            # f.seek(0)                           # Jump to start, so we overwrite instead of appending

            updates = 0

            # loop through lines of file
            # add parameters if found
            for line in in_file:
                for values in extracted_values:
                    if '"' + values[0] + '"' in line:
                        line_update = update_params[current_file_id](line, values)
                        if line_update != line:
                            updates += 1
                        break
                    else:
                        line_update = line

                out_file.write(line_update)

            in_file.close()
            out_file.close()
            old_file_mode = os.popen("stat --format '%a' " + current_file).read().replace("\n", "")
            os.system('mv ' + current_file + '.tmp' + ' ' + current_file)
            os.system('chmod ' + old_file_mode + ' ' + current_file)

            print "Updated values for " + str(updates) + " parameter occurrences in file \'../" + file_name + file_types[current_file_id + 1] + "\'"


# Find missing params by comparing all lists of found parameters
list_a_id = 0
missing_params = [Set([]), Set([]), Set([]), Set([])]

for list_a_id, list_a in enumerate(found_params_all[:2]):
    list_b_id = list_a_id + 1

    for list_b in found_params_all[list_b_id:]:
        # Find differences of lists
        diff_a = list_a - list_b
        diff_b = list_b - list_a

        missing_params[list_b_id].update(diff_a)
        missing_params[list_a_id].update(diff_b)

        list_b_id += 1


# Print missing parameters
found_missing_params = False
for i, missing_param_list in enumerate(missing_params):
    if len(missing_param_list) > 0:
        if not found_missing_params:
            print
        print "\nParameters not in file '../" + file_name + file_types[i] + "':"
    for missing_param in missing_param_list:
        print "- " + missing_param
        found_missing_params = True


# Get missing parameter lines for adding
if found_missing_params:
    print "\n\n[1]        Print lines of all missing values on screen"
    print "[2]        Save lines of all missing values to file ('" + file_name + ".missing')"
    print "[Any key]  Print nothing\n"

    user_input = raw_input("Choice: ")

    user_input_ok = True

    try:
        user_input = int(user_input)
    except ValueError:
        print "Parameter lines will not be printed.\n"
        user_input_ok = False

    if user_input_ok:
        if user_input == 2:
            file = open(file_name + '.missing', 'w+')

        # Construct lines for specific files
        for current_file_id, current_file in enumerate(missing_params):
            if user_input == 1 and len(current_file) > 0:
                print "\n\nMissing parameters (values) for file: \'" + file_types[current_file_id] + "\':\n"
            elif user_input == 2 and len(current_file) > 0:
                file.write("Missing parameters (values) for file: \'" + file_types[current_file_id] + "\':\n\n")
            for param in sorted(current_file):
                # Find param in value list
                for values in extracted_values:
                    if param == values[0]:
                        if user_input == 1:
                            # print out the constructed line for file
                            print extract_values[current_file_id](values)
                        elif user_input == 2:
                            # write the constructed line to a file
                            file.write(extract_values[current_file_id](values) + '\n')
            if user_input == 2:
                file.write('\n\n')

        print
        if user_input == 2:
            print 'Missing parameters were written to local file \'' + file_name + '.missing\''
            file.close()


# Open files in sublime
user_input = raw_input('\nOpen changed files in sublime? [y/n]  ')
if user_input == 'y' or user_input == 'Y':
    os.system(editor + ' ' + file_paths[FILE_COMMON] + ' ' + file_paths[FILE_ROS] +
              ' ' + file_paths[FILE_CFG] + ' ' + file_paths[FILE_YAML])