# Codelet Tuning Infrastructure
# Copyright (C) 2010-2015 Intel Corporation, CEA, GENCI, and UVSQ
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#************************************************************************

# Authors: Franck Talbart, Mathieu Bordet, Nicolas Petit

""" Entry module provides facilities to work with the CTI entries.
"""

import cti, ctr

import util, types, util_uid, alias, database, repository, database_manager

import os, copy, json, time, datetime, shutil, distutils.dir_util, UserDict

DataEntryMetaFiles = [cti.cti_plugin_config_get_value(v) for
                      v in [cti.PLUGIN_OUTPUT_FILENAME,
                            cti.PLUGIN_INPUT_FILENAME,
                            cti.DATA_INFO_FILENAME]]

class odict(UserDict.DictMixin):
    def __init__(self):
        self._keys = []
        self._data = {}
    #------------------------------------------------------------------------

    def __setitem__(self, key, value):
        if key not in self._data:
            self._keys.append(key)
        self._data[key] = value
    #------------------------------------------------------------------------

    def __getitem__(self, key):
        return self._data[key]
    #------------------------------------------------------------------------

    def __delitem__(self, key):
        del self._data[key]
        self._keys.remove(key)
    #------------------------------------------------------------------------

    def keys(self):
        return list(self._keys)
    #------------------------------------------------------------------------

    def copy(self):
        copyDict = odict()
        copyDict._data = self._data.copy()
        copyDict._keys = self._keys[:]
        return copyDict
#------------------------------------------------------------------------

class CmdNode(object):
    """ Command node class represent a simple node.

    The typicall CTI command has some attributes, characterize the command itself,
    and params which is an input for the command.

    For ordered iteration use ordered_params list.
    """
    def __init__(self, attributes, params):
        if cti.META_ATTRIBUTE_REP in attributes:
            if attributes[cti.META_ATTRIBUTE_REP] == cti.LOCAL_REPOSITORY:
                attributes[cti.META_ATTRIBUTE_REP] = cti.CTR_REP_LOCAL
            elif attributes[cti.META_ATTRIBUTE_REP] == cti.COMMON_REPOSITORY:
                attributes[cti.META_ATTRIBUTE_REP] = cti.CTR_REP_COMMON
            elif attributes[cti.META_ATTRIBUTE_REP] == cti.TEMP_REPOSITORY:
                attributes[cti.META_ATTRIBUTE_REP] = cti.CTR_REP_TEMP
        self.attributes = attributes
        self.params = params

#------------------------------------------------------------------------

class Commands(odict):
    """CTI commands dictionary.

    This class represents an extension for the standard dictionary class.
    With Commands we are able to use CTI files as a structured dictionaries.
    Here is a structure of such a dictionary.
        Commands         CmdNode
       +-------+      +------------+
       | cmd1  |----->| attributes |
       | cmd2  |      |   params   |
       | ...   |      +------------+
       +-------+
    """

    def __init__(self, datatype, uid, basename, default_data = None, no_none_value = False, remove_outdated_params=False):
        odict.__init__(self)
        self.datatype = datatype
        self.uid = None
        self.basename = basename
        if basename is None:
            return None
        filename = None
        if uid is None:
            filename = basename
        else:
            filedir = ctr.ctr_plugin_get_path_by_uid(datatype, uid)
            if filedir is not None:
                filename = os.path.join(filedir, basename)
            else:
                filename = basename
            self.uid = uid
        try:
            self.load(filename, default_data, no_none_value, remove_outdated_params)
        except Exception as e:
            print("Can't load the commands of \"%s\" (wrong input or output files)" % uid)
            if uid is None:
                print("Plugin probably not found.")
            raise e
    
    #------------------------------------------------------------------------
    
    def __str__(self):
        """ The Commands class pretty printer. """
        result = ""
        for cmd in self.keys():
            result += "****************************************************\n"
            result += "Command: %s\n" % (cmd)
            result += "Attributes: %s \n" % (self[cmd].attributes)
            for p in self[cmd].params:
                result += "Param: %s\n" % p
                val = "NONE"
                if cti.META_ATTRIBUTE_VALUE in self[cmd].params[p]:
                    val = self[cmd].params[p][cti.META_ATTRIBUTE_VALUE]
                result += "Value = %s \n" % (val)
            result += "\n"
        return result
    
    #------------------------------------------------------------------------
    
    def record_output(self, command, path):
        """ Records data to the output file
            Args:
                command: the set of parameters
                path: the path to the output file
        """
        try:
            output_name = cti.cti_plugin_config_get_value(cti.PLUGIN_OUTPUT_FILENAME)
            if output_name:
                filename = os.path.join(path, output_name)
            else:
                util.hapi_fail("Can't get value from config file")
        except OSError, e:
            util.hapi_fail("Failed to concat path: %s" % e)
        return self.record(command, filename, only_values=True)
    
    #------------------------------------------------------------------------
    
    def record_input(self, command, path):
        """ Records data to the input file
            Args:
                command: the set of parameters
                path: the path to the output file
        """
        try:
            output_name = cti.cti_plugin_config_get_value(cti.PLUGIN_INPUT_FILENAME)
            if output_name:
                filename = os.path.join(path, output_name)
            else:
                util.hapi_fail("Can't get value from config file")
        except OSError, e:
            util.hapi_fail("Failed to concat path: %s" % e)
        return self.record(command, filename, only_values=True)
    
    #------------------------------------------------------------------------
    
    def record(self, command, filename, only_values=False):
        """ Record all values for a given command in a given file.
        
        Args:
          command: for which command
          filename: an optional filename to which the data should be recorded
        """
        # JSON begin
        d = self[command]
        jd = {}
        jd[command] = {}
        if only_values:
            jd[command]["attributes"] = {cti.META_ATTRIBUTE_NAME:command}
        else:
            jd[command]["attributes"] = d.attributes
            
        
        params_list = []
        # marshal dict
        for k in d.params:
            if d.params[k][cti.META_ATTRIBUTE_NAME] != cti.META_ATTRIBUTE_REP_PRODUCE:
                params_list.append(d.params[k])
        new_params_list = []
        for l in params_list:
            # Copy the dict to avoid modification it
            p = dict(l)
            #Setting the correct empty value for lists
            if cti.META_ATTRIBUTE_LIST in p and p[cti.META_ATTRIBUTE_LIST] and (cti.META_ATTRIBUTE_VALUE not in p or p[cti.META_ATTRIBUTE_VALUE] is None):
                p[cti.META_ATTRIBUTE_VALUE]=[]
            
            # For data entries remove everything but name and values
            if only_values:
                allowed = [cti.META_ATTRIBUTE_NAME, cti.META_ATTRIBUTE_VALUE]
                to_remove = []
                for k in p:
                    if k not in allowed: to_remove.append(k)
                for k in to_remove: del(p[k])
            new_params_list.append(p)
            
        jd[command]["params"] = new_params_list
        f = open(filename, 'w')
        json.dump(types.marshall(jd), f, indent=4, allow_nan=False)
        f.close()
    
    #------------------------------------------------------------------------
    
    def load(self, filename, default_data=None, no_none_value = False, remove_outdated_params=False):
        filename = os.path.abspath(filename)
        try:
            f = open(filename, 'r')
            try:
                jd = json.load(f, encoding="utf_8")
            except ValueError, e:
                print filename
                util.fatal("JSON file is incorrect. {0}".
                              format(e),
                              cti.CTI_ERROR_UNEXPECTED)
            
            # When plugin_uid is defined, load the
            # type, list and other meta attributes
            # from the plugin default input file.
            if default_data:
                # if the entry does not contain all the parameters (could happen if CTI has been updated), we add them
                if not no_none_value:
                    for cname, command in default_data.iteritems():
                        if cname in jd:
                            for param in command.params:
                                corresponding_params = [d[cti.META_ATTRIBUTE_NAME] for d in jd[cname]["params"]]
                                
                                if param not in corresponding_params:
                                    none_value = None
                                    #Matrix none value for new parameters
                                    if command.params[param][cti.META_ATTRIBUTE_TYPE] == cti.META_CONTENT_ATTRIBUTE_TYPE_MATRIX:
                                        none_value = dict([(c,None) for c in command.params[param][cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]])
                                    #List none value for new parameters
                                    elif cti.META_ATTRIBUTE_LIST in (command.params[param]) and\
                                                    command.params[param][cti.META_ATTRIBUTE_LIST]:
                                        none_value = []
                                    
                                    #Adding the new parameter
                                    jd[cname]["params"].append({cti.META_ATTRIBUTE_NAME: param,
                                                                cti.META_ATTRIBUTE_VALUE: none_value,
                                                                cti.META_ATTRIBUTE_TYPE: command.params[param][cti.META_ATTRIBUTE_TYPE]})
                                else:
                                    corresponding_param_index = [d[cti.META_ATTRIBUTE_NAME] for d in jd[cname]["params"]].index(param)
                                    #Processing matrices updates
                                    if command.params[param][cti.META_ATTRIBUTE_TYPE] == cti.META_CONTENT_ATTRIBUTE_TYPE_MATRIX:
                                        if cti.META_ATTRIBUTE_VALUE in jd[cname]["params"][corresponding_param_index]:
                                                old_values = jd[cname]["params"][corresponding_param_index][cti.META_ATTRIBUTE_VALUE]
                                        else:
                                            jd[cname]["params"][corresponding_param_index][cti.META_ATTRIBUTE_VALUE] = {}
                                            old_values = {}
                                        old_columns = old_values.keys()
                                        
                                        #Warning on old params
                                        for column_name in old_columns:
                                            if column_name not in command.params[param][cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]:
                                                util.cti_plugin_print_warning("Matrix parameter '%s' doesn't have a column named '%s'. UID: %s"%(param, column_name, self.uid))
                                        
                                        #Creating a default void value to fill the eventual new columns of the matrix
                                        default_column_value = []
                                        if old_columns:
                                            default_column_value = len(old_values[old_columns[0]]) * [None]
                                        
                                        #Generating missing columns with default values
                                        for column_name in command.params[param][cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]:
                                            if column_name not in old_columns:
                                                jd[cname]["params"][corresponding_param_index][cti.META_ATTRIBUTE_VALUE][column_name] = default_column_value
                                        
                for cname, command in jd.iteritems():
                    #Validating command existance.
                    if not cname in default_data:
                        util.hapi_fail("Command %s doesn't exist in ctr_default file."%cname)
                    outdated_params = []
                    for v in command["params"]:
                        if not v[cti.META_ATTRIBUTE_NAME] in default_data[cname].params:
                            util.cti_plugin_print_warning("Command %s doesn't accept parameter %s. UID: %s"%(cname, v[cti.META_ATTRIBUTE_NAME], self.uid))
                            if remove_outdated_params:
                                outdated_params.append(command["params"].index(v))
                                
                        if default_data[cname].params.has_key(v[cti.META_ATTRIBUTE_NAME]):
                            if cti.META_ATTRIBUTE_LIST in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_LIST] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_LIST]
                                
                            if cti.META_ATTRIBUTE_TYPE in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_TYPE] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_TYPE]
                            else:
                                util.hapi_fail("Filename %s: can't get value type for parameter %s."%(filename, v[cti.META_ATTRIBUTE_NAME]))
                                
                            if cti.META_ATTRIBUTE_DESC in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_DESC] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_DESC]
                            if cti.META_ATTRIBUTE_LONG_DESC in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_LONG_DESC] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_LONG_DESC]
                                
                            if cti.META_ATTRIBUTE_PASSWORD in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_PASSWORD] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_PASSWORD]
                        if v.has_key(cti.META_ATTRIBUTE_TYPE) and v[cti.META_ATTRIBUTE_TYPE] == cti.META_CONTENT_ATTRIBUTE_TYPE_MATRIX:
                            if cti.META_ATTRIBUTE_LIST in v and v[cti.META_ATTRIBUTE_LIST]:
                                util.hapi_fail("Filename %s: illegal list attribute for MATRIX parameter %s."%filename, v[cti.META_ATTRIBUTE_NAME])
                            if cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES in default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]:
                                v[cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]
                            else:
                                util.hapi_fail("Filename %s: can't get column names for MATRIX parameter %s."%(filename, v[cti.META_ATTRIBUTE_NAME]))
                                
                            if cti.META_ATTRIBUTE_MATRIX_COLUMN_TYPES in default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]:
                                v[cti.META_ATTRIBUTE_MATRIX_COLUMN_TYPES] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_MATRIX_COLUMN_TYPES]
                            else:
                                util.hapi_fail("Filename %s: can't get column types for MATRIX parameter %s."%(filename, v[cti.META_ATTRIBUTE_NAME]))
                            
                            if cti.META_ATTRIBUTE_MATRIX_COLUMN_DESCS in default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]:
                                v[cti.META_ATTRIBUTE_MATRIX_COLUMN_DESCS] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_MATRIX_COLUMN_DESCS]
                            else:
                                util.hapi_fail("Filename %s: can't get column desc for MATRIX parameter %s."%(filename, v[cti.META_ATTRIBUTE_NAME]))
                            
                            if cti.META_ATTRIBUTE_MATRIX_COLUMN_LONG_DESCS in default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]:
                                v[cti.META_ATTRIBUTE_MATRIX_COLUMN_LONG_DESCS] = default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_MATRIX_COLUMN_LONG_DESCS]
                            else:
                                util.hapi_fail("Filename %s: can't get column long_desc for MATRIX parameter %s."%(filename, v[cti.META_ATTRIBUTE_NAME]))
                        if default_data[cname].params.has_key(v[cti.META_ATTRIBUTE_NAME]):
                            if cti.META_ATTRIBUTE_PRODUCED_BY in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_PRODUCED_BY] = (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_PRODUCED_BY])
                                
                            if cti.META_ATTRIBUTE_TARGET in (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]]):
                                v[cti.META_ATTRIBUTE_TARGET] = (default_data[cname].params[v[cti.META_ATTRIBUTE_NAME]][cti.META_ATTRIBUTE_TARGET])
                                
                    if remove_outdated_params:
                        outdated_params.reverse()
                        for op in outdated_params:
                            del command["params"][op]
                            
            #Formatting values
            for command in jd.values():
                for v in command["params"]:
                    islist = (cti.META_ATTRIBUTE_LIST in v and v[cti.META_ATTRIBUTE_LIST])
                    #Trigger a critical failure on missing value
                    if not cti.META_ATTRIBUTE_VALUE in v:
                        #If it's a plugin with no default value, we skip
                        if self.datatype == cti.CTR_ENTRY_PLUGIN:
                            continue
                        if islist:
                            v[cti.META_ATTRIBUTE_VALUE] = []
                        else:
                            v[cti.META_ATTRIBUTE_VALUE] = None
                            
                    if not cti.META_ATTRIBUTE_TYPE in v:
                        ptype='TEXT'
                    else:
                        ptype = v[cti.META_ATTRIBUTE_TYPE]
                    if islist:
                        #Trigger a critical failure on wrongly formated list
                        if not isinstance(v[cti.META_ATTRIBUTE_VALUE], (list, type(None))):
                            util.hapi_fail("CORRUPTED file '{0}: parameter '{1}' should be a list, but contains '{2}' of type '{3}' instead "\
                                       .format(filename, v[cti.META_ATTRIBUTE_NAME], v[cti.META_ATTRIBUTE_VALUE], type(v[cti.META_ATTRIBUTE_VALUE])))
                    matrix_types = None
                    if ptype == cti.META_CONTENT_ATTRIBUTE_TYPE_MATRIX:
                        matrix_types = dict([(v[cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES][i], v[cti.META_ATTRIBUTE_MATRIX_COLUMN_TYPES][i]) for i in range(len(v[cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]))])
                    
                    v[cti.META_ATTRIBUTE_VALUE] = types.from_json(v[cti.META_ATTRIBUTE_VALUE], ptype,
                                                 islist, matrix_types)
                    
            for name in jd.keys():
                params = odict()
                for p in jd[name]["params"]:
                    params[p[cti.META_ATTRIBUTE_NAME]] = p
                    
                self[name] = CmdNode(jd[name]["attributes"], params)
                
        except IOError as e:
            util.cti_plugin_print_error("Could not load %s"%filename)
            raise e
        except Exception as e:
            util.cti_plugin_print_error("Error decoding %s"%filename)
            util.cti_plugin_print_error(str(e))
            raise e
    
    #------------------------------------------------------------------------
    
    def update_references(self, command, data_uid, old_value=None):
        """ Update references
            Args:
                command: the set of parameters
                data_uid: the entry UID
                old_value: if provided, contains a dict of the old params of the entry
        """
        
        plugin_uid = load_data_info(data_uid)[cti.DATA_INFO_PLUGIN_UID]
        plugin_name = util_uid.uid_visualization(util_uid.CTI_UID(plugin_uid, cti.CTR_ENTRY_PLUGIN), cti.CTR_ENTRY_PLUGIN)
        
        data_uid = str(data_uid)
        
        params_list = self[command].params
        for p in params_list:
            if cti.META_ATTRIBUTE_TYPE in params_list[p] and \
                   params_list[p][cti.META_ATTRIBUTE_TYPE] == cti.META_CONTENT_ATTRIBUTE_TYPE_DATA_UID and \
                   cti.META_ATTRIBUTE_VALUE in params_list[p]:
                
                #Coming from an update
                if old_value is not None:
                    #If the field was not updated
                    if p not in old_value:
                        continue
                #Coming from init with empty value/list
                elif params_list[p][cti.META_ATTRIBUTE_VALUE] is None or \
                        (isinstance(params_list[p][cti.META_ATTRIBUTE_VALUE],list) and params_list[p][cti.META_ATTRIBUTE_VALUE] == []):
                    continue
                
                if cti.META_ATTRIBUTE_TARGET in params_list[p]:
                    old_values_list = []
                    if old_value and \
                       old_value.has_key(params_list[p][cti.META_ATTRIBUTE_NAME]) and \
                       old_value[params_list[p][cti.META_ATTRIBUTE_NAME]]:
                            # Get old value
                            old_values_list = old_value[params_list[p][cti.META_ATTRIBUTE_NAME]]
                    
                    params_list_str = map(str, params_list[p][cti.META_ATTRIBUTE_VALUE])
                    old_values_list = map(str, old_values_list)
                    
                    # If old list and new list are the same, do nothing
                    if params_list_str != old_values_list:
                        add_list = []
                        # Search the values to delete (old_values_list) and
                        #        the values to add (add_list)
                        for v in params_list_str:
                            if old_values_list.count(v) != 0:
                                try:
                                    old_values_list.remove(v)
                                except:
                                    util.hapi_error("Error with list in entry <%s>" % data_uid)
                            else:
                                add_list.append(v)
                                
                        old_values_list = map(util_uid.CTI_UID, old_values_list)
                        # Update values on the "delete" list
                        for d in old_values_list:
                            if d:
                                # Check the delete is not already done
                                (_, out_entry) = load_data(d)
                                old_value_del = out_entry[command].params[params_list[p][cti.META_ATTRIBUTE_TARGET]][cti.META_ATTRIBUTE_VALUE]
                                if str(old_value_del) == data_uid:
                                    update_entry_parameter(d, {params_list[p][cti.META_ATTRIBUTE_TARGET] :{"value": ""}})
                                    
                        add_list = map(util_uid.CTI_UID, add_list)
                        # Update values on the "add" list
                        for a in add_list:
                            if a:
                                # Check the add is not already done
                                (_, out_entry) = load_data(a)
                                old_value_add = out_entry[command].params[params_list[p][cti.META_ATTRIBUTE_TARGET]][cti.META_ATTRIBUTE_VALUE]
                                if str(old_value_add) != data_uid:
                                    update_entry_parameter(a, {params_list[p][cti.META_ATTRIBUTE_TARGET] :{"value": data_uid}})
                                    
                elif cti.META_ATTRIBUTE_PRODUCED_BY in params_list[p]:
                    table_target = util_uid.uid_visualization(
                                              util_uid.CTI_UID(str(params_list[p][cti.META_ATTRIBUTE_PRODUCED_BY]), cti.CTR_ENTRY_PLUGIN),
                                              cti.CTR_ENTRY_PLUGIN)
                    # Search the name_source on the link_table
                    res = database_manager.search(
                                                  { 
                                                   'L':{'NAME':["source"], 'TYPE':"=", 'VAL': table_target},
                                                   'LOGIC':'AND',
                                                   'R':{
                                                        'L':{'NAME':["target"], 'TYPE':"=", 'VAL': plugin_name},
                                                       'LOGIC':'AND',
                                                       'R':{'NAME':["name_target"], 'TYPE':"=", 'VAL': params_list[p][cti.META_ATTRIBUTE_NAME]}
                                                       }
                                                  },
                                                 database.Database(),
                                                 "link_table",
                                                 ["name_source"]
                                                )
                    target_name = ""
                    for r in res:
                        target_name = r[0]
                    if target_name:
                        if old_value and \
                           old_value.has_key(params_list[p][cti.META_ATTRIBUTE_NAME]) and \
                           old_value[params_list[p][cti.META_ATTRIBUTE_NAME]]:
                            if str(old_value[params_list[p][cti.META_ATTRIBUTE_NAME]]) != str(params_list[p][cti.META_ATTRIBUTE_VALUE]):
                                # Load old value and check the update is not already done
                                (_, out_entry) = load_data(old_value[params_list[p][cti.META_ATTRIBUTE_NAME]])
                                old_values_list = out_entry[command].params[target_name][cti.META_ATTRIBUTE_VALUE]
                                
                                old_values_list = map(str, old_values_list)
                                
                                # Check the update is not already done
                                if old_values_list.count(data_uid) != 0:
                                    # Update the list
                                    old_values_list.remove(data_uid)
                                    # Update the old value
                                    update_entry_parameter(old_value[params_list[p][cti.META_ATTRIBUTE_NAME]],
                                                           {target_name: {"value": old_values_list}})
                                        
                        if params_list[p][cti.META_ATTRIBUTE_VALUE]:
                            # Load new value and check the update is not already done
                            (_, out_entry) = load_data(util_uid.CTI_UID(str(params_list[p][cti.META_ATTRIBUTE_VALUE])))
                            new_value = out_entry[command].params[target_name][cti.META_ATTRIBUTE_VALUE]
                            
                            new_value = map(str, new_value)
                            
                            if data_uid not in new_value:
                                # Update the new value
                                update_entry_parameter(params_list[p][cti.META_ATTRIBUTE_VALUE],
                                                       {target_name : {"value": [data_uid], "append": True}})

#------------------------------------------------------------------------

def load_defaults(plugin_uid):
    """Load default values for a given plugin.
    Args:
      plugin_uid: a plugin UID for which to load defaults.
    
    Returns:
      A tuple with input/output dictionaries.
    """
    datatype = cti.CTR_ENTRY_PLUGIN
    input_file = cti.cti_plugin_config_get_value(cti.PLUGIN_DEFAULT_INPUT_FILENAME)
    output_file = cti.cti_plugin_config_get_value(cti.PLUGIN_DEFAULT_OUTPUT_FILENAME)
    
    try:
        return (Commands(datatype, plugin_uid, input_file), Commands(datatype, plugin_uid, output_file))
    except Exception as e:
        raise e

#------------------------------------------------------------------------

def load_data_info(uid):
    """ Parses ctr_info.txt file and returns a dictionary for it.
    Args:
      uid: an uid or alias for data
    Returns:
      A dictionary on success. This dictionary represents the contents of
      the ctr_info.txt file. None value is returend on failure.
    """
    
    info_file = ctr.ctr_plugin_info_file_load_by_uid(uid)
    if info_file is None:
        util.hapi_error("Can't load the entry %s\n" % uid)
        
    result = {
                cti.DATA_INFO_PLUGIN_UID : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_PLUGIN_UID),
                cti.DATA_INFO_ADDITIONAL_FILES : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_ADDITIONAL_FILES),
                cti.DATA_INFO_ALIAS : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_ALIAS),
                cti.DATA_INFO_DATE_TIME_END : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_DATE_TIME_END),
                cti.DATA_INFO_DATE_TIME_START : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_DATE_TIME_START),
                cti.DATA_INFO_PLUGIN_EXIT_CODE : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_PLUGIN_EXIT_CODE),
                cti.DATA_INFO_NOTE : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_NOTE),
                cti.DATA_INFO_TAG : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_TAG),
                cti.DATA_INFO_USER_UID : ctr.ctr_plugin_info_get_value(info_file, cti.DATA_INFO_USER_UID)
            }
    return result

#------------------------------------------------------------------------

def load_data(data_uid, remove_outdated_params=False):
    """ Loads data for a given uid.
    Args:
      data_uid: UID
    Returns:
      A tuple with Commands dictionaries (Commands(input), Commands(output)).
    """
    
    entry_path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, data_uid)
    if not entry_path:
        util.fatal("Cannot find entry <{0}>".
                  format(data_uid),
                  cti.CTI_ERROR_UNEXPECTED)
        
    if not util_uid.is_valid_uid(data_uid):
        util.cti_plugin_print_error("Wrong UID or alias: %s" % data_uid)
        exit(cti.CTI_PLUGIN_ERROR_INVALID_ARGUMENTS)
    # When working with data entry, we need the creator plugin_uid
    # so we can get "list", "optional" and "type" meta attributes
    # we can get this info from the ctr_info file.
    info = load_data_info(data_uid)
    if not info:
        util.cti_plugin_print_error("Can't find a data entry for given data uid %s " % data_uid)
        exit(cti.CTI_PLUGIN_ERROR_INVALID_ARGUMENTS)
        
    plugin_uid = util_uid.CTI_UID(info[cti.DATA_INFO_PLUGIN_UID], cti.CTR_ENTRY_PLUGIN)
    inp, out = load_defaults(plugin_uid)
    
    # process input
    in_basename = cti.cti_plugin_config_get_value(cti.PLUGIN_INPUT_FILENAME)
    input_data = Commands(cti.CTR_ENTRY_DATA, data_uid, in_basename, inp, remove_outdated_params=remove_outdated_params)
    # process output
    out_basename = cti.cti_plugin_config_get_value(cti.PLUGIN_OUTPUT_FILENAME)
    output_data = Commands(cti.CTR_ENTRY_DATA, data_uid, out_basename, out, remove_outdated_params=remove_outdated_params)
    return (input_data, output_data)

#------------------------------------------------------------------------

class Entry(object):
    """ Represents an entry. """
    def __init__(self, entry, path, uid):
        self.path = path
        self.uid = uid
        self.entry = entry

#------------------------------------------------------------------------

def get_file_from_entry(uid, file_path, safe=True, destname=None):
    """ Get <file_path> inside entry <uid>
        
    Args:
      uid: CTI_UID of the entry (data entry by default)
      file_path: the file to get
      safe: if safe is True, we check that a file with
            the same name does not already exists.
      destname: destination filename
      plugin: if set then uid considered as plugin entry
    Effect:
      copies <file_path> to the current directory.
    """
    if not util_uid.is_valid_uid(uid):
        util.fatal("The uid <{0}> is not a valid UID".
                  format(uid),
                  cti.CTI_ERROR_UNEXPECTED)
    if file_path in DataEntryMetaFiles:
        util.hapi_error("metafile get is forbidden")
        return
    path_type = ctr.ctr_plugin_get_path_and_type_by_uid(uid)
    fdir = cti.cti_plugin_config_get_value(cti.CTR_ENTRY_FILES_DIR)
    if path_type:
        path = path_type.key
        src = os.path.join(path, fdir, file_path)
        dest = os.path.join(os.getcwd(), file_path)
        if safe and os.path.exists(dest):
            util.hapi_error("File <%s> already exists" % dest)
            return
        if not destname:
            shutil.copy(src, dest)
        else:
            shutil.copy(src, destname)
    else:
        util.hapi_error("Could not find entry <%s>" % uid)

#------------------------------------------------------------------------

def get_dir_from_entry(uid, dst):
    """ Get files inside entry <uid>
    Args:
      uid: CTI_UID of the entry
        
    Effect:
      copies everything from entry's files subdir to the dst
    """
    if not util_uid.is_valid_uid(uid):
        util.fatal("The uid <{0}> is not a valid UID".
                  format(uid),
                  cti.CTI_ERROR_UNEXPECTED)
    entry_path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, uid)
    if not entry_path:
        util.fatal("Cannot find entry <{0}>".
                  format(uid),
                  cti.CTI_ERROR_UNEXPECTED)
        
    fdir = cti.cti_plugin_config_get_value(cti.CTR_ENTRY_FILES_DIR)
    path = os.path.join(entry_path, fdir)
    try:
        distutils.dir_util.copy_tree(path, dst)
    except distutils.dir_util.DistutilsFileError, why:
        util.hapi_fail("%s " % str(why))

#------------------------------------------------------------------------

def update_entry_parameter(entry_uid, values, command="init"):
    """ Update an entry
    
    Args:
      entry_uid: the entry UID
      values: a dictionary of keys and values to update. for example: {"a": {"value":"toto"}, "b":{"value": ["titi"], "append": True}}
                append is not mandatory, and should be used with list only. If append is false, the previous list is fully replaced.
      command: the command of the parameter to update
        
    Returns: 0 if it fails, 1 if it succeeds
    """
    def update_info(key, value):
        data_info = ctr.ctr_plugin_info_file_load_by_uid(entry_uid)
        ctr.ctr_plugin_info_put_value(data_info, key, value)
        ctr_info_name = os.path.join(ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA,
                                                                    entry_uid),
                                    cti.cti_plugin_config_get_value(cti.DATA_INFO_FILENAME))
        ctr.ctr_plugin_info_record_in_file(data_info, ctr_info_name)
    
    def update_db(key, value, entry_uid, plugin_uid, append, db, type_v):
        if key in ["repository", "path_repository", cti.DATA_INFO_DATE_TIME_START]:
            if database_manager.update("entry_info", {key: value}, {'NAME':["entry_uid"], 'TYPE':"=", 'VAL':str(entry_uid)}, db) is False:
                return False
        else:
            plugin_alias = alias.get_plugin_alias(plugin_uid)
            id_entry = database_manager.uid2id(entry_uid, db)
            _,output = load_defaults(plugin_uid)
            key_defaults = output["init"].params[key]
            if cti.META_ATTRIBUTE_LIST in key_defaults and key_defaults[cti.META_ATTRIBUTE_LIST]:
                if type_v != cti.META_CONTENT_ATTRIBUTE_TYPE_DATA_UID:
                    table_temp = "%s_%s" % (plugin_alias, key)
                    id_temp = "id_{0}".format(plugin_alias)
                    if not append:
                        database_manager.delete(table_temp, {'NAME':[id_temp], 'TYPE':"=", 'VAL':id_entry}, db)
                    if len(value) > 0 and not util_uid.is_valid_uid(value[0]):
                        rows = []
                        for v in value:
                            rows.append({key:v, id_temp:id_entry})
                        if database_manager.insert_rows(table_temp, rows, db) is False:
                            return False
            elif type_v == cti.META_CONTENT_ATTRIBUTE_TYPE_MATRIX:
                table_temp = "%s_%s" % (plugin_alias, key)
                id_table_temp = "id_{0}".format(plugin_alias)
                matrix_columns = key_defaults[cti.META_ATTRIBUTE_MATRIX_COLUMN_NAMES]
                
                if not append:
                    database_manager.delete(table_temp, {'NAME':[id_table_temp], 'TYPE':"=", 'VAL':id_entry}, db)
                if value and value[matrix_columns[0]]:
                    for line in range(len(value[matrix_columns[0]])):
                        line_dict = dict([(column, value[column][line]) for column in matrix_columns])
                        line_dict[id_table_temp] = id_entry
                        if database_manager.insert(table_temp, line_dict, db) is False:
                            return False
            else:
                if util_uid.is_valid_uid(value):
                    value = database_manager.uid2id(value, db)
                if database_manager.update(plugin_alias, 
                                      {key: value},
                                      {'NAME':["id_%s" % plugin_alias], 'TYPE':"=", 'VAL':str(id_entry)}, db) is False:
                    return False
        return True
    
    db = database.Database()
    if entry_uid is None:
        return 0
    res = load_data(entry_uid)
    if res is None:
        return 0
    (_, out) = res
    
    old_value = {}
    for k in values.keys():
        #Setting default "undefined" old value
        old_value[k] = None
        if isinstance(values[k]["value"], list):
            old_value[k] = []
        
        append = False
        if values[k].has_key("append"):
            append = values[k]["append"]
           
        if k in [cti.DATA_INFO_DATE_TIME_START]:
            update_info(k, values[k]["value"])
        else:
            if out[command].params.has_key(k):
                old_value[k] = copy.copy(out[command].params[k][cti.META_ATTRIBUTE_VALUE])
                
            if isinstance(values[k]["value"], list) and append:
                if not out[command].params[k].has_key(cti.META_ATTRIBUTE_VALUE):
                    out[command].params[k][cti.META_ATTRIBUTE_VALUE] = []
                out[command].params[k][cti.META_ATTRIBUTE_VALUE] += values[k]["value"]
            elif k not in ["repository", "path_repository"]:
                out[command].params[k][cti.META_ATTRIBUTE_VALUE] = values[k]["value"]
                
        info = load_data_info(entry_uid)
        
        type_v = ""
        if out[command].params.has_key(k) and out[command].params[k].has_key(cti.META_ATTRIBUTE_TYPE):
            type_v = out[command].params[k][cti.META_ATTRIBUTE_TYPE]
        if not update_db(k, values[k]["value"], entry_uid, cti.CTI_UID(info[cti.DATA_INFO_PLUGIN_UID]), append, db, type_v):
            return 0
    path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, entry_uid)
    out.record_output(command, path)
    out.update_references("init", entry_uid, old_value)
    
    # Update the end date
    date_end = str(time.strftime('%Y/%m/%d %H:%M:%S',time.localtime()))
    if database_manager.update("entry_info", {"date_time_end": date_end}, {'NAME':["entry_uid"], 'TYPE':"=", 'VAL':str(entry_uid)}, db) is False:
        return 0
    update_info(cti.DATA_INFO_DATE_TIME_END, date_end)
    return 1

#------------------------------------------------------------------------

def create_data_entry(data_id, rep, username, alias_data=None):
    return create_entry(data_id, rep, cti.CTR_ENTRY_DATA, username, alias_data)

#------------------------------------------------------------------------

def create_plugin_entry(plugin_id, rep, username, alias_plugin=None):
    return create_entry(plugin_id, rep, cti.CTR_ENTRY_PLUGIN, username, alias_plugin)

#------------------------------------------------------------------------

def create_entry(uid, rep_type, datatype, username, alias_entry=None):
    """General wrapper for create_entry.
    
    Args:
      uid: CTI_UID
      rep_type: the repostitory type
      datatype: the CTR entry data type (CTR_ENTRY_PLUGIN or CTR_ENTRY_DATA)
      username: the username
      alias_entry: the alias
    Returns:
      An Entry object or None if failed
    """
    rep_type = str(rep_type).strip()
    local_repository = None
    
    # recognize repository type and convert it
    if rep_type == cti.COMMON_REPOSITORY or rep_type == "1":
        rep = cti.CTR_REP_COMMON
        
    elif cti.cti_plugin_is_UID(rep_type):
        rep = cti.CTR_REP_LOCAL
        local_repository = util_uid.CTI_UID(rep_type, cti.CTR_ENTRY_REPOSITORY)
        # We update the last_use date of the repository
        now = datetime.datetime.now()
        date = now.strftime('%Y/%m/%d %H:%M:%S')
        try:
            update_entry_parameter(local_repository, {"last_use": {"value": date}})
        except:
            print "Repository entry not found"
            
    elif rep_type == cti.LOCAL_REPOSITORY or rep_type == "0":
        rep = cti.CTR_REP_LOCAL
        if not repository.local_exist():
            print "Can't create data entry."
            print "%s \n %s \n %s\n" % (cti.CTI_ERROR_MSG_REP_DOESNT_EXISTS,
                                        cti.CTI_ERROR_MSG_CREATE_REP,
                                        cti.CTI_ERROR_MSG_IMPORT_REP)
            exit(cti.CTI_PLUGIN_ERROR_LOCAL_REP_DOESNT_EXISTS)
            
        # We update the last_use date of the repository
        repository_path = repository.get_local_rep()
        uid_rep = ctr.ctr_plugin_global_index_file_get_uid_by_ctr(repository_path)
        now = datetime.datetime.now()
        date = now.strftime('%Y/%m/%d %H:%M:%S')
        if uid_rep is not None:
            update_entry_parameter(uid_rep, {"last_use": {"value": date}})
    elif rep_type == cti.TEMP_REPOSITORY or rep_type == "2":
        rep = cti.CTR_REP_TEMP
    else:
        #trying to see if it is a repository alias
        local_repository = alias.get_repository_uid(rep_type)
        if local_repository is None:
            #<NEED-FIX The following redundant line has been added because hapi_error() doesn't print anything on the console, leaving the user confused.
            util.cti_plugin_print_error("Unkown repository type {0}".format(rep_type))
            #NEED-FIX>
            util.hapi_error("Unkown repository type")
            return None
        else:
            rep = cti.CTR_REP_LOCAL
            now = datetime.datetime.now()
            date = now.strftime('%Y/%m/%d %H:%M:%S')
            update_entry_parameter(local_repository, {"last_use": {"value": date}})
    
    db = database.Database()
    result = list(database_manager.search_uids(
                                                  {'NAME':["username"], 'TYPE':"=", 'VAL':username},
                                                  db,
                                                  "user"
                                              ))
    
    if len(result)  == 1:
        user_uid = result[0]
    else:
        util.hapi_error("Error while converting username to user_uid.")
        return None
    
    if user_uid is None:
        util.hapi_error("Error with username_to_user_uid.")
        return None
    
    # create entry
    x = ctr.ctr_plugin_create_entry(uid, rep, cti.CTR_ENTRY_DATA, local_repository, user_uid)
    if x is None:
        print(cti.CTI_ERROR_MSG_CANT_CREATE_ENTRY)
        exit(cti.CTI_ERROR_UNEXPECTED)
    output_uid = cti.CTI_UID(x.key)
    output_dir = str(x)
    # check alias
    if alias_entry is not None:
        if datatype == cti.CTR_ENTRY_PLUGIN:
            if alias.set_plugin_alias(output_uid, alias_entry) == 0:
                util.cti_plugin_print_error("Cannot set the alias %s (already used?)" % (alias_entry))
        elif datatype == cti.CTR_ENTRY_DATA:
            if alias.set_data_alias(output_uid, alias_entry) == 0:
                util.cti_plugin_print_error("Cannot set the alias %s (already used?)"%(alias_entry))
        else:
            util.hapi_error("Can't set alias %s. Unkown data type. " % alias_entry)
            return None    

    if datatype == cti.CTR_ENTRY_DATA:
        update_entry_parameter(user_uid, {"last_entry_created": {"value": output_uid}})
    return Entry(x, output_dir, output_uid)

#------------------------------------------------------------------------

def put_file_in_entry(uid, file_path, safe=True, filename=None):
    """ Put <file_path> inside entry <uid>.
        
    Args:
      uid: CTI_UID of the entry
      file_path: the file to put, path is relative to the
            current working directory. It could be a list of files
      safe: if safe is True, we check that a file with
            the same name does not already exists.
      filename: the filename of the file to put. For the moment, work only if file_path is NOT a list @todo
    """
    if not(isinstance(file_path, list)):
        file_path=[file_path]

    dir_append = cti.cti_plugin_config_get_value(cti.CTR_ENTRY_FILES_DIR)
    path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, uid)
    if path:
        path_dest = os.path.join(path, dir_append)
        if (not os.path.exists(path_dest)):
            os.makedirs(path_dest)

        filename_tmp = None
        for f in file_path:
            if f in DataEntryMetaFiles:
                util.cti_plugin_print_error("metafile put is forbidden")
                return
            if not filename:
                filename_tmp = os.path.basename(f)
            else:
                filename_tmp = filename
            dest = os.path.join(path_dest, filename_tmp)
           
            if safe and os.path.exists(dest):
                util.cti_plugin_print_error("File <%s> already exists" % dest)
                return
            shutil.copy(f, dest)
        return filename_tmp
        
    else:
        util.cti_plugin_print_error("Could not find entry <%s>" % uid)


#------------------------------------------------------------------------

def put_dir_in_entry(uid, path=None, dir_dest=""):
    """ Put <path> inside entry <uid>.
    
    Args:
      uid: CTI_UID of the entry
      path: the directory to put
      dir_dest: the destination directory
    """
    
    pdine_log_names = {"log":[]}
    if not path:
        path = os.getcwd()

    entry_path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, uid)
    if not entry_path:
        util.hapi_fail("Can't find \"%s\" UID. \n"
                        "Hint: try to import your current repository with \"cti repository import\""
                       % uid)
        return []
    
    path = os.path.abspath(path)
    fdir = cti.cti_plugin_config_get_value(cti.CTR_ENTRY_FILES_DIR)
    # log filenames
    # TODO: refactor, add custom exclude patterns as a parameter
    #------------------------------------------------------------------------
    def save_list(src, names):
        toignore = []
        l = pdine_log_names["log"]
        for n in names:
            if n.find('.ctr') != -1:
                toignore.append(n)
            else:
                src = os.path.abspath(src)
                src_d = src[len(path)+1:]
                if n != "." and n != "./":
                    if not os.path.isdir(os.path.join(src, n)):
                        if n.startswith("./"):
                            l.append(str(os.path.join(src_d, n[2:])))
                        else:
                            l.append(str(os.path.join(src_d, n)))
        if src.find('.ctr') != -1:
            toignore.append(src)
        return toignore
    #------------------------------------------------------------------------
    
    #------------------------------------------------------------------------
    def copytree(src, dst, symlinks=False, ignore=None, prefix=""):
        for item in os.listdir(src):
            s = os.path.join(src, item)
            d = os.path.join(dst, item)
            if os.path.isdir(s):
                # if directory already exists...
                if os.path.isdir(d):
                    copytree(s,d,symlinks,ignore, prefix=item)
                else:
                    shutil.copytree(s, d, symlinks, ignore)
            else:
                shutil.copy2(s, d)
                pdine_log_names["log"].append(os.path.join(prefix, item))
    #------------------------------------------------------------------------
    try:
        dest = os.path.join(entry_path, fdir, dir_dest)
        if not os.path.isdir(dest):
            os.makedirs(dest)
        copytree(path, dest, ignore=save_list)
    except (IOError, os.error), why:
        util.hapi_fail("%s " % str(why))
    return pdine_log_names["log"]

#------------------------------------------------------------------------

def rm_all_files_from_entry(uid):
    """ Remove all the files from an entry
    
    Args:
      uid: CTI_UID of the entry
    Return
    """
    if not util_uid.is_valid_uid(uid):
        util.fatal("The uid <{0}> is not a valid UID".
                  format(uid),
                  cti.CTI_ERROR_UNEXPECTED)
    path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, uid)
    
    if path:
        src = os.path.join(path, "files")
        try:
            shutil.rmtree(src)
        except:
            print "Error when removing the files"
            return -1
        
    else:
        util.hapi_error("Could not find entry <%s>" % uid)
        return -1
    return 0

#------------------------------------------------------------------------

def rm_file_from_entry(uid, filename):
    """ Remove a file from an entry
    
    Args:
      uid: CTI_UID of the entry
      filename: filename of the file to remove
    Return
    """
    if not util_uid.is_valid_uid(uid):
        util.fatal("The uid <{0}> is not a valid UID".
                  format(uid),
                  cti.CTI_ERROR_UNEXPECTED)
    path = ctr.ctr_plugin_get_path_by_uid(cti.CTR_ENTRY_DATA, uid)
    
    if path:
        src = os.path.join(path, "files", filename)
        try:
            if os.path.isfile(src):
                os.remove(src)
            else:
                shutil.rmtree(src)
        except:
            print "Error when removing the file or the directory"
            return -1
        
    else:
        util.hapi_error("Could not find entry <%s>" % uid)
        return -1
    return 0
#------------------------------------------------------------------------
