(self, results, source_url=None, technique="")
return cleaned_results
Allows standardizing extracted contents, at this time:

        1. removes multiple whitespaces
        2. rewrite relative URLs as absolute URLs if source_url is specified
        3. filter out duplicate values
        4. marks the technique that produced the result
        5. returns only specified text_types and url_types depending on self.strict_types
Allows standardizing extracted contents, at this time:
Allows standardizing extracted contents at this time :
def cleanup(self, results, source_url=None, technique=""):
        """
        Allows standardizing extracted contents, at this time:

        1. removes multiple whitespaces
        2. rewrite relative URLs as absolute URLs if source_url is specified
        3. filter out duplicate values
        4. marks the technique that produced the result
        5. returns only specified text_types and url_types depending on self.strict_types
        """
        cleaned_results = {}
        mark = MARK_TECHNIQUE and u"#" + technique.split('.')[-1]

        for data_type, data_values in results.items():
            if data_type in self.text_types:
                data_values = [self.cleanup_text(x, mark) for x in filter(None, data_values)]
            elif data_type in self.url_types:
                data_values = [self.cleanup_url(x, source_url, mark) for x in data_values]
            elif self.strict_types:
                continue

            # filter out duplicate values
            unique_values = []
            for data_value in data_values:
                if data_value not in unique_values:
                    unique_values.append(data_value)

            cleaned_results[data_type] = unique_values

        return cleaned_results
def cleanup ( self , results , source_url = None , technique = "" ) : cleaned_results = { } mark = MARK_TECHNIQUE and u"#" + technique . split ( '.' ) [ - 1 ] for data_type , data_values in results . items ( ) : if data_type in self . text_types : data_values = [ self . cleanup_text ( x , mark ) for x in filter ( None , data_values ) ] elif data_type in self . url_types : data_values = [ self . cleanup_url ( x , source_url , mark ) for x in data_values ] elif self . strict_types : continue # filter out duplicate values unique_values = [ ] for data_value in data_values : if data_value not in unique_values : unique_values . append ( data_value ) cleaned_results [ data_type ] = unique_values return cleaned_results