(self, para)
增量学习模板
            如果没匹配上，会生成新的模板，然后返回新的模板号
            每条日志单条学习，流式数据学习
增量学习模板
            如果没匹配上，会生成新的模板，然后返回新的模板号
            每条日志单条学习，流式数据学习
增量学习模板 如果没匹配上，会生成新的模板，然后返回新的模板号 每条日志单条学习，流式数据学习
def matchLogsAndLearnTemplateOneByOne(self, para):
        '''
            增量学习模板
            如果没匹配上，会生成新的模板，然后返回新的模板号
            每条日志单条学习，流式数据学习
        '''
        template_path = para['template_path']
        new_logs_path = para['log_path']
        out_seq_path = para['out_seq_path']
        short_threshold = para['short_threshold']
        match_model = para['match_model']

        f = open(out_seq_path, 'w')
        short_log = 0
        # short_threshold = 5
        count_zero = 0
        total_num = 0
        with open(new_logs_path) as IN:
            for line in IN:
                total_num +=1
                timestamp = line.strip().split()[0]
                log_words = ft_tree.getMsgFromNewSyslog(line)[1]
                tag, cur_match = self.match(log_words, match_model)
                # print (line.strip())
                # print ('~~cur_match:',cur_match)
                # print ('')
                if len(log_words)< short_threshold:#过滤长度小于5的日志
                    short_log+=1
                    tag = -1


                #如果匹配不上，则增量学习模板
                if tag == 0:
                    print ('learned a new template:')
                    count_zero += 1
                    #增量学习
                    # temp_tree=self.tree
                    print(line)
                    cur_log_once_list=[['', log_words]]
                    self.tree.auto_temp(cur_log_once_list, self.words_frequency, para)
                    new_tag = len(self.template_tag_dir)+1

                    #添加完新的模板之后，重新匹配日志，把新的模板match到的文本输出出来
                    tag, cur_match = self.match(log_words)
                    self.template_tag_dir[cur_match]=new_tag
                    self.tag_template_dir[new_tag]=cur_match
                    #第三次匹配模板，输出目前匹配的tag
                    tag, cur_match = self.match(log_words)
#                    self.drawTree()
                    print (tag, cur_match)
                    # print ('')
                    #保存新的模板
                    ff = open(template_path,'a')
                    ff.writelines(str(tag)+' '+cur_match+'\n')
                    ff.close()
                #匹配到了输出1~n，没匹配到输出新增量学习的模板号，日志小于过滤长度输出-1
                f.writelines(timestamp+' '+str(tag)+'\n')

        print ('filting # short logs:',short_log,'| threshold =',short_threshold)
        print ('# of unmatched log (except filting):', count_zero)
        print ('# of total logs:',total_num)
        print ('seq_file_path:',out_seq_path)

        if para['plot_flag'] == 1:
            self.drawTree()
def matchLogsAndLearnTemplateOneByOne ( self , para ) : template_path = para [ 'template_path' ] new_logs_path = para [ 'log_path' ] out_seq_path = para [ 'out_seq_path' ] short_threshold = para [ 'short_threshold' ] match_model = para [ 'match_model' ] f = open ( out_seq_path , 'w' ) short_log = 0 # short_threshold = 5 count_zero = 0 total_num = 0 with open ( new_logs_path ) as IN : for line in IN : total_num += 1 timestamp = line . strip ( ) . split ( ) [ 0 ] log_words = ft_tree . getMsgFromNewSyslog ( line ) [ 1 ] tag , cur_match = self . match ( log_words , match_model ) # print (line.strip()) # print ('~~cur_match:',cur_match) # print ('') if len ( log_words ) < short_threshold : #过滤长度小于5的日志 short_log += 1 tag = - 1 #如果匹配不上，则增量学习模板 if tag == 0 : print ( 'learned a new template:' ) count_zero += 1 #增量学习 # temp_tree=self.tree print ( line ) cur_log_once_list = [ [ '' , log_words ] ] self . tree . auto_temp ( cur_log_once_list , self . words_frequency , para ) new_tag = len ( self . template_tag_dir ) + 1 #添加完新的模板之后，重新匹配日志，把新的模板match到的文本输出出来 tag , cur_match = self . match ( log_words ) self . template_tag_dir [ cur_match ] = new_tag self . tag_template_dir [ new_tag ] = cur_match #第三次匹配模板，输出目前匹配的tag tag , cur_match = self . match ( log_words ) #                    self.drawTree() print ( tag , cur_match ) # print ('') #保存新的模板 ff = open ( template_path , 'a' ) ff . writelines ( str ( tag ) + ' ' + cur_match + '\n' ) ff . close ( ) #匹配到了输出1~n，没匹配到输出新增量学习的模板号，日志小于过滤长度输出-1 f . writelines ( timestamp + ' ' + str ( tag ) + '\n' ) print ( 'filting # short logs:' , short_log , '| threshold =' , short_threshold ) print ( '# of unmatched log (except filting):' , count_zero ) print ( '# of total logs:' , total_num ) print ( 'seq_file_path:' , out_seq_path ) if para [ 'plot_flag' ] == 1 : self . drawTree ( )