(self, para)
增量学习模板
            每一时段增量学习一次
增量学习模板
            每一时段增量学习一次
增量学习模板 每一时段增量学习一次
def LearnTemplateByIntervals(self, para):
        '''
            增量学习模板
            每一时段增量学习一次
        '''
        # print (para)
        template_path = para['template_path']
        new_logs_path = para['log_path']
        leaf_num = para['leaf_num']
        short_threshold = para['short_threshold']
        match_model = para['match_model']


        f = open(template_path, 'a')
        short_log = 0
        count_zero = 0
        total_num = 0
        # print('template_tag_dir:',self.template_tag_dir)


        with open(new_logs_path) as IN:
            for line in IN:
                total_num +=1
                timestamp = line.strip().split()[0]
                log_words = ft_tree.getMsgFromNewSyslog(line)[1]
                tag, cur_match = self.match(log_words, match_model)
                # print (line.strip())
                # print ('~~cur_match:',cur_match)
                # print ('')
                if len(log_words)< short_threshold:#过滤长度小于5的日志
                    short_log+=1
                    tag = -1


                #如果匹配不上，则增量学习模板
                if tag == 0:
                    # print ('learned a new template:')
                    count_zero += 1
                    #增量学习
                    # temp_tree=self.tree
                    cur_log_once_list=[['', log_words]]
                    self.tree.auto_temp(cur_log_once_list, self.words_frequency, para)



       # 遍历特征树,每条路径作为一个模板
        all_paths = {}

        for pid in self.tree.tree_list:
            all_paths[pid] = []
            path = self.tree.traversal_tree(self.tree.tree_list[pid])

            for template in path[1]:
                all_paths[pid].append(template)

            # 大集合优先
            # 有的模板是另外一个模板的子集,此时要保证大集合优先`
            all_paths[pid].sort(key=lambda x: len(x), reverse=True)
        # count=0

        typeList = []
        # 将每条模板存储到对应的pid文件夹中

        i = 1
        print ('new templates:')
        for pid in all_paths:
            for path in all_paths[pid]:
                print (i, pid, end=' ')
                # 首先把pid保存下来
                cur_match =' '.join(path)
                for w in path:
                    print (w, end=' ')
                print ( '')
                i += 1
                # if True:
                if cur_match not in self.template_tag_dir:
                    tag = len(self.template_tag_dir)+1
                    self.template_tag_dir[cur_match]=tag
                    f.writelines(str(tag)+' '+cur_match+'\n')
                    print (cur_match)



        with open(new_logs_path) as IN:
            for line in IN:
                total_num +=1
                timestamp = line.strip().split()[0]
                log_words = ft_tree.getMsgFromNewSyslog(line)[1]
                tag, cur_match = self.match(log_words)
                # print (tag, cur_match)
        if para['plot_flag'] == 1:
            self.drawTree()

        print ('filting # short logs:',short_log,'| threshold =',short_threshold)
        print ('# of unmatched log (except filting):', count_zero)
        print ('# of total logs:',total_num)
def LearnTemplateByIntervals ( self , para ) : # print (para) template_path = para [ 'template_path' ] new_logs_path = para [ 'log_path' ] leaf_num = para [ 'leaf_num' ] short_threshold = para [ 'short_threshold' ] match_model = para [ 'match_model' ] f = open ( template_path , 'a' ) short_log = 0 count_zero = 0 total_num = 0 # print('template_tag_dir:',self.template_tag_dir) with open ( new_logs_path ) as IN : for line in IN : total_num += 1 timestamp = line . strip ( ) . split ( ) [ 0 ] log_words = ft_tree . getMsgFromNewSyslog ( line ) [ 1 ] tag , cur_match = self . match ( log_words , match_model ) # print (line.strip()) # print ('~~cur_match:',cur_match) # print ('') if len ( log_words ) < short_threshold : #过滤长度小于5的日志 short_log += 1 tag = - 1 #如果匹配不上，则增量学习模板 if tag == 0 : # print ('learned a new template:') count_zero += 1 #增量学习 # temp_tree=self.tree cur_log_once_list = [ [ '' , log_words ] ] self . tree . auto_temp ( cur_log_once_list , self . words_frequency , para ) # 遍历特征树,每条路径作为一个模板 all_paths = { } for pid in self . tree . tree_list : all_paths [ pid ] = [ ] path = self . tree . traversal_tree ( self . tree . tree_list [ pid ] ) for template in path [ 1 ] : all_paths [ pid ] . append ( template ) # 大集合优先 # 有的模板是另外一个模板的子集,此时要保证大集合优先` all_paths [ pid ] . sort ( key = lambda x : len ( x ) , reverse = True ) # count=0 typeList = [ ] # 将每条模板存储到对应的pid文件夹中 i = 1 print ( 'new templates:' ) for pid in all_paths : for path in all_paths [ pid ] : print ( i , pid , end = ' ' ) # 首先把pid保存下来 cur_match = ' ' . join ( path ) for w in path : print ( w , end = ' ' ) print ( '' ) i += 1 # if True: if cur_match not in self . template_tag_dir : tag = len ( self . template_tag_dir ) + 1 self . template_tag_dir [ cur_match ] = tag f . writelines ( str ( tag ) + ' ' + cur_match + '\n' ) print ( cur_match ) with open ( new_logs_path ) as IN : for line in IN : total_num += 1 timestamp = line . strip ( ) . split ( ) [ 0 ] log_words = ft_tree . getMsgFromNewSyslog ( line ) [ 1 ] tag , cur_match = self . match ( log_words ) # print (tag, cur_match) if para [ 'plot_flag' ] == 1 : self . drawTree ( ) print ( 'filting # short logs:' , short_log , '| threshold =' , short_threshold ) print ( '# of unmatched log (except filting):' , count_zero ) print ( '# of total logs:' , total_num )