parameters: (self)
docstring: run DMR, creating an output file divided by time
docstring_summary: run DMR, creating an output file divided by time
docstring_tokens: run DMR creating an output file divided by time
function: def process(self):
          """
          run DMR, creating an output file divided by time
          """
  
          if self.named_args is not None:
              self.tfidf = self.named_args['tfidf']
              self.min_df = int(self.named_args['min_df'])
              self.stemming = self.named_args['stemming']
              self.topics = int(self.named_args['topics'])
              self.lang = self.named_args['lang']
          else:
              self.tfidf = True
              self.min_df = 5
              self.topics = 50
              self.stemming = True
              self.lang = 'en'
  
          self._setup_mallet_instances(tfidf=self.tfidf,
                  stemming=self.stemming)
  
          os.chdir(self.mallet_out_dir)
  
          # from cc.mallet.topics.DMRTopicModel import main as DMRTopicModel
          from cc.mallet.topics import DMRTopicModel
          process_args = [self.instance_file, str(self.topics)]
          logging.info('begin DMR')
  
          start_time = time.time()
          self.parameter_file = os.path.join(self.mallet_out_dir,
                           'dmr.parameters')
          self.state_file = os.path.join(self.mallet_out_dir,
                           'dmr.state.gz')
          if not self.dry_run:
              # DMRTopicModel(process_args)
              from java.io import File, PrintStream, FileOutputStream
              from java.lang import System
  
              self.progress_file.close()
              progress_file = File(self.progress_filename)
              System.setOut(PrintStream(FileOutputStream(progress_file)))
  
              from cc.mallet.types import InstanceList
              training = InstanceList.load(File(self.instance_file))
              numTopics = int(self.topics)
              lda = DMRTopicModel(numTopics)
              lda.setOptimizeInterval(100)
              lda.setTopicDisplay(100, 10)
              lda.addInstances(training)
              lda.estimate()
              lda.writeParameters(File(self.parameter_file))
              lda.printState(File(self.state_file))
  
          logging.info('DMR complete in ' + str(time.time() - start_time)
                       + ' seconds')
  
          self.topic_features = {}
          with codecs.open(self.parameter_file, 'r', encoding='utf-8') as f:
              topic = 0
              for line in f:
                  new_topic = re.match('FEATURES FOR CLASS topic([0-9]+)'
                          , line)
                  if new_topic is not None:
                      topic = int(new_topic.group(1))
                  else:
                      if not topic in self.topic_features:
                          self.topic_features[topic] = {}
                      this_line = line.split(' ')
                      feature = this_line[1]
                      self.topic_features[topic][feature] = \
                          float(this_line[2])
  
          self.progress_file = file(self.progress_filename, 'r')
          self.progress_file.seek(0, os.SEEK_SET)
          self.alphas = {}
          for line in self.progress_file:
              if re.match('[0-9]+\t[0-9.]+', line) is not None:
                  this_line = line.split('\t')
                  topic = int(this_line[0])
                  alpha = float(this_line[1])
                  tokens = int(this_line[2])
  
                  self.alphas[topic] = alpha
  
          self.alpha_sum = sum(self.alphas.values())
  
          self.topic_words = {}
          self.doc_topics = {}
  
          with gzip.open(self.state_file, 'rb') as state_file:
              state_file.next()
              for line in state_file:
                  this_line = line.split(' ')
                  topic = int(this_line[5])
                  word = this_line[4]
                  doc = int(this_line[0])
                  position = int(this_line[2])
  
                  if not doc in self.doc_topics:
                      self.doc_topics[doc] = {}
                  if not topic in self.doc_topics[doc]:
                      self.doc_topics[doc][topic] = 0
                  self.doc_topics[doc][topic] += 1
  
                  if not topic in self.topic_words:
                      self.topic_words[topic] = {}
                  if not word in self.topic_words[topic]:
                      self.topic_words[topic][word] = 0
                  self.topic_words[topic][word] += 1
  
          # total_tokens = float(sum([sum(y.values()) for x, y in self.topic_words.iteritems()]))
  
          for topic in self.topic_words.keys():
              total = float(sum(self.topic_words[topic].values()))
              for k in self.topic_words[topic].keys():
                  self.topic_words[topic][k] /= total
  
          top_N = 20
          top_topic_words = dict((x, dict((word, y[word]) for word in
                                 self.argsort(y, reverse=True)[:top_N]))
                                 for (x, y) in
                                 self.topic_words.iteritems())
          wordProbs = [[{'text': word, 'prob': prob} for (word, prob) in
                       y.iteritems()] for (x, y) in
                       top_topic_words.iteritems()]
  
          DEFAULT_DOC_PROPORTIONS = [
              0.01,
              0.02,
              0.05,
              0.1,
              0.2,
              0.3,
              0.5,
              ]
          numDocumentsAtProportions = dict((topic, dict((k, 0.0) for k in
                  DEFAULT_DOC_PROPORTIONS)) for topic in
                  self.topic_words.keys())
          for (doc, topics) in self.doc_topics.iteritems():
              doc_length = sum(topics.values())
              for (topic, count) in topics.iteritems():
                  proportion = (self.alphas[topic] + count) \
                      / (self.alpha_sum + doc_length)
                  for min_proportion in DEFAULT_DOC_PROPORTIONS:
                      if proportion < min_proportion:
                          break
                      numDocumentsAtProportions[topic][min_proportion] += \
                          1
  
          allocationRatios = dict((topic, proportions[0.5]
                                  / proportions[0.02]) for (topic,
                                  proportions) in
                                  numDocumentsAtProportions.iteritems()
                                  if proportions[0.02] > 0.0)
  
          labels = dict((topic, {'label': self.argsort(words,
                        reverse=True)[:3], 'fulltopic': wordProbs[topic],
                        'allocation_ratio': allocationRatios.get(topic,0)})
                        for (topic, words) in top_topic_words.iteritems())
  
          doc_metadata = {}
  
          for doc in self.doc_topics.keys():
              total = float(sum(self.doc_topics[doc].values()))
              for k in self.doc_topics[doc].keys():
                  self.doc_topics[doc][k] /= total
  
          for (id, topics) in self.doc_topics.iteritems():
              try:
                  filename = self.docs[int(id)]
  
                  itemid = self.metadata[filename]['itemID']
  
                  doc_metadata[itemid] = \
                      {'label': self.metadata[filename]['label'],
                       'title': self.metadata[filename]['title']}
  
                  freqs = topics
                  main_topic = None
                  topic_max = 0.0
                  for i in freqs.keys():
                      if freqs[i] > topic_max:
                          main_topic = i
                          topic_max = freqs[i]
                  doc_metadata[itemid]['main_topic'] = main_topic
                  self.metadata[filename]["topics"] = freqs
              except KeyboardInterrupt:
                  sys.exit(1)
              except:
                  logging.error(traceback.format_exc())
  
          self.template_filename = os.path.join(self.cwd, 'templates',
                  self.template_name + '.html')
  
          if getattr(self, "index", None) is not None:
              for term in self.index:
                  if isinstance(self.index[term], set):
                      self.index[term] = list(self.index[term])
              self.index = dict(self.index)
  
          params = {"CATEGORICAL": self.categorical,
                          "TOPIC_LABELS": labels,
                          "TOPIC_COHERENCE": {},
                          "TAGS": getattr(self, "tags", {}),
                          "INDEX": getattr(self, "index", {})
          }
  
          self.write_html(params)
function_tokens: def process ( self ) : if self . named_args is not None : self . tfidf = self . named_args [ 'tfidf' ] self . min_df = int ( self . named_args [ 'min_df' ] ) self . stemming = self . named_args [ 'stemming' ] self . topics = int ( self . named_args [ 'topics' ] ) self . lang = self . named_args [ 'lang' ] else : self . tfidf = True self . min_df = 5 self . topics = 50 self . stemming = True self . lang = 'en' self . _setup_mallet_instances ( tfidf = self . tfidf , stemming = self . stemming ) os . chdir ( self . mallet_out_dir ) # from cc.mallet.topics.DMRTopicModel import main as DMRTopicModel from cc . mallet . topics import DMRTopicModel process_args = [ self . instance_file , str ( self . topics ) ] logging . info ( 'begin DMR' ) start_time = time . time ( ) self . parameter_file = os . path . join ( self . mallet_out_dir , 'dmr.parameters' ) self . state_file = os . path . join ( self . mallet_out_dir , 'dmr.state.gz' ) if not self . dry_run : # DMRTopicModel(process_args) from java . io import File , PrintStream , FileOutputStream from java . lang import System self . progress_file . close ( ) progress_file = File ( self . progress_filename ) System . setOut ( PrintStream ( FileOutputStream ( progress_file ) ) ) from cc . mallet . types import InstanceList training = InstanceList . load ( File ( self . instance_file ) ) numTopics = int ( self . topics ) lda = DMRTopicModel ( numTopics ) lda . setOptimizeInterval ( 100 ) lda . setTopicDisplay ( 100 , 10 ) lda . addInstances ( training ) lda . estimate ( ) lda . writeParameters ( File ( self . parameter_file ) ) lda . printState ( File ( self . state_file ) ) logging . info ( 'DMR complete in ' + str ( time . time ( ) - start_time ) + ' seconds' ) self . topic_features = { } with codecs . open ( self . parameter_file , 'r' , encoding = 'utf-8' ) as f : topic = 0 for line in f : new_topic = re . match ( 'FEATURES FOR CLASS topic([0-9]+)' , line ) if new_topic is not None : topic = int ( new_topic . group ( 1 ) ) else : if not topic in self . topic_features : self . topic_features [ topic ] = { } this_line = line . split ( ' ' ) feature = this_line [ 1 ] self . topic_features [ topic ] [ feature ] = float ( this_line [ 2 ] ) self . progress_file = file ( self . progress_filename , 'r' ) self . progress_file . seek ( 0 , os . SEEK_SET ) self . alphas = { } for line in self . progress_file : if re . match ( '[0-9]+\t[0-9.]+' , line ) is not None : this_line = line . split ( '\t' ) topic = int ( this_line [ 0 ] ) alpha = float ( this_line [ 1 ] ) tokens = int ( this_line [ 2 ] ) self . alphas [ topic ] = alpha self . alpha_sum = sum ( self . alphas . values ( ) ) self . topic_words = { } self . doc_topics = { } with gzip . open ( self . state_file , 'rb' ) as state_file : state_file . next ( ) for line in state_file : this_line = line . split ( ' ' ) topic = int ( this_line [ 5 ] ) word = this_line [ 4 ] doc = int ( this_line [ 0 ] ) position = int ( this_line [ 2 ] ) if not doc in self . doc_topics : self . doc_topics [ doc ] = { } if not topic in self . doc_topics [ doc ] : self . doc_topics [ doc ] [ topic ] = 0 self . doc_topics [ doc ] [ topic ] += 1 if not topic in self . topic_words : self . topic_words [ topic ] = { } if not word in self . topic_words [ topic ] : self . topic_words [ topic ] [ word ] = 0 self . topic_words [ topic ] [ word ] += 1 # total_tokens = float(sum([sum(y.values()) for x, y in self.topic_words.iteritems()])) for topic in self . topic_words . keys ( ) : total = float ( sum ( self . topic_words [ topic ] . values ( ) ) ) for k in self . topic_words [ topic ] . keys ( ) : self . topic_words [ topic ] [ k ] /= total top_N = 20 top_topic_words = dict ( ( x , dict ( ( word , y [ word ] ) for word in self . argsort ( y , reverse = True ) [ : top_N ] ) ) for ( x , y ) in self . topic_words . iteritems ( ) ) wordProbs = [ [ { 'text' : word , 'prob' : prob } for ( word , prob ) in y . iteritems ( ) ] for ( x , y ) in top_topic_words . iteritems ( ) ] DEFAULT_DOC_PROPORTIONS = [ 0.01 , 0.02 , 0.05 , 0.1 , 0.2 , 0.3 , 0.5 , ] numDocumentsAtProportions = dict ( ( topic , dict ( ( k , 0.0 ) for k in DEFAULT_DOC_PROPORTIONS ) ) for topic in self . topic_words . keys ( ) ) for ( doc , topics ) in self . doc_topics . iteritems ( ) : doc_length = sum ( topics . values ( ) ) for ( topic , count ) in topics . iteritems ( ) : proportion = ( self . alphas [ topic ] + count ) / ( self . alpha_sum + doc_length ) for min_proportion in DEFAULT_DOC_PROPORTIONS : if proportion < min_proportion : break numDocumentsAtProportions [ topic ] [ min_proportion ] += 1 allocationRatios = dict ( ( topic , proportions [ 0.5 ] / proportions [ 0.02 ] ) for ( topic , proportions ) in numDocumentsAtProportions . iteritems ( ) if proportions [ 0.02 ] > 0.0 ) labels = dict ( ( topic , { 'label' : self . argsort ( words , reverse = True ) [ : 3 ] , 'fulltopic' : wordProbs [ topic ] , 'allocation_ratio' : allocationRatios . get ( topic , 0 ) } ) for ( topic , words ) in top_topic_words . iteritems ( ) ) doc_metadata = { } for doc in self . doc_topics . keys ( ) : total = float ( sum ( self . doc_topics [ doc ] . values ( ) ) ) for k in self . doc_topics [ doc ] . keys ( ) : self . doc_topics [ doc ] [ k ] /= total for ( id , topics ) in self . doc_topics . iteritems ( ) : try : filename = self . docs [ int ( id ) ] itemid = self . metadata [ filename ] [ 'itemID' ] doc_metadata [ itemid ] = { 'label' : self . metadata [ filename ] [ 'label' ] , 'title' : self . metadata [ filename ] [ 'title' ] } freqs = topics main_topic = None topic_max = 0.0 for i in freqs . keys ( ) : if freqs [ i ] > topic_max : main_topic = i topic_max = freqs [ i ] doc_metadata [ itemid ] [ 'main_topic' ] = main_topic self . metadata [ filename ] [ "topics" ] = freqs except KeyboardInterrupt : sys . exit ( 1 ) except : logging . error ( traceback . format_exc ( ) ) self . template_filename = os . path . join ( self . cwd , 'templates' , self . template_name + '.html' ) if getattr ( self , "index" , None ) is not None : for term in self . index : if isinstance ( self . index [ term ] , set ) : self . index [ term ] = list ( self . index [ term ] ) self . index = dict ( self . index ) params = { "CATEGORICAL" : self . categorical , "TOPIC_LABELS" : labels , "TOPIC_COHERENCE" : { } , "TAGS" : getattr ( self , "tags" , { } ) , "INDEX" : getattr ( self , "index" , { } ) } self . write_html ( params )