(self,args)
self.INP.vocab.stoi['<pad>']=0
    self.INP.vocab.stoi['<unk>']=1
    self.INP.vocab.itos = ['<pad>','<unk>']+self.INP.vocab.itos[2:]
self.INP.vocab.stoi['<pad>']=0
    self.INP.vocab.stoi['<unk>']=1
    self.INP.vocab.itos = ['<pad>','<unk>']+self.INP.vocab.itos[2:]
self . INP . vocab . stoi [ <pad > ] = 0 self . INP . vocab . stoi [ <unk > ] = 1 self . INP . vocab . itos = [ <pad > <unk > ] + self . INP . vocab . itos [ 2 : ]
def mkVocabs(self,args):
    args.path = args.datadir + args.data
    self.INP = data.Field(sequential=True, batch_first=True,init_token="<start>", eos_token="<eos>",include_lengths=True)
    self.OUTP = data.Field(sequential=True, batch_first=True,init_token="<start>", eos_token="<eos>",include_lengths=True)
    self.TGT = data.Field(sequential=True, batch_first=True,init_token="<start>", eos_token="<eos>")
    self.ENT = data.RawField()
    self.REL = data.RawField()
    self.REL.is_target = False 
    self.ENT.is_target = False 
    self.fields=[("src",self.INP),("ent",self.ENT),("rel",self.REL),("tgt",self.TGT),("out",self.OUTP)]
    train = data.TabularDataset(path=args.path, format='tsv',fields=self.fields)

    print('building vocab')
    self.OUTP.build_vocab(train, min_freq=args.outunk)   
    self.TGT.vocab = copy(self.OUTP.vocab)
    specials = zip("method material otherscientificterm metric task".split(" "),range(40))
    for x in specials:
      s = "<"+x[0]+"_"+str(x[1])+">"
      self.TGT.vocab.stoi[s] = len(self.TGT.vocab.itos)+x[1]

    self.INP.build_vocab(train, min_freq=args.outunk)   
    '''
    self.INP.vocab.stoi['<pad>']=0
    self.INP.vocab.stoi['<unk>']=1
    self.INP.vocab.itos = ['<pad>','<unk>']+self.INP.vocab.itos[2:]
    '''

    self.REL.special = ['<pad>','<unk>','ROOT']
    with open(args.datadir+"/"+args.relvocab) as f:
      rvocab = [x.strip() for x in f.readlines()]
      self.REL.size = len(rvocab)
      rvocab += [x+"_inv" for x in rvocab]
      relvocab = self.REL.special + rvocab
    self.REL.itos = relvocab

    self.ENT.itos,self.ENT.stoi = self.build_ent_vocab(args.path)

    print('done')
def mkVocabs ( self , args ) : args . path = args . datadir + args . data self . INP = data . Field ( sequential = True , batch_first = True , init_token = "<start>" , eos_token = "<eos>" , include_lengths = True ) self . OUTP = data . Field ( sequential = True , batch_first = True , init_token = "<start>" , eos_token = "<eos>" , include_lengths = True ) self . TGT = data . Field ( sequential = True , batch_first = True , init_token = "<start>" , eos_token = "<eos>" ) self . ENT = data . RawField ( ) self . REL = data . RawField ( ) self . REL . is_target = False self . ENT . is_target = False self . fields = [ ( "src" , self . INP ) , ( "ent" , self . ENT ) , ( "rel" , self . REL ) , ( "tgt" , self . TGT ) , ( "out" , self . OUTP ) ] train = data . TabularDataset ( path = args . path , format = 'tsv' , fields = self . fields ) print ( 'building vocab' ) self . OUTP . build_vocab ( train , min_freq = args . outunk ) self . TGT . vocab = copy ( self . OUTP . vocab ) specials = zip ( "method material otherscientificterm metric task" . split ( " " ) , range ( 40 ) ) for x in specials : s = "<" + x [ 0 ] + "_" + str ( x [ 1 ] ) + ">" self . TGT . vocab . stoi [ s ] = len ( self . TGT . vocab . itos ) + x [ 1 ] self . INP . build_vocab ( train , min_freq = args . outunk ) self . REL . special = [ '<pad>' , '<unk>' , 'ROOT' ] with open ( args . datadir + "/" + args . relvocab ) as f : rvocab = [ x . strip ( ) for x in f . readlines ( ) ] self . REL . size = len ( rvocab ) rvocab += [ x + "_inv" for x in rvocab ] relvocab = self . REL . special + rvocab self . REL . itos = relvocab self . ENT . itos , self . ENT . stoi = self . build_ent_vocab ( args . path ) print ( 'done' )