(self, tasks_dir)
Init tries to read from pickle file, if it doesn't exist it creates it.
        Creates separate dictionaries of train and test sets, data and labels

        :param tasks_dir: relative directory of bAbI tasks
Init tries to read from pickle file, if it doesn't exist it creates it.
        Creates separate dictionaries of train and test sets, data and labels
Init tries to read from pickle file if it doesn t exist it creates it . Creates separate dictionaries of train and test sets data and labels
def __init__(self, tasks_dir):
        """
        Init tries to read from pickle file, if it doesn't exist it creates it.
        Creates separate dictionaries of train and test sets, data and labels

        :param tasks_dir: relative directory of bAbI tasks
        """
        self.tasks_dir = os.path.join(bAbITask.base, tasks_dir)
        self.processed_dir = self.tasks_dir + bAbITask.processed_append
        self.files_path = []

        for f in os.listdir(self.tasks_dir):
            f_path = os.path.join(self.tasks_dir, f)
            if os.path.isfile(f_path):
                self.files_path.append(f_path)

        if not os.path.isfile(self.processed_dir):
            pickle.dump(self.preprocess_files(), open(self.processed_dir, "wb"))
            print("Pickled!", self.processed_dir)
        self.word_to_ind, all_input_stories, all_output_stories = pickle.load(open(self.processed_dir, "rb"))
        self.ind_to_word = {ind: word for word, ind in self.word_to_ind.items()}

        from natsort import natsorted
        self.train_list = natsorted([k for k, v in all_input_stories.items() if k[-9:] == "train.txt"])
        self.test_list = natsorted([k for k, v in all_input_stories.items() if k[-8:] == "test.txt"])

        self.vector_size = len(self.word_to_ind)
        self.n_tasks = 20

        self.x_train_stories = {k: v for k, v in all_input_stories.items() if k in self.train_list}
        self.y_train_stories = {k: v for k, v in all_output_stories.items() if k in self.train_list}

        self.x_test_stories = {k: v for k, v in all_input_stories.items() if k in self.test_list}
        self.y_test_stories = {k: v for k, v in all_output_stories.items() if k in self.test_list}

        assert len(self.x_train_stories.keys()) == len(self.y_train_stories.keys())

        # shape [batch_size, length, vector_size]
        self.x_shape = [None, None, self.vector_size]
        self.y_shape = [None, None, self.vector_size]
        self.mask = [None, None, 1]

        self.mean_test_errors = []
def __init__ ( self , tasks_dir ) : self . tasks_dir = os . path . join ( bAbITask . base , tasks_dir ) self . processed_dir = self . tasks_dir + bAbITask . processed_append self . files_path = [ ] for f in os . listdir ( self . tasks_dir ) : f_path = os . path . join ( self . tasks_dir , f ) if os . path . isfile ( f_path ) : self . files_path . append ( f_path ) if not os . path . isfile ( self . processed_dir ) : pickle . dump ( self . preprocess_files ( ) , open ( self . processed_dir , "wb" ) ) print ( "Pickled!" , self . processed_dir ) self . word_to_ind , all_input_stories , all_output_stories = pickle . load ( open ( self . processed_dir , "rb" ) ) self . ind_to_word = { ind : word for word , ind in self . word_to_ind . items ( ) } from natsort import natsorted self . train_list = natsorted ( [ k for k , v in all_input_stories . items ( ) if k [ - 9 : ] == "train.txt" ] ) self . test_list = natsorted ( [ k for k , v in all_input_stories . items ( ) if k [ - 8 : ] == "test.txt" ] ) self . vector_size = len ( self . word_to_ind ) self . n_tasks = 20 self . x_train_stories = { k : v for k , v in all_input_stories . items ( ) if k in self . train_list } self . y_train_stories = { k : v for k , v in all_output_stories . items ( ) if k in self . train_list } self . x_test_stories = { k : v for k , v in all_input_stories . items ( ) if k in self . test_list } self . y_test_stories = { k : v for k , v in all_output_stories . items ( ) if k in self . test_list } assert len ( self . x_train_stories . keys ( ) ) == len ( self . y_train_stories . keys ( ) ) # shape [batch_size, length, vector_size] self . x_shape = [ None , None , self . vector_size ] self . y_shape = [ None , None , self . vector_size ] self . mask = [ None , None , 1 ] self . mean_test_errors = [ ]