(self, sess, outputs_tf, fd, batch_size)
Evaluates the performance of the network on the whole test set.

        :param sess: tensorflow session to be used
        :param outputs_tf: object that represents callable tensorflow outputs
        :param fd: list of feed_dict variables [x, y, lenghts, mask]
        :param batch_size:
        :return:
Evaluates the performance of the network on the whole test set.
Evaluates the performance of the network on the whole test set .
def test(self, sess, outputs_tf, fd, batch_size):
        """
        Evaluates the performance of the network on the whole test set.

        :param sess: tensorflow session to be used
        :param outputs_tf: object that represents callable tensorflow outputs
        :param fd: list of feed_dict variables [x, y, lenghts, mask]
        :param batch_size:
        :return:
        """
        from time import time
        t = time()
        print("Testing...")
        num_passed_tasks = 0
        num_tasks = len(self.x_test_stories)
        task_errors = []
        for ind, (inp, output) in enumerate(zip(self.x_test_stories.items(), self.y_test_stories.items())):
            # if inp[0] != self.test_list[0]:
            #     continue
            correct_questions = 0
            total_questions = 0
            num_stories = len(inp[1])
            # num_stories = 30

            """
            Processing each story with other batch - 1 stories. Stupid hack because I can't have variable batch 
            size because I'm using tf.unstack and for some reason it doesn't work.
            """
            x = [bAbITask.to_onehot(i, self.vector_size) for i in inp[1][:batch_size]]
            y = [bAbITask.to_onehot(i, self.vector_size) for i in output[1][:batch_size]]
            for index in range(num_stories):
                x = list(x)
                y = list(y)
                x[0] = bAbITask.to_onehot(inp[1][index], self.vector_size)
                y[0] = bAbITask.to_onehot(output[1][index], self.vector_size)
                x, y, lengths = self.pad_stories(x, y)
                m = x[:, :, :1]
                outputs = sess.run(outputs_tf, feed_dict={fd[0]: x, fd[1]: y, fd[2]: lengths, fd[3]: m})
                """
                Each story (one whole sequence) has several questions and each of those questions might have several
                words as an answer.
                So a list of answer indices like [23, 56, 52, 45, 68] might only correspond to three questions.
                In the code below we loop through the answers, check if the question indices in the story are adjacent
                and count the adjacent words as one question only.
                Sanity check is that total number of questions turns out to be 1000 in all tasks :)
                """

                outputs_list = bAbITask.tensor_to_indices(outputs[:1], m[:1])
                correct_list = bAbITask.tensor_to_indices(y[:1], m[:1])
                answers = y[0] * m[0, :, :1]
                locations = np.argwhere(answers > 0)[:, 0]
                i = 0
                while i < len(locations):
                    all_words_correct = True
                    j = 0
                    while i + j < len(locations) and locations[i] + j == locations[i + j]:
                        if outputs_list[i + j] != correct_list[i + j]:
                            all_words_correct = False
                        j += 1
                    total_questions += 1
                    correct_questions += all_words_correct
                    i += j

            task_name = inp[0].split("/")[-1]
            task_error = 1 - correct_questions / total_questions
            print(ind, task_name, " Total_correct:", correct_questions, " total questions:", total_questions,
                  " task error:", task_error * 100, "%")
            num_passed_tasks += task_error <= 0.05
            task_errors.append(task_error)
        mean_error = np.mean(task_errors)
        print("TOTAL PASSED TASKS:", num_passed_tasks, "TOTAL TASKS:", num_tasks, " MEAN ERROR", mean_error)
        self.mean_test_errors.append(mean_error)
        print("ALL ERRORS: ", self.mean_test_errors)
        print("Time == ", time() - t)
def test ( self , sess , outputs_tf , fd , batch_size ) : from time import time t = time ( ) print ( "Testing..." ) num_passed_tasks = 0 num_tasks = len ( self . x_test_stories ) task_errors = [ ] for ind , ( inp , output ) in enumerate ( zip ( self . x_test_stories . items ( ) , self . y_test_stories . items ( ) ) ) : # if inp[0] != self.test_list[0]: #     continue correct_questions = 0 total_questions = 0 num_stories = len ( inp [ 1 ] ) # num_stories = 30 """
            Processing each story with other batch - 1 stories. Stupid hack because I can't have variable batch 
            size because I'm using tf.unstack and for some reason it doesn't work.
            """ x = [ bAbITask . to_onehot ( i , self . vector_size ) for i in inp [ 1 ] [ : batch_size ] ] y = [ bAbITask . to_onehot ( i , self . vector_size ) for i in output [ 1 ] [ : batch_size ] ] for index in range ( num_stories ) : x = list ( x ) y = list ( y ) x [ 0 ] = bAbITask . to_onehot ( inp [ 1 ] [ index ] , self . vector_size ) y [ 0 ] = bAbITask . to_onehot ( output [ 1 ] [ index ] , self . vector_size ) x , y , lengths = self . pad_stories ( x , y ) m = x [ : , : , : 1 ] outputs = sess . run ( outputs_tf , feed_dict = { fd [ 0 ] : x , fd [ 1 ] : y , fd [ 2 ] : lengths , fd [ 3 ] : m } ) """
                Each story (one whole sequence) has several questions and each of those questions might have several
                words as an answer.
                So a list of answer indices like [23, 56, 52, 45, 68] might only correspond to three questions.
                In the code below we loop through the answers, check if the question indices in the story are adjacent
                and count the adjacent words as one question only.
                Sanity check is that total number of questions turns out to be 1000 in all tasks :)
                """ outputs_list = bAbITask . tensor_to_indices ( outputs [ : 1 ] , m [ : 1 ] ) correct_list = bAbITask . tensor_to_indices ( y [ : 1 ] , m [ : 1 ] ) answers = y [ 0 ] * m [ 0 , : , : 1 ] locations = np . argwhere ( answers > 0 ) [ : , 0 ] i = 0 while i < len ( locations ) : all_words_correct = True j = 0 while i + j < len ( locations ) and locations [ i ] + j == locations [ i + j ] : if outputs_list [ i + j ] != correct_list [ i + j ] : all_words_correct = False j += 1 total_questions += 1 correct_questions += all_words_correct i += j task_name = inp [ 0 ] . split ( "/" ) [ - 1 ] task_error = 1 - correct_questions / total_questions print ( ind , task_name , " Total_correct:" , correct_questions , " total questions:" , total_questions , " task error:" , task_error * 100 , "%" ) num_passed_tasks += task_error <= 0.05 task_errors . append ( task_error ) mean_error = np . mean ( task_errors ) print ( "TOTAL PASSED TASKS:" , num_passed_tasks , "TOTAL TASKS:" , num_tasks , " MEAN ERROR" , mean_error ) self . mean_test_errors . append ( mean_error ) print ( "ALL ERRORS: " , self . mean_test_errors ) print ( "Time == " , time ( ) - t )