(model, vocab, data_path, seed=1234, evaltest=False, vocab_size_layer=20002)
Run experiment
Run experiment
Run experiment
def evaluate(model, vocab, data_path, seed=1234, evaltest=False, vocab_size_layer=20002):
    """
    Run experiment
    """
    neon_logger.display('Preparing SICK evaluation data...')
    # Check if SICK data exists in specified directory, otherwise download
    sick_data = SICK(path=data_path)
    train, dev, test, scores = sick_data.load_eval_data()

    np.random.seed(seed)
    shuf_idxs = np.random.permutation(range(len(train[0])))
    train_A_shuf = train[0][shuf_idxs]
    train_B_shuf = train[1][shuf_idxs]
    scores_shuf = scores[0][shuf_idxs]

    train_A_tok = tokenize_input(train_A_shuf, vocab=vocab)
    train_B_tok = tokenize_input(train_B_shuf, vocab=vocab)
    dev_A_tok = tokenize_input(dev[0], vocab=vocab)
    dev_B_tok = tokenize_input(dev[1], vocab=vocab)

    # Get iterator from tokenized data
    train_set_A = SentenceEncode(train_A_tok, [], len(train_A_tok), vocab_size_layer,
                                 max_len=30, index_from=2)
    train_set_B = SentenceEncode(train_B_tok, [], len(train_B_tok), vocab_size_layer,
                                 max_len=30, index_from=2)

    # Compute embeddings using iterator
    trainA = model.get_outputs(train_set_A)
    trainB = model.get_outputs(train_set_B)

    dev_set_A = SentenceEncode(dev_A_tok, [], len(dev_A_tok), vocab_size_layer,
                               max_len=30, index_from=2)
    dev_set_B = SentenceEncode(dev_B_tok, [], len(dev_B_tok), vocab_size_layer,
                               max_len=30, index_from=2)

    devA = model.get_outputs(dev_set_A)
    devB = model.get_outputs(dev_set_B)

    trainF = np.c_[np.abs(trainA - trainB), trainA * trainB]
    devF = np.c_[np.abs(devA - devB), devA * devB]

    trainY = encode_labels(scores_shuf, ndata=len(trainF))
    devY = encode_labels(scores[1], ndata=len(devF))

    lrmodel, opt, cost = prepare_model(ninputs=trainF.shape[1])

    neon_logger.display('Training the regression model...')
    bestlrmodel = train_model(lrmodel, opt, cost, trainF,
                              trainY, devF, devY, scores[1][:len(devF)])

    if evaltest:
        test_A_tok = tokenize_input(test[0], vocab=vocab)
        test_B_tok = tokenize_input(test[1], vocab=vocab)

        test_set_A = SentenceEncode(test_A_tok, [], len(test_A_tok), vocab_size_layer,
                                    max_len=30, index_from=2)
        test_set_B = SentenceEncode(test_B_tok, [], len(test_B_tok), vocab_size_layer,
                                    max_len=30, index_from=2)

        testA = model.get_outputs(test_set_A)
        testB = model.get_outputs(test_set_B)

        testF = np.c_[np.abs(testA - testB), testA * testB]

        neon_logger.display('Evaluating using vectors and linear regression model')
        r = np.arange(1, 6)
        yhat = np.dot(bestlrmodel.get_outputs(ArrayIterator(testF)), r)
        pr = pearsonr(yhat, scores[2][:len(yhat)])[0]
        sr = spearmanr(yhat, scores[2][:len(yhat)])[0]
        se = np.mean((yhat - scores[2][:len(yhat)]) ** 2)
        neon_logger.display('Test Pearson: ' + str(pr))
        neon_logger.display('Test Spearman: ' + str(sr))
        neon_logger.display('Test MSE: ' + str(se))

        return yhat
def evaluate ( model , vocab , data_path , seed = 1234 , evaltest = False , vocab_size_layer = 20002 ) : neon_logger . display ( 'Preparing SICK evaluation data...' ) # Check if SICK data exists in specified directory, otherwise download sick_data = SICK ( path = data_path ) train , dev , test , scores = sick_data . load_eval_data ( ) np . random . seed ( seed ) shuf_idxs = np . random . permutation ( range ( len ( train [ 0 ] ) ) ) train_A_shuf = train [ 0 ] [ shuf_idxs ] train_B_shuf = train [ 1 ] [ shuf_idxs ] scores_shuf = scores [ 0 ] [ shuf_idxs ] train_A_tok = tokenize_input ( train_A_shuf , vocab = vocab ) train_B_tok = tokenize_input ( train_B_shuf , vocab = vocab ) dev_A_tok = tokenize_input ( dev [ 0 ] , vocab = vocab ) dev_B_tok = tokenize_input ( dev [ 1 ] , vocab = vocab ) # Get iterator from tokenized data train_set_A = SentenceEncode ( train_A_tok , [ ] , len ( train_A_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) train_set_B = SentenceEncode ( train_B_tok , [ ] , len ( train_B_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) # Compute embeddings using iterator trainA = model . get_outputs ( train_set_A ) trainB = model . get_outputs ( train_set_B ) dev_set_A = SentenceEncode ( dev_A_tok , [ ] , len ( dev_A_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) dev_set_B = SentenceEncode ( dev_B_tok , [ ] , len ( dev_B_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) devA = model . get_outputs ( dev_set_A ) devB = model . get_outputs ( dev_set_B ) trainF = np . c_ [ np . abs ( trainA - trainB ) , trainA * trainB ] devF = np . c_ [ np . abs ( devA - devB ) , devA * devB ] trainY = encode_labels ( scores_shuf , ndata = len ( trainF ) ) devY = encode_labels ( scores [ 1 ] , ndata = len ( devF ) ) lrmodel , opt , cost = prepare_model ( ninputs = trainF . shape [ 1 ] ) neon_logger . display ( 'Training the regression model...' ) bestlrmodel = train_model ( lrmodel , opt , cost , trainF , trainY , devF , devY , scores [ 1 ] [ : len ( devF ) ] ) if evaltest : test_A_tok = tokenize_input ( test [ 0 ] , vocab = vocab ) test_B_tok = tokenize_input ( test [ 1 ] , vocab = vocab ) test_set_A = SentenceEncode ( test_A_tok , [ ] , len ( test_A_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) test_set_B = SentenceEncode ( test_B_tok , [ ] , len ( test_B_tok ) , vocab_size_layer , max_len = 30 , index_from = 2 ) testA = model . get_outputs ( test_set_A ) testB = model . get_outputs ( test_set_B ) testF = np . c_ [ np . abs ( testA - testB ) , testA * testB ] neon_logger . display ( 'Evaluating using vectors and linear regression model' ) r = np . arange ( 1 , 6 ) yhat = np . dot ( bestlrmodel . get_outputs ( ArrayIterator ( testF ) ) , r ) pr = pearsonr ( yhat , scores [ 2 ] [ : len ( yhat ) ] ) [ 0 ] sr = spearmanr ( yhat , scores [ 2 ] [ : len ( yhat ) ] ) [ 0 ] se = np . mean ( ( yhat - scores [ 2 ] [ : len ( yhat ) ] ) ** 2 ) neon_logger . display ( 'Test Pearson: ' + str ( pr ) ) neon_logger . display ( 'Test Spearman: ' + str ( sr ) ) neon_logger . display ( 'Test MSE: ' + str ( se ) ) return yhat