Spaces:
Runtime error
Runtime error
increasing figure size. adding names to the publication. adding submission title and using it for scores.
Browse files- app.py +31 -8
- details.html +1 -1
- score.py +6 -4
app.py
CHANGED
@@ -24,14 +24,16 @@ sent_model = SentenceTransformer('sentence-transformers/gtr-t5-base')
|
|
24 |
sent_model.to(device)
|
25 |
|
26 |
def get_similar_paper(
|
|
|
27 |
abstract_text_input,
|
28 |
author_id_input,
|
29 |
results={}, # this state variable will be updated and returned
|
30 |
#progress=gr.Progress()
|
31 |
-
):
|
32 |
-
|
33 |
progress = gr.Progress()
|
34 |
num_papers_show = 10 # number of top papers to show from the reviewer
|
|
|
|
|
35 |
print('retrieving similar papers...')
|
36 |
start = time.time()
|
37 |
input_sentences = sent_tokenize(abstract_text_input)
|
@@ -47,6 +49,7 @@ def get_similar_paper(
|
|
47 |
titles, abstracts, paper_urls, doc_scores = compute_document_score(
|
48 |
doc_model,
|
49 |
tokenizer,
|
|
|
50 |
abstract_text_input,
|
51 |
papers,
|
52 |
batch=10
|
@@ -256,9 +259,9 @@ with gr.Blocks(css='style.css') as demo:
|
|
256 |
# R2P2: Reviewer TO Paper in Peer review
|
257 |
|
258 |
#### Who is it for?
|
259 |
-
It is for meta-reviewers, area chairs, program chairs, or anyone who oversees the submission-reviewer matching process in peer review for
|
260 |
|
261 |
-
<center><img src="file/tool.gif" width="
|
262 |
|
263 |
#### How does it help?
|
264 |
A typical meta-reviewer workflow lacks supportive information on **what makes the pre-selected candidate reviewers a good fit** for the submission. Only affinity scores between the reviewer and the paper are shown, without additional detail.
|
@@ -267,7 +270,7 @@ R2P2 provides more information about each reviewer. It searches for the most rel
|
|
267 |
"""
|
268 |
# TODO add instruction video link
|
269 |
# More details (video, addendum)
|
270 |
-
more_details_instruction = """Check out <a href="", target="_blank">this video</a> for a quick demo of what R2P2 is and how it can help.
|
271 |
|
272 |
gr.Markdown(general_instruction)
|
273 |
gr.HTML(more_details_instruction)
|
@@ -275,20 +278,37 @@ R2P2 provides more information about each reviewer. It searches for the most rel
|
|
275 |
|
276 |
### INPUT
|
277 |
with gr.Row() as input_row:
|
278 |
-
with gr.Column():
|
279 |
-
|
280 |
-
|
|
|
|
|
|
|
281 |
with gr.Row():
|
282 |
author_id_input = gr.Textbox(label='Reviewer Profile Link (Semantic Scholar)', info="Paste in the reviewer's Semantic Scholar link")
|
283 |
with gr.Row():
|
284 |
name = gr.Textbox(label='Confirm Reviewer Name', info='This will be automatically updated based on the reviewer profile link above', interactive=False)
|
285 |
author_id_input.change(fn=update_name, inputs=author_id_input, outputs=name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
286 |
with gr.Row():
|
287 |
compute_btn = gr.Button('What Makes This a Good Match?')
|
288 |
|
289 |
with gr.Row():
|
290 |
search_status = gr.Textbox(label='Search Status', interactive=False, visible=False)
|
291 |
|
|
|
|
|
292 |
### OVERVIEW
|
293 |
# Paper title, score, and top-ranking sentence pairs -- two sentence pairs per paper, three papers
|
294 |
## ONE BLOCK OF INFO FOR A SINGLE PAPER
|
@@ -425,6 +445,8 @@ R2P2 provides more information about each reviewer. It searches for the most rel
|
|
425 |
with gr.Row(visible=False) as hl_row:
|
426 |
# highlighted text from paper
|
427 |
highlight = gr.components.Interpretation(paper_abstract)
|
|
|
|
|
428 |
|
429 |
### EVENT LISTENERS
|
430 |
|
@@ -438,6 +460,7 @@ R2P2 provides more information about each reviewer. It searches for the most rel
|
|
438 |
compute_btn.click(
|
439 |
fn=get_similar_paper,
|
440 |
inputs=[
|
|
|
441 |
abstract_text_input,
|
442 |
author_id_input,
|
443 |
info
|
|
|
24 |
sent_model.to(device)
|
25 |
|
26 |
def get_similar_paper(
|
27 |
+
title_input,
|
28 |
abstract_text_input,
|
29 |
author_id_input,
|
30 |
results={}, # this state variable will be updated and returned
|
31 |
#progress=gr.Progress()
|
32 |
+
):
|
|
|
33 |
progress = gr.Progress()
|
34 |
num_papers_show = 10 # number of top papers to show from the reviewer
|
35 |
+
if title_input == None:
|
36 |
+
title_input = '' # if no title is given, just focus on abstract.
|
37 |
print('retrieving similar papers...')
|
38 |
start = time.time()
|
39 |
input_sentences = sent_tokenize(abstract_text_input)
|
|
|
49 |
titles, abstracts, paper_urls, doc_scores = compute_document_score(
|
50 |
doc_model,
|
51 |
tokenizer,
|
52 |
+
title_input,
|
53 |
abstract_text_input,
|
54 |
papers,
|
55 |
batch=10
|
|
|
259 |
# R2P2: Reviewer TO Paper in Peer review
|
260 |
|
261 |
#### Who is it for?
|
262 |
+
It is for meta-reviewers, area chairs, program chairs, or anyone who oversees the submission-reviewer matching process in peer review for academic conferences, journals, and grants.
|
263 |
|
264 |
+
<center><img src="file/tool.gif" width="70%" alt="general workflow"></center>
|
265 |
|
266 |
#### How does it help?
|
267 |
A typical meta-reviewer workflow lacks supportive information on **what makes the pre-selected candidate reviewers a good fit** for the submission. Only affinity scores between the reviewer and the paper are shown, without additional detail.
|
|
|
270 |
"""
|
271 |
# TODO add instruction video link
|
272 |
# More details (video, addendum)
|
273 |
+
more_details_instruction = """Check out <a href="", target="_blank">this video</a> for a quick demo of what R2P2 is and how it can help. You can find more details <a href="file/details.html", target="_blank">here</a> about R2P2, along with our privacy policy and disclaimer."""
|
274 |
|
275 |
gr.Markdown(general_instruction)
|
276 |
gr.HTML(more_details_instruction)
|
|
|
278 |
|
279 |
### INPUT
|
280 |
with gr.Row() as input_row:
|
281 |
+
with gr.Column(scale=3):
|
282 |
+
with gr.Row():
|
283 |
+
title_input = gr.Textbox(label='Submission Title', info='Paste in the title of the submission.')
|
284 |
+
with gr.Row():
|
285 |
+
abstract_text_input = gr.Textbox(label='Submission Abstract', info='Paste in the abstract of the submission.')
|
286 |
+
with gr.Column(scale=2):
|
287 |
with gr.Row():
|
288 |
author_id_input = gr.Textbox(label='Reviewer Profile Link (Semantic Scholar)', info="Paste in the reviewer's Semantic Scholar link")
|
289 |
with gr.Row():
|
290 |
name = gr.Textbox(label='Confirm Reviewer Name', info='This will be automatically updated based on the reviewer profile link above', interactive=False)
|
291 |
author_id_input.change(fn=update_name, inputs=author_id_input, outputs=name)
|
292 |
+
|
293 |
+
# Add examples
|
294 |
+
example_title ="The Toronto Paper Matching System: An automated paper-reviewer assignment system"
|
295 |
+
example_submission = """One of the most important tasks of conference organizers is the assignment of papers to reviewers. Reviewers' assessments of papers is a crucial step in determining the conference program, and in a certain sense to shape the direction of a field. However this is not a simple task: large conferences typically have to assign hundreds of papers to hundreds of reviewers, and time constraints make the task impossible for one person to accomplish. Furthermore other constraints, such as reviewer load have to be taken into account, preventing the process from being completely distributed. We built the first version of a system to suggest reviewer assignments for the NIPS 2010 conference, followed, in 2012, by a release that better integrated our system with Microsoft's popular Conference Management Toolkit (CMT). Since then our system has been widely adopted by the leading conferences in both the machine learning and computer vision communities. This paper provides an overview of the system, a summary of learning models and methods of evaluation that we have been using, as well as some of the recent progress and open issues."""
|
296 |
+
example_reviewer = "https://www.semanticscholar.org/author/Nihar-B.-Shah/1737249"
|
297 |
+
gr.Examples(
|
298 |
+
examples=[[example_title, example_submission, example_reviewer]],
|
299 |
+
inputs=[title_input, abstract_text_input, author_id_input],
|
300 |
+
cache_examples=False,
|
301 |
+
label="Click to try out the example input."
|
302 |
+
)
|
303 |
+
|
304 |
with gr.Row():
|
305 |
compute_btn = gr.Button('What Makes This a Good Match?')
|
306 |
|
307 |
with gr.Row():
|
308 |
search_status = gr.Textbox(label='Search Status', interactive=False, visible=False)
|
309 |
|
310 |
+
|
311 |
+
|
312 |
### OVERVIEW
|
313 |
# Paper title, score, and top-ranking sentence pairs -- two sentence pairs per paper, three papers
|
314 |
## ONE BLOCK OF INFO FOR A SINGLE PAPER
|
|
|
445 |
with gr.Row(visible=False) as hl_row:
|
446 |
# highlighted text from paper
|
447 |
highlight = gr.components.Interpretation(paper_abstract)
|
448 |
+
|
449 |
+
|
450 |
|
451 |
### EVENT LISTENERS
|
452 |
|
|
|
460 |
compute_btn.click(
|
461 |
fn=get_similar_paper,
|
462 |
inputs=[
|
463 |
+
title_input,
|
464 |
abstract_text_input,
|
465 |
author_id_input,
|
466 |
info
|
details.html
CHANGED
@@ -4,7 +4,7 @@
|
|
4 |
|
5 |
<h1>Relevant Work</h1>
|
6 |
|
7 |
-
The tool is based on the findings from
|
8 |
|
9 |
<h1>What Happens Behind the Scenes</h1>
|
10 |
<ul>
|
|
|
4 |
|
5 |
<h1>Relevant Work</h1>
|
6 |
|
7 |
+
The tool is developed by <a href="https://wnstlr.github.io", target="_blank">Joon Sik Kim</a>, based on the findings from our paper <a href="https://arxiv.org/abs/2302.08450", target="_blank">Assisting Human Decisions in Document Matching</a>, with <a href="https://valeriechen.github.io/", target="_blank">Valerie Chen</a>, <a href="https://danishpruthi.com/", target="_blank">Danish Pruthi</a>, <a href="https://www.cs.cmu.edu/~nihars/", target="_blank">Nihar B. Shah</a>, and <a href="https://www.cs.cmu.edu/~atalwalk/", target="_blank">Ameet Talwalkar</a>.
|
8 |
|
9 |
<h1>What Happens Behind the Scenes</h1>
|
10 |
<ul>
|
score.py
CHANGED
@@ -251,7 +251,6 @@ def get_highlight_info(model, text1, text2, K=None):
|
|
251 |
return sent_ids, sent_scores, info, top_pairs_info
|
252 |
|
253 |
### Document-level operations
|
254 |
-
# TODO Use specter_MFR
|
255 |
def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
|
256 |
# compute document scores for each papers
|
257 |
|
@@ -259,8 +258,7 @@ def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
|
|
259 |
title_abs = []
|
260 |
for t, a in zip(titles, abstracts):
|
261 |
if t is not None and a is not None:
|
262 |
-
|
263 |
-
title_abs.append(a)
|
264 |
|
265 |
num_docs = len(title_abs)
|
266 |
no_iter = int(np.ceil(num_docs / batch))
|
@@ -292,7 +290,7 @@ def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
|
|
292 |
|
293 |
return scores
|
294 |
|
295 |
-
def compute_document_score(doc_model, tokenizer,
|
296 |
scores = []
|
297 |
titles = []
|
298 |
abstracts = []
|
@@ -302,6 +300,10 @@ def compute_document_score(doc_model, tokenizer, query, papers, batch=5):
|
|
302 |
titles.append(p['title'])
|
303 |
abstracts.append(p['abstract'])
|
304 |
urls.append(p['url'])
|
|
|
|
|
|
|
|
|
305 |
scores = predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=batch)
|
306 |
assert(len(scores) == len(abstracts))
|
307 |
idx_sorted = np.argsort(scores)[::-1]
|
|
|
251 |
return sent_ids, sent_scores, info, top_pairs_info
|
252 |
|
253 |
### Document-level operations
|
|
|
254 |
def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
|
255 |
# compute document scores for each papers
|
256 |
|
|
|
258 |
title_abs = []
|
259 |
for t, a in zip(titles, abstracts):
|
260 |
if t is not None and a is not None:
|
261 |
+
title_abs.append(t + ' [SEP] ' + a) # title + abstract
|
|
|
262 |
|
263 |
num_docs = len(title_abs)
|
264 |
no_iter = int(np.ceil(num_docs / batch))
|
|
|
290 |
|
291 |
return scores
|
292 |
|
293 |
+
def compute_document_score(doc_model, tokenizer, query_title, query_abs, papers, batch=5):
|
294 |
scores = []
|
295 |
titles = []
|
296 |
abstracts = []
|
|
|
300 |
titles.append(p['title'])
|
301 |
abstracts.append(p['abstract'])
|
302 |
urls.append(p['url'])
|
303 |
+
if query_title == '':
|
304 |
+
query = query_abs
|
305 |
+
else:
|
306 |
+
query = query_title + ' [SEP] ' + query_abs # feed in submission title and abstract
|
307 |
scores = predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=batch)
|
308 |
assert(len(scores) == len(abstracts))
|
309 |
idx_sorted = np.argsort(scores)[::-1]
|