jskim commited on
Commit
e7933f3
·
1 Parent(s): 573dc49

increasing figure size. adding names to the publication. adding submission title and using it for scores.

Browse files
Files changed (3) hide show
  1. app.py +31 -8
  2. details.html +1 -1
  3. score.py +6 -4
app.py CHANGED
@@ -24,14 +24,16 @@ sent_model = SentenceTransformer('sentence-transformers/gtr-t5-base')
24
  sent_model.to(device)
25
 
26
  def get_similar_paper(
 
27
  abstract_text_input,
28
  author_id_input,
29
  results={}, # this state variable will be updated and returned
30
  #progress=gr.Progress()
31
- ):
32
-
33
  progress = gr.Progress()
34
  num_papers_show = 10 # number of top papers to show from the reviewer
 
 
35
  print('retrieving similar papers...')
36
  start = time.time()
37
  input_sentences = sent_tokenize(abstract_text_input)
@@ -47,6 +49,7 @@ def get_similar_paper(
47
  titles, abstracts, paper_urls, doc_scores = compute_document_score(
48
  doc_model,
49
  tokenizer,
 
50
  abstract_text_input,
51
  papers,
52
  batch=10
@@ -256,9 +259,9 @@ with gr.Blocks(css='style.css') as demo:
256
  # R2P2: Reviewer TO Paper in Peer review
257
 
258
  #### Who is it for?
259
- It is for meta-reviewers, area chairs, program chairs, or anyone who oversees the submission-reviewer matching process in peer review for acadmeic conferences, journals, and grants.
260
 
261
- <center><img src="file/tool.gif" width="50%" alt="general workflow"></center>
262
 
263
  #### How does it help?
264
  A typical meta-reviewer workflow lacks supportive information on **what makes the pre-selected candidate reviewers a good fit** for the submission. Only affinity scores between the reviewer and the paper are shown, without additional detail.
@@ -267,7 +270,7 @@ R2P2 provides more information about each reviewer. It searches for the most rel
267
  """
268
  # TODO add instruction video link
269
  # More details (video, addendum)
270
- more_details_instruction = """Check out <a href="", target="_blank">this video</a> for a quick demo of what R2P2 is and how it can help. For more details (e.g., relevant work, privacy policy, disclaimer), refer to <a href="file/details.html", target="_blank">here</a>."""
271
 
272
  gr.Markdown(general_instruction)
273
  gr.HTML(more_details_instruction)
@@ -275,20 +278,37 @@ R2P2 provides more information about each reviewer. It searches for the most rel
275
 
276
  ### INPUT
277
  with gr.Row() as input_row:
278
- with gr.Column():
279
- abstract_text_input = gr.Textbox(label='Submission Abstract', info='Paste in the abstract of the submission.')
280
- with gr.Column():
 
 
 
281
  with gr.Row():
282
  author_id_input = gr.Textbox(label='Reviewer Profile Link (Semantic Scholar)', info="Paste in the reviewer's Semantic Scholar link")
283
  with gr.Row():
284
  name = gr.Textbox(label='Confirm Reviewer Name', info='This will be automatically updated based on the reviewer profile link above', interactive=False)
285
  author_id_input.change(fn=update_name, inputs=author_id_input, outputs=name)
 
 
 
 
 
 
 
 
 
 
 
 
286
  with gr.Row():
287
  compute_btn = gr.Button('What Makes This a Good Match?')
288
 
289
  with gr.Row():
290
  search_status = gr.Textbox(label='Search Status', interactive=False, visible=False)
291
 
 
 
292
  ### OVERVIEW
293
  # Paper title, score, and top-ranking sentence pairs -- two sentence pairs per paper, three papers
294
  ## ONE BLOCK OF INFO FOR A SINGLE PAPER
@@ -425,6 +445,8 @@ R2P2 provides more information about each reviewer. It searches for the most rel
425
  with gr.Row(visible=False) as hl_row:
426
  # highlighted text from paper
427
  highlight = gr.components.Interpretation(paper_abstract)
 
 
428
 
429
  ### EVENT LISTENERS
430
 
@@ -438,6 +460,7 @@ R2P2 provides more information about each reviewer. It searches for the most rel
438
  compute_btn.click(
439
  fn=get_similar_paper,
440
  inputs=[
 
441
  abstract_text_input,
442
  author_id_input,
443
  info
 
24
  sent_model.to(device)
25
 
26
  def get_similar_paper(
27
+ title_input,
28
  abstract_text_input,
29
  author_id_input,
30
  results={}, # this state variable will be updated and returned
31
  #progress=gr.Progress()
32
+ ):
 
33
  progress = gr.Progress()
34
  num_papers_show = 10 # number of top papers to show from the reviewer
35
+ if title_input == None:
36
+ title_input = '' # if no title is given, just focus on abstract.
37
  print('retrieving similar papers...')
38
  start = time.time()
39
  input_sentences = sent_tokenize(abstract_text_input)
 
49
  titles, abstracts, paper_urls, doc_scores = compute_document_score(
50
  doc_model,
51
  tokenizer,
52
+ title_input,
53
  abstract_text_input,
54
  papers,
55
  batch=10
 
259
  # R2P2: Reviewer TO Paper in Peer review
260
 
261
  #### Who is it for?
262
+ It is for meta-reviewers, area chairs, program chairs, or anyone who oversees the submission-reviewer matching process in peer review for academic conferences, journals, and grants.
263
 
264
+ <center><img src="file/tool.gif" width="70%" alt="general workflow"></center>
265
 
266
  #### How does it help?
267
  A typical meta-reviewer workflow lacks supportive information on **what makes the pre-selected candidate reviewers a good fit** for the submission. Only affinity scores between the reviewer and the paper are shown, without additional detail.
 
270
  """
271
  # TODO add instruction video link
272
  # More details (video, addendum)
273
+ more_details_instruction = """Check out <a href="", target="_blank">this video</a> for a quick demo of what R2P2 is and how it can help. You can find more details <a href="file/details.html", target="_blank">here</a> about R2P2, along with our privacy policy and disclaimer."""
274
 
275
  gr.Markdown(general_instruction)
276
  gr.HTML(more_details_instruction)
 
278
 
279
  ### INPUT
280
  with gr.Row() as input_row:
281
+ with gr.Column(scale=3):
282
+ with gr.Row():
283
+ title_input = gr.Textbox(label='Submission Title', info='Paste in the title of the submission.')
284
+ with gr.Row():
285
+ abstract_text_input = gr.Textbox(label='Submission Abstract', info='Paste in the abstract of the submission.')
286
+ with gr.Column(scale=2):
287
  with gr.Row():
288
  author_id_input = gr.Textbox(label='Reviewer Profile Link (Semantic Scholar)', info="Paste in the reviewer's Semantic Scholar link")
289
  with gr.Row():
290
  name = gr.Textbox(label='Confirm Reviewer Name', info='This will be automatically updated based on the reviewer profile link above', interactive=False)
291
  author_id_input.change(fn=update_name, inputs=author_id_input, outputs=name)
292
+
293
+ # Add examples
294
+ example_title ="The Toronto Paper Matching System: An automated paper-reviewer assignment system"
295
+ example_submission = """One of the most important tasks of conference organizers is the assignment of papers to reviewers. Reviewers' assessments of papers is a crucial step in determining the conference program, and in a certain sense to shape the direction of a field. However this is not a simple task: large conferences typically have to assign hundreds of papers to hundreds of reviewers, and time constraints make the task impossible for one person to accomplish. Furthermore other constraints, such as reviewer load have to be taken into account, preventing the process from being completely distributed. We built the first version of a system to suggest reviewer assignments for the NIPS 2010 conference, followed, in 2012, by a release that better integrated our system with Microsoft's popular Conference Management Toolkit (CMT). Since then our system has been widely adopted by the leading conferences in both the machine learning and computer vision communities. This paper provides an overview of the system, a summary of learning models and methods of evaluation that we have been using, as well as some of the recent progress and open issues."""
296
+ example_reviewer = "https://www.semanticscholar.org/author/Nihar-B.-Shah/1737249"
297
+ gr.Examples(
298
+ examples=[[example_title, example_submission, example_reviewer]],
299
+ inputs=[title_input, abstract_text_input, author_id_input],
300
+ cache_examples=False,
301
+ label="Click to try out the example input."
302
+ )
303
+
304
  with gr.Row():
305
  compute_btn = gr.Button('What Makes This a Good Match?')
306
 
307
  with gr.Row():
308
  search_status = gr.Textbox(label='Search Status', interactive=False, visible=False)
309
 
310
+
311
+
312
  ### OVERVIEW
313
  # Paper title, score, and top-ranking sentence pairs -- two sentence pairs per paper, three papers
314
  ## ONE BLOCK OF INFO FOR A SINGLE PAPER
 
445
  with gr.Row(visible=False) as hl_row:
446
  # highlighted text from paper
447
  highlight = gr.components.Interpretation(paper_abstract)
448
+
449
+
450
 
451
  ### EVENT LISTENERS
452
 
 
460
  compute_btn.click(
461
  fn=get_similar_paper,
462
  inputs=[
463
+ title_input,
464
  abstract_text_input,
465
  author_id_input,
466
  info
details.html CHANGED
@@ -4,7 +4,7 @@
4
 
5
  <h1>Relevant Work</h1>
6
 
7
- The tool is based on the findings from the paper <a href="https://arxiv.org/abs/2302.08450", target="_blank">Assisting Human Decisions in Document Matching</a>.
8
 
9
  <h1>What Happens Behind the Scenes</h1>
10
  <ul>
 
4
 
5
  <h1>Relevant Work</h1>
6
 
7
+ The tool is developed by <a href="https://wnstlr.github.io", target="_blank">Joon Sik Kim</a>, based on the findings from our paper <a href="https://arxiv.org/abs/2302.08450", target="_blank">Assisting Human Decisions in Document Matching</a>, with <a href="https://valeriechen.github.io/", target="_blank">Valerie Chen</a>, <a href="https://danishpruthi.com/", target="_blank">Danish Pruthi</a>, <a href="https://www.cs.cmu.edu/~nihars/", target="_blank">Nihar B. Shah</a>, and <a href="https://www.cs.cmu.edu/~atalwalk/", target="_blank">Ameet Talwalkar</a>.
8
 
9
  <h1>What Happens Behind the Scenes</h1>
10
  <ul>
score.py CHANGED
@@ -251,7 +251,6 @@ def get_highlight_info(model, text1, text2, K=None):
251
  return sent_ids, sent_scores, info, top_pairs_info
252
 
253
  ### Document-level operations
254
- # TODO Use specter_MFR
255
  def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
256
  # compute document scores for each papers
257
 
@@ -259,8 +258,7 @@ def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
259
  title_abs = []
260
  for t, a in zip(titles, abstracts):
261
  if t is not None and a is not None:
262
- # title_abs.append(t + ' [SEP] ' + a)
263
- title_abs.append(a)
264
 
265
  num_docs = len(title_abs)
266
  no_iter = int(np.ceil(num_docs / batch))
@@ -292,7 +290,7 @@ def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
292
 
293
  return scores
294
 
295
- def compute_document_score(doc_model, tokenizer, query, papers, batch=5):
296
  scores = []
297
  titles = []
298
  abstracts = []
@@ -302,6 +300,10 @@ def compute_document_score(doc_model, tokenizer, query, papers, batch=5):
302
  titles.append(p['title'])
303
  abstracts.append(p['abstract'])
304
  urls.append(p['url'])
 
 
 
 
305
  scores = predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=batch)
306
  assert(len(scores) == len(abstracts))
307
  idx_sorted = np.argsort(scores)[::-1]
 
251
  return sent_ids, sent_scores, info, top_pairs_info
252
 
253
  ### Document-level operations
 
254
  def predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=20):
255
  # compute document scores for each papers
256
 
 
258
  title_abs = []
259
  for t, a in zip(titles, abstracts):
260
  if t is not None and a is not None:
261
+ title_abs.append(t + ' [SEP] ' + a) # title + abstract
 
262
 
263
  num_docs = len(title_abs)
264
  no_iter = int(np.ceil(num_docs / batch))
 
290
 
291
  return scores
292
 
293
+ def compute_document_score(doc_model, tokenizer, query_title, query_abs, papers, batch=5):
294
  scores = []
295
  titles = []
296
  abstracts = []
 
300
  titles.append(p['title'])
301
  abstracts.append(p['abstract'])
302
  urls.append(p['url'])
303
+ if query_title == '':
304
+ query = query_abs
305
+ else:
306
+ query = query_title + ' [SEP] ' + query_abs # feed in submission title and abstract
307
  scores = predict_docscore(doc_model, tokenizer, query, titles, abstracts, batch=batch)
308
  assert(len(scores) == len(abstracts))
309
  idx_sorted = np.argsort(scores)[::-1]