Prgckwb commited on
Commit
2494b9d
·
1 Parent(s): 78a5cec
Files changed (2) hide show
  1. app.py +8 -2
  2. data/sample_submission.csv +7 -0
app.py CHANGED
@@ -8,6 +8,8 @@ import torch
8
 
9
  from metric import PerplexityCalculator
10
 
 
 
11
  os.environ['OMP_NUM_THREADS'] = '1'
12
  os.environ['TOKENIZERS_PARALLELISM'] = 'false'
13
  PAD_TOKEN_LABEL_ID = torch.nn.CrossEntropyLoss().ignore_index
@@ -18,12 +20,16 @@ text_list = df_sample_submission.get_column('text').to_list()
18
  text_counters = [Counter(text.split()) for text in text_list]
19
 
20
  # Model Loading
21
- scorer = PerplexityCalculator('google/gemma-2-9b')
 
22
 
23
 
24
  @spaces.GPU()
25
  def inference(text: str, progress=gr.Progress(track_tqdm=True)):
26
- score = scorer.get_perplexity(text)
 
 
 
27
 
28
  input_counter = Counter(text.split())
29
  is_match_list = [input_counter == text_counter for text_counter in text_counters]
 
8
 
9
  from metric import PerplexityCalculator
10
 
11
+ IS_DEBUG = False
12
+
13
  os.environ['OMP_NUM_THREADS'] = '1'
14
  os.environ['TOKENIZERS_PARALLELISM'] = 'false'
15
  PAD_TOKEN_LABEL_ID = torch.nn.CrossEntropyLoss().ignore_index
 
20
  text_counters = [Counter(text.split()) for text in text_list]
21
 
22
  # Model Loading
23
+ if not IS_DEBUG:
24
+ scorer = PerplexityCalculator('google/gemma-2-9b')
25
 
26
 
27
  @spaces.GPU()
28
  def inference(text: str, progress=gr.Progress(track_tqdm=True)):
29
+ if IS_DEBUG:
30
+ score = -1
31
+ else:
32
+ score = scorer.get_perplexity(text)
33
 
34
  input_counter = Counter(text.split())
35
  is_match_list = [input_counter == text_counter for text_counter in text_counters]
data/sample_submission.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ id,text
2
+ 0,advent chimney elf family fireplace gingerbread mistletoe ornament reindeer scrooge
3
+ 1,"advent chimney elf family fireplace gingerbread mistletoe ornament reindeer scrooge walk give jump drive bake the sleep night laugh and"
4
+ 2,"yuletide decorations gifts cheer holiday carol magi nutcracker polar grinch sleigh chimney workshop stocking ornament holly jingle beard naughty nice"
5
+ 3,"yuletide decorations gifts cheer holiday carol magi nutcracker polar grinch sleigh chimney workshop stocking ornament holly jingle beard naughty nice sing cheer and of the is eat visit relax unwrap"
6
+ 4,"hohoho candle poinsettia snowglobe peppermint eggnog fruitcake chocolate candy puzzle game doll toy workshop wonder believe dream hope peace joy merry season greeting card wrapping paper bow fireplace night cookie milk star wish wreath angel the to of and in that have it not with as you from we kaggle"
7
+ 5,"advent chimney elf family fireplace gingerbread mistletoe ornament reindeer scrooge walk give jump drive bake the sleep night laugh and yuletide decorations gifts cheer holiday carol magi nutcracker polar grinch sleigh chimney workshop stocking ornament holly jingle beard naughty nice sing cheer and of the is eat visit relax unwrap hohoho candle poinsettia snowglobe peppermint eggnog fruitcake chocolate candy puzzle game doll toy workshop wonder believe dream hope peace joy merry season greeting card wrapping paper bow fireplace night cookie milk star wish wreath angel the to of and in that have it not with as you from we kaggle"