Bram Vanroy commited on
Commit
8b74974
·
1 Parent(s): cd55f6a

revert model add. Experiencing some bugs with geitje ultra so have to investigate first

Browse files
evals/arc/arc_nl_GEITje-ultra-sft.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "results": {
3
- "arc_nl": {
4
- "acc": 0.38751069289991447,
5
- "acc_stderr": 0.014255080678869606,
6
- "acc_norm": 0.39093242087254065,
7
- "acc_norm_stderr": 0.014277828606329127
8
- }
9
- },
10
- "versions": {
11
- "arc_nl": 0
12
- },
13
- "config": {
14
- "model": "hf-auto",
15
- "model_args": "pretrained=BramVanroy/GEITje-ultra-sft,use_accelerate=True,device_map_option=auto,dtype=auto,trust_remote_code=True",
16
- "batch_size": 2,
17
- "device": "cuda",
18
- "no_cache": false,
19
- "limit": null,
20
- "bootstrap_iters": 100000,
21
- "description_dict": {}
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
evals/hellaswag/hellaswag_nl_GEITje-ultra-sft.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "results": {
3
- "hellaswag_nl": {
4
- "acc": 0.5017808958445764,
5
- "acc_stderr": 0.005194789766470539,
6
- "acc_norm": 0.6382083108472747,
7
- "acc_norm_stderr": 0.004992421532295554
8
- }
9
- },
10
- "versions": {
11
- "hellaswag_nl": 1
12
- },
13
- "config": {
14
- "model": "hf-auto",
15
- "model_args": "pretrained=BramVanroy/GEITje-ultra-sft,use_accelerate=True,device_map_option=auto,dtype=auto,trust_remote_code=True",
16
- "batch_size": 2,
17
- "device": "cuda",
18
- "no_cache": false,
19
- "limit": null,
20
- "bootstrap_iters": 100000,
21
- "description_dict": {}
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
evals/models.json CHANGED
@@ -15,14 +15,6 @@
15
  "num_parameters": 7241732096,
16
  "quantization": "8-bit"
17
  },
18
- "geitje-ultra-sft": {
19
- "compute_dtype": "auto",
20
- "dutch_coverage": "not-given",
21
- "model_name": "BramVanroy/GEITje-ultra-sft",
22
- "model_type": "not-given",
23
- "num_parameters": 7241732096,
24
- "quantization": null
25
- },
26
  "gpt-neo-1.3b-dutch": {
27
  "compute_dtype": "bfloat16",
28
  "dutch_coverage": "pretrained",
@@ -207,4 +199,4 @@
207
  "num_parameters": 7241732096,
208
  "quantization": "8-bit"
209
  }
210
- }
 
15
  "num_parameters": 7241732096,
16
  "quantization": "8-bit"
17
  },
 
 
 
 
 
 
 
 
18
  "gpt-neo-1.3b-dutch": {
19
  "compute_dtype": "bfloat16",
20
  "dutch_coverage": "pretrained",
 
199
  "num_parameters": 7241732096,
200
  "quantization": "8-bit"
201
  }
202
+ }
evals/truthfulqa/truthfulqa_nl_GEITje-ultra-sft.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "results": {
3
- "truthfulqa_nl": {
4
- "mc1": 0.2840764331210191,
5
- "mc1_stderr": 0.01610619308516951,
6
- "mc2": 0.451556591245413,
7
- "mc2_stderr": 0.015176365000200752
8
- }
9
- },
10
- "versions": {
11
- "truthfulqa_nl": 1
12
- },
13
- "config": {
14
- "model": "hf-auto",
15
- "model_args": "pretrained=BramVanroy/GEITje-ultra-sft,use_accelerate=True,device_map_option=auto,dtype=auto,trust_remote_code=True",
16
- "batch_size": 2,
17
- "device": "cuda",
18
- "no_cache": false,
19
- "limit": null,
20
- "bootstrap_iters": 100000,
21
- "description_dict": {}
22
- }
23
- }