fix several errors

ShaderEval.py

@@ -158,7 +158,7 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
 class Suite(evaluate.EvaluationSuite):
 
 
-    def __init__(self, name, snippet: int = ""):
+    def __init__(self, name):
         super().__init__(name)
         self.preprocessor = lambda x: {"return_statement": x["return_statement"].split(";")[0]} #like this? refactored to RetrunGenerationEvaluator
         self.suite = [
@@ -167,7 +167,7 @@ class Suite(evaluate.EvaluationSuite):
                 task_type="text-generation", #this call an evaluator, but can you specify your own custom evaluator instead?
                 data="Vipitis/Shadertoys-fine",
                 subset="return_completion",
-                split=f"test[:{snippet}]", # use this to select a subset of the data during testing, perhaps remove later?
+                split="test", # use this to select a subset of the data during testing, perhaps remove later?
                 args_for_task={
                     # "metric": "exact_match",
                     "input_column": "body",
@@ -178,7 +178,8 @@ class Suite(evaluate.EvaluationSuite):
 
     # from: https://github.com/huggingface/evaluate/blob/v0.4.0/src/evaluate/evaluation_suite/__init__.py#LL103C5-L129C27
     def run(
-        self, model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"] = "Vipitis/CodeGPT-small-java-adaptedGPT2-transfer-shadertoys" # noqa: F821 not so useful default model?
+        self, model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"] = "Vipitis/CodeGPT-small-java-adaptedGPT2-transfer-shadertoys", #not so useful default model?
+        snippet: int = "" # noqa: F821 
     ) -> Dict[str, float]:
 
         self.assert_suite_nonempty()
@@ -189,7 +190,7 @@ class Suite(evaluate.EvaluationSuite):
             task_name = task.data
 
             if task.data_preprocessor:  # task requires extra preprocessing is all done inside the Evaluator
-                ds = load_dataset(task.data, name=task.subset, split=task.split)
+                ds = load_dataset(task.data, name=task.subset, split=(task.split + f"[:{snippet}]"))
                 task.data = ds.map(task.data_preprocessor)
 
             task_evaluator = ReturnGenerationEvaluator() #this is the change we make: specify our custom evaluator from above.
@@ -197,7 +198,7 @@ class Suite(evaluate.EvaluationSuite):
             args_for_task["model_or_pipeline"] = model_or_pipeline
             args_for_task["data"] = task.data
             args_for_task["subset"] = task.subset
-            args_for_task["split"] = task.split
+            args_for_task["split"] = (task.split + f"[:{snippet}]") #make a downselection of the split via keywordarg in the .run() call?
             results = task_evaluator.compute(**args_for_task)
 
             results["task_name"] = task_name + "/" + task.subset if task.subset else task_name

app.py

@@ -1,13 +1,15 @@
 import gradio as gr
 import evaluate
-from ShaderEval import Suite
 
-# suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval") #downloads it
-suite = Suite("Vipitis/ShaderEval") #local to the space?
+suite = evaluate.EvaluationSuite.load("Vipitis/ShaderEval") #downloads it
+
+#TODO: can you import it locally instead?
+# from ShaderEval import Suite
+# suite = Suite("Vipitis/ShaderEval")
 
 def run_suite(model_cp, snippet):
-    suite = Suite("Vipitis/ShaderEval", snippet)
-    results = suite.run(model_cp)
+    # print(model_cp, snippet)
+    results = suite.run(model_cp, snippet)
     return results[0]["exact_match"]
 
 with gr.Blocks() as site:
@@ -30,7 +32,7 @@ with gr.Blocks() as site:
             - Click **Run** to run the suite
             - The results will be displayed in the **Output** box
             """)
-    model_cp = gr.Textbox(label="Model Checkpoint")
+    model_cp = gr.Textbox(value="gpt2", label="Model Checkpoint", interactive=True)
     first_n = gr.Slider(minimum=1, maximum=100, default=10, label="num_samples", step=1.0)
     output = gr.Textbox(label="Output")
     run_button = gr.Button(label="Run")