prepare dataset

scripts/prepare_contrain_dataset.py

@@ -153,10 +153,11 @@ def batch_iterator(name=None):
         dataset = load_dataset('datatab/open-orca-slim-serbian', split='train')
         role_map = {'system': 'system', 'human': 'user', 'gpt': 'assistant'}
 
-        for row in dataset:
+        for row in dataset['conversations']:
             text = [
                 f"<|im_start|>{role_map[n['from']]}\n{n['value']}<|im_end|>"
-                for n in row['conversations']
+                for n in row
+                if n
             ]
 
             text = '\n'.join(text) + '\n'