pretrain fixed bigcode/the-stack-smol-xl dataset

scripts/prepare_pretrain_dataset.py

@@ -94,7 +94,7 @@ datasets_configs = [
     ],
     [
         # ~3 GB, 4,976,850
-        {'path': 'saillab/taco-datasets', 'name': name, 'split': 'train', 'format': '{instruction} {input} {output}'}
+        {'path': 'saillab/taco-datasets', 'data_dir': name, 'split': 'train', 'format': '{instruction} {input} {output}'}
         for name in [
             # 'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4',
             'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k',

scripts/train_tokenizer.py

@@ -23,24 +23,25 @@ def batch_iterator():
 
     # code
     dataset = (
-        load_dataset('bigcode/the-stack-smol-xs', lang, split='train', trust_remote_code=True)
-        for lang in [
-            'python', 'javascript', 'typescript', 'html', 'css', 'c', 'c++', 'markdown',
+        load_dataset('bigcode/the-stack-smol-xs', data_dir=f'data/{name}', split='train', trust_remote_code=True)
+        for name in [
+            # 'batchfile' - unsafe
+            # 'powershell' - unsafe
             'ada', 'agda', 'alloy', 'antlr', 'applescript', 'assembly',
-            'augeas', 'awk', 'batchfile', 'bison', 'bluespec',
-            'c-sharp', 'clojure', 'cmake', 'coffeescript', 'common-lisp',
-            'cuda', 'dart', 'dockerfile', 'elixir',
+            'augeas', 'awk', 'bison', 'bluespec', 'c',
+            'c++', 'c-sharp', 'clojure', 'cmake', 'coffeescript', 'common-lisp',
+            'css', 'cuda', 'dart', 'dockerfile', 'elixir',
             'elm', 'emacs-lisp','erlang', 'f-sharp', 'fortran', 'glsl', 'go',
-            'groovy', 'haskell', 'idris', 'isabelle', 'java',
-            'java-server-pages', 'julia', 'kotlin', 'lean',
+            'groovy', 'haskell','html', 'idris', 'isabelle', 'java',
+            'java-server-pages', 'javascript', 'julia', 'kotlin', 'lean',
             'literate-agda', 'literate-coffeescript', 'literate-haskell',
-            'lua', 'makefile', 'maple', 'mathematica', 'matlab',
-            'ocaml', 'pascal', 'perl', 'php', 'powershell', 'prolog',
-            'protocol-buffer', 'r', 'racket', 'restructuredtext',
+            'lua', 'makefile', 'maple', 'markdown', 'mathematica', 'matlab',
+            'ocaml', 'pascal', 'perl', 'php', 'prolog',
+            'protocol-buffer', 'python', 'r', 'racket', 'restructuredtext',
             'rmarkdown', 'ruby', 'rust', 'sas', 'scala', 'scheme',
             'shell', 'smalltalk', 'solidity', 'sparql', 'sql', 'stan',
             'standard-ml', 'stata', 'systemverilog', 'tcl', 'tcsh', 'tex',
-            'thrift', 'verilog', 'vhdl', 'visual-basic', 'xslt',
+            'thrift', 'typescript', 'verilog', 'vhdl', 'visual-basic', 'xslt',
             'yacc', 'zig',
         ]
     )