# Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset:

```python
import multiprocessing
from typing import Union

from datasets import DatasetDict, load_dataset

from transformers import LlamaTokenizer

# Find your available cores
num_cores = multiprocessing.cpu_count()

cutoff_len = 2048

tokenizer = LlamaTokenizer.from_pretrained("chainyo/alpaca-lora-7b")
tokenizer.padding_side = "left"
tokenizer.pad_token_id = (0)

prompt_template = {
    "prompt": "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
    "response": "### Response:"    
}

class Prompter(object):
    __slots__ = ("template", "_verbose")

    def __init__(self, verbose: bool = False):
        self._verbose = verbose

    def generate_prompt(
        self,
        definition: str,
        inputs: str,
        targets: Union[None, str] = None,
    ) -> str:
        """Generate a prompt from instruction and input."""
        res = prompt_template["prompt"].format(
            instruction=definition, input=inputs
        )

        if targets:
            res = f"{res}{targets}"

        return res

    def get_response(self, output: str) -> str:
        return output.split(self.template["response"])[1].strip()


prompter = Prompter()

def tokenize(prompt, add_eos_token=True):
    # there's probably a way to do this with the tokenizer settings
    # but again, gotta move fast
    result = tokenizer(
        prompt,
        truncation=True,
        max_length=cutoff_len,
        padding=False,
        return_tensors=None,
    )
    if (
        result["input_ids"][-1] != tokenizer.eos_token_id
        and len(result["input_ids"]) < cutoff_len
        and add_eos_token
    ):
        result["input_ids"].append(tokenizer.eos_token_id)
        result["attention_mask"].append(1)

    result["labels"] = result["input_ids"].copy()

    return result

def generate_and_tokenize_prompt(data_point):
    full_prompt = prompter.generate_prompt(
        data_point["definition"],
        data_point["inputs"],
        data_point["targets"],
    )
    tokenized_full_prompt = tokenize(full_prompt)

    return tokenized_full_prompt


data_path = "Muennighoff/natural-instructions"
data = load_dataset(data_path)

train_data = data["train"].shuffle().map(generate_and_tokenize_prompt, num_proc=num_cores)
val_data = data["validation"].map(generate_and_tokenize_prompt, num_proc=num_cores)
test_data = data["test"].map(generate_and_tokenize_prompt, num_proc=num_cores)

tokenized_dataset = DatasetDict({"train": train_data, "validation": val_data, "test": test_data})
tokenized_dataset.save_to_disk("natural-instructions-tokenized")
```
region:us
{"dataset_info": {"features": [{"name": "task_name", "dtype": "string"}, {"name": "id", "dtype": "string"}, {"name": "definition", "dtype": "string"}, {"name": "inputs", "dtype": "string"}, {"name": "targets", "dtype": "string"}, {"name": "input_ids", "sequence": "int32"}, {"name": "attention_mask", "sequence": "int8"}, {"name": "labels", "sequence": "int64"}], "splits": [{"name": "train", "num_bytes": 30103878026, "num_examples": 6164188}, {"name": "validation", "num_bytes": 10504992, "num_examples": 5995}, {"name": "test", "num_bytes": 3937289042, "num_examples": 982285}], "download_size": 10233494905, "dataset_size": 34051672060}}
TAGS
#region-us 

# Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset:


# Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset:
TAGS
#region-us 
 # Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset:
passage: TAGS
#region-us 
# Dataset Card for "natural-instructions-tokenized"

Here is the script used to tokenize the dataset: