# Dataset Card for "human_genome_GCF_009914755.1"

how to build this data:
human full genome data from:
https://www.ncbi.nlm.nih.gov/datasets/genome/GCF_009914755.1/

Preprocess:
1 download data use ncbi data set tools:
curl -o datasets 'https://ftp.ncbi.nlm.nih.gov/pub/datasets/command-line/LATEST/linux-amd64/datasets'
chmod +x datasets
./datasets download genome accession GCF_000001405.40 --filename genomes/human_genome_dataset.zip
then move the gene data to human2.fra

2 write the origin data into pure dna data, one line 1000 bp/letters:

```

filename = "human2.fna"
data_file = open(filename, 'r')

out_filename = "human2.fna.line"
out_file = open(out_filename, 'w')

max_line_len = 1000 #1000个字母一行数据

text = ""
for line in data_file:
    line = line.strip()
    if line.find(">") != 0: #去掉标题行
        line = line.upper()
        line = line.replace(" ","").replace("N","") #去掉N和空格
        text = text + line
        if len(text) > max_line_len:
            text = text.strip()
            out_file.write(text+"\n")
            text = "" #clear text
#last line
if len(text) <= max_line_len:
    pass #不要了


3 split data into train and valid dataset:

```

filename = "human2.fna.line"
data_file = open(filename, 'r')

out_train_filename = "human2.fna.line.train"
out_train_file = open(out_train_filename, 'w')

out_valid_filename = "human2.fna.line.valid"
out_valid_file = open(out_valid_filename, 'w')

line_num = 0
select_line_num = 0

for line in data_file:
    if 0==line_num%3: #取1/3数据
        if select_line_num%100: #取1%做校验
            out_train_file.write(line)
        else:
            out_valid_file.write(line)
        select_line_num = select_line_num + 1

    line_num = line_num + 1



4 the we could use it in local, or push it to hub:

```

from datasets import load_dataset
data_files = {"train": "human2.fna.line.train", "test": "human2.fna.line.valid"}
train_dataset = load_dataset("text", data_files=data_files)
train_dataset.push_to_hub("dnagpt/human_genome_GCF_009914755.1",token="hf_*******")




[More Information needed](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)
license:apache-2.0 region:us
{"license": "apache-2.0", "configs": [{"config_name": "default", "data_files": [{"split": "train", "path": "data/train-*"}, {"split": "test", "path": "data/test-*"}]}], "dataset_info": {"features": [{"name": "text", "dtype": "string"}], "splits": [{"name": "train", "num_bytes": 1032653672, "num_examples": 989132}, {"name": "test", "num_bytes": 10431648, "num_examples": 9992}], "download_size": 472762984, "dataset_size": 1043085320}}
TAGS
#license-apache-2.0 #region-us 

# Dataset Card for "human_genome_GCF_009914755.1"

how to build this data:
human full genome data from:
URL

Preprocess:
1 download data use ncbi data set tools:
curl -o datasets 'URL
chmod +x datasets
./datasets download genome accession GCF_000001405.40 --filename genomes/human_genome_dataset.zip
then move the gene data to URL

2 write the origin data into pure dna data, one line 1000 bp/letters:



filename = "URL"
data_file = open(filename, 'r')

out_train_filename = "URL"
out_train_file = open(out_train_filename, 'w')

out_valid_filename = "URL"
out_valid_file = open(out_valid_filename, 'w')

line_num = 0
select_line_num = 0

for line in data_file:
    if 0==line_num%3: #取1/3数据
        if select_line_num%100: #取1%做校验
            out_train_file.write(line)
        else:
            out_valid_file.write(line)
        select_line_num = select_line_num + 1

    line_num = line_num + 1



4 the we could use it in local, or push it to hub:

'''

from datasets import load_dataset
data_files = {"train": "URL", "test": "URL"}
train_dataset = load_dataset("text", data_files=data_files)
train_dataset.push_to_hub("dnagpt/human_genome_GCF_009914755.1",token="hf_*")




More Information needed
# Dataset Card for "human_genome_GCF_009914755.1"

how to build this data:
human full genome data from:
URL

Preprocess:
1 download data use ncbi data set tools:
curl -o datasets 'URL
chmod +x datasets
./datasets download genome accession GCF_000001405.40 --filename genomes/human_genome_dataset.zip
then move the gene data to URL

2 write the origin data into pure dna data, one line 1000 bp/letters:



filename = "URL"
data_file = open(filename, 'r')

out_train_filename = "URL"
out_train_file = open(out_train_filename, 'w')

out_valid_filename = "URL"
out_valid_file = open(out_valid_filename, 'w')

line_num = 0
select_line_num = 0

for line in data_file:
    if 0==line_num%3: #取1/3数据
        if select_line_num%100: #取1%做校验
            out_train_file.write(line)
        else:
            out_valid_file.write(line)
        select_line_num = select_line_num + 1

    line_num = line_num + 1



4 the we could use it in local, or push it to hub:

'''

from datasets import load_dataset
data_files = {"train": "URL", "test": "URL"}
train_dataset = load_dataset("text", data_files=data_files)
train_dataset.push_to_hub("dnagpt/human_genome_GCF_009914755.1",token="hf_*")




More Information needed
TAGS
#license-apache-2.0 #region-us 
 # Dataset Card for "human_genome_GCF_009914755.1"

how to build this data:
human full genome data from:
URL

Preprocess:
1 download data use ncbi data set tools:
curl -o datasets 'URL
chmod +x datasets
./datasets download genome accession GCF_000001405.40 --filename genomes/human_genome_dataset.zip
then move the gene data to URL

2 write the origin data into pure dna data, one line 1000 bp/letters:



filename = "URL"
data_file = open(filename, 'r')

out_train_filename = "URL"
out_train_file = open(out_train_filename, 'w')

out_valid_filename = "URL"
out_valid_file = open(out_valid_filename, 'w')

line_num = 0
select_line_num = 0

for line in data_file:
    if 0==line_num%3: #取1/3数据
        if select_line_num%100: #取1%做校验
            out_train_file.write(line)
        else:
            out_valid_file.write(line)
        select_line_num = select_line_num + 1

    line_num = line_num + 1



4 the we could use it in local, or push it to hub:

'''

from datasets import load_dataset
data_files = {"train": "URL", "test": "URL"}
train_dataset = load_dataset("text", data_files=data_files)
train_dataset.push_to_hub("dnagpt/human_genome_GCF_009914755.1",token="hf_*")




More Information needed