add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Dec15_20-25-53_fd3f948fdbfb/1639599988.9528167/events.out.tfevents.1639599988.fd3f948fdbfb.107.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42b61358166b4d6975766a8c133e5cb19bf14bbae3a4781636bbff9399942fce
+size 4564

runs/Dec15_20-25-53_fd3f948fdbfb/events.out.tfevents.1639599988.fd3f948fdbfb.107.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b4b2d15e79cf8f44c8c3c29117c5f919b231efb4a992f3c5405932af8a172a1
+size 4438

runs/Dec15_20-27-30_fd3f948fdbfb/1639600065.7345753/events.out.tfevents.1639600065.fd3f948fdbfb.107.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dbedd6e23ee4721ba6ede3b7cfcfd43e58b3358bb59b25ab982816afad8ff72
+size 4564

runs/Dec15_20-27-30_fd3f948fdbfb/events.out.tfevents.1639600065.fd3f948fdbfb.107.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:884694a40a687f2b13c001ee2bc886d5a89a01472259ffdc1819abe40e6b79e1
+size 4435

runs/Dec15_20-28-23_fd3f948fdbfb/1639600117.9638968/events.out.tfevents.1639600117.fd3f948fdbfb.107.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac19964edcec7e99822bbd2aaacc9822696936daa546b720a1184210dcfe78dc
+size 4564

runs/Dec15_20-28-23_fd3f948fdbfb/events.out.tfevents.1639600117.fd3f948fdbfb.107.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7711b17efb519a98de89eeb6e2a989911e5ca45a19a0de946d23cd56f8edaa95
+size 4434

vocab.json

@@ -1 +1 @@
-{"ू": 0, "ऑ": 1, "ल": 2, "श": 3, "ध": 4, "फ": 5, "ं": 6, "ै": 7, "ग": 8, "a": 9, "ब": 10, "आ": 11, "ख": 12, "p": 13, "म": 14, "े": 15, "औ": 16, "ष": 17, "ः": 18, "r": 19, "ो": 20, "न": 21, "भ": 22, "अ": 23, "ज": 24, "।": 25, "ठ": 26, "ड़": 27, "र": 28, "इ": 29, "l": 30, "ह": 31, "त": 32, "ई": 33, "m": 34, "य": 35, "ण": 36, "व": 37, "स": 38, "थ": 39, "ए": 40, "ँ": 41, "ौ": 42, "झ": 44, "ृ": 45, "w": 46, "ऐ": 47, "'": 48, "ी": 49, "च": 50, "घ": 51, "्": 52, "ड": 53, "ु": 54, "e": 55, "़": 56, "u": 57, "ट": 58, "ऊ": 59, "छ": 60, "f": 61, "क": 62, "ढ": 63, "ओ": 64, "द": 65, "उ": 66, "प": 67, "ा": 68, "ॉ": 69, "ि": 70, "i": 71, "|": 43, "[UNK]": 72, "[PAD]": 73}
+{"ृ": 0, "प": 1, "ि": 2, "ू": 3, "उ": 4, "झ": 5, "ठ": 6, "u": 7, "फ": 9, "p": 10, "औ": 11, "'": 12, "f": 13, "क": 14, "भ": 15, "्": 16, "ँ": 17, "ऊ": 18, "ढ": 19, "ष": 20, "इ": 21, "स": 22, "ो": 23, "l": 24, "छ": 25, "ख": 26, "ु": 27, "य": 28, "ड": 29, "े": 30, "ं": 31, "ई": 32, "ॉ": 33, "अ": 34, "w": 35, "ज": 36, "ा": 37, "ध": 38, "आ": 39, "ल": 40, "श": 41, "ट": 42, "च": 43, "ौ": 44, "म": 45, "ए": 46, "i": 47, "द": 48, "m": 49, "ऑ": 50, "त": 51, "।": 52, "़": 53, "थ": 54, "व": 55, "ऐ": 56, "घ": 57, "ी": 58, "e": 59, "ब": 60, "र": 61, "न": 62, "ः": 63, "ण": 64, "ग": 65, "a": 66, "ड़": 67, "ै": 68, "ओ": 69, "r": 70, "ह": 71, "|": 8, "[UNK]": 72, "[PAD]": 73}