Hugging Face's logo

TrevorJS
/
mtg-phi-1_5-sft-merged

Text Generation
Transformers
PyTorch
mixformer-sequential
custom_code
Model card Files Community
TrevorJS commited on
Commit
640847e
•
1 Parent(s): 3e00dfa

Upload tokenizer

Browse files
Files changed (6) hide show
  1. added_tokens.json +40 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +4 -28
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +7 -42
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "\t\t": 50294,
3
+ "\t\t\t": 50293,
4
+ "\t\t\t\t": 50292,
5
+ "\t\t\t\t\t": 50291,
6
+ "\t\t\t\t\t\t": 50290,
7
+ "\t\t\t\t\t\t\t": 50289,
8
+ "\t\t\t\t\t\t\t\t": 50288,
9
+ "\t\t\t\t\t\t\t\t\t": 50287,
10
+ " ": 50286,
11
+ " ": 50285,
12
+ " ": 50284,
13
+ " ": 50283,
14
+ " ": 50282,
15
+ " ": 50281,
16
+ " ": 50280,
17
+ " ": 50279,
18
+ " ": 50278,
19
+ " ": 50277,
20
+ " ": 50276,
21
+ " ": 50275,
22
+ " ": 50274,
23
+ " ": 50273,
24
+ " ": 50272,
25
+ " ": 50271,
26
+ " ": 50270,
27
+ " ": 50269,
28
+ " ": 50268,
29
+ " ": 50267,
30
+ " ": 50266,
31
+ " ": 50265,
32
+ " ": 50264,
33
+ " ": 50263,
34
+ " ": 50262,
35
+ " ": 50261,
36
+ " ": 50260,
37
+ " ": 50259,
38
+ " ": 50258,
39
+ " ": 50257
40
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json CHANGED
@@ -1,30 +1,6 @@
1
  {
2
- "additional_special_tokens": [
3
- "▁<PRE>",
4
- "▁<MID>",
5
- "▁<SUF>",
6
- "▁<EOT>"
7
- ],
8
- "bos_token": {
9
- "content": "<s>",
10
- "lstrip": false,
11
- "normalized": true,
12
- "rstrip": false,
13
- "single_word": false
14
- },
15
- "eos_token": {
16
- "content": "</s>",
17
- "lstrip": false,
18
- "normalized": true,
19
- "rstrip": false,
20
- "single_word": false
21
- },
22
- "pad_token": "</s>",
23
- "unk_token": {
24
- "content": "<unk>",
25
- "lstrip": false,
26
- "normalized": true,
27
- "rstrip": false,
28
- "single_word": false
29
- }
30
  }
 
1
  {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  }
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,44 +1,9 @@
1
  {
2
- "additional_special_tokens": [
3
- "▁<PRE>",
4
- "▁<MID>",
5
- "▁<SUF>",
6
- "▁<EOT>"
7
- ],
8
- "bos_token": {
9
- "__type": "AddedToken",
10
- "content": "<s>",
11
- "lstrip": false,
12
- "normalized": true,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "clean_up_tokenization_spaces": false,
17
- "eos_token": {
18
- "__type": "AddedToken",
19
- "content": "</s>",
20
- "lstrip": false,
21
- "normalized": true,
22
- "rstrip": false,
23
- "single_word": false
24
- },
25
- "eot_token": "▁<EOT>",
26
- "fill_token": "<FILL_ME>",
27
- "legacy": null,
28
- "middle_token": "▁<MID>",
29
- "model_max_length": 1000000000000000019884624838656,
30
- "pad_token": null,
31
- "prefix_token": "▁<PRE>",
32
- "sp_model_kwargs": {},
33
- "suffix_token": "▁<SUF>",
34
- "tokenizer_class": "CodeLlamaTokenizer",
35
- "unk_token": {
36
- "__type": "AddedToken",
37
- "content": "<unk>",
38
- "lstrip": false,
39
- "normalized": true,
40
- "rstrip": false,
41
- "single_word": false
42
- },
43
- "use_default_system_prompt": false
44
  }
 
1
  {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|endoftext|>",
6
+ "model_max_length": 2048,
7
+ "tokenizer_class": "CodeGenTokenizer",
8
+ "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  }
vocab.json ADDED
The diff for this file is too large to render. See raw diff