gijs commited on
Commit
f04445e
·
verified ·
1 Parent(s): 7fc7ba8

Bundle the r50k_base tokenizer files (OpenCLIP push_to_hf_hub layout)

Browse files
open_clip_config.json CHANGED
@@ -50,7 +50,9 @@
50
  "attention_bias": false,
51
  "mlp_bias": false,
52
  "tokenizer_type": "tiktoken",
53
- "tiktoken_name": "r50k_base"
 
 
54
  }
55
  }
56
  }
 
50
  "attention_bias": false,
51
  "mlp_bias": false,
52
  "tokenizer_type": "tiktoken",
53
+ "tiktoken_name": "r50k_base",
54
+ "tiktoken_bpe_path": "open_clip_tiktoken/r50k_base.tiktoken",
55
+ "tiktoken_config_path": "open_clip_tiktoken/r50k_base.json"
56
  }
57
  }
58
  }
open_clip_tiktoken/r50k_base.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "r50k_base",
3
+ "pat_str": "'(?:[sdmt]|ll|ve|re)| ?\\p{L}++| ?\\p{N}++| ?[^\\s\\p{L}\\p{N}]++|\\s++$|\\s+(?!\\S)|\\s",
4
+ "special_tokens": {
5
+ "<|endoftext|>": 50256
6
+ },
7
+ "explicit_n_vocab": 50257
8
+ }
open_clip_tiktoken/r50k_base.tiktoken ADDED
The diff for this file is too large to render. See raw diff