up

tokenizer.json

@@ -50,7 +50,7 @@
     },
     {
       "id": 5,
-      "content": "[TOOL_CALLS]",
+      "content": "<tool_call>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 6,
-      "content": "[AVAILABLE_TOOLS]",
+      "content": "<available_tools>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
     },
     {
       "id": 7,
-      "content": "[/AVAILABLE_TOOLS]",
+      "content": "</available_tools>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "[TOOL_RESULTS]",
+      "content": "<tool_response>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -86,7 +86,7 @@
     },
     {
       "id": 9,
-      "content": "[/TOOL_RESULTS]",
+      "content": "</tool_response>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "[control_8]",
+      "content": "<available_documents>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "[control_9]",
+      "content": "</available_documents>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 12,
-      "content": "[control_10]",
+      "content": "<reference>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 13,
-      "content": "[control_11]",
+      "content": "</reference>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,7 +131,7 @@
     },
     {
       "id": 14,
-      "content": "[control_12]",
+      "content": "<|system|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -140,7 +140,7 @@
     },
     {
       "id": 15,
-      "content": "[control_13]",
+      "content": "<|user|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -149,7 +149,7 @@
     },
     {
       "id": 16,
-      "content": "[control_14]",
+      "content": "<|assistant|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -7043,18 +7043,18 @@
       "</s>": 2,
       "[INST]": 3,
       "[/INST]": 4,
-      "[TOOL_CALLS]": 5,
-      "[AVAILABLE_TOOLS]": 6,
-      "[/AVAILABLE_TOOLS]": 7,
-      "[TOOL_RESULTS]": 8,
-      "[/TOOL_RESULTS]": 9,
-      "[control_8]": 10,
-      "[control_9]": 11,
-      "[control_10]": 12,
-      "[control_11]": 13,
-      "[control_12]": 14,
-      "[control_13]": 15,
-      "[control_14]": 16,
+      "<tool_call>": 5,
+      "<available_tools>": 6,
+      "</available_tools>": 7,
+      "<tool_response>": 8,
+      "</tool_response>": 9,
+      "<available_documents>": 10,
+      "</available_documents>": 11,
+      "<reference>": 12,
+      "</reference>": 13,
+      "<|system|>": 14,
+      "<|user|>": 15,
+      "<|assistant|>": 16,
       "[control_15]": 17,
       "[control_16]": 18,
       "[control_17]": 19,
@@ -98790,4 +98790,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
-size 587404
+oid sha256:7c6f8a481c33393f0799a3fbb8b4905efda071b90a3c852b134ef4959890190a
+size 587420

tokenizer_config.json

@@ -44,7 +44,7 @@
       "special": true
     },
     "5": {
-      "content": "[TOOL_CALLS]",
+      "content": "<tool_call>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -52,7 +52,7 @@
       "special": true
     },
     "6": {
-      "content": "[AVAILABLE_TOOLS]",
+      "content": "<available_tools>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -60,7 +60,7 @@
       "special": true
     },
     "7": {
-      "content": "[/AVAILABLE_TOOLS]",
+      "content": "</available_tools>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
       "special": true
     },
     "8": {
-      "content": "[TOOL_RESULTS]",
+      "content": "<tool_response>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -76,7 +76,7 @@
       "special": true
     },
     "9": {
-      "content": "[/TOOL_RESULTS]",
+      "content": "</tool_response>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -84,7 +84,7 @@
       "special": true
     },
     "10": {
-      "content": "[control_8]",
+      "content": "<available_documents>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -92,7 +92,7 @@
       "special": true
     },
     "11": {
-      "content": "[control_9]",
+      "content": "</available_documents>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -100,7 +100,7 @@
       "special": true
     },
     "12": {
-      "content": "[control_10]",
+      "content": "<reference>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -108,7 +108,7 @@
       "special": true
     },
     "13": {
-      "content": "[control_11]",
+      "content": "</reference>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -116,7 +116,7 @@
       "special": true
     },
     "14": {
-      "content": "[control_12]",
+      "content": "<|system|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -124,7 +124,7 @@
       "special": true
     },
     "15": {
-      "content": "[control_13]",
+      "content": "<|user|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -132,7 +132,7 @@
       "special": true
     },
     "16": {
-      "content": "[control_14]",
+      "content": "<|assistant|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -6173,6 +6173,7 @@
     }
   },
   "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": false,