lykeven commited on
Commit
54abf22
1 Parent(s): c20d693

first model commit

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. LICENSE +201 -0
  2. MODEL_LICENSE +33 -0
  3. README.md +85 -0
  4. config.json +28 -0
  5. configuration_chatglm.py +103 -0
  6. glms.py +28 -0
  7. ice_text.model +3 -0
  8. modeling_chatglm.py +1435 -0
  9. ptuning/.gitattributes +1 -0
  10. ptuning/AdvertiseGen/dev.json +0 -0
  11. ptuning/AdvertiseGen/train.json +0 -0
  12. ptuning/README.md +254 -0
  13. ptuning/README_en.md +115 -0
  14. ptuning/__pycache__/arguments.cpython-39.pyc +0 -0
  15. ptuning/__pycache__/trainer.cpython-39.pyc +0 -0
  16. ptuning/__pycache__/trainer_seq2seq.cpython-39.pyc +0 -0
  17. ptuning/arguments.py +224 -0
  18. ptuning/deepspeed.json +21 -0
  19. ptuning/ds_train_finetune.sh +28 -0
  20. ptuning/evaluate.sh +21 -0
  21. ptuning/evaluate_finetune.sh +18 -0
  22. ptuning/main.py +431 -0
  23. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/all_results.json +8 -0
  24. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/config.json +31 -0
  25. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/configuration_chatglm.py +103 -0
  26. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/generation_config.json +7 -0
  27. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/ice_text.model +3 -0
  28. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/modeling_chatglm.py +1435 -0
  29. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/optimizer.pt +3 -0
  30. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/pytorch_model.bin +3 -0
  31. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/quantization.py +201 -0
  32. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/rng_state.pth +3 -0
  33. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/scheduler.pt +3 -0
  34. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/special_tokens_map.json +7 -0
  35. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/tokenization_chatglm.py +430 -0
  36. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/tokenizer_config.json +22 -0
  37. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/trainer_state.json +76 -0
  38. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/training_args.bin +3 -0
  39. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/config.json +31 -0
  40. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/configuration_chatglm.py +103 -0
  41. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/generation_config.json +7 -0
  42. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/ice_text.model +3 -0
  43. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/modeling_chatglm.py +1435 -0
  44. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/optimizer.pt +3 -0
  45. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/pytorch_model.bin +3 -0
  46. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/quantization.py +201 -0
  47. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/rng_state.pth +3 -0
  48. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/scheduler.pt +3 -0
  49. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/special_tokens_map.json +7 -0
  50. ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/tokenization_chatglm.py +430 -0
LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright Zhengxiao Du
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
MODEL_LICENSE ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ The GLM-130B License
2
+
3
+ 1. Definitions
4
+
5
+ “Licensor” means the GLM-130B Model Team that distributes its Software.
6
+
7
+ “Software” means the GLM-130B model parameters made available under this license.
8
+
9
+ 2. License Grant
10
+
11
+ Subject to the terms and conditions of this License, the Licensor hereby grants to you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty-free copyright license to use the Software solely for your non-commercial research purposes.
12
+
13
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
14
+
15
+ 3. Restriction
16
+
17
+ You will not use, copy, modify, merge, publish, distribute, reproduce, or create derivative works of the Software, in whole or in part, for any commercial, military, or illegal purposes.
18
+
19
+ You will not use the Software for any act that may undermine China's national security and national unity, harm the public interest of society, or infringe upon the rights and interests of human beings.
20
+
21
+ 4. Disclaimer
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24
+
25
+ 5. Limitation of Liability
26
+
27
+ EXCEPT TO THE EXTENT PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER BASED IN TORT, NEGLIGENCE, CONTRACT, LIABILITY, OR OTHERWISE WILL ANY LICENSOR BE LIABLE TO YOU FOR ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES, OR ANY OTHER COMMERCIAL LOSSES, EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
28
+
29
+ 6. Dispute Resolution
30
+
31
+ This license shall be governed and construed in accordance with the laws of People’s Republic of China. Any dispute arising from or in connection with this License shall be submitted to Haidian District People's Court in Beijing.
32
+
33
+ Note that the license is subject to update to a more comprehensive version. For any questions related to the license and copyright, please contact us at [email protected].
README.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - zh
4
+ - en
5
+ tags:
6
+ - glm
7
+ - chatglm
8
+ - thudm
9
+ ---
10
+ # ChatGLM-6B
11
+ <p align="center">
12
+ 🌐 <a href="https://chatglm.cn/blog" target="_blank">Blog</a> • 💻 <a href="https://github.com/THUDM/ChatGLM-6B" target="_blank">Github Repo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a> • 📃 <a href="https://arxiv.org/abs/2103.10360" target="_blank">[GLM@ACL 22]</a> <a href="https://github.com/THUDM/GLM" target="_blank">[GitHub]</a> • 📃 <a href="https://arxiv.org/abs/2210.02414" target="_blank">[GLM-130B@ICLR 23]</a> <a href="https://github.com/THUDM/GLM-130B" target="_blank">[GitHub]</a> <br>
13
+ </p>
14
+
15
+ <p align="center">
16
+ 👋 Join our <a href="https://join.slack.com/t/chatglm/shared_invite/zt-1th2q5u69-7tURzFuOPanmuHy9hsZnKA" target="_blank">Slack</a> and <a href="https://github.com/THUDM/ChatGLM-6B/blob/main/resources/WECHAT.md" target="_blank">WeChat</a>
17
+ </p>
18
+
19
+ ## 介绍
20
+ ChatGLM-6B 是一个开源的、支持中英双语问答的对话语言模型,基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构,具有 62 亿参数。结合模型量化技术,用户可以在消费级的显卡上进行本地部署(INT4 量化级别下最低只需 6GB 显存)。ChatGLM-6B 使用了和 [ChatGLM](https://chatglm.cn) 相同的技术,针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练,辅以监督微调、反馈自助、人类反馈强化学习等技术的加持,62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。
21
+
22
+ ChatGLM-6B is an open bilingual language model based on [General Language Model (GLM)](https://github.com/THUDM/GLM) framework, with 6.2 billion parameters. With the quantization technique, users can deploy locally on consumer-grade graphics cards (only 6GB of GPU memory is required at the INT4 quantization level). ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dialogue. The model is trained for about 1T tokens of Chinese and English corpus, supplemented by supervised fine-tuning, feedback bootstrap, and reinforcement learning wit human feedback. With only about 6.2 billion parameters, the model is able to generate answers that are in line with human preference.
23
+
24
+ ## 软件依赖
25
+
26
+ ```shell
27
+ pip install protobuf==3.20.0 transformers==4.27.1 icetk cpm_kernels
28
+ ```
29
+
30
+ ## 代码调用
31
+
32
+ 可以通过如下代码调用 ChatGLM-6B 模型来生成对话:
33
+
34
+ ```ipython
35
+ >>> from transformers import AutoTokenizer, AutoModel
36
+ >>> tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
37
+ >>> model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda()
38
+ >>> response, history = model.chat(tokenizer, "你好", history=[])
39
+ >>> print(response)
40
+ 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。
41
+ >>> response, history = model.chat(tokenizer, "晚上睡不着应该怎么办", history=history)
42
+ >>> print(response)
43
+ 晚上睡不着可能会让你感到焦虑或不舒服,但以下是一些可以帮助你入睡的方法:
44
+
45
+ 1. 制定规律的睡眠时间表:保持规律的睡眠时间表可以帮助你建立健康的睡眠习惯,使你更容易入睡。尽量在每天的相同时间上床,并在同一时间起床。
46
+ 2. 创造一个舒适的睡眠环境:确保睡眠环境舒适,安静,黑暗且温度适宜。可以使用舒适的床上用品,并保持房间通风。
47
+ 3. 放松身心:在睡前做些放松的活动,例如泡个热水澡,听些轻柔的音乐,阅读一些有趣的书籍等,有助于缓解紧张和焦虑,使你更容易入睡。
48
+ 4. 避免饮用含有咖啡因的饮料:咖啡因是一种刺激性物质,会影响你的睡眠质量。尽量避免在睡前饮用含有咖啡因的饮料,例如咖啡,茶和可乐。
49
+ 5. 避免在床上做与睡眠无关的事情:在床上做些与睡眠无关的事情,例如看电影,玩游戏或工作等,可能会干扰你的睡眠。
50
+ 6. 尝试呼吸技巧:深呼吸是一种放松技巧,可以帮助你缓解紧张和焦虑,使你更容易入睡。试着慢慢吸气,保持几秒钟,然后缓慢呼气。
51
+
52
+ 如果这些方法无法帮助你入睡,你可以考虑咨询医生或睡眠专家,寻求进一步的建议。
53
+ ```
54
+
55
+ 关于更多的使用说明,包括如何运行命令行和网页版本的 DEMO,以及使用模型量化以节省显存,请参考我们的 [Github Repo](https://github.com/THUDM/ChatGLM-6B)。
56
+
57
+ For more instructions, including how to run CLI and web demos, and model quantization, please refer to our [Github Repo](https://github.com/THUDM/ChatGLM-6B).
58
+
59
+ ## 协议
60
+
61
+ 本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源,ChatGLM-6B 模型的权重的使用则需要遵循 [Model License](MODEL_LICENSE)。
62
+
63
+ ## 引用
64
+
65
+ 如果你觉得我们的工作有帮助的话,请考虑引用下列论文:
66
+
67
+ ```
68
+ @inproceedings{
69
+ zeng2023glm-130b,
70
+ title={{GLM}-130B: An Open Bilingual Pre-trained Model},
71
+ author={Aohan Zeng and Xiao Liu and Zhengxiao Du and Zihan Wang and Hanyu Lai and Ming Ding and Zhuoyi Yang and Yifan Xu and Wendi Zheng and Xiao Xia and Weng Lam Tam and Zixuan Ma and Yufei Xue and Jidong Zhai and Wenguang Chen and Zhiyuan Liu and Peng Zhang and Yuxiao Dong and Jie Tang},
72
+ booktitle={The Eleventh International Conference on Learning Representations (ICLR)},
73
+ year={2023},
74
+ url={https://openreview.net/forum?id=-Aw0rrrPUF}
75
+ }
76
+ ```
77
+ ```
78
+ @inproceedings{du2022glm,
79
+ title={GLM: General Language Model Pretraining with Autoregressive Blank Infilling},
80
+ author={Du, Zhengxiao and Qian, Yujie and Liu, Xiao and Ding, Ming and Qiu, Jiezhong and Yang, Zhilin and Tang, Jie},
81
+ booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
82
+ pages={320--335},
83
+ year={2022}
84
+ }
85
+ ```
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "THUDM/chatglm-6b",
3
+ "architectures": [
4
+ "ChatGLMModel"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_chatglm.ChatGLMConfig",
8
+ "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
9
+ "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
10
+ },
11
+ "bos_token_id": 130004,
12
+ "eos_token_id": 130005,
13
+ "mask_token_id": 130000,
14
+ "gmask_token_id": 130001,
15
+ "pad_token_id": 3,
16
+ "hidden_size": 4096,
17
+ "inner_hidden_size": 16384,
18
+ "layernorm_epsilon": 1e-05,
19
+ "max_sequence_length": 2048,
20
+ "model_type": "chatglm",
21
+ "num_attention_heads": 32,
22
+ "num_layers": 28,
23
+ "position_encoding_2d": true,
24
+ "torch_dtype": "float16",
25
+ "transformers_version": "4.23.1",
26
+ "use_cache": true,
27
+ "vocab_size": 130528
28
+ }
configuration_chatglm.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ ChatGLM model configuration """
2
+
3
+ from transformers.configuration_utils import PretrainedConfig
4
+ from transformers.utils import logging
5
+
6
+ logger = logging.get_logger(__name__)
7
+
8
+
9
+ class ChatGLMConfig(PretrainedConfig):
10
+ r"""
11
+ This is the configuration class to store the configuration of a [`~ChatGLMModel`].
12
+ It is used to instantiate an ChatGLM model according to the specified arguments, defining the model
13
+ architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
14
+ the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture.
15
+
16
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
17
+ to control the model outputs. Read the documentation from [`PretrainedConfig`]
18
+ for more information.
19
+
20
+
21
+ Args:
22
+ vocab_size (`int`, *optional*, defaults to 150528):
23
+ Vocabulary size of the ChatGLM-6B model. Defines the number of different tokens that can be represented by the
24
+ `inputs_ids` passed when calling [`~ChatGLMModel`] or
25
+ [`~TFChatGLMModel`].
26
+ hidden_size (`int`, *optional*, defaults to 4096):
27
+ Dimension of the encoder layers and the pooler layer.
28
+ num_hidden_layers (`int`, *optional*, defaults to 28):
29
+ Number of hidden layers in the Transformer encoder.
30
+ num_attention_heads (`int`, *optional*, defaults to 32):
31
+ Number of attention heads for each attention layer in the Transformer encoder.
32
+ inner_hidden_size (`int`, *optional*, defaults to 16384):
33
+ Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
34
+ max_sequence_length (`int`, *optional*, defaults to 512):
35
+ The maximum sequence length that this model might ever be used with.
36
+ Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
37
+ layernorm_epsilon (`float`, *optional*, defaults to 1e-5):
38
+ The epsilon used by the layer normalization layers.
39
+ use_cache (`bool`, *optional*, defaults to `True`):
40
+ Whether the model should return the last key/values attentions (not used by all models).
41
+ Example:
42
+
43
+ ```python
44
+ >>> from configuration_chatglm import ChatGLMConfig
45
+ >>> from modeling_chatglm import ChatGLMModel
46
+
47
+ >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration
48
+ >>> configuration = ChatGLMConfig()
49
+
50
+ >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration
51
+ >>> model = ChatGLMModel(configuration)
52
+
53
+ >>> # Accessing the model configuration
54
+ >>> configuration = model.config
55
+ ```
56
+ """
57
+ model_type = "chatglm"
58
+
59
+ def __init__(
60
+ self,
61
+ vocab_size=150528,
62
+ hidden_size=4096,
63
+ num_layers=28,
64
+ num_attention_heads=32,
65
+ layernorm_epsilon=1e-5,
66
+ use_cache=False,
67
+ bos_token_id=150004,
68
+ eos_token_id=150005,
69
+ mask_token_id=150000,
70
+ gmask_token_id=150001,
71
+ pad_token_id=0,
72
+ max_sequence_length=2048,
73
+ inner_hidden_size=16384,
74
+ position_encoding_2d=True,
75
+ quantization_bit=0,
76
+ pre_seq_len=None,
77
+ prefix_projection=False,
78
+ **kwargs
79
+ ):
80
+ self.num_layers = num_layers
81
+ self.vocab_size = vocab_size
82
+ self.hidden_size = hidden_size
83
+ self.num_attention_heads = num_attention_heads
84
+ self.max_sequence_length = max_sequence_length
85
+ self.layernorm_epsilon = layernorm_epsilon
86
+ self.inner_hidden_size = inner_hidden_size
87
+ self.use_cache = use_cache
88
+ self.bos_token_id = bos_token_id
89
+ self.eos_token_id = eos_token_id
90
+ self.pad_token_id = pad_token_id
91
+ self.mask_token_id = mask_token_id
92
+ self.gmask_token_id = gmask_token_id
93
+ self.position_encoding_2d = position_encoding_2d
94
+ self.quantization_bit = quantization_bit
95
+ self.pre_seq_len = pre_seq_len
96
+ self.prefix_projection = prefix_projection
97
+
98
+ super().__init__(
99
+ pad_token_id=pad_token_id,
100
+ bos_token_id=bos_token_id,
101
+ eos_token_id=eos_token_id,
102
+ **kwargs
103
+ )
glms.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ # os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7"
3
+ os.environ["CUDA_VISIBLE_DEVICES"] = "0"
4
+
5
+
6
+ import torch
7
+ import numpy as np
8
+
9
+ from transformers import AutoModelForSequenceClassification, AutoModelForMultipleChoice, AutoModel
10
+ from transformers import TrainingArguments, Trainer
11
+ from transformers import AutoTokenizer
12
+ from transformers import DataCollatorWithPadding
13
+ from datasets import load_dataset
14
+ from datasets import load_metric
15
+ import torchsnooper
16
+
17
+ model_name = "THUDM/chatglm-6b"
18
+
19
+ if __name__ == "__main__":
20
+ device = "cuda" if torch.cuda.is_available() else "cpu"
21
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
22
+ model = AutoModel.from_pretrained(model_name, trust_remote_code=True).half().cuda()
23
+ response, history = model.chat(tokenizer, "你好", history=[])
24
+ print(response)
25
+ response, history = model.chat(tokenizer, "晚上睡不着应该怎么办", history=history)
26
+ print(response)
27
+
28
+
ice_text.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e974d9a69c242ce014c88c2b26089270f6198f3c0b700a887666cd3e816f17e
3
+ size 2706249
modeling_chatglm.py ADDED
@@ -0,0 +1,1435 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ PyTorch ChatGLM model. """
2
+
3
+ import math
4
+ import copy
5
+ import os
6
+ import warnings
7
+ import re
8
+ import sys
9
+
10
+ import torch
11
+ import torch.utils.checkpoint
12
+ import torch.nn.functional as F
13
+ from torch import nn
14
+ from torch.nn import CrossEntropyLoss, LayerNorm
15
+ from torch.nn.utils import skip_init
16
+ from typing import Optional, Tuple, Union, List, Callable, Dict, Any
17
+
18
+ from transformers.utils import (
19
+ add_code_sample_docstrings,
20
+ add_start_docstrings,
21
+ add_start_docstrings_to_model_forward,
22
+ )
23
+ from transformers.modeling_outputs import (
24
+ BaseModelOutputWithPast,
25
+ CausalLMOutputWithPast,
26
+ BaseModelOutputWithPastAndCrossAttentions,
27
+ )
28
+ from transformers.modeling_utils import PreTrainedModel
29
+ from transformers.utils import logging
30
+ from transformers.generation.logits_process import LogitsProcessor
31
+ from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationConfig, ModelOutput
32
+
33
+ from .configuration_chatglm import ChatGLMConfig
34
+
35
+ # flags required to enable jit fusion kernels
36
+
37
+ if sys.platform != 'darwin':
38
+ torch._C._jit_set_profiling_mode(False)
39
+ torch._C._jit_set_profiling_executor(False)
40
+ torch._C._jit_override_can_fuse_on_cpu(True)
41
+ torch._C._jit_override_can_fuse_on_gpu(True)
42
+
43
+ logger = logging.get_logger(__name__)
44
+
45
+ _CHECKPOINT_FOR_DOC = "THUDM/ChatGLM-6B"
46
+ _CONFIG_FOR_DOC = "ChatGLM6BConfig"
47
+
48
+ CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [
49
+ "THUDM/chatglm-6b",
50
+ # See all ChatGLM-6B models at https://huggingface.co/models?filter=chatglm
51
+ ]
52
+
53
+
54
+ class InvalidScoreLogitsProcessor(LogitsProcessor):
55
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
56
+ if torch.isnan(scores).any() or torch.isinf(scores).any():
57
+ scores.zero_()
58
+ scores[..., 5] = 5e4
59
+ return scores
60
+
61
+
62
+ def load_tf_weights_in_chatglm_6b(model, config, tf_checkpoint_path):
63
+ """Load tf checkpoints in a pytorch model."""
64
+ try:
65
+ import re
66
+
67
+ import numpy as np
68
+ import tensorflow as tf
69
+ except ImportError:
70
+ logger.error(
71
+ "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
72
+ "https://www.tensorflow.org/install/ for installation instructions."
73
+ )
74
+ raise
75
+ tf_path = os.path.abspath(tf_checkpoint_path)
76
+ logger.info(f"Converting TensorFlow checkpoint from {tf_path}")
77
+ # Load weights from TF model
78
+ init_vars = tf.train.list_variables(tf_path)
79
+ names = []
80
+ arrays = []
81
+ for name, shape in init_vars:
82
+ logger.info(f"Loading TF weight {name} with shape {shape}")
83
+ array = tf.train.load_variable(tf_path, name)
84
+ names.append(name)
85
+ arrays.append(array)
86
+
87
+ for name, array in zip(names, arrays):
88
+ name = name.split("/")
89
+ # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
90
+ # which are not required for using pretrained model
91
+ if any(
92
+ n in ["adam_v", "adam_m", "AdamWeightDecayOptimizer", "AdamWeightDecayOptimizer_1", "global_step"]
93
+ for n in name
94
+ ):
95
+ logger.info(f"Skipping {'/'.join(name)}")
96
+ continue
97
+ pointer = model
98
+ for m_name in name:
99
+ if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
100
+ scope_names = re.split(r"_(\d+)", m_name)
101
+ else:
102
+ scope_names = [m_name]
103
+ if scope_names[0] == "kernel" or scope_names[0] == "gamma":
104
+ pointer = getattr(pointer, "weight")
105
+ elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
106
+ pointer = getattr(pointer, "bias")
107
+ elif scope_names[0] == "output_weights":
108
+ pointer = getattr(pointer, "weight")
109
+ elif scope_names[0] == "squad":
110
+ pointer = getattr(pointer, "classifier")
111
+ else:
112
+ try:
113
+ pointer = getattr(pointer, scope_names[0])
114
+ except AttributeError:
115
+ logger.info(f"Skipping {'/'.join(name)}")
116
+ continue
117
+ if len(scope_names) >= 2:
118
+ num = int(scope_names[1])
119
+ pointer = pointer[num]
120
+ if m_name[-11:] == "_embeddings":
121
+ pointer = getattr(pointer, "weight")
122
+ elif m_name == "kernel":
123
+ array = np.transpose(array)
124
+ try:
125
+ assert (
126
+ pointer.shape == array.shape
127
+ ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"
128
+ except AssertionError as e:
129
+ e.args += (pointer.shape, array.shape)
130
+ raise
131
+ logger.info(f"Initialize PyTorch weight {name}")
132
+ pointer.data = torch.from_numpy(array)
133
+ return model
134
+
135
+
136
+ class PrefixEncoder(torch.nn.Module):
137
+ """
138
+ The torch.nn model to encode the prefix
139
+ Input shape: (batch-size, prefix-length)
140
+ Output shape: (batch-size, prefix-length, 2*layers*hidden)
141
+ """
142
+
143
+ def __init__(self, config):
144
+ super().__init__()
145
+ self.prefix_projection = config.prefix_projection
146
+ if self.prefix_projection:
147
+ # Use a two-layer MLP to encode the prefix
148
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.hidden_size)
149
+ self.trans = torch.nn.Sequential(
150
+ torch.nn.Linear(config.hidden_size, config.hidden_size),
151
+ torch.nn.Tanh(),
152
+ torch.nn.Linear(config.hidden_size, config.num_layers * config.hidden_size * 2)
153
+ )
154
+ else:
155
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.num_layers * config.hidden_size * 2)
156
+
157
+ def forward(self, prefix: torch.Tensor):
158
+ if self.prefix_projection:
159
+ prefix_tokens = self.embedding(prefix)
160
+ past_key_values = self.trans(prefix_tokens)
161
+ else:
162
+ past_key_values = self.embedding(prefix)
163
+ return past_key_values
164
+
165
+
166
+ @torch.jit.script
167
+ def gelu_impl(x):
168
+ """OpenAI's gelu implementation."""
169
+ return 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x *
170
+ (1.0 + 0.044715 * x * x)))
171
+
172
+
173
+ def gelu(x):
174
+ return gelu_impl(x)
175
+
176
+
177
+ class RotaryEmbedding(torch.nn.Module):
178
+ def __init__(self, dim, base=10000, precision=torch.half, learnable=False):
179
+ super().__init__()
180
+ inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim))
181
+ inv_freq = inv_freq.half()
182
+ self.learnable = learnable
183
+ if learnable:
184
+ self.inv_freq = torch.nn.Parameter(inv_freq)
185
+ self.max_seq_len_cached = None
186
+ else:
187
+ self.register_buffer('inv_freq', inv_freq)
188
+ self.max_seq_len_cached = None
189
+ self.cos_cached = None
190
+ self.sin_cached = None
191
+ self.precision = precision
192
+
193
+ def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys,
194
+ error_msgs):
195
+ pass
196
+
197
+ def forward(self, x, seq_dim=1, seq_len=None):
198
+ if seq_len is None:
199
+ seq_len = x.shape[seq_dim]
200
+ if self.max_seq_len_cached is None or (seq_len > self.max_seq_len_cached):
201
+ self.max_seq_len_cached = None if self.learnable else seq_len
202
+ t = torch.arange(seq_len, device=x.device, dtype=self.inv_freq.dtype)
203
+ freqs = torch.einsum('i,j->ij', t, self.inv_freq)
204
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
205
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
206
+ if self.precision == torch.bfloat16:
207
+ emb = emb.float()
208
+
209
+ # [sx, 1 (b * np), hn]
210
+ cos_cached = emb.cos()[:, None, :]
211
+ sin_cached = emb.sin()[:, None, :]
212
+ if self.precision == torch.bfloat16:
213
+ cos_cached = cos_cached.bfloat16()
214
+ sin_cached = sin_cached.bfloat16()
215
+ if self.learnable:
216
+ return cos_cached, sin_cached
217
+ self.cos_cached, self.sin_cached = cos_cached, sin_cached
218
+ return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
219
+
220
+ def _apply(self, fn):
221
+ if self.cos_cached is not None:
222
+ self.cos_cached = fn(self.cos_cached)
223
+ if self.sin_cached is not None:
224
+ self.sin_cached = fn(self.sin_cached)
225
+ return super()._apply(fn)
226
+
227
+
228
+ def rotate_half(x):
229
+ x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
230
+ return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions
231
+
232
+
233
+ @torch.jit.script
234
+ def apply_rotary_pos_emb_index(q, k, cos, sin, position_id):
235
+ # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn]
236
+ cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \
237
+ F.embedding(position_id, sin.squeeze(1)).unsqueeze(2)
238
+ q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
239
+ return q, k
240
+
241
+
242
+ def attention_fn(
243
+ self,
244
+ query_layer,
245
+ key_layer,
246
+ value_layer,
247
+ attention_mask,
248
+ hidden_size_per_partition,
249
+ layer_id,
250
+ layer_past=None,
251
+ scaling_attention_score=True,
252
+ use_cache=False,
253
+ ):
254
+ if layer_past is not None:
255
+ past_key, past_value = layer_past[0], layer_past[1]
256
+ key_layer = torch.cat((past_key, key_layer), dim=0)
257
+ value_layer = torch.cat((past_value, value_layer), dim=0)
258
+
259
+ # seqlen, batch, num_attention_heads, hidden_size_per_attention_head
260
+ seq_len, b, nh, hidden_size = key_layer.shape
261
+
262
+ if use_cache:
263
+ present = (key_layer, value_layer)
264
+ else:
265
+ present = None
266
+
267
+ query_key_layer_scaling_coeff = float(layer_id + 1)
268
+ if scaling_attention_score:
269
+ query_layer = query_layer / (math.sqrt(hidden_size) * query_key_layer_scaling_coeff)
270
+
271
+ # ===================================
272
+ # Raw attention scores. [b, np, s, s]
273
+ # ===================================
274
+
275
+ # [b, np, sq, sk]
276
+ output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0))
277
+
278
+ # [sq, b, np, hn] -> [sq, b * np, hn]
279
+ query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
280
+ # [sk, b, np, hn] -> [sk, b * np, hn]
281
+ key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
282
+
283
+ matmul_result = torch.zeros(
284
+ 1, 1, 1,
285
+ dtype=query_layer.dtype,
286
+ device=query_layer.device,
287
+ )
288
+
289
+ matmul_result = torch.baddbmm(
290
+ matmul_result,
291
+ query_layer.transpose(0, 1), # [b * np, sq, hn]
292
+ key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk]
293
+ beta=0.0,
294
+ alpha=1.0,
295
+ )
296
+
297
+ # change view to [b, np, sq, sk]
298
+ attention_scores = matmul_result.view(*output_size)
299
+
300
+ if self.scale_mask_softmax:
301
+ self.scale_mask_softmax.scale = query_key_layer_scaling_coeff
302
+ attention_probs = self.scale_mask_softmax(attention_scores, attention_mask.contiguous())
303
+ else:
304
+ if not (attention_mask == 0).all():
305
+ # if auto-regressive, skip
306
+ attention_scores.masked_fill_(attention_mask, -10000.0)
307
+ dtype = attention_scores.dtype
308
+ attention_scores = attention_scores.float()
309
+ attention_scores = attention_scores * query_key_layer_scaling_coeff
310
+
311
+ attention_probs = F.softmax(attention_scores, dim=-1)
312
+
313
+ attention_probs = attention_probs.type(dtype)
314
+
315
+ # =========================
316
+ # Context layer. [sq, b, hp]
317
+ # =========================
318
+
319
+ # value_layer -> context layer.
320
+ # [sk, b, np, hn] --> [b, np, sq, hn]
321
+
322
+ # context layer shape: [b, np, sq, hn]
323
+ output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3))
324
+
325
+ # change view [sk, b * np, hn]
326
+ value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1)
327
+
328
+ # change view [b * np, sq, sk]
329
+ attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1)
330
+
331
+ # matmul: [b * np, sq, hn]
332
+ context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1))
333
+
334
+ # change view [b, np, sq, hn]
335
+ context_layer = context_layer.view(*output_size)
336
+
337
+ # [b, np, sq, hn] --> [sq, b, np, hn]
338
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
339
+
340
+ # [sq, b, np, hn] --> [sq, b, hp]
341
+ new_context_layer_shape = context_layer.size()[:-2] + (hidden_size_per_partition,)
342
+ context_layer = context_layer.view(*new_context_layer_shape)
343
+
344
+ outputs = (context_layer, present, attention_probs)
345
+
346
+ return outputs
347
+
348
+
349
+ def default_init(cls, *args, **kwargs):
350
+ return cls(*args, **kwargs)
351
+
352
+
353
+ class SelfAttention(torch.nn.Module):
354
+ def __init__(self, hidden_size, num_attention_heads,
355
+ layer_id, hidden_size_per_attention_head=None, bias=True,
356
+ params_dtype=torch.float, position_encoding_2d=True, empty_init=True):
357
+ if empty_init:
358
+ init_method = skip_init
359
+ else:
360
+ init_method = default_init
361
+ super(SelfAttention, self).__init__()
362
+
363
+ self.layer_id = layer_id
364
+ self.hidden_size = hidden_size
365
+ self.hidden_size_per_partition = hidden_size
366
+ self.num_attention_heads = num_attention_heads
367
+ self.num_attention_heads_per_partition = num_attention_heads
368
+ self.position_encoding_2d = position_encoding_2d
369
+ self.rotary_emb = RotaryEmbedding(
370
+ self.hidden_size // (self.num_attention_heads * 2)
371
+ if position_encoding_2d
372
+ else self.hidden_size // self.num_attention_heads,
373
+ base=10000,
374
+ precision=torch.half,
375
+ learnable=False,
376
+ )
377
+
378
+ self.scale_mask_softmax = None
379
+
380
+ if hidden_size_per_attention_head is None:
381
+ self.hidden_size_per_attention_head = hidden_size // num_attention_heads
382
+ else:
383
+ self.hidden_size_per_attention_head = hidden_size_per_attention_head
384
+
385
+ self.inner_hidden_size = num_attention_heads * self.hidden_size_per_attention_head
386
+
387
+ # Strided linear layer.
388
+ self.query_key_value = init_method(
389
+ torch.nn.Linear,
390
+ hidden_size,
391
+ 3 * self.inner_hidden_size,
392
+ bias=bias,
393
+ dtype=params_dtype,
394
+ )
395
+
396
+ self.dense = init_method(
397
+ torch.nn.Linear,
398
+ self.inner_hidden_size,
399
+ hidden_size,
400
+ bias=bias,
401
+ dtype=params_dtype,
402
+ )
403
+
404
+ @staticmethod
405
+ def attention_mask_func(attention_scores, attention_mask):
406
+ attention_scores.masked_fill_(attention_mask, -10000.0)
407
+ return attention_scores
408
+
409
+ def split_tensor_along_last_dim(self, tensor, num_partitions,
410
+ contiguous_split_chunks=False):
411
+ """Split a tensor along its last dimension.
412
+ Arguments:
413
+ tensor: input tensor.
414
+ num_partitions: number of partitions to split the tensor
415
+ contiguous_split_chunks: If True, make each chunk contiguous
416
+ in memory.
417
+ """
418
+ # Get the size and dimension.
419
+ last_dim = tensor.dim() - 1
420
+ last_dim_size = tensor.size()[last_dim] // num_partitions
421
+ # Split.
422
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
423
+ # Note: torch.split does not create contiguous tensors by default.
424
+ if contiguous_split_chunks:
425
+ return tuple(chunk.contiguous() for chunk in tensor_list)
426
+
427
+ return tensor_list
428
+
429
+ def forward(
430
+ self,
431
+ hidden_states: torch.Tensor,
432
+ position_ids,
433
+ attention_mask: torch.Tensor,
434
+ layer_id,
435
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
436
+ use_cache: bool = False,
437
+ output_attentions: bool = False,
438
+ ):
439
+ """
440
+ hidden_states: [seq_len, batch, hidden_size]
441
+ attention_mask: [(1, 1), seq_len, seq_len]
442
+ """
443
+
444
+ # [seq_len, batch, 3 * hidden_size]
445
+ mixed_raw_layer = self.query_key_value(hidden_states)
446
+
447
+ # [seq_len, batch, 3 * hidden_size] --> [seq_len, batch, num_attention_heads, 3 * hidden_size_per_attention_head]
448
+ new_tensor_shape = mixed_raw_layer.size()[:-1] + (
449
+ self.num_attention_heads_per_partition,
450
+ 3 * self.hidden_size_per_attention_head,
451
+ )
452
+ mixed_raw_layer = mixed_raw_layer.view(*new_tensor_shape)
453
+
454
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
455
+ (query_layer, key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_raw_layer, 3)
456
+
457
+ if self.position_encoding_2d:
458
+ q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1))
459
+ k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1))
460
+ cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1)
461
+ position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \
462
+ position_ids[:, 1, :].transpose(0, 1).contiguous()
463
+ q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids)
464
+ q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, block_position_ids)
465
+ query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1))
466
+ key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1))
467
+ else:
468
+ position_ids = position_ids.transpose(0, 1)
469
+ cos, sin = self.rotary_emb(value_layer, seq_len=position_ids.max() + 1)
470
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
471
+ query_layer, key_layer = apply_rotary_pos_emb_index(query_layer, key_layer, cos, sin, position_ids)
472
+
473
+ # [seq_len, batch, hidden_size]
474
+ context_layer, present, attention_probs = attention_fn(
475
+ self=self,
476
+ query_layer=query_layer,
477
+ key_layer=key_layer,
478
+ value_layer=value_layer,
479
+ attention_mask=attention_mask,
480
+ hidden_size_per_partition=self.hidden_size_per_partition,
481
+ layer_id=layer_id,
482
+ layer_past=layer_past,
483
+ use_cache=use_cache
484
+ )
485
+
486
+ output = self.dense(context_layer)
487
+
488
+ outputs = (output, present)
489
+
490
+ if output_attentions:
491
+ outputs += (attention_probs,)
492
+
493
+ return outputs # output, present, attention_probs
494
+
495
+
496
+ class GEGLU(torch.nn.Module):
497
+ def __init__(self):
498
+ super().__init__()
499
+ self.activation_fn = F.gelu
500
+
501
+ def forward(self, x):
502
+ # dim=-1 breaks in jit for pt<1.10
503
+ x1, x2 = x.chunk(2, dim=(x.ndim - 1))
504
+ return x1 * self.activation_fn(x2)
505
+
506
+
507
+ class GLU(torch.nn.Module):
508
+ def __init__(self, hidden_size, inner_hidden_size=None,
509
+ layer_id=None, bias=True, activation_func=gelu, params_dtype=torch.float, empty_init=True):
510
+ super(GLU, self).__init__()
511
+ if empty_init:
512
+ init_method = skip_init
513
+ else:
514
+ init_method = default_init
515
+ self.layer_id = layer_id
516
+ self.activation_func = activation_func
517
+
518
+ # Project to 4h.
519
+ self.hidden_size = hidden_size
520
+ if inner_hidden_size is None:
521
+ inner_hidden_size = 4 * hidden_size
522
+ self.inner_hidden_size = inner_hidden_size
523
+ self.dense_h_to_4h = init_method(
524
+ torch.nn.Linear,
525
+ self.hidden_size,
526
+ self.inner_hidden_size,
527
+ bias=bias,
528
+ dtype=params_dtype,
529
+ )
530
+ # Project back to h.
531
+ self.dense_4h_to_h = init_method(
532
+ torch.nn.Linear,
533
+ self.inner_hidden_size,
534
+ self.hidden_size,
535
+ bias=bias,
536
+ dtype=params_dtype,
537
+ )
538
+
539
+ def forward(self, hidden_states):
540
+ """
541
+ hidden_states: [seq_len, batch, hidden_size]
542
+ """
543
+
544
+ # [seq_len, batch, inner_hidden_size]
545
+ intermediate_parallel = self.dense_h_to_4h(hidden_states)
546
+
547
+ intermediate_parallel = self.activation_func(intermediate_parallel)
548
+
549
+ output = self.dense_4h_to_h(intermediate_parallel)
550
+
551
+ return output
552
+
553
+
554
+ class GLMBlock(torch.nn.Module):
555
+ def __init__(
556
+ self,
557
+ hidden_size,
558
+ num_attention_heads,
559
+ layernorm_epsilon,
560
+ layer_id,
561
+ inner_hidden_size=None,
562
+ hidden_size_per_attention_head=None,
563
+ layernorm=LayerNorm,
564
+ use_bias=True,
565
+ params_dtype=torch.float,
566
+ num_layers=28,
567
+ position_encoding_2d=True,
568
+ empty_init=True
569
+ ):
570
+ super(GLMBlock, self).__init__()
571
+ # Set output layer initialization if not provided.
572
+
573
+ self.layer_id = layer_id
574
+
575
+ # Layernorm on the input data.
576
+ self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
577
+
578
+ self.position_encoding_2d = position_encoding_2d
579
+
580
+ # Self attention.
581
+ self.attention = SelfAttention(
582
+ hidden_size,
583
+ num_attention_heads,
584
+ layer_id,
585
+ hidden_size_per_attention_head=hidden_size_per_attention_head,
586
+ bias=use_bias,
587
+ params_dtype=params_dtype,
588
+ position_encoding_2d=self.position_encoding_2d,
589
+ empty_init=empty_init
590
+ )
591
+
592
+ # Layernorm on the input data.
593
+ self.post_attention_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
594
+
595
+ self.num_layers = num_layers
596
+
597
+ # GLU
598
+ self.mlp = GLU(
599
+ hidden_size,
600
+ inner_hidden_size=inner_hidden_size,
601
+ bias=use_bias,
602
+ layer_id=layer_id,
603
+ params_dtype=params_dtype,
604
+ empty_init=empty_init
605
+ )
606
+
607
+ def forward(
608
+ self,
609
+ hidden_states: torch.Tensor,
610
+ position_ids,
611
+ attention_mask: torch.Tensor,
612
+ layer_id,
613
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
614
+ use_cache: bool = False,
615
+ output_attentions: bool = False,
616
+ ):
617
+ """
618
+ hidden_states: [seq_len, batch, hidden_size]
619
+ attention_mask: [(1, 1), seq_len, seq_len]
620
+ """
621
+
622
+ # Layer norm at the begining of the transformer layer.
623
+ # [seq_len, batch, hidden_size]
624
+ attention_input = self.input_layernorm(hidden_states)
625
+
626
+ # Self attention.
627
+ attention_outputs = self.attention(
628
+ attention_input,
629
+ position_ids,
630
+ attention_mask=attention_mask,
631
+ layer_id=layer_id,
632
+ layer_past=layer_past,
633
+ use_cache=use_cache,
634
+ output_attentions=output_attentions
635
+ )
636
+
637
+ attention_output = attention_outputs[0]
638
+
639
+ outputs = attention_outputs[1:]
640
+
641
+ # Residual connection.
642
+ alpha = (2 * self.num_layers) ** 0.5
643
+ hidden_states = attention_input * alpha + attention_output
644
+
645
+ mlp_input = self.post_attention_layernorm(hidden_states)
646
+
647
+ # MLP.
648
+ mlp_output = self.mlp(mlp_input)
649
+
650
+ # Second residual connection.
651
+ output = mlp_input * alpha + mlp_output
652
+
653
+ if use_cache:
654
+ outputs = (output,) + outputs
655
+ else:
656
+ outputs = (output,) + outputs[1:]
657
+
658
+ return outputs # hidden_states, present, attentions
659
+
660
+
661
+ class ChatGLMPreTrainedModel(PreTrainedModel):
662
+ """
663
+ An abstract class to handle weights initialization and
664
+ a simple interface for downloading and loading pretrained models.
665
+ """
666
+
667
+ is_parallelizable = False
668
+ supports_gradient_checkpointing = True
669
+ config_class = ChatGLMConfig
670
+ base_model_prefix = "transformer"
671
+ _no_split_modules = ["GLMBlock"]
672
+
673
+ def __init__(self, *inputs, **kwargs):
674
+ super().__init__(*inputs, **kwargs)
675
+
676
+ def _init_weights(self, module: nn.Module):
677
+ """Initialize the weights."""
678
+ return
679
+
680
+ def get_masks(self, input_ids, device):
681
+ batch_size, seq_length = input_ids.shape
682
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
683
+ attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device)
684
+ attention_mask.tril_()
685
+ for i, context_length in enumerate(context_lengths):
686
+ attention_mask[i, :, :context_length] = 1
687
+ attention_mask.unsqueeze_(1)
688
+ attention_mask = (attention_mask < 0.5).bool()
689
+
690
+ return attention_mask
691
+
692
+ def get_position_ids(self, input_ids, mask_positions, device, use_gmasks=None):
693
+ batch_size, seq_length = input_ids.shape
694
+ if use_gmasks is None:
695
+ use_gmasks = [False] * batch_size
696
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
697
+ if self.position_encoding_2d:
698
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
699
+ for i, context_length in enumerate(context_lengths):
700
+ position_ids[i, context_length:] = mask_positions[i]
701
+ block_position_ids = [torch.cat((
702
+ torch.zeros(context_length, dtype=torch.long, device=device),
703
+ torch.arange(seq_length - context_length, dtype=torch.long, device=device) + 1
704
+ )) for context_length in context_lengths]
705
+ block_position_ids = torch.stack(block_position_ids, dim=0)
706
+ position_ids = torch.stack((position_ids, block_position_ids), dim=1)
707
+ else:
708
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
709
+ for i, context_length in enumerate(context_lengths):
710
+ if not use_gmasks[i]:
711
+ position_ids[context_length:] = mask_positions[i]
712
+
713
+ return position_ids
714
+
715
+ def _set_gradient_checkpointing(self, module, value=False):
716
+ if isinstance(module, ChatGLMModel):
717
+ module.gradient_checkpointing = value
718
+
719
+
720
+ CHATGLM_6B_START_DOCSTRING = r"""
721
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
722
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
723
+ usage and behavior.
724
+
725
+ Parameters:
726
+ config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model.
727
+ Initializing with a config file does not load the weights associated with the model, only the configuration.
728
+ Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
729
+ """
730
+
731
+ CHATGLM_6B_INPUTS_DOCSTRING = r"""
732
+ Args:
733
+ input_ids (`torch.LongTensor` of shape `({0})`):
734
+ Indices of input sequence tokens in the vocabulary.
735
+
736
+ Indices can be obtained using [`ChatGLM6BTokenizer`].
737
+ See [`PreTrainedTokenizer.encode`] and
738
+ [`PreTrainedTokenizer.__call__`] for details.
739
+
740
+ [What are input IDs?](../glossary#input-ids)
741
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
742
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
743
+
744
+ - 1 for tokens that are **not masked**,
745
+ - 0 for tokens that are **masked**.
746
+
747
+ [What are attention masks?](../glossary#attention-mask)
748
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
749
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
750
+
751
+ - 0 corresponds to a *sentence A* token,
752
+ - 1 corresponds to a *sentence B* token.
753
+
754
+ [What are token type IDs?](../glossary#token-type-ids)
755
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
756
+ Indices of positions of each input sequence tokens in the position embeddings.
757
+ Selected in the range `[0, config.max_position_embeddings - 1]`.
758
+
759
+ [What are position IDs?](../glossary#position-ids)
760
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
761
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
762
+
763
+ - 1 indicates the head is **not masked**,
764
+ - 0 indicates the head is **masked**.
765
+
766
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
767
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
768
+ This is useful if you want more control over how to convert *input_ids* indices into associated vectors
769
+ than the model's internal embedding lookup matrix.
770
+ output_attentions (`bool`, *optional*):
771
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
772
+ tensors for more detail.
773
+ output_hidden_states (`bool`, *optional*):
774
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
775
+ more detail.
776
+ return_dict (`bool`, *optional*):
777
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
778
+ """
779
+
780
+
781
+ @add_start_docstrings(
782
+ "The bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.",
783
+ CHATGLM_6B_START_DOCSTRING,
784
+ )
785
+ class ChatGLMModel(ChatGLMPreTrainedModel):
786
+ """
787
+
788
+ The model can behave as an encoder (with only self-attention) as well
789
+ as a decoder, in which case a layer of cross-attention is added between
790
+ the self-attention layers, following the architecture described in [Attention is
791
+ all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
792
+ Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
793
+
794
+ To behave as an decoder the model needs to be initialized with the
795
+ `is_decoder` argument of the configuration set to `True`.
796
+ To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
797
+ argument and `add_cross_attention` set to `True`; an
798
+ `encoder_hidden_states` is then expected as an input to the forward pass.
799
+ """
800
+
801
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
802
+ super().__init__(config)
803
+ if empty_init:
804
+ init_method = skip_init
805
+ else:
806
+ init_method = default_init
807
+ # recording parameters
808
+ self.max_sequence_length = config.max_sequence_length
809
+ self.hidden_size = config.hidden_size
810
+ self.params_dtype = torch.half
811
+ self.num_attention_heads = config.num_attention_heads
812
+ self.vocab_size = config.vocab_size
813
+ self.num_layers = config.num_layers
814
+ self.layernorm_epsilon = config.layernorm_epsilon
815
+ self.inner_hidden_size = config.inner_hidden_size
816
+ self.hidden_size_per_attention_head = self.hidden_size // self.num_attention_heads
817
+ self.position_encoding_2d = config.position_encoding_2d
818
+ self.pre_seq_len = config.pre_seq_len
819
+ self.prefix_projection = config.prefix_projection
820
+
821
+ self.word_embeddings = init_method(
822
+ torch.nn.Embedding,
823
+ num_embeddings=self.vocab_size, embedding_dim=self.hidden_size,
824
+ dtype=self.params_dtype
825
+ )
826
+ self.gradient_checkpointing = False
827
+
828
+ def get_layer(layer_id):
829
+ return GLMBlock(
830
+ self.hidden_size,
831
+ self.num_attention_heads,
832
+ self.layernorm_epsilon,
833
+ layer_id,
834
+ inner_hidden_size=self.inner_hidden_size,
835
+ hidden_size_per_attention_head=self.hidden_size_per_attention_head,
836
+ layernorm=LayerNorm,
837
+ use_bias=True,
838
+ params_dtype=self.params_dtype,
839
+ position_encoding_2d=self.position_encoding_2d,
840
+ empty_init=empty_init
841
+ )
842
+
843
+ self.layers = torch.nn.ModuleList(
844
+ [get_layer(layer_id) for layer_id in range(self.num_layers)]
845
+ )
846
+
847
+ # Final layer norm before output.
848
+ self.final_layernorm = LayerNorm(self.hidden_size, eps=self.layernorm_epsilon)
849
+
850
+ if self.pre_seq_len is not None:
851
+ for param in self.parameters():
852
+ param.requires_grad = False
853
+ self.prefix_tokens = torch.arange(self.pre_seq_len).long()
854
+ self.prefix_encoder = PrefixEncoder(config)
855
+ self.dropout = torch.nn.Dropout(0.1)
856
+
857
+ # total_params = sum(p.numel() for p in self.parameters())
858
+ # trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad)
859
+ # print("Using p-tuning v2: # trainable_params = {} / {}".format(trainable_params, total_params))
860
+
861
+ def get_input_embeddings(self):
862
+ return self.word_embeddings
863
+
864
+ def set_input_embeddings(self, new_embeddings: torch.Tensor):
865
+ self.word_embeddings = new_embeddings
866
+
867
+ def get_prompt(self, batch_size, device, dtype=torch.half):
868
+ prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
869
+ past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)
870
+ past_key_values = past_key_values.view(
871
+ batch_size,
872
+ self.pre_seq_len,
873
+ self.num_layers * 2,
874
+ self.num_attention_heads,
875
+ self.hidden_size // self.num_attention_heads
876
+ )
877
+ # seq_len, b, nh, hidden_size
878
+ past_key_values = self.dropout(past_key_values)
879
+ past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2)
880
+ # past_key_values = [(v[0], v[1]) for v in past_key_values]
881
+ return past_key_values
882
+
883
+ @add_start_docstrings_to_model_forward(CHATGLM_6B_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
884
+ @add_code_sample_docstrings(
885
+ checkpoint=_CHECKPOINT_FOR_DOC,
886
+ output_type=BaseModelOutputWithPastAndCrossAttentions,
887
+ config_class=_CONFIG_FOR_DOC,
888
+ )
889
+ def forward(
890
+ self,
891
+ input_ids: Optional[torch.LongTensor] = None,
892
+ position_ids: Optional[torch.LongTensor] = None,
893
+ attention_mask: Optional[torch.Tensor] = None,
894
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
895
+ inputs_embeds: Optional[torch.LongTensor] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPast]:
901
+
902
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
903
+ output_hidden_states = (
904
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
905
+ )
906
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
907
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
908
+
909
+ if self.gradient_checkpointing and self.training:
910
+ if use_cache:
911
+ logger.warning_once(
912
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
913
+ )
914
+ use_cache = False
915
+
916
+ if input_ids is not None and inputs_embeds is not None:
917
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
918
+ elif input_ids is not None:
919
+ batch_size, seq_length = input_ids.shape[:2]
920
+ elif inputs_embeds is not None:
921
+ batch_size, seq_length = inputs_embeds.shape[:2]
922
+ else:
923
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
924
+
925
+ if inputs_embeds is None:
926
+ inputs_embeds = self.word_embeddings(input_ids)
927
+
928
+ if past_key_values is None:
929
+ if self.pre_seq_len is not None:
930
+ past_key_values = self.get_prompt(batch_size=input_ids.shape[0], device=input_ids.device,
931
+ dtype=inputs_embeds.dtype)
932
+ else:
933
+ past_key_values = tuple([None] * len(self.layers))
934
+
935
+ if attention_mask is None:
936
+ attention_mask = self.get_masks(
937
+ input_ids,
938
+ device=input_ids.device
939
+ )
940
+
941
+
942
+ if position_ids is None:
943
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
944
+ seqs = input_ids.tolist()
945
+
946
+ mask_positions, use_gmasks = [], []
947
+ for seq in seqs:
948
+ mask_token = gMASK if gMASK in seq else MASK
949
+ use_gmask = mask_token == gMASK
950
+ mask_positions.append(seq.index(mask_token))
951
+ use_gmasks.append(use_gmask)
952
+
953
+ position_ids = self.get_position_ids(
954
+ input_ids,
955
+ mask_positions=mask_positions,
956
+ device=input_ids.device,
957
+ use_gmasks=use_gmasks
958
+ )
959
+
960
+ if self.pre_seq_len is not None and attention_mask is not None:
961
+ prefix_attention_mask = torch.ones(batch_size, 1, input_ids.size(-1), self.pre_seq_len).to(
962
+ attention_mask.device)
963
+ prefix_attention_mask = (prefix_attention_mask < 0.5).bool()
964
+ attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=3)
965
+
966
+ # [seq_len, batch, hidden_size]
967
+ hidden_states = inputs_embeds.transpose(0, 1)
968
+
969
+ presents = () if use_cache else None
970
+ all_self_attentions = () if output_attentions else None
971
+ all_hidden_states = () if output_hidden_states else None
972
+
973
+ if attention_mask is None:
974
+ attention_mask = torch.zeros(1, 1, device=input_ids.device).bool()
975
+ else:
976
+ attention_mask = attention_mask.to(hidden_states.device)
977
+
978
+ for i, layer in enumerate(self.layers):
979
+
980
+ if output_hidden_states:
981
+ all_hidden_states = all_hidden_states + (hidden_states,)
982
+ layer_past = past_key_values[i]
983
+
984
+ if self.gradient_checkpointing and self.training:
985
+ layer_ret = torch.utils.checkpoint.checkpoint(
986
+ layer,
987
+ hidden_states,
988
+ position_ids,
989
+ attention_mask,
990
+ torch.tensor(i),
991
+ layer_past,
992
+ use_cache,
993
+ output_attentions
994
+ )
995
+ else:
996
+ layer_ret = layer(
997
+ hidden_states,
998
+ position_ids=position_ids,
999
+ attention_mask=attention_mask,
1000
+ layer_id=torch.tensor(i),
1001
+ layer_past=layer_past,
1002
+ use_cache=use_cache,
1003
+ output_attentions=output_attentions
1004
+ )
1005
+
1006
+ hidden_states = layer_ret[0]
1007
+
1008
+ if use_cache:
1009
+ presents = presents + (layer_ret[1],)
1010
+
1011
+ if output_attentions:
1012
+ all_self_attentions = all_self_attentions + (layer_ret[2 if use_cache else 1],)
1013
+
1014
+ # Final layer norm.
1015
+ hidden_states = self.final_layernorm(hidden_states)
1016
+
1017
+ if output_hidden_states:
1018
+ all_hidden_states = all_hidden_states + (hidden_states,)
1019
+
1020
+ if not return_dict:
1021
+ return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
1022
+
1023
+ return BaseModelOutputWithPast(
1024
+ last_hidden_state=hidden_states,
1025
+ past_key_values=presents,
1026
+ hidden_states=all_hidden_states,
1027
+ attentions=all_self_attentions,
1028
+ )
1029
+
1030
+
1031
+ class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
1032
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
1033
+ super().__init__(config)
1034
+ if empty_init:
1035
+ init_method = skip_init
1036
+ else:
1037
+ init_method = default_init
1038
+
1039
+ # self.hidden_size = config.hidden_size
1040
+ # self.params_dtype = torch.half
1041
+ # self.vocab_size = config.vocab_size
1042
+ self.max_sequence_length = config.max_sequence_length
1043
+
1044
+ self.position_encoding_2d = config.position_encoding_2d
1045
+
1046
+ self.transformer = ChatGLMModel(config, empty_init=empty_init)
1047
+
1048
+ self.lm_head = init_method(
1049
+ nn.Linear,
1050
+ config.hidden_size,
1051
+ config.vocab_size,
1052
+ bias=False,
1053
+ dtype=torch.half
1054
+ )
1055
+
1056
+ self.config = config
1057
+
1058
+ self.quantized = False
1059
+
1060
+ if self.config.quantization_bit:
1061
+ self.quantize(self.config.quantization_bit, empty_init=True)
1062
+
1063
+ def get_output_embeddings(self):
1064
+ return self.lm_head
1065
+
1066
+ def set_output_embeddings(self, new_embeddings):
1067
+ self.lm_head = new_embeddings
1068
+
1069
+ def _update_model_kwargs_for_generation(
1070
+ self,
1071
+ outputs: ModelOutput,
1072
+ model_kwargs: Dict[str, Any],
1073
+ is_encoder_decoder: bool = False,
1074
+ standardize_cache_format: bool = False,
1075
+ ) -> Dict[str, Any]:
1076
+ # update past_key_values
1077
+ model_kwargs["past_key_values"] = self._extract_past_from_model_output(
1078
+ outputs, standardize_cache_format=standardize_cache_format
1079
+ )
1080
+
1081
+ # update attention mask
1082
+ if "attention_mask" in model_kwargs:
1083
+ attention_mask = model_kwargs["attention_mask"]
1084
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1085
+ attention_mask = torch.cat(
1086
+ [attention_mask, attention_mask.new_ones((*attention_mask.shape[:3], 1))], dim=3)
1087
+ new_attention_mask = attention_mask[:, :, -1:].clone()
1088
+ new_attention_mask[..., -1] = False
1089
+ model_kwargs["attention_mask"] = torch.cat(
1090
+ [attention_mask, new_attention_mask], dim=2
1091
+ )
1092
+
1093
+ # update position ids
1094
+ if "position_ids" in model_kwargs:
1095
+ position_ids = model_kwargs["position_ids"]
1096
+ new_position_id = position_ids[..., -1:].clone()
1097
+ new_position_id[:, 1, :] += 1
1098
+ model_kwargs["position_ids"] = torch.cat(
1099
+ [position_ids, new_position_id], dim=-1
1100
+ )
1101
+
1102
+ return model_kwargs
1103
+
1104
+ def prepare_inputs_for_generation(
1105
+ self,
1106
+ input_ids: torch.LongTensor,
1107
+ past: Optional[torch.Tensor] = None,
1108
+ past_key_values: Optional[torch.Tensor] = None,
1109
+ attention_mask: Optional[torch.Tensor] = None,
1110
+ position_ids: Optional[torch.Tensor] = None,
1111
+ **kwargs
1112
+ ) -> dict:
1113
+ batch_size, seq_length = input_ids.shape
1114
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
1115
+ seqs = input_ids.tolist()
1116
+ mask_positions, use_gmasks = [], []
1117
+ for seq in seqs:
1118
+ mask_token = gMASK if gMASK in seq else MASK
1119
+ use_gmask = mask_token == gMASK
1120
+ mask_positions.append(seq.index(mask_token))
1121
+ use_gmasks.append(use_gmask)
1122
+
1123
+ # only last token for input_ids if past is not None
1124
+ if past is not None or past_key_values is not None:
1125
+ last_token = input_ids[:, -1].unsqueeze(-1)
1126
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1127
+ attention_mask = attention_mask[:, :, -1:]
1128
+ else:
1129
+ attention_mask = None
1130
+ if position_ids is not None:
1131
+ position_ids = position_ids[..., -1:]
1132
+ else:
1133
+ context_lengths = [seq.index(self.config.bos_token_id) for seq in seqs]
1134
+ if self.position_encoding_2d:
1135
+ position_ids = torch.tensor(
1136
+ [[mask_position, seq_length - context_length] for mask_position, context_length in
1137
+ zip(mask_positions, context_lengths)], dtype=torch.long, device=input_ids.device).unsqueeze(-1)
1138
+ else:
1139
+ position_ids = torch.tensor([mask_position for mask_position in mask_positions], dtype=torch.long,
1140
+ device=input_ids.device).unsqueeze(-1)
1141
+
1142
+ if past is None:
1143
+ past = past_key_values
1144
+ return {
1145
+ "input_ids": last_token,
1146
+ "past_key_values": past,
1147
+ "position_ids": position_ids,
1148
+ "attention_mask": attention_mask
1149
+ }
1150
+ else:
1151
+ if attention_mask is not None and attention_mask.dtype != torch.bool:
1152
+ logger.warning_once(f"The dtype of attention mask ({attention_mask.dtype}) is not bool")
1153
+ attention_mask = None
1154
+ if attention_mask is None:
1155
+ attention_mask = self.get_masks(
1156
+ input_ids,
1157
+ device=input_ids.device
1158
+ )
1159
+ if position_ids is None:
1160
+ position_ids = self.get_position_ids(
1161
+ input_ids,
1162
+ device=input_ids.device,
1163
+ mask_positions=mask_positions,
1164
+ use_gmasks=use_gmasks
1165
+ )
1166
+
1167
+ return {
1168
+ "input_ids": input_ids,
1169
+ "past_key_values": past,
1170
+ "position_ids": position_ids,
1171
+ "attention_mask": attention_mask
1172
+ }
1173
+
1174
+ def forward(
1175
+ self,
1176
+ input_ids: Optional[torch.Tensor] = None,
1177
+ position_ids: Optional[torch.Tensor] = None,
1178
+ attention_mask: Optional[torch.Tensor] = None,
1179
+ past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
1180
+ inputs_embeds: Optional[torch.Tensor] = None,
1181
+ labels: Optional[torch.Tensor] = None,
1182
+ use_cache: Optional[bool] = None,
1183
+ output_attentions: Optional[bool] = None,
1184
+ output_hidden_states: Optional[bool] = None,
1185
+ return_dict: Optional[bool] = None,
1186
+ ):
1187
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1188
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1189
+
1190
+ transformer_outputs = self.transformer(
1191
+ input_ids=input_ids,
1192
+ position_ids=position_ids,
1193
+ attention_mask=attention_mask,
1194
+ past_key_values=past_key_values,
1195
+ inputs_embeds=inputs_embeds,
1196
+ use_cache=use_cache,
1197
+ output_attentions=output_attentions,
1198
+ output_hidden_states=output_hidden_states,
1199
+ return_dict=return_dict,
1200
+ )
1201
+
1202
+ hidden_states = transformer_outputs[0]
1203
+
1204
+ lm_logits = self.lm_head(hidden_states).permute(1, 0, 2).contiguous()
1205
+
1206
+ loss = None
1207
+ if labels is not None:
1208
+ lm_logits = lm_logits.to(torch.float32)
1209
+
1210
+ # Shift so that tokens < n predict n
1211
+ shift_logits = lm_logits[..., :-1, :].contiguous()
1212
+ shift_labels = labels[..., 1:].contiguous()
1213
+ # Flatten the tokens
1214
+ loss_fct = CrossEntropyLoss(ignore_index=-100)
1215
+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
1216
+
1217
+ lm_logits = lm_logits.to(hidden_states.dtype)
1218
+ loss = loss.to(hidden_states.dtype)
1219
+
1220
+ if not return_dict:
1221
+ output = (lm_logits,) + transformer_outputs[1:]
1222
+ return ((loss,) + output) if loss is not None else output
1223
+
1224
+ return CausalLMOutputWithPast(
1225
+ loss=loss,
1226
+ logits=lm_logits,
1227
+ past_key_values=transformer_outputs.past_key_values,
1228
+ hidden_states=transformer_outputs.hidden_states,
1229
+ attentions=transformer_outputs.attentions,
1230
+ )
1231
+
1232
+ @staticmethod
1233
+ def _reorder_cache(
1234
+ past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor
1235
+ ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]:
1236
+ """
1237
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
1238
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
1239
+ beam_idx at every generation step.
1240
+
1241
+ Output shares the same memory storage as `past`.
1242
+ """
1243
+ return tuple(
1244
+ (
1245
+ layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)),
1246
+ layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)),
1247
+ )
1248
+ for layer_past in past
1249
+ )
1250
+
1251
+ def process_response(self, response):
1252
+ response = response.strip()
1253
+ response = response.replace("[[训练时间]]", "2023年")
1254
+ punkts = [
1255
+ [",", ","],
1256
+ ["!", "!"],
1257
+ [":", ":"],
1258
+ [";", ";"],
1259
+ ["\?", "?"],
1260
+ ]
1261
+ for item in punkts:
1262
+ response = re.sub(r"([\u4e00-\u9fff])%s" % item[0], r"\1%s" % item[1], response)
1263
+ response = re.sub(r"%s([\u4e00-\u9fff])" % item[0], r"%s\1" % item[1], response)
1264
+ return response
1265
+
1266
+ @torch.no_grad()
1267
+ def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048, num_beams=1,
1268
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1269
+ if history is None:
1270
+ history = []
1271
+ if logits_processor is None:
1272
+ logits_processor = LogitsProcessorList()
1273
+ logits_processor.append(InvalidScoreLogitsProcessor())
1274
+ gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p,
1275
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1276
+ if not history:
1277
+ prompt = query
1278
+ else:
1279
+ prompt = ""
1280
+ for i, (old_query, response) in enumerate(history):
1281
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1282
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1283
+ inputs = tokenizer([prompt], return_tensors="pt")
1284
+ inputs = inputs.to(self.device)
1285
+ outputs = self.generate(**inputs, **gen_kwargs)
1286
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1287
+ response = tokenizer.decode(outputs)
1288
+ response = self.process_response(response)
1289
+ history = history + [(query, response)]
1290
+ return response, history
1291
+
1292
+ @torch.no_grad()
1293
+ def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048,
1294
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1295
+ if history is None:
1296
+ history = []
1297
+ if logits_processor is None:
1298
+ logits_processor = LogitsProcessorList()
1299
+ logits_processor.append(InvalidScoreLogitsProcessor())
1300
+ gen_kwargs = {"max_length": max_length, "do_sample": do_sample, "top_p": top_p,
1301
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1302
+ if not history:
1303
+ prompt = query
1304
+ else:
1305
+ prompt = ""
1306
+ for i, (old_query, response) in enumerate(history):
1307
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1308
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1309
+ inputs = tokenizer([prompt], return_tensors="pt")
1310
+ inputs = inputs.to(self.device)
1311
+ for outputs in self.stream_generate(**inputs, **gen_kwargs):
1312
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1313
+ response = tokenizer.decode(outputs)
1314
+ response = self.process_response(response)
1315
+ new_history = history + [(query, response)]
1316
+ yield response, new_history
1317
+
1318
+ @torch.no_grad()
1319
+ def stream_generate(
1320
+ self,
1321
+ input_ids,
1322
+ generation_config: Optional[GenerationConfig] = None,
1323
+ logits_processor: Optional[LogitsProcessorList] = None,
1324
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1325
+ prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
1326
+ **kwargs,
1327
+ ):
1328
+ batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
1329
+
1330
+ if generation_config is None:
1331
+ generation_config = self.generation_config
1332
+ generation_config = copy.deepcopy(generation_config)
1333
+ model_kwargs = generation_config.update(**kwargs)
1334
+ bos_token_id, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id
1335
+
1336
+ if isinstance(eos_token_id, int):
1337
+ eos_token_id = [eos_token_id]
1338
+
1339
+ has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
1340
+ if has_default_max_length and generation_config.max_new_tokens is None:
1341
+ warnings.warn(
1342
+ f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
1343
+ "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
1344
+ " recommend using `max_new_tokens` to control the maximum length of the generation.",
1345
+ UserWarning,
1346
+ )
1347
+ elif generation_config.max_new_tokens is not None:
1348
+ generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
1349
+ if not has_default_max_length:
1350
+ logger.warn(
1351
+ f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
1352
+ f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
1353
+ "Please refer to the documentation for more information. "
1354
+ "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
1355
+ UserWarning,
1356
+ )
1357
+
1358
+ if input_ids_seq_length >= generation_config.max_length:
1359
+ input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
1360
+ logger.warning(
1361
+ f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to"
1362
+ f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
1363
+ " increasing `max_new_tokens`."
1364
+ )
1365
+
1366
+ # 2. Set generation parameters if not already defined
1367
+ logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
1368
+ stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
1369
+
1370
+ logits_processor = self._get_logits_processor(
1371
+ generation_config=generation_config,
1372
+ input_ids_seq_length=input_ids_seq_length,
1373
+ encoder_input_ids=input_ids,
1374
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1375
+ logits_processor=logits_processor,
1376
+ )
1377
+
1378
+ stopping_criteria = self._get_stopping_criteria(
1379
+ generation_config=generation_config, stopping_criteria=stopping_criteria
1380
+ )
1381
+ logits_warper = self._get_logits_warper(generation_config)
1382
+
1383
+ unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
1384
+ scores = None
1385
+ while True:
1386
+ model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
1387
+ # forward pass to get next token
1388
+ outputs = self(
1389
+ **model_inputs,
1390
+ return_dict=True,
1391
+ output_attentions=False,
1392
+ output_hidden_states=False,
1393
+ )
1394
+
1395
+ next_token_logits = outputs.logits[:, -1, :]
1396
+
1397
+ # pre-process distribution
1398
+ next_token_scores = logits_processor(input_ids, next_token_logits)
1399
+ next_token_scores = logits_warper(input_ids, next_token_scores)
1400
+
1401
+ # sample
1402
+ probs = nn.functional.softmax(next_token_scores, dim=-1)
1403
+ if generation_config.do_sample:
1404
+ next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
1405
+ else:
1406
+ next_tokens = torch.argmax(probs, dim=-1)
1407
+
1408
+ # update generated ids, model inputs, and length for next step
1409
+ input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
1410
+ model_kwargs = self._update_model_kwargs_for_generation(
1411
+ outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
1412
+ )
1413
+ unfinished_sequences = unfinished_sequences.mul((sum(next_tokens != i for i in eos_token_id)).long())
1414
+
1415
+ # stop when each sentence is finished, or if we exceed the maximum length
1416
+ if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
1417
+ break
1418
+ yield input_ids
1419
+
1420
+ def quantize(self, bits: int, empty_init=False, **kwargs):
1421
+ if bits == 0:
1422
+ return
1423
+
1424
+ from .quantization import quantize
1425
+
1426
+ if self.quantized:
1427
+ logger.info("Already quantized.")
1428
+ return self
1429
+
1430
+ self.quantized = True
1431
+
1432
+ self.config.quantization_bit = bits
1433
+
1434
+ self.transformer = quantize(self.transformer, bits, empty_init=empty_init, **kwargs)
1435
+ return self
ptuning/.gitattributes ADDED
@@ -0,0 +1 @@
 
 
1
+ *.bin filter=lfs diff=lfs merge=lfs -text
ptuning/AdvertiseGen/dev.json ADDED
The diff for this file is too large to render. See raw diff
 
ptuning/AdvertiseGen/train.json ADDED
Binary file (53.8 MB). View file
 
ptuning/README.md ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ChatGLM-6B-PT
2
+ 本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2 将需要微调的参数量减少到原来的 0.1%,再通过模型量化、Gradient Checkpoint 等方法,最低只需要 7GB 显存即可运行。
3
+
4
+ 下面以 [ADGEN](https://aclanthology.org/D19-1321.pdf) (广告生成) 数据集为例介绍代码的使用方法。
5
+
6
+ *Read this in [English](README_en.md).*
7
+
8
+ ## 软件依赖
9
+ 运行微调需要4.27.1版本的`transformers`。除 ChatGLM-6B 的依赖之外,还需要安装以下依赖
10
+ ```
11
+ pip install rouge_chinese nltk jieba datasets
12
+ ```
13
+ ## 使用方法
14
+
15
+ ### 下载数据集
16
+ ADGEN 数据集任务为根据输入(content)生成一段广告词(summary)。
17
+
18
+ ```json
19
+ {
20
+ "content": "类型#上衣*版型#宽松*版型#显瘦*图案#线条*衣样式#衬衫*衣袖型#泡泡袖*衣款式#抽绳",
21
+ "summary": "这件衬衫的款式非常的宽松,利落的线条可以很好的隐藏身材上的小缺点,穿在身上有着很好的显瘦效果。领口装饰了一个可爱的抽绳,漂亮的绳结展现出了十足的个性,配合时尚的泡泡袖型,尽显女性甜美可爱的气息。"
22
+ }
23
+ ```
24
+
25
+ 从 [Google Drive](https://drive.google.com/file/d/13_vf0xRTQsyneRKdD1bZIr93vBGOczrk/view?usp=sharing) 或者 [Tsinghua Cloud](https://cloud.tsinghua.edu.cn/f/b3f119a008264b1cabd1/?dl=1) 下载处理好的 ADGEN 数据集,将解压后的 `AdvertiseGen` 目录放到本目录下。
26
+
27
+ ### 训练
28
+
29
+ #### P-tuning v2
30
+
31
+ 运行以下指令进行训练:
32
+ ```shell
33
+ bash train.sh
34
+ ```
35
+ `train.sh` 中的 `PRE_SEQ_LEN` 和 `LR` 分别是 soft prompt 长度和训练的学习率,可以进行调节以取得最佳的效果。P-Tuning-v2 方法会冻结全部的模型参数,可通过调整 `quantization_bit` 来被原始模型的量化等级,不加此选项则为 FP16 精度加载。
36
+
37
+ 在默认配置 `quantization_bit=4`、`per_device_train_batch_size=1`、`gradient_accumulation_steps=16` 下,INT4 的模型参数被冻结,一次训练迭代会以 1 的批处理大小进行 16 次累加的前后向传播,等效为 16 的总批处理大小,此时最低只需 6.7G 显存。若想在同等批处理大小下提升训练效率,可在二者乘积不变的情况下,加大 `per_device_train_batch_size` 的值,但也会带来更多的显存消耗,请根据实际情况酌情调整。
38
+
39
+ 如果你想要[从本地加载模型](https://github.com/THUDM/ChatGLM-6B#%E4%BB%8E%E6%9C%AC%E5%9C%B0%E5%8A%A0%E8%BD%BD%E6%A8%A1%E5%9E%8B),可以将 `train.sh` 中的 `THUDM/chatglm-6b` 改为你本地的模型路径。
40
+
41
+ #### Finetune
42
+
43
+ 如果需要进行全参数的 Finetune,需要安装 [Deepspeed](https://github.com/microsoft/DeepSpeed),然后运行以下指令:
44
+
45
+ ```shell
46
+ bash ds_train_finetune.sh
47
+ ```
48
+
49
+ ### 推理
50
+
51
+ 将 `evaluate.sh` 中的 `CHECKPOINT` 更改为训练时保存的 checkpoint 名称,运行以下指令进行模型推理和评测:
52
+ ```shell
53
+ bash evaluate.sh
54
+ ```
55
+ **[2023/04/10更新]** 在 P-tuning v2 训练时模型只保存 PrefixEncoder 部分的参数,所以在推理时需要同时加载原 ChatGLM-6B 模型以及 PrefixEncoder 的权重,因此需要指定参数(已更新 `evaluate.sh`) :
56
+
57
+ ```shell
58
+ --model_name_or_path THUDM/chatglm-6b
59
+ --ptuning_checkpoint $CHECKPOINT_PATH
60
+ ```
61
+
62
+ 仍然兼容旧版全参保存的 Checkpoint,只需要跟之前一样设定 `model_name_or_path`:
63
+
64
+ ```shell
65
+ --model_name_or_path $CHECKPOINT_PATH
66
+ ```
67
+
68
+ 评测指标为中文 Rouge score 和 BLEU-4。生成的结果保存在
69
+ `./output/adgen-chatglm-6b-pt-8-1e-2/generated_predictions.txt`。
70
+
71
+ ### 例子
72
+ #### 示例1
73
+ * Input: 类型#上衣\*材质#牛仔布\*颜色#白色\*风格#简约\*图案#刺绣\*衣样式#外套\*衣款式#破洞
74
+ * Label: 简约而不简单的牛仔外套,白色的衣身十分百搭。衣身多处有做旧破洞设计,打破单调乏味,增加一丝造型看点。衣身后背处有趣味刺绣装饰,丰富层次感,彰显别样时尚。
75
+ * Output[微调前]: 这件上衣的材质是牛仔布,颜色是白色,风格是简约,图案是刺绣,衣样式是外套,衣款式是破洞。
76
+ * Output[微调后]: 这是一款简约的牛仔外套,破洞设计,将牛仔布破洞,带来一种随意与个性。破洞的牛仔外套,展现出时尚气息,带来一种休闲感。同时,刺绣图案,让整件外套更加立体。
77
+
78
+ #### 示例2
79
+
80
+ * Input: 类型#裙\*版型#显瘦\*风格#文艺\*风格#简约\*图案#印花\*图案#撞色\*裙下摆#压褶\*裙长#连衣裙\*裙领型#圆领
81
+ * Label: 文艺个性的印花连衣裙,藏青色底蕴,低调又大气,撞色太阳花分布整个裙身,绚丽而美好,带来时尚减龄的气质。基础款的舒适圆领,简约不失大方,勾勒精致脸庞。领后是一粒包布扣固定,穿脱十分方便。前片立体的打褶设计,搭配后片压褶的做工,增添层次和空间感,显瘦又有型。
82
+ * Output[微调前]: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、富有层次感。
83
+ * Output[微调后]: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。
84
+
85
+ ### 评估结果
86
+
87
+ | | Finetune | P-tuning v2 | LoRA |
88
+ | ------------- | ----------- | ----- | ------------- |
89
+ | BLEU-4 | 8.01 | 8.10 | 7.62 |
90
+ | Rouge-1 | 31.23 | 31.12 | 30.60 |
91
+ | Rouge-2 | 7.36 | 7.11 | 6.96 |
92
+ | Rouge-l | 25.08 | 24.97 | 24.80 |
93
+ | Training Loss | 3.00 | 3.74 | 3.32 |
94
+
95
+
96
+
97
+ #### 实验设置
98
+
99
+ ```
100
+ max_source_length=64
101
+ max_target_length=64
102
+ max_steps=3000
103
+ ```
104
+
105
+ ##### P-tuning v2
106
+
107
+ ```
108
+ pre_seq_len=128
109
+ learning_rate=2e-2
110
+ quantization_bit=4
111
+ per_device_train_batch_size=16
112
+ gradient_accumulation_steps=1
113
+ ```
114
+
115
+ ##### Finetune
116
+
117
+ ```
118
+ learning_rate=1e-4
119
+ fp16
120
+ num_gpus=4
121
+ per_device_train_batch_size=4
122
+ gradient_accumulation_steps=1
123
+ ```
124
+
125
+ ##### LoRA
126
+
127
+ 实现采用的是 [simple_thu_chatglm6b](https://github.com/yuanzhoulvpi2017/zero_nlp/tree/main/simple_thu_chatglm6b)
128
+
129
+ ```
130
+ learning_rate=5e-4
131
+ per_device_train_batch_size=16
132
+ gradient_accumulation_steps=1
133
+ ```
134
+
135
+
136
+
137
+ ## 模型部署
138
+ 首先载入Tokenizer:
139
+
140
+ ```python
141
+ import os
142
+ import torch
143
+ from transformers import AutoConfig, AutoModel, AutoTokenizer
144
+
145
+ # 载入Tokenizer
146
+ tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
147
+ ```
148
+
149
+ 1. 如果需要加载的是新 Checkpoint(只包含 PrefixEncoder 参数):
150
+
151
+ ```python
152
+ config = AutoConfig.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True, pre_seq_len=128)
153
+ model = AutoModel.from_pretrained("THUDM/chatglm-6b", config=config, trust_remote_code=True)
154
+ prefix_state_dict = torch.load(os.path.join(CHECKPOINT_PATH, "pytorch_model.bin"))
155
+ new_prefix_state_dict = {}
156
+ for k, v in prefix_state_dict.items():
157
+ if k.startswith("transformer.prefix_encoder."):
158
+ new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
159
+ model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)
160
+ ```
161
+ 注意你可能需要将 `pre_seq_len` 改成你训练时的实际值。如果你是[从本地加载模型](https://github.com/THUDM/ChatGLM-6B#%E4%BB%8E%E6%9C%AC%E5%9C%B0%E5%8A%A0%E8%BD%BD%E6%A8%A1%E5%9E%8B)的话,需要将 `THUDM/chatglm-6b` 改成本地的模型路径(注意不是checkpoint路径)。
162
+
163
+ 2. 如果需要加载的是旧 Checkpoint(包含 ChatGLM-6B 以及 PrefixEncoder 参数),或者进行的是全参数微调,则直接加载整个 Checkpoint:
164
+
165
+ ```python
166
+ model = AutoModel.from_pretrained(CHECKPOINT_PATH, trust_remote_code=True)
167
+ ```
168
+
169
+ 之后根据需求可以进行量化,也可以直接使用:
170
+
171
+ ```python
172
+ # Comment out the following line if you don't use quantization
173
+ model = model.quantize(4)
174
+ model = model.half().cuda()
175
+ model.transformer.prefix_encoder.float()
176
+ model = model.eval()
177
+
178
+ response, history = model.chat(tokenizer, "你好", history=[])
179
+ ```
180
+
181
+ **[23/04/19]** 你也可以直接运行支持加载 P-Tuning v2 checkpoint 的 [web demo](./web_demo.py)
182
+ ```shell
183
+ bash web_demo.sh
184
+ ```
185
+ 可能需要修改 [web_demo.sh](./web_demo.sh) 的内容以符合你实际的 checkpoint 情况。
186
+
187
+ ## 使用自己的数据集
188
+ 修改 `train.sh` 和 `evaluate.sh` 中的 `train_file`、`validation_file`和`test_file`为你自己的 JSON 格式数据集路径,并将 `prompt_column` 和 `response_column` 改为 JSON 文件中输入文本和输出文本对应的 KEY。可能还需要增大 `max_source_length` 和 `max_target_length` 来匹配你自己的数据集中的最大输入输出长度。
189
+
190
+ ## 对话数据集
191
+
192
+ 如需要使用多轮对话数据对模型进行微调,可以提供聊天历史,例如
193
+
194
+ ```json
195
+ {
196
+ "prompt": "是的。上下水管都好的",
197
+ "response": "那就要检查线路了,一般风扇继电器是由电脑控制吸合的,如果电路存在断路,或者电脑坏了的话会出现继电器不吸合的情况!",
198
+ "history": [
199
+ [
200
+ "长城h3风扇不转。继电器好的。保险丝好的传感器新的风扇也新的这是为什么。就是继电器缺一个信号线",
201
+ "用电脑能读数据流吗?水温多少"
202
+ ],
203
+ [
204
+ "95",
205
+ "上下水管温差怎么样啊?空气是不是都排干净了呢?"
206
+ ]
207
+ ]
208
+ }
209
+ ```
210
+
211
+ 训练时需要指定 `--history_column` 为数据中聊天历史的 key(在此例子中是 `history`),将自动把聊天历史拼接,例如:
212
+
213
+ - Input
214
+
215
+ ```
216
+ [Round 0]
217
+ 问:长城h3风扇不转。继电器好的。保险丝好的传感器新的风扇也新的这是为什么。就是继电器缺一个信号线
218
+ 答:用电脑能读数据流吗?水温多少
219
+ [Round 1]
220
+ 问:95
221
+ 答:上下水管温差怎么样啊?空气是不是都排干净了呢?
222
+ [Round 2]
223
+ 问:是的。上下水管都好的
224
+ 答:
225
+ ```
226
+
227
+ - Label
228
+
229
+ ```
230
+ 那就要检查线路了,一般风扇继电器是由电脑控制吸合的,如果电路存在断路,或者电脑坏了的话会出现继电器不吸合的情况!
231
+ ```
232
+
233
+ 要注意超过输入长度 `max_source_length` 的内容会被截。
234
+
235
+ 可以参考以下指令:
236
+
237
+ ```shell
238
+ bash train_chat.sh
239
+ ```
240
+
241
+ ## 引用
242
+
243
+ ```
244
+ @inproceedings{liu2022p,
245
+ title={P-tuning: Prompt tuning can be comparable to fine-tuning across scales and tasks},
246
+ author={Liu, Xiao and Ji, Kaixuan and Fu, Yicheng and Tam, Weng and Du, Zhengxiao and Yang, Zhilin and Tang, Jie},
247
+ booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
248
+ pages={61--68},
249
+ year={2022}
250
+ }
251
+ ```
252
+
253
+
254
+
ptuning/README_en.md ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ChatGLM-6B-PT
2
+ This repository implements tuning of the ChatGLM-6B model based on [P-Tuning v2](https://github.com/THUDM/P-tuning-v2). P-Tuning v2 reduces the amount of parameters that need to be optimized to 0.1% of the full fine-tuning, and then through model quantization, Gradient Checkpoint and other methods, it only needs a minimum of 7GB of video memory to run.
3
+
4
+ The following uses the [ADGEN](https://aclanthology.org/D19-1321.pdf) (advertising generation) dataset as an example to introduce how to use the code.
5
+
6
+ ## Software dependencies
7
+ Running p-tuning requires version 4.27.1 of `transformers`. In addition to the dependencies of ChatGLM-6B, the following dependencies are required
8
+ ```
9
+ pip install rouge_chinese nltk jieba datasets
10
+ ```
11
+ ## Instructions
12
+
13
+ ### Download the dataset
14
+ The task of the ADGEN dataset is to generate an advertisement word (summary) based on the input (content).
15
+
16
+ ```json
17
+ {
18
+ "content": "类型#上衣*版型#宽松*版型#显瘦*图案#线条*衣样式#衬衫*衣袖型#泡泡袖*衣款式#抽绳",
19
+ "summary": "这件衬衫的款式非常的宽松,利落的线条可以很好的隐藏身材上的小缺点,穿在身上有着很好的显瘦效果。领口装饰了一个可爱的抽绳,漂亮的绳结展现出了十足的个性,配合时尚的泡泡袖型,尽显女性甜美可爱的气息。"
20
+ }
21
+ ```
22
+
23
+ From [Google Drive](https://drive.google.com/file/d/13_vf0xRTQsyneRKdD1bZIr93vBGOczrk/view?usp=sharing) or [Tsinghua Cloud](https://cloud.tsinghua.edu.cn/f/b3f119a008264b1cabd1/?dl=1) Download the processed ADGEN dataset, and put the decompressed `AdvertiseGen` directory into this directory.
24
+
25
+ ### Training
26
+ Run the following commands for training:
27
+ ```shell
28
+ bash train.sh
29
+ ```
30
+ `PRE_SEQ_LEN` and `LR` in `train.sh` are soft prompt length and training learning rate respectively, which can be adjusted to achieve the best results. The P-Tuning-v2 method will freeze all model parameters, and the quantization level of the original model can be adjusted by adjusting `quantization_bit`. If this option is not added, it will be loaded with FP16 precision.
31
+
32
+ Under the default configuration of `per_device_train_batch_size=1`, `gradient_accumulation_steps=16`, the model parameters of INT4 are frozen, and a training iteration will perform 16 cumulative forward and backward propagations with a batch size of 1, which is equivalent to the total batch size of 16, and only 6.7G GPU memory is required at this time with `quantization_bit=4`. If you want to improve the training efficiency under the same batch size, you can increase the value of `per_device_train_batch_size` while keeping the product of the two unchanged, but it will also bring more GPU memory consumption, please adjust it according to the actual situation.
33
+
34
+ ### Inference
35
+
36
+ Change `CHECKPOINT` in `evaluate.sh` to the checkpoint name saved during training, and run the following commands for model inference and evaluation:
37
+ ```shell
38
+ bash evaluate.sh
39
+ ```
40
+
41
+ The evaluation indicators are Chinese Rouge score and BLEU-4. The generated results are saved in
42
+ `./output/adgen-chatglm-6b-pt-8-1e-2/generated_predictions.txt`.
43
+
44
+ ### Example
45
+ #### Example 1
46
+ * Input: 类型#上衣\*材质#牛仔布\*颜色#白色\*风格#简约\*图案#刺绣\*衣样式#外套\*衣款式#破洞
47
+ * Label: 简约而不简单的牛仔外套,白色的衣身十分百搭。衣身多处有做旧破洞设计,打破单调乏味,增加一丝造型看点。衣身后背处有趣味刺绣装饰,丰富层次感,彰显别样时尚。
48
+ * Output[微调前]: 这件上衣的材质是牛仔布,颜色是白色,风格是简约,图案是刺绣,衣样式是外套,衣款式是破洞。
49
+ * Output[微调后]: 这是一款简约的牛仔外套,破洞设计,将牛仔布破洞,带来一种随意与个性。破洞的牛仔外套,展现出时尚气息,带来一种休闲感。同时,刺绣图案,让整件外套更加立体。
50
+
51
+ #### Example 2
52
+
53
+ * Input: 类型#裙\*版型#显瘦\*风格#文艺\*风格#简约\*图案#印花\*图案#撞色\*裙下摆#压褶\*裙长#连衣裙\*裙领型#圆领
54
+ * Label: 文艺个性的印花连衣裙,藏青色底蕴,低调又大气,撞色太阳花分布整个裙身,绚丽而美好,带来时尚减龄的气质。基础款的舒适圆领,简约不失大方,勾勒精致脸庞。领后是一粒包布扣固定,穿脱十分方便。前片立体的打褶设计,搭配后片压褶的做工,增添层次和空间感,显瘦又有型。
55
+ * Output[微调前]: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、���有层次感。
56
+ * Output[微调后]: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。
57
+
58
+ ### evaluation result
59
+
60
+ | | P-tuning v2 | LoRA |
61
+ | ------- | ----------- | ----- |
62
+ | BLEU-4 | 7.71 | 6.13 |
63
+ | Rouge-1 | 31.35 | 28.36 |
64
+ | Rouge-2 | 7.19 | 4.38 |
65
+ | Rouge-l | 25.17 | 17.54 |
66
+
67
+ #### Experiment Settings
68
+
69
+ ```
70
+ max_source_length=64
71
+ max_target_length=64
72
+ per_device_train_batch_size=1
73
+ gradient_accumulation_steps=16
74
+ max_steps=3000
75
+ ```
76
+
77
+ ##### P-tuning v2
78
+
79
+ ```
80
+ pre_seq_len=128
81
+ learning_rate=2e-2
82
+ quantization_bit=4
83
+ ```
84
+
85
+ ##### LoRA
86
+
87
+ ```
88
+ learning_rate=5e-4
89
+ ```
90
+
91
+ The implementation uses [simple_thu_chatglm6b](https://github.com/yuanzhoulvpi2017/zero_nlp/tree/main/simple_thu_chatglm6b)
92
+
93
+
94
+
95
+ ## Model Deployment
96
+ Replace `THUDM/chatglm-6b` in the corresponding demo or code with the path of the checkpoint after P-Tuning(in the example, `./output/adgen-chatglm-6b-pt-8-1e-2/ checkpoint-3000`). Note that the current fine-tuning does not support multiple rounds of data, so only the responses from the first round of the conversation are fine-tuned.
97
+
98
+ ## Use your own dataset
99
+ Modify `train_file`, `validation_file` and `test_file` in `train.sh` and `evaluate.sh` to your own JSON format dataset paths, and change `prompt_column` and `response_column` to the keys in the JSON file corresponding to input text and output text.
100
+
101
+ ## TODO
102
+ * [ ] Support for chat data
103
+ * [ ] Support for full finetuning
104
+
105
+ ## quoting
106
+
107
+ ```
108
+ @inproceedings{liu2022p,
109
+ title={P-tuning: Prompt tuning can be comparable to fine-tuning across scales and tasks},
110
+ author={Liu, Xiao and Ji, Kaixuan and Fu, Yicheng and Tam, Weng and Du, Zhengxiao and Yang, Zhilin and Tang, Jie},
111
+ booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
112
+ pages={61--68},
113
+ year={2022}
114
+ }
115
+ ```
ptuning/__pycache__/arguments.cpython-39.pyc ADDED
Binary file (6.61 kB). View file
 
ptuning/__pycache__/trainer.cpython-39.pyc ADDED
Binary file (103 kB). View file
 
ptuning/__pycache__/trainer_seq2seq.cpython-39.pyc ADDED
Binary file (8.44 kB). View file
 
ptuning/arguments.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+
4
+
5
+ @dataclass
6
+ class ModelArguments:
7
+ """
8
+ Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
9
+ """
10
+
11
+ model_name_or_path: str = field(
12
+ metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
13
+ )
14
+ ptuning_checkpoint: str = field(
15
+ default=None, metadata={"help": "Path to p-tuning v2 checkpoints"}
16
+ )
17
+ config_name: Optional[str] = field(
18
+ default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
19
+ )
20
+ tokenizer_name: Optional[str] = field(
21
+ default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
22
+ )
23
+ cache_dir: Optional[str] = field(
24
+ default=None,
25
+ metadata={"help": "Where to store the pretrained models downloaded from huggingface.co"},
26
+ )
27
+ use_fast_tokenizer: bool = field(
28
+ default=True,
29
+ metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
30
+ )
31
+ model_revision: str = field(
32
+ default="main",
33
+ metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
34
+ )
35
+ use_auth_token: bool = field(
36
+ default=False,
37
+ metadata={
38
+ "help": (
39
+ "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
40
+ "with private models)."
41
+ )
42
+ },
43
+ )
44
+ resize_position_embeddings: Optional[bool] = field(
45
+ default=None,
46
+ metadata={
47
+ "help": (
48
+ "Whether to automatically resize the position embeddings if `max_source_length` exceeds "
49
+ "the model's position embeddings."
50
+ )
51
+ },
52
+ )
53
+ quantization_bit: Optional[int] = field(
54
+ default=None
55
+ )
56
+ pre_seq_len: Optional[int] = field(
57
+ default=None
58
+ )
59
+ prefix_projection: bool = field(
60
+ default=False
61
+ )
62
+
63
+
64
+ @dataclass
65
+ class DataTrainingArguments:
66
+ """
67
+ Arguments pertaining to what data we are going to input our model for training and eval.
68
+ """
69
+
70
+ lang: Optional[str] = field(default=None, metadata={"help": "Language id for summarization."})
71
+
72
+ dataset_name: Optional[str] = field(
73
+ default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
74
+ )
75
+ dataset_config_name: Optional[str] = field(
76
+ default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
77
+ )
78
+ prompt_column: Optional[str] = field(
79
+ default=None,
80
+ metadata={"help": "The name of the column in the datasets containing the full texts (for summarization)."},
81
+ )
82
+ response_column: Optional[str] = field(
83
+ default=None,
84
+ metadata={"help": "The name of the column in the datasets containing the summaries (for summarization)."},
85
+ )
86
+ history_column: Optional[str] = field(
87
+ default=None,
88
+ metadata={"help": "The name of the column in the datasets containing the history of chat."},
89
+ )
90
+ train_file: Optional[str] = field(
91
+ default=None, metadata={"help": "The input training data file (a jsonlines or csv file)."}
92
+ )
93
+ validation_file: Optional[str] = field(
94
+ default=None,
95
+ metadata={
96
+ "help": (
97
+ "An optional input evaluation data file to evaluate the metrics (rouge) on (a jsonlines or csv file)."
98
+ )
99
+ },
100
+ )
101
+ test_file: Optional[str] = field(
102
+ default=None,
103
+ metadata={
104
+ "help": "An optional input test data file to evaluate the metrics (rouge) on (a jsonlines or csv file)."
105
+ },
106
+ )
107
+ overwrite_cache: bool = field(
108
+ default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
109
+ )
110
+ preprocessing_num_workers: Optional[int] = field(
111
+ default=None,
112
+ metadata={"help": "The number of processes to use for the preprocessing."},
113
+ )
114
+ max_source_length: Optional[int] = field(
115
+ default=1024,
116
+ metadata={
117
+ "help": (
118
+ "The maximum total input sequence length after tokenization. Sequences longer "
119
+ "than this will be truncated, sequences shorter will be padded."
120
+ )
121
+ },
122
+ )
123
+ max_target_length: Optional[int] = field(
124
+ default=128,
125
+ metadata={
126
+ "help": (
127
+ "The maximum total sequence length for target text after tokenization. Sequences longer "
128
+ "than this will be truncated, sequences shorter will be padded."
129
+ )
130
+ },
131
+ )
132
+ val_max_target_length: Optional[int] = field(
133
+ default=None,
134
+ metadata={
135
+ "help": (
136
+ "The maximum total sequence length for validation target text after tokenization. Sequences longer "
137
+ "than this will be truncated, sequences shorter will be padded. Will default to `max_target_length`."
138
+ "This argument is also used to override the ``max_length`` param of ``model.generate``, which is used "
139
+ "during ``evaluate`` and ``predict``."
140
+ )
141
+ },
142
+ )
143
+ pad_to_max_length: bool = field(
144
+ default=False,
145
+ metadata={
146
+ "help": (
147
+ "Whether to pad all samples to model maximum sentence length. "
148
+ "If False, will pad the samples dynamically when batching to the maximum length in the batch. More "
149
+ "efficient on GPU but very bad for TPU."
150
+ )
151
+ },
152
+ )
153
+ max_train_samples: Optional[int] = field(
154
+ default=None,
155
+ metadata={
156
+ "help": (
157
+ "For debugging purposes or quicker training, truncate the number of training examples to this "
158
+ "value if set."
159
+ )
160
+ },
161
+ )
162
+ max_eval_samples: Optional[int] = field(
163
+ default=None,
164
+ metadata={
165
+ "help": (
166
+ "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
167
+ "value if set."
168
+ )
169
+ },
170
+ )
171
+ max_predict_samples: Optional[int] = field(
172
+ default=None,
173
+ metadata={
174
+ "help": (
175
+ "For debugging purposes or quicker training, truncate the number of prediction examples to this "
176
+ "value if set."
177
+ )
178
+ },
179
+ )
180
+ num_beams: Optional[int] = field(
181
+ default=None,
182
+ metadata={
183
+ "help": (
184
+ "Number of beams to use for evaluation. This argument will be passed to ``model.generate``, "
185
+ "which is used during ``evaluate`` and ``predict``."
186
+ )
187
+ },
188
+ )
189
+ ignore_pad_token_for_loss: bool = field(
190
+ default=True,
191
+ metadata={
192
+ "help": "Whether to ignore the tokens corresponding to padded labels in the loss computation or not."
193
+ },
194
+ )
195
+ source_prefix: Optional[str] = field(
196
+ default="", metadata={"help": "A prefix to add before every source text (useful for T5 models)."}
197
+ )
198
+
199
+ forced_bos_token: Optional[str] = field(
200
+ default=None,
201
+ metadata={
202
+ "help": (
203
+ "The token to force as the first generated token after the decoder_start_token_id."
204
+ "Useful for multilingual models like mBART where the first generated token"
205
+ "needs to be the target language token (Usually it is the target language token)"
206
+ )
207
+ },
208
+ )
209
+
210
+
211
+
212
+ def __post_init__(self):
213
+ if self.dataset_name is None and self.train_file is None and self.validation_file is None and self.test_file is None:
214
+ raise ValueError("Need either a dataset name or a training/validation/test file.")
215
+ else:
216
+ if self.train_file is not None:
217
+ extension = self.train_file.split(".")[-1]
218
+ assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
219
+ if self.validation_file is not None:
220
+ extension = self.validation_file.split(".")[-1]
221
+ assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
222
+ if self.val_max_target_length is None:
223
+ self.val_max_target_length = self.max_target_length
224
+
ptuning/deepspeed.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_micro_batch_size_per_gpu": "auto",
3
+ "zero_allow_untested_optimizer": true,
4
+ "fp16": {
5
+ "enabled": "auto",
6
+ "loss_scale": 0,
7
+ "initial_scale_power": 16,
8
+ "loss_scale_window": 1000,
9
+ "hysteresis": 2,
10
+ "min_loss_scale": 1
11
+ },
12
+ "zero_optimization": {
13
+ "stage": 2,
14
+ "allgather_partitions": true,
15
+ "allgather_bucket_size": 5e8,
16
+ "overlap_comm": false,
17
+ "reduce_scatter": true,
18
+ "reduce_bucket_size": 5e8,
19
+ "contiguous_gradients" : true
20
+ }
21
+ }
ptuning/ds_train_finetune.sh ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ LR=1e-4
3
+
4
+ MASTER_PORT=$(shuf -n 1 -i 10000-65535)
5
+
6
+ deepspeed --num_gpus=4 --master_port $MASTER_PORT main.py \
7
+ --deepspeed deepspeed.json \
8
+ --do_train \
9
+ --train_file AdvertiseGen/train.json \
10
+ --test_file AdvertiseGen/dev.json \
11
+ --prompt_column content \
12
+ --response_column summary \
13
+ --overwrite_cache \
14
+ --model_name_or_path THUDM/chatglm-6b \
15
+ --output_dir ./output/adgen-chatglm-6b-ft-$LR \
16
+ --overwrite_output_dir \
17
+ --max_source_length 64 \
18
+ --max_target_length 64 \
19
+ --per_device_train_batch_size 4 \
20
+ --per_device_eval_batch_size 1 \
21
+ --gradient_accumulation_steps 1 \
22
+ --predict_with_generate \
23
+ --max_steps 5000 \
24
+ --logging_steps 10 \
25
+ --save_steps 1000 \
26
+ --learning_rate $LR \
27
+ --fp16
28
+
ptuning/evaluate.sh ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PRE_SEQ_LEN=128
2
+ CHECKPOINT=adgen-chatglm-6b-pt-128-2e-2
3
+ STEP=100
4
+
5
+ CUDA_VISIBLE_DEVICES=0 python3 main.py \
6
+ --do_predict \
7
+ --validation_file AdvertiseGen/dev.json \
8
+ --test_file AdvertiseGen/dev.json \
9
+ --overwrite_cache \
10
+ --prompt_column content \
11
+ --response_column summary \
12
+ --model_name_or_path /home/wangyan/project/hft/uptest \
13
+ --ptuning_checkpoint ./output/$CHECKPOINT/checkpoint-$STEP \
14
+ --output_dir ./output/$CHECKPOINT \
15
+ --overwrite_output_dir \
16
+ --max_source_length 64 \
17
+ --max_target_length 64 \
18
+ --per_device_eval_batch_size 1 \
19
+ --predict_with_generate \
20
+ --pre_seq_len $PRE_SEQ_LEN \
21
+ --quantization_bit 4
ptuning/evaluate_finetune.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CHECKPOINT=adgen-chatglm-6b-ft-1e-4
2
+ STEP=3000
3
+
4
+ CUDA_VISIBLE_DEVICES=0 python3 main.py \
5
+ --do_predict \
6
+ --validation_file AdvertiseGen/dev.json \
7
+ --test_file AdvertiseGen/dev.json \
8
+ --overwrite_cache \
9
+ --prompt_column content \
10
+ --response_column summary \
11
+ --model_name_or_path ./output/$CHECKPOINT/checkpoint-$STEP \
12
+ --output_dir ./output/$CHECKPOINT \
13
+ --overwrite_output_dir \
14
+ --max_source_length 256 \
15
+ --max_target_length 256 \
16
+ --per_device_eval_batch_size 1 \
17
+ --predict_with_generate \
18
+ --fp16_full_eval
ptuning/main.py ADDED
@@ -0,0 +1,431 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ # coding=utf-8
3
+ # Copyright 2021 The HuggingFace Team. All rights reserved.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ Fine-tuning the library models for sequence to sequence.
18
+ """
19
+ # You can also adapt this script on your own sequence to sequence task. Pointers for this are left as comments.
20
+
21
+ import logging
22
+ import os
23
+ import sys
24
+ import json
25
+
26
+ import numpy as np
27
+ from datasets import load_dataset
28
+ import jieba
29
+ from rouge_chinese import Rouge
30
+ from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
31
+ import torch
32
+
33
+ import transformers
34
+ from transformers import (
35
+ AutoConfig,
36
+ AutoModel,
37
+ AutoTokenizer,
38
+ AutoTokenizer,
39
+ DataCollatorForSeq2Seq,
40
+ HfArgumentParser,
41
+ Seq2SeqTrainingArguments,
42
+ set_seed,
43
+ )
44
+ from trainer_seq2seq import Seq2SeqTrainer
45
+
46
+ from arguments import ModelArguments, DataTrainingArguments
47
+
48
+ logger = logging.getLogger(__name__)
49
+
50
+ def main():
51
+
52
+ parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))
53
+ if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
54
+ # If we pass only one argument to the script and it's the path to a json file,
55
+ # let's parse it to get our arguments.
56
+ model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
57
+ else:
58
+ model_args, data_args, training_args = parser.parse_args_into_dataclasses()
59
+
60
+ # Setup logging
61
+ logging.basicConfig(
62
+ format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
63
+ datefmt="%m/%d/%Y %H:%M:%S",
64
+ handlers=[logging.StreamHandler(sys.stdout)],
65
+ )
66
+
67
+ if training_args.should_log:
68
+ # The default of training_args.log_level is passive, so we set log level at info here to have that default.
69
+ transformers.utils.logging.set_verbosity_info()
70
+
71
+ log_level = training_args.get_process_log_level()
72
+ logger.setLevel(log_level)
73
+ # datasets.utils.logging.set_verbosity(log_level)
74
+ transformers.utils.logging.set_verbosity(log_level)
75
+ transformers.utils.logging.enable_default_handler()
76
+ transformers.utils.logging.enable_explicit_format()
77
+
78
+ # Log on each process the small summary:
79
+ logger.warning(
80
+ f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
81
+ + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
82
+ )
83
+ logger.info(f"Training/evaluation parameters {training_args}")
84
+
85
+ # Set seed before initializing model.
86
+ set_seed(training_args.seed)
87
+
88
+ # Load dataset
89
+ data_files = {}
90
+ if data_args.train_file is not None:
91
+ data_files["train"] = data_args.train_file
92
+ extension = data_args.train_file.split(".")[-1]
93
+ if data_args.validation_file is not None:
94
+ data_files["validation"] = data_args.validation_file
95
+ extension = data_args.validation_file.split(".")[-1]
96
+ if data_args.test_file is not None:
97
+ data_files["test"] = data_args.test_file
98
+ extension = data_args.test_file.split(".")[-1]
99
+
100
+ raw_datasets = load_dataset(
101
+ extension,
102
+ data_files=data_files,
103
+ cache_dir=model_args.cache_dir,
104
+ use_auth_token=True if model_args.use_auth_token else None,
105
+ )
106
+
107
+ # Load pretrained model and tokenizer
108
+ config = AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)
109
+ config.pre_seq_len = model_args.pre_seq_len
110
+ config.prefix_projection = model_args.prefix_projection
111
+
112
+ tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)
113
+
114
+ if model_args.ptuning_checkpoint is not None:
115
+ # Evaluation
116
+ # Loading extra state dict of prefix encoder
117
+ model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True)
118
+ prefix_state_dict = torch.load(os.path.join(model_args.ptuning_checkpoint, "pytorch_model.bin"))
119
+ new_prefix_state_dict = {}
120
+ for k, v in prefix_state_dict.items():
121
+ if k.startswith("transformer.prefix_encoder."):
122
+ new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
123
+ model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)
124
+ else:
125
+ model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True)
126
+
127
+ if model_args.quantization_bit is not None:
128
+ print(f"Quantized to {model_args.quantization_bit} bit")
129
+ model = model.quantize(model_args.quantization_bit)
130
+ if model_args.pre_seq_len is not None:
131
+ # P-tuning v2
132
+ model = model.half()
133
+ model.transformer.prefix_encoder.float()
134
+ else:
135
+ # Finetune
136
+ model = model.float()
137
+
138
+ prefix = data_args.source_prefix if data_args.source_prefix is not None else ""
139
+
140
+ # Preprocessing the datasets.
141
+ # We need to tokenize inputs and targets.
142
+ if training_args.do_train:
143
+ column_names = raw_datasets["train"].column_names
144
+ elif training_args.do_eval:
145
+ column_names = raw_datasets["validation"].column_names
146
+ elif training_args.do_predict:
147
+ column_names = raw_datasets["test"].column_names
148
+ else:
149
+ logger.info("There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.")
150
+ return
151
+
152
+ # Get the column names for input/target.
153
+ prompt_column = data_args.prompt_column
154
+ response_column = data_args.response_column
155
+ history_column = data_args.history_column
156
+
157
+ # Temporarily set max_target_length for training.
158
+ max_target_length = data_args.max_target_length
159
+
160
+ def preprocess_function_eval(examples):
161
+ inputs, targets = [], []
162
+ for i in range(len(examples[prompt_column])):
163
+ if examples[prompt_column][i] and examples[response_column][i]:
164
+ query = examples[prompt_column][i]
165
+ if history_column is None or len(examples[history_column][i]) == 0:
166
+ prompt = query
167
+ else:
168
+ prompt = ""
169
+ history = examples[history_column][i]
170
+ for turn_idx, (old_query, response) in enumerate(history):
171
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(turn_idx, old_query, response)
172
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
173
+ inputs.append(prompt)
174
+ targets.append(examples[response_column][i])
175
+
176
+ inputs = [prefix + inp for inp in inputs]
177
+ model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, truncation=True, padding=True)
178
+ labels = tokenizer(text_target=targets, max_length=max_target_length, truncation=True)
179
+
180
+ if data_args.ignore_pad_token_for_loss:
181
+ labels["input_ids"] = [
182
+ [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
183
+ ]
184
+ model_inputs["labels"] = labels["input_ids"]
185
+
186
+ return model_inputs
187
+
188
+ def preprocess_function_train(examples):
189
+ max_seq_length = data_args.max_source_length + data_args.max_target_length
190
+
191
+ model_inputs = {
192
+ "input_ids": [],
193
+ "labels": [],
194
+ }
195
+ for i in range(len(examples[prompt_column])):
196
+ if examples[prompt_column][i] and examples[response_column][i]:
197
+ query, answer = examples[prompt_column][i], examples[response_column][i]
198
+
199
+ if history_column is None:
200
+ prompt = query
201
+ else:
202
+ prompt = ""
203
+ history = examples[history_column][i]
204
+ for turn_idx, (old_query, response) in enumerate(history):
205
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(turn_idx, old_query, response)
206
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
207
+
208
+ prompt = prefix + prompt
209
+ a_ids = tokenizer.encode(text=prompt, add_special_tokens=False)
210
+ b_ids = tokenizer.encode(text=answer, add_special_tokens=False)
211
+
212
+ if len(a_ids) > data_args.max_source_length - 1:
213
+ a_ids = a_ids[: data_args.max_source_length - 1]
214
+
215
+ if len(b_ids) > data_args.max_target_length - 2:
216
+ b_ids = b_ids[: data_args.max_target_length - 2]
217
+
218
+ input_ids = tokenizer.build_inputs_with_special_tokens(a_ids, b_ids)
219
+
220
+ context_length = input_ids.index(tokenizer.bos_token_id)
221
+ mask_position = context_length - 1
222
+ labels = [-100] * context_length + input_ids[mask_position+1:]
223
+
224
+ pad_len = max_seq_length - len(input_ids)
225
+ input_ids = input_ids + [tokenizer.pad_token_id] * pad_len
226
+ labels = labels + [tokenizer.pad_token_id] * pad_len
227
+ if data_args.ignore_pad_token_for_loss:
228
+ labels = [(l if l != tokenizer.pad_token_id else -100) for l in labels]
229
+
230
+ model_inputs["input_ids"].append(input_ids)
231
+ model_inputs["labels"].append(labels)
232
+
233
+ return model_inputs
234
+
235
+ def print_dataset_example(example):
236
+ print("input_ids",example["input_ids"])
237
+ print("inputs", tokenizer.decode(example["input_ids"]))
238
+ print("label_ids", example["labels"])
239
+ print("labels", tokenizer.decode(example["labels"]))
240
+
241
+ if training_args.do_train:
242
+ if "train" not in raw_datasets:
243
+ raise ValueError("--do_train requires a train dataset")
244
+ train_dataset = raw_datasets["train"]
245
+ if data_args.max_train_samples is not None:
246
+ max_train_samples = min(len(train_dataset), data_args.max_train_samples)
247
+ train_dataset = train_dataset.select(range(max_train_samples))
248
+ with training_args.main_process_first(desc="train dataset map pre-processing"):
249
+ train_dataset = train_dataset.map(
250
+ preprocess_function_train,
251
+ batched=True,
252
+ num_proc=data_args.preprocessing_num_workers,
253
+ remove_columns=column_names,
254
+ load_from_cache_file=not data_args.overwrite_cache,
255
+ desc="Running tokenizer on train dataset",
256
+ )
257
+ print_dataset_example(train_dataset[0])
258
+
259
+ if training_args.do_eval:
260
+ max_target_length = data_args.val_max_target_length
261
+ if "validation" not in raw_datasets:
262
+ raise ValueError("--do_eval requires a validation dataset")
263
+ eval_dataset = raw_datasets["validation"]
264
+ if data_args.max_eval_samples is not None:
265
+ max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
266
+ eval_dataset = eval_dataset.select(range(max_eval_samples))
267
+ with training_args.main_process_first(desc="validation dataset map pre-processing"):
268
+ eval_dataset = eval_dataset.map(
269
+ preprocess_function_eval,
270
+ batched=True,
271
+ num_proc=data_args.preprocessing_num_workers,
272
+ remove_columns=column_names,
273
+ load_from_cache_file=not data_args.overwrite_cache,
274
+ desc="Running tokenizer on validation dataset",
275
+ )
276
+ print_dataset_example(eval_dataset[0])
277
+
278
+ if training_args.do_predict:
279
+ max_target_length = data_args.val_max_target_length
280
+ if "test" not in raw_datasets:
281
+ raise ValueError("--do_predict requires a test dataset")
282
+ predict_dataset = raw_datasets["test"]
283
+ if data_args.max_predict_samples is not None:
284
+ max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)
285
+ predict_dataset = predict_dataset.select(range(max_predict_samples))
286
+ with training_args.main_process_first(desc="prediction dataset map pre-processing"):
287
+ predict_dataset = predict_dataset.map(
288
+ preprocess_function_eval,
289
+ batched=True,
290
+ num_proc=data_args.preprocessing_num_workers,
291
+ remove_columns=column_names,
292
+ load_from_cache_file=not data_args.overwrite_cache,
293
+ desc="Running tokenizer on prediction dataset",
294
+ )
295
+ print_dataset_example(predict_dataset[0])
296
+
297
+ # Data collator
298
+ label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id
299
+ data_collator = DataCollatorForSeq2Seq(
300
+ tokenizer,
301
+ model=model,
302
+ label_pad_token_id=label_pad_token_id,
303
+ pad_to_multiple_of=None,
304
+ padding=False
305
+ )
306
+
307
+ # Metric
308
+ def compute_metrics(eval_preds):
309
+ preds, labels = eval_preds
310
+ if isinstance(preds, tuple):
311
+ preds = preds[0]
312
+ decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
313
+ if data_args.ignore_pad_token_for_loss:
314
+ # Replace -100 in the labels as we can't decode them.
315
+ labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
316
+ decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
317
+
318
+ score_dict = {
319
+ "rouge-1": [],
320
+ "rouge-2": [],
321
+ "rouge-l": [],
322
+ "bleu-4": []
323
+ }
324
+ for pred, label in zip(decoded_preds, decoded_labels):
325
+ hypothesis = list(jieba.cut(pred))
326
+ reference = list(jieba.cut(label))
327
+ rouge = Rouge()
328
+ scores = rouge.get_scores(' '.join(hypothesis) , ' '.join(reference))
329
+ result = scores[0]
330
+
331
+ for k, v in result.items():
332
+ score_dict[k].append(round(v["f"] * 100, 4))
333
+ bleu_score = sentence_bleu([list(label)], list(pred), smoothing_function=SmoothingFunction().method3)
334
+ score_dict["bleu-4"].append(round(bleu_score * 100, 4))
335
+
336
+ for k, v in score_dict.items():
337
+ score_dict[k] = float(np.mean(v))
338
+ return score_dict
339
+
340
+ # Override the decoding parameters of Seq2SeqTrainer
341
+ training_args.generation_max_length = (
342
+ training_args.generation_max_length
343
+ if training_args.generation_max_length is not None
344
+ else data_args.val_max_target_length
345
+ )
346
+ training_args.generation_num_beams = (
347
+ data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams
348
+ )
349
+ # Initialize our Trainer
350
+ trainer = Seq2SeqTrainer(
351
+ model=model,
352
+ args=training_args,
353
+ train_dataset=train_dataset if training_args.do_train else None,
354
+ eval_dataset=eval_dataset if training_args.do_eval else None,
355
+ tokenizer=tokenizer,
356
+ data_collator=data_collator,
357
+ compute_metrics=compute_metrics if training_args.predict_with_generate else None,
358
+ save_prefixencoder=model_args.pre_seq_len is not None
359
+ )
360
+
361
+ # Training
362
+ if training_args.do_train:
363
+ checkpoint = None
364
+ if training_args.resume_from_checkpoint is not None:
365
+ checkpoint = training_args.resume_from_checkpoint
366
+ # elif last_checkpoint is not None:
367
+ # checkpoint = last_checkpoint
368
+ model.gradient_checkpointing_enable()
369
+ model.enable_input_require_grads()
370
+ train_result = trainer.train(resume_from_checkpoint=checkpoint)
371
+ # trainer.save_model() # Saves the tokenizer too for easy upload
372
+
373
+ metrics = train_result.metrics
374
+ max_train_samples = (
375
+ data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
376
+ )
377
+ metrics["train_samples"] = min(max_train_samples, len(train_dataset))
378
+
379
+ trainer.log_metrics("train", metrics)
380
+ trainer.save_metrics("train", metrics)
381
+ trainer.save_state()
382
+
383
+ # Evaluation
384
+ results = {}
385
+ if training_args.do_eval:
386
+ logger.info("*** Evaluate ***")
387
+ metrics = trainer.evaluate(metric_key_prefix="eval", do_sample=True, top_p=0.7, max_length=512, temperature=0.95)
388
+ max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
389
+ metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
390
+
391
+ trainer.log_metrics("eval", metrics)
392
+ trainer.save_metrics("eval", metrics)
393
+
394
+ if training_args.do_predict:
395
+ logger.info("*** Predict ***")
396
+
397
+ predict_results = trainer.predict(predict_dataset, metric_key_prefix="predict", max_length=512, do_sample=True, top_p=0.7, temperature=0.95)
398
+ metrics = predict_results.metrics
399
+ max_predict_samples = (
400
+ data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
401
+ )
402
+ metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
403
+
404
+ trainer.log_metrics("predict", metrics)
405
+ trainer.save_metrics("predict", metrics)
406
+
407
+ if trainer.is_world_process_zero():
408
+ if training_args.predict_with_generate:
409
+ predictions = tokenizer.batch_decode(
410
+ predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
411
+ )
412
+ predictions = [pred.strip() for pred in predictions]
413
+ labels = tokenizer.batch_decode(
414
+ predict_results.label_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True
415
+ )
416
+ labels = [label.strip() for label in labels]
417
+ output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt")
418
+ with open(output_prediction_file, "w", encoding="utf-8") as writer:
419
+ for p, l in zip(predictions, labels):
420
+ res = json.dumps({"labels": l, "predict": p}, ensure_ascii=False)
421
+ writer.write(f"{res}\n")
422
+ return results
423
+
424
+
425
+ def _mp_fn(index):
426
+ # For xla_spawn (TPUs)
427
+ main()
428
+
429
+
430
+ if __name__ == "__main__":
431
+ main()
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.01,
3
+ "train_loss": 4.456654052734375,
4
+ "train_runtime": 395.1154,
5
+ "train_samples": 114599,
6
+ "train_samples_per_second": 4.049,
7
+ "train_steps_per_second": 0.253
8
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/wangyan/project/hft/uptest",
3
+ "architectures": [
4
+ "ChatGLMForConditionalGeneration"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_chatglm.ChatGLMConfig",
8
+ "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
9
+ "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
10
+ },
11
+ "bos_token_id": 130004,
12
+ "eos_token_id": 130005,
13
+ "gmask_token_id": 130001,
14
+ "hidden_size": 4096,
15
+ "inner_hidden_size": 16384,
16
+ "layernorm_epsilon": 1e-05,
17
+ "mask_token_id": 130000,
18
+ "max_sequence_length": 2048,
19
+ "model_type": "chatglm",
20
+ "num_attention_heads": 32,
21
+ "num_layers": 28,
22
+ "pad_token_id": 3,
23
+ "position_encoding_2d": true,
24
+ "pre_seq_len": 128,
25
+ "prefix_projection": false,
26
+ "quantization_bit": 4,
27
+ "torch_dtype": "float16",
28
+ "transformers_version": "4.27.1",
29
+ "use_cache": true,
30
+ "vocab_size": 130528
31
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/configuration_chatglm.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ ChatGLM model configuration """
2
+
3
+ from transformers.configuration_utils import PretrainedConfig
4
+ from transformers.utils import logging
5
+
6
+ logger = logging.get_logger(__name__)
7
+
8
+
9
+ class ChatGLMConfig(PretrainedConfig):
10
+ r"""
11
+ This is the configuration class to store the configuration of a [`~ChatGLMModel`].
12
+ It is used to instantiate an ChatGLM model according to the specified arguments, defining the model
13
+ architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
14
+ the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture.
15
+
16
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
17
+ to control the model outputs. Read the documentation from [`PretrainedConfig`]
18
+ for more information.
19
+
20
+
21
+ Args:
22
+ vocab_size (`int`, *optional*, defaults to 150528):
23
+ Vocabulary size of the ChatGLM-6B model. Defines the number of different tokens that can be represented by the
24
+ `inputs_ids` passed when calling [`~ChatGLMModel`] or
25
+ [`~TFChatGLMModel`].
26
+ hidden_size (`int`, *optional*, defaults to 4096):
27
+ Dimension of the encoder layers and the pooler layer.
28
+ num_hidden_layers (`int`, *optional*, defaults to 28):
29
+ Number of hidden layers in the Transformer encoder.
30
+ num_attention_heads (`int`, *optional*, defaults to 32):
31
+ Number of attention heads for each attention layer in the Transformer encoder.
32
+ inner_hidden_size (`int`, *optional*, defaults to 16384):
33
+ Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
34
+ max_sequence_length (`int`, *optional*, defaults to 512):
35
+ The maximum sequence length that this model might ever be used with.
36
+ Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
37
+ layernorm_epsilon (`float`, *optional*, defaults to 1e-5):
38
+ The epsilon used by the layer normalization layers.
39
+ use_cache (`bool`, *optional*, defaults to `True`):
40
+ Whether the model should return the last key/values attentions (not used by all models).
41
+ Example:
42
+
43
+ ```python
44
+ >>> from configuration_chatglm import ChatGLMConfig
45
+ >>> from modeling_chatglm import ChatGLMModel
46
+
47
+ >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration
48
+ >>> configuration = ChatGLMConfig()
49
+
50
+ >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration
51
+ >>> model = ChatGLMModel(configuration)
52
+
53
+ >>> # Accessing the model configuration
54
+ >>> configuration = model.config
55
+ ```
56
+ """
57
+ model_type = "chatglm"
58
+
59
+ def __init__(
60
+ self,
61
+ vocab_size=150528,
62
+ hidden_size=4096,
63
+ num_layers=28,
64
+ num_attention_heads=32,
65
+ layernorm_epsilon=1e-5,
66
+ use_cache=False,
67
+ bos_token_id=150004,
68
+ eos_token_id=150005,
69
+ mask_token_id=150000,
70
+ gmask_token_id=150001,
71
+ pad_token_id=0,
72
+ max_sequence_length=2048,
73
+ inner_hidden_size=16384,
74
+ position_encoding_2d=True,
75
+ quantization_bit=0,
76
+ pre_seq_len=None,
77
+ prefix_projection=False,
78
+ **kwargs
79
+ ):
80
+ self.num_layers = num_layers
81
+ self.vocab_size = vocab_size
82
+ self.hidden_size = hidden_size
83
+ self.num_attention_heads = num_attention_heads
84
+ self.max_sequence_length = max_sequence_length
85
+ self.layernorm_epsilon = layernorm_epsilon
86
+ self.inner_hidden_size = inner_hidden_size
87
+ self.use_cache = use_cache
88
+ self.bos_token_id = bos_token_id
89
+ self.eos_token_id = eos_token_id
90
+ self.pad_token_id = pad_token_id
91
+ self.mask_token_id = mask_token_id
92
+ self.gmask_token_id = gmask_token_id
93
+ self.position_encoding_2d = position_encoding_2d
94
+ self.quantization_bit = quantization_bit
95
+ self.pre_seq_len = pre_seq_len
96
+ self.prefix_projection = prefix_projection
97
+
98
+ super().__init__(
99
+ pad_token_id=pad_token_id,
100
+ bos_token_id=bos_token_id,
101
+ eos_token_id=eos_token_id,
102
+ **kwargs
103
+ )
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 130004,
4
+ "eos_token_id": 130005,
5
+ "pad_token_id": 3,
6
+ "transformers_version": "4.27.1"
7
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/ice_text.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e974d9a69c242ce014c88c2b26089270f6198f3c0b700a887666cd3e816f17e
3
+ size 2706249
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/modeling_chatglm.py ADDED
@@ -0,0 +1,1435 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ PyTorch ChatGLM model. """
2
+
3
+ import math
4
+ import copy
5
+ import os
6
+ import warnings
7
+ import re
8
+ import sys
9
+
10
+ import torch
11
+ import torch.utils.checkpoint
12
+ import torch.nn.functional as F
13
+ from torch import nn
14
+ from torch.nn import CrossEntropyLoss, LayerNorm
15
+ from torch.nn.utils import skip_init
16
+ from typing import Optional, Tuple, Union, List, Callable, Dict, Any
17
+
18
+ from transformers.utils import (
19
+ add_code_sample_docstrings,
20
+ add_start_docstrings,
21
+ add_start_docstrings_to_model_forward,
22
+ )
23
+ from transformers.modeling_outputs import (
24
+ BaseModelOutputWithPast,
25
+ CausalLMOutputWithPast,
26
+ BaseModelOutputWithPastAndCrossAttentions,
27
+ )
28
+ from transformers.modeling_utils import PreTrainedModel
29
+ from transformers.utils import logging
30
+ from transformers.generation.logits_process import LogitsProcessor
31
+ from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationConfig, ModelOutput
32
+
33
+ from .configuration_chatglm import ChatGLMConfig
34
+
35
+ # flags required to enable jit fusion kernels
36
+
37
+ if sys.platform != 'darwin':
38
+ torch._C._jit_set_profiling_mode(False)
39
+ torch._C._jit_set_profiling_executor(False)
40
+ torch._C._jit_override_can_fuse_on_cpu(True)
41
+ torch._C._jit_override_can_fuse_on_gpu(True)
42
+
43
+ logger = logging.get_logger(__name__)
44
+
45
+ _CHECKPOINT_FOR_DOC = "THUDM/ChatGLM-6B"
46
+ _CONFIG_FOR_DOC = "ChatGLM6BConfig"
47
+
48
+ CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [
49
+ "THUDM/chatglm-6b",
50
+ # See all ChatGLM-6B models at https://huggingface.co/models?filter=chatglm
51
+ ]
52
+
53
+
54
+ class InvalidScoreLogitsProcessor(LogitsProcessor):
55
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
56
+ if torch.isnan(scores).any() or torch.isinf(scores).any():
57
+ scores.zero_()
58
+ scores[..., 5] = 5e4
59
+ return scores
60
+
61
+
62
+ def load_tf_weights_in_chatglm_6b(model, config, tf_checkpoint_path):
63
+ """Load tf checkpoints in a pytorch model."""
64
+ try:
65
+ import re
66
+
67
+ import numpy as np
68
+ import tensorflow as tf
69
+ except ImportError:
70
+ logger.error(
71
+ "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
72
+ "https://www.tensorflow.org/install/ for installation instructions."
73
+ )
74
+ raise
75
+ tf_path = os.path.abspath(tf_checkpoint_path)
76
+ logger.info(f"Converting TensorFlow checkpoint from {tf_path}")
77
+ # Load weights from TF model
78
+ init_vars = tf.train.list_variables(tf_path)
79
+ names = []
80
+ arrays = []
81
+ for name, shape in init_vars:
82
+ logger.info(f"Loading TF weight {name} with shape {shape}")
83
+ array = tf.train.load_variable(tf_path, name)
84
+ names.append(name)
85
+ arrays.append(array)
86
+
87
+ for name, array in zip(names, arrays):
88
+ name = name.split("/")
89
+ # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
90
+ # which are not required for using pretrained model
91
+ if any(
92
+ n in ["adam_v", "adam_m", "AdamWeightDecayOptimizer", "AdamWeightDecayOptimizer_1", "global_step"]
93
+ for n in name
94
+ ):
95
+ logger.info(f"Skipping {'/'.join(name)}")
96
+ continue
97
+ pointer = model
98
+ for m_name in name:
99
+ if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
100
+ scope_names = re.split(r"_(\d+)", m_name)
101
+ else:
102
+ scope_names = [m_name]
103
+ if scope_names[0] == "kernel" or scope_names[0] == "gamma":
104
+ pointer = getattr(pointer, "weight")
105
+ elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
106
+ pointer = getattr(pointer, "bias")
107
+ elif scope_names[0] == "output_weights":
108
+ pointer = getattr(pointer, "weight")
109
+ elif scope_names[0] == "squad":
110
+ pointer = getattr(pointer, "classifier")
111
+ else:
112
+ try:
113
+ pointer = getattr(pointer, scope_names[0])
114
+ except AttributeError:
115
+ logger.info(f"Skipping {'/'.join(name)}")
116
+ continue
117
+ if len(scope_names) >= 2:
118
+ num = int(scope_names[1])
119
+ pointer = pointer[num]
120
+ if m_name[-11:] == "_embeddings":
121
+ pointer = getattr(pointer, "weight")
122
+ elif m_name == "kernel":
123
+ array = np.transpose(array)
124
+ try:
125
+ assert (
126
+ pointer.shape == array.shape
127
+ ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"
128
+ except AssertionError as e:
129
+ e.args += (pointer.shape, array.shape)
130
+ raise
131
+ logger.info(f"Initialize PyTorch weight {name}")
132
+ pointer.data = torch.from_numpy(array)
133
+ return model
134
+
135
+
136
+ class PrefixEncoder(torch.nn.Module):
137
+ """
138
+ The torch.nn model to encode the prefix
139
+ Input shape: (batch-size, prefix-length)
140
+ Output shape: (batch-size, prefix-length, 2*layers*hidden)
141
+ """
142
+
143
+ def __init__(self, config):
144
+ super().__init__()
145
+ self.prefix_projection = config.prefix_projection
146
+ if self.prefix_projection:
147
+ # Use a two-layer MLP to encode the prefix
148
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.hidden_size)
149
+ self.trans = torch.nn.Sequential(
150
+ torch.nn.Linear(config.hidden_size, config.hidden_size),
151
+ torch.nn.Tanh(),
152
+ torch.nn.Linear(config.hidden_size, config.num_layers * config.hidden_size * 2)
153
+ )
154
+ else:
155
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.num_layers * config.hidden_size * 2)
156
+
157
+ def forward(self, prefix: torch.Tensor):
158
+ if self.prefix_projection:
159
+ prefix_tokens = self.embedding(prefix)
160
+ past_key_values = self.trans(prefix_tokens)
161
+ else:
162
+ past_key_values = self.embedding(prefix)
163
+ return past_key_values
164
+
165
+
166
+ @torch.jit.script
167
+ def gelu_impl(x):
168
+ """OpenAI's gelu implementation."""
169
+ return 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x *
170
+ (1.0 + 0.044715 * x * x)))
171
+
172
+
173
+ def gelu(x):
174
+ return gelu_impl(x)
175
+
176
+
177
+ class RotaryEmbedding(torch.nn.Module):
178
+ def __init__(self, dim, base=10000, precision=torch.half, learnable=False):
179
+ super().__init__()
180
+ inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim))
181
+ inv_freq = inv_freq.half()
182
+ self.learnable = learnable
183
+ if learnable:
184
+ self.inv_freq = torch.nn.Parameter(inv_freq)
185
+ self.max_seq_len_cached = None
186
+ else:
187
+ self.register_buffer('inv_freq', inv_freq)
188
+ self.max_seq_len_cached = None
189
+ self.cos_cached = None
190
+ self.sin_cached = None
191
+ self.precision = precision
192
+
193
+ def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys,
194
+ error_msgs):
195
+ pass
196
+
197
+ def forward(self, x, seq_dim=1, seq_len=None):
198
+ if seq_len is None:
199
+ seq_len = x.shape[seq_dim]
200
+ if self.max_seq_len_cached is None or (seq_len > self.max_seq_len_cached):
201
+ self.max_seq_len_cached = None if self.learnable else seq_len
202
+ t = torch.arange(seq_len, device=x.device, dtype=self.inv_freq.dtype)
203
+ freqs = torch.einsum('i,j->ij', t, self.inv_freq)
204
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
205
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
206
+ if self.precision == torch.bfloat16:
207
+ emb = emb.float()
208
+
209
+ # [sx, 1 (b * np), hn]
210
+ cos_cached = emb.cos()[:, None, :]
211
+ sin_cached = emb.sin()[:, None, :]
212
+ if self.precision == torch.bfloat16:
213
+ cos_cached = cos_cached.bfloat16()
214
+ sin_cached = sin_cached.bfloat16()
215
+ if self.learnable:
216
+ return cos_cached, sin_cached
217
+ self.cos_cached, self.sin_cached = cos_cached, sin_cached
218
+ return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
219
+
220
+ def _apply(self, fn):
221
+ if self.cos_cached is not None:
222
+ self.cos_cached = fn(self.cos_cached)
223
+ if self.sin_cached is not None:
224
+ self.sin_cached = fn(self.sin_cached)
225
+ return super()._apply(fn)
226
+
227
+
228
+ def rotate_half(x):
229
+ x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
230
+ return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions
231
+
232
+
233
+ @torch.jit.script
234
+ def apply_rotary_pos_emb_index(q, k, cos, sin, position_id):
235
+ # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn]
236
+ cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \
237
+ F.embedding(position_id, sin.squeeze(1)).unsqueeze(2)
238
+ q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
239
+ return q, k
240
+
241
+
242
+ def attention_fn(
243
+ self,
244
+ query_layer,
245
+ key_layer,
246
+ value_layer,
247
+ attention_mask,
248
+ hidden_size_per_partition,
249
+ layer_id,
250
+ layer_past=None,
251
+ scaling_attention_score=True,
252
+ use_cache=False,
253
+ ):
254
+ if layer_past is not None:
255
+ past_key, past_value = layer_past[0], layer_past[1]
256
+ key_layer = torch.cat((past_key, key_layer), dim=0)
257
+ value_layer = torch.cat((past_value, value_layer), dim=0)
258
+
259
+ # seqlen, batch, num_attention_heads, hidden_size_per_attention_head
260
+ seq_len, b, nh, hidden_size = key_layer.shape
261
+
262
+ if use_cache:
263
+ present = (key_layer, value_layer)
264
+ else:
265
+ present = None
266
+
267
+ query_key_layer_scaling_coeff = float(layer_id + 1)
268
+ if scaling_attention_score:
269
+ query_layer = query_layer / (math.sqrt(hidden_size) * query_key_layer_scaling_coeff)
270
+
271
+ # ===================================
272
+ # Raw attention scores. [b, np, s, s]
273
+ # ===================================
274
+
275
+ # [b, np, sq, sk]
276
+ output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0))
277
+
278
+ # [sq, b, np, hn] -> [sq, b * np, hn]
279
+ query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
280
+ # [sk, b, np, hn] -> [sk, b * np, hn]
281
+ key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
282
+
283
+ matmul_result = torch.zeros(
284
+ 1, 1, 1,
285
+ dtype=query_layer.dtype,
286
+ device=query_layer.device,
287
+ )
288
+
289
+ matmul_result = torch.baddbmm(
290
+ matmul_result,
291
+ query_layer.transpose(0, 1), # [b * np, sq, hn]
292
+ key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk]
293
+ beta=0.0,
294
+ alpha=1.0,
295
+ )
296
+
297
+ # change view to [b, np, sq, sk]
298
+ attention_scores = matmul_result.view(*output_size)
299
+
300
+ if self.scale_mask_softmax:
301
+ self.scale_mask_softmax.scale = query_key_layer_scaling_coeff
302
+ attention_probs = self.scale_mask_softmax(attention_scores, attention_mask.contiguous())
303
+ else:
304
+ if not (attention_mask == 0).all():
305
+ # if auto-regressive, skip
306
+ attention_scores.masked_fill_(attention_mask, -10000.0)
307
+ dtype = attention_scores.dtype
308
+ attention_scores = attention_scores.float()
309
+ attention_scores = attention_scores * query_key_layer_scaling_coeff
310
+
311
+ attention_probs = F.softmax(attention_scores, dim=-1)
312
+
313
+ attention_probs = attention_probs.type(dtype)
314
+
315
+ # =========================
316
+ # Context layer. [sq, b, hp]
317
+ # =========================
318
+
319
+ # value_layer -> context layer.
320
+ # [sk, b, np, hn] --> [b, np, sq, hn]
321
+
322
+ # context layer shape: [b, np, sq, hn]
323
+ output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3))
324
+
325
+ # change view [sk, b * np, hn]
326
+ value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1)
327
+
328
+ # change view [b * np, sq, sk]
329
+ attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1)
330
+
331
+ # matmul: [b * np, sq, hn]
332
+ context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1))
333
+
334
+ # change view [b, np, sq, hn]
335
+ context_layer = context_layer.view(*output_size)
336
+
337
+ # [b, np, sq, hn] --> [sq, b, np, hn]
338
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
339
+
340
+ # [sq, b, np, hn] --> [sq, b, hp]
341
+ new_context_layer_shape = context_layer.size()[:-2] + (hidden_size_per_partition,)
342
+ context_layer = context_layer.view(*new_context_layer_shape)
343
+
344
+ outputs = (context_layer, present, attention_probs)
345
+
346
+ return outputs
347
+
348
+
349
+ def default_init(cls, *args, **kwargs):
350
+ return cls(*args, **kwargs)
351
+
352
+
353
+ class SelfAttention(torch.nn.Module):
354
+ def __init__(self, hidden_size, num_attention_heads,
355
+ layer_id, hidden_size_per_attention_head=None, bias=True,
356
+ params_dtype=torch.float, position_encoding_2d=True, empty_init=True):
357
+ if empty_init:
358
+ init_method = skip_init
359
+ else:
360
+ init_method = default_init
361
+ super(SelfAttention, self).__init__()
362
+
363
+ self.layer_id = layer_id
364
+ self.hidden_size = hidden_size
365
+ self.hidden_size_per_partition = hidden_size
366
+ self.num_attention_heads = num_attention_heads
367
+ self.num_attention_heads_per_partition = num_attention_heads
368
+ self.position_encoding_2d = position_encoding_2d
369
+ self.rotary_emb = RotaryEmbedding(
370
+ self.hidden_size // (self.num_attention_heads * 2)
371
+ if position_encoding_2d
372
+ else self.hidden_size // self.num_attention_heads,
373
+ base=10000,
374
+ precision=torch.half,
375
+ learnable=False,
376
+ )
377
+
378
+ self.scale_mask_softmax = None
379
+
380
+ if hidden_size_per_attention_head is None:
381
+ self.hidden_size_per_attention_head = hidden_size // num_attention_heads
382
+ else:
383
+ self.hidden_size_per_attention_head = hidden_size_per_attention_head
384
+
385
+ self.inner_hidden_size = num_attention_heads * self.hidden_size_per_attention_head
386
+
387
+ # Strided linear layer.
388
+ self.query_key_value = init_method(
389
+ torch.nn.Linear,
390
+ hidden_size,
391
+ 3 * self.inner_hidden_size,
392
+ bias=bias,
393
+ dtype=params_dtype,
394
+ )
395
+
396
+ self.dense = init_method(
397
+ torch.nn.Linear,
398
+ self.inner_hidden_size,
399
+ hidden_size,
400
+ bias=bias,
401
+ dtype=params_dtype,
402
+ )
403
+
404
+ @staticmethod
405
+ def attention_mask_func(attention_scores, attention_mask):
406
+ attention_scores.masked_fill_(attention_mask, -10000.0)
407
+ return attention_scores
408
+
409
+ def split_tensor_along_last_dim(self, tensor, num_partitions,
410
+ contiguous_split_chunks=False):
411
+ """Split a tensor along its last dimension.
412
+ Arguments:
413
+ tensor: input tensor.
414
+ num_partitions: number of partitions to split the tensor
415
+ contiguous_split_chunks: If True, make each chunk contiguous
416
+ in memory.
417
+ """
418
+ # Get the size and dimension.
419
+ last_dim = tensor.dim() - 1
420
+ last_dim_size = tensor.size()[last_dim] // num_partitions
421
+ # Split.
422
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
423
+ # Note: torch.split does not create contiguous tensors by default.
424
+ if contiguous_split_chunks:
425
+ return tuple(chunk.contiguous() for chunk in tensor_list)
426
+
427
+ return tensor_list
428
+
429
+ def forward(
430
+ self,
431
+ hidden_states: torch.Tensor,
432
+ position_ids,
433
+ attention_mask: torch.Tensor,
434
+ layer_id,
435
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
436
+ use_cache: bool = False,
437
+ output_attentions: bool = False,
438
+ ):
439
+ """
440
+ hidden_states: [seq_len, batch, hidden_size]
441
+ attention_mask: [(1, 1), seq_len, seq_len]
442
+ """
443
+
444
+ # [seq_len, batch, 3 * hidden_size]
445
+ mixed_raw_layer = self.query_key_value(hidden_states)
446
+
447
+ # [seq_len, batch, 3 * hidden_size] --> [seq_len, batch, num_attention_heads, 3 * hidden_size_per_attention_head]
448
+ new_tensor_shape = mixed_raw_layer.size()[:-1] + (
449
+ self.num_attention_heads_per_partition,
450
+ 3 * self.hidden_size_per_attention_head,
451
+ )
452
+ mixed_raw_layer = mixed_raw_layer.view(*new_tensor_shape)
453
+
454
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
455
+ (query_layer, key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_raw_layer, 3)
456
+
457
+ if self.position_encoding_2d:
458
+ q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1))
459
+ k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1))
460
+ cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1)
461
+ position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \
462
+ position_ids[:, 1, :].transpose(0, 1).contiguous()
463
+ q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids)
464
+ q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, block_position_ids)
465
+ query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1))
466
+ key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1))
467
+ else:
468
+ position_ids = position_ids.transpose(0, 1)
469
+ cos, sin = self.rotary_emb(value_layer, seq_len=position_ids.max() + 1)
470
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
471
+ query_layer, key_layer = apply_rotary_pos_emb_index(query_layer, key_layer, cos, sin, position_ids)
472
+
473
+ # [seq_len, batch, hidden_size]
474
+ context_layer, present, attention_probs = attention_fn(
475
+ self=self,
476
+ query_layer=query_layer,
477
+ key_layer=key_layer,
478
+ value_layer=value_layer,
479
+ attention_mask=attention_mask,
480
+ hidden_size_per_partition=self.hidden_size_per_partition,
481
+ layer_id=layer_id,
482
+ layer_past=layer_past,
483
+ use_cache=use_cache
484
+ )
485
+
486
+ output = self.dense(context_layer)
487
+
488
+ outputs = (output, present)
489
+
490
+ if output_attentions:
491
+ outputs += (attention_probs,)
492
+
493
+ return outputs # output, present, attention_probs
494
+
495
+
496
+ class GEGLU(torch.nn.Module):
497
+ def __init__(self):
498
+ super().__init__()
499
+ self.activation_fn = F.gelu
500
+
501
+ def forward(self, x):
502
+ # dim=-1 breaks in jit for pt<1.10
503
+ x1, x2 = x.chunk(2, dim=(x.ndim - 1))
504
+ return x1 * self.activation_fn(x2)
505
+
506
+
507
+ class GLU(torch.nn.Module):
508
+ def __init__(self, hidden_size, inner_hidden_size=None,
509
+ layer_id=None, bias=True, activation_func=gelu, params_dtype=torch.float, empty_init=True):
510
+ super(GLU, self).__init__()
511
+ if empty_init:
512
+ init_method = skip_init
513
+ else:
514
+ init_method = default_init
515
+ self.layer_id = layer_id
516
+ self.activation_func = activation_func
517
+
518
+ # Project to 4h.
519
+ self.hidden_size = hidden_size
520
+ if inner_hidden_size is None:
521
+ inner_hidden_size = 4 * hidden_size
522
+ self.inner_hidden_size = inner_hidden_size
523
+ self.dense_h_to_4h = init_method(
524
+ torch.nn.Linear,
525
+ self.hidden_size,
526
+ self.inner_hidden_size,
527
+ bias=bias,
528
+ dtype=params_dtype,
529
+ )
530
+ # Project back to h.
531
+ self.dense_4h_to_h = init_method(
532
+ torch.nn.Linear,
533
+ self.inner_hidden_size,
534
+ self.hidden_size,
535
+ bias=bias,
536
+ dtype=params_dtype,
537
+ )
538
+
539
+ def forward(self, hidden_states):
540
+ """
541
+ hidden_states: [seq_len, batch, hidden_size]
542
+ """
543
+
544
+ # [seq_len, batch, inner_hidden_size]
545
+ intermediate_parallel = self.dense_h_to_4h(hidden_states)
546
+
547
+ intermediate_parallel = self.activation_func(intermediate_parallel)
548
+
549
+ output = self.dense_4h_to_h(intermediate_parallel)
550
+
551
+ return output
552
+
553
+
554
+ class GLMBlock(torch.nn.Module):
555
+ def __init__(
556
+ self,
557
+ hidden_size,
558
+ num_attention_heads,
559
+ layernorm_epsilon,
560
+ layer_id,
561
+ inner_hidden_size=None,
562
+ hidden_size_per_attention_head=None,
563
+ layernorm=LayerNorm,
564
+ use_bias=True,
565
+ params_dtype=torch.float,
566
+ num_layers=28,
567
+ position_encoding_2d=True,
568
+ empty_init=True
569
+ ):
570
+ super(GLMBlock, self).__init__()
571
+ # Set output layer initialization if not provided.
572
+
573
+ self.layer_id = layer_id
574
+
575
+ # Layernorm on the input data.
576
+ self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
577
+
578
+ self.position_encoding_2d = position_encoding_2d
579
+
580
+ # Self attention.
581
+ self.attention = SelfAttention(
582
+ hidden_size,
583
+ num_attention_heads,
584
+ layer_id,
585
+ hidden_size_per_attention_head=hidden_size_per_attention_head,
586
+ bias=use_bias,
587
+ params_dtype=params_dtype,
588
+ position_encoding_2d=self.position_encoding_2d,
589
+ empty_init=empty_init
590
+ )
591
+
592
+ # Layernorm on the input data.
593
+ self.post_attention_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
594
+
595
+ self.num_layers = num_layers
596
+
597
+ # GLU
598
+ self.mlp = GLU(
599
+ hidden_size,
600
+ inner_hidden_size=inner_hidden_size,
601
+ bias=use_bias,
602
+ layer_id=layer_id,
603
+ params_dtype=params_dtype,
604
+ empty_init=empty_init
605
+ )
606
+
607
+ def forward(
608
+ self,
609
+ hidden_states: torch.Tensor,
610
+ position_ids,
611
+ attention_mask: torch.Tensor,
612
+ layer_id,
613
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
614
+ use_cache: bool = False,
615
+ output_attentions: bool = False,
616
+ ):
617
+ """
618
+ hidden_states: [seq_len, batch, hidden_size]
619
+ attention_mask: [(1, 1), seq_len, seq_len]
620
+ """
621
+
622
+ # Layer norm at the begining of the transformer layer.
623
+ # [seq_len, batch, hidden_size]
624
+ attention_input = self.input_layernorm(hidden_states)
625
+
626
+ # Self attention.
627
+ attention_outputs = self.attention(
628
+ attention_input,
629
+ position_ids,
630
+ attention_mask=attention_mask,
631
+ layer_id=layer_id,
632
+ layer_past=layer_past,
633
+ use_cache=use_cache,
634
+ output_attentions=output_attentions
635
+ )
636
+
637
+ attention_output = attention_outputs[0]
638
+
639
+ outputs = attention_outputs[1:]
640
+
641
+ # Residual connection.
642
+ alpha = (2 * self.num_layers) ** 0.5
643
+ hidden_states = attention_input * alpha + attention_output
644
+
645
+ mlp_input = self.post_attention_layernorm(hidden_states)
646
+
647
+ # MLP.
648
+ mlp_output = self.mlp(mlp_input)
649
+
650
+ # Second residual connection.
651
+ output = mlp_input * alpha + mlp_output
652
+
653
+ if use_cache:
654
+ outputs = (output,) + outputs
655
+ else:
656
+ outputs = (output,) + outputs[1:]
657
+
658
+ return outputs # hidden_states, present, attentions
659
+
660
+
661
+ class ChatGLMPreTrainedModel(PreTrainedModel):
662
+ """
663
+ An abstract class to handle weights initialization and
664
+ a simple interface for downloading and loading pretrained models.
665
+ """
666
+
667
+ is_parallelizable = False
668
+ supports_gradient_checkpointing = True
669
+ config_class = ChatGLMConfig
670
+ base_model_prefix = "transformer"
671
+ _no_split_modules = ["GLMBlock"]
672
+
673
+ def __init__(self, *inputs, **kwargs):
674
+ super().__init__(*inputs, **kwargs)
675
+
676
+ def _init_weights(self, module: nn.Module):
677
+ """Initialize the weights."""
678
+ return
679
+
680
+ def get_masks(self, input_ids, device):
681
+ batch_size, seq_length = input_ids.shape
682
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
683
+ attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device)
684
+ attention_mask.tril_()
685
+ for i, context_length in enumerate(context_lengths):
686
+ attention_mask[i, :, :context_length] = 1
687
+ attention_mask.unsqueeze_(1)
688
+ attention_mask = (attention_mask < 0.5).bool()
689
+
690
+ return attention_mask
691
+
692
+ def get_position_ids(self, input_ids, mask_positions, device, use_gmasks=None):
693
+ batch_size, seq_length = input_ids.shape
694
+ if use_gmasks is None:
695
+ use_gmasks = [False] * batch_size
696
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
697
+ if self.position_encoding_2d:
698
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
699
+ for i, context_length in enumerate(context_lengths):
700
+ position_ids[i, context_length:] = mask_positions[i]
701
+ block_position_ids = [torch.cat((
702
+ torch.zeros(context_length, dtype=torch.long, device=device),
703
+ torch.arange(seq_length - context_length, dtype=torch.long, device=device) + 1
704
+ )) for context_length in context_lengths]
705
+ block_position_ids = torch.stack(block_position_ids, dim=0)
706
+ position_ids = torch.stack((position_ids, block_position_ids), dim=1)
707
+ else:
708
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
709
+ for i, context_length in enumerate(context_lengths):
710
+ if not use_gmasks[i]:
711
+ position_ids[context_length:] = mask_positions[i]
712
+
713
+ return position_ids
714
+
715
+ def _set_gradient_checkpointing(self, module, value=False):
716
+ if isinstance(module, ChatGLMModel):
717
+ module.gradient_checkpointing = value
718
+
719
+
720
+ CHATGLM_6B_START_DOCSTRING = r"""
721
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
722
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
723
+ usage and behavior.
724
+
725
+ Parameters:
726
+ config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model.
727
+ Initializing with a config file does not load the weights associated with the model, only the configuration.
728
+ Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
729
+ """
730
+
731
+ CHATGLM_6B_INPUTS_DOCSTRING = r"""
732
+ Args:
733
+ input_ids (`torch.LongTensor` of shape `({0})`):
734
+ Indices of input sequence tokens in the vocabulary.
735
+
736
+ Indices can be obtained using [`ChatGLM6BTokenizer`].
737
+ See [`PreTrainedTokenizer.encode`] and
738
+ [`PreTrainedTokenizer.__call__`] for details.
739
+
740
+ [What are input IDs?](../glossary#input-ids)
741
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
742
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
743
+
744
+ - 1 for tokens that are **not masked**,
745
+ - 0 for tokens that are **masked**.
746
+
747
+ [What are attention masks?](../glossary#attention-mask)
748
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
749
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
750
+
751
+ - 0 corresponds to a *sentence A* token,
752
+ - 1 corresponds to a *sentence B* token.
753
+
754
+ [What are token type IDs?](../glossary#token-type-ids)
755
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
756
+ Indices of positions of each input sequence tokens in the position embeddings.
757
+ Selected in the range `[0, config.max_position_embeddings - 1]`.
758
+
759
+ [What are position IDs?](../glossary#position-ids)
760
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
761
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
762
+
763
+ - 1 indicates the head is **not masked**,
764
+ - 0 indicates the head is **masked**.
765
+
766
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
767
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
768
+ This is useful if you want more control over how to convert *input_ids* indices into associated vectors
769
+ than the model's internal embedding lookup matrix.
770
+ output_attentions (`bool`, *optional*):
771
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
772
+ tensors for more detail.
773
+ output_hidden_states (`bool`, *optional*):
774
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
775
+ more detail.
776
+ return_dict (`bool`, *optional*):
777
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
778
+ """
779
+
780
+
781
+ @add_start_docstrings(
782
+ "The bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.",
783
+ CHATGLM_6B_START_DOCSTRING,
784
+ )
785
+ class ChatGLMModel(ChatGLMPreTrainedModel):
786
+ """
787
+
788
+ The model can behave as an encoder (with only self-attention) as well
789
+ as a decoder, in which case a layer of cross-attention is added between
790
+ the self-attention layers, following the architecture described in [Attention is
791
+ all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
792
+ Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
793
+
794
+ To behave as an decoder the model needs to be initialized with the
795
+ `is_decoder` argument of the configuration set to `True`.
796
+ To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
797
+ argument and `add_cross_attention` set to `True`; an
798
+ `encoder_hidden_states` is then expected as an input to the forward pass.
799
+ """
800
+
801
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
802
+ super().__init__(config)
803
+ if empty_init:
804
+ init_method = skip_init
805
+ else:
806
+ init_method = default_init
807
+ # recording parameters
808
+ self.max_sequence_length = config.max_sequence_length
809
+ self.hidden_size = config.hidden_size
810
+ self.params_dtype = torch.half
811
+ self.num_attention_heads = config.num_attention_heads
812
+ self.vocab_size = config.vocab_size
813
+ self.num_layers = config.num_layers
814
+ self.layernorm_epsilon = config.layernorm_epsilon
815
+ self.inner_hidden_size = config.inner_hidden_size
816
+ self.hidden_size_per_attention_head = self.hidden_size // self.num_attention_heads
817
+ self.position_encoding_2d = config.position_encoding_2d
818
+ self.pre_seq_len = config.pre_seq_len
819
+ self.prefix_projection = config.prefix_projection
820
+
821
+ self.word_embeddings = init_method(
822
+ torch.nn.Embedding,
823
+ num_embeddings=self.vocab_size, embedding_dim=self.hidden_size,
824
+ dtype=self.params_dtype
825
+ )
826
+ self.gradient_checkpointing = False
827
+
828
+ def get_layer(layer_id):
829
+ return GLMBlock(
830
+ self.hidden_size,
831
+ self.num_attention_heads,
832
+ self.layernorm_epsilon,
833
+ layer_id,
834
+ inner_hidden_size=self.inner_hidden_size,
835
+ hidden_size_per_attention_head=self.hidden_size_per_attention_head,
836
+ layernorm=LayerNorm,
837
+ use_bias=True,
838
+ params_dtype=self.params_dtype,
839
+ position_encoding_2d=self.position_encoding_2d,
840
+ empty_init=empty_init
841
+ )
842
+
843
+ self.layers = torch.nn.ModuleList(
844
+ [get_layer(layer_id) for layer_id in range(self.num_layers)]
845
+ )
846
+
847
+ # Final layer norm before output.
848
+ self.final_layernorm = LayerNorm(self.hidden_size, eps=self.layernorm_epsilon)
849
+
850
+ if self.pre_seq_len is not None:
851
+ for param in self.parameters():
852
+ param.requires_grad = False
853
+ self.prefix_tokens = torch.arange(self.pre_seq_len).long()
854
+ self.prefix_encoder = PrefixEncoder(config)
855
+ self.dropout = torch.nn.Dropout(0.1)
856
+
857
+ # total_params = sum(p.numel() for p in self.parameters())
858
+ # trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad)
859
+ # print("Using p-tuning v2: # trainable_params = {} / {}".format(trainable_params, total_params))
860
+
861
+ def get_input_embeddings(self):
862
+ return self.word_embeddings
863
+
864
+ def set_input_embeddings(self, new_embeddings: torch.Tensor):
865
+ self.word_embeddings = new_embeddings
866
+
867
+ def get_prompt(self, batch_size, device, dtype=torch.half):
868
+ prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
869
+ past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)
870
+ past_key_values = past_key_values.view(
871
+ batch_size,
872
+ self.pre_seq_len,
873
+ self.num_layers * 2,
874
+ self.num_attention_heads,
875
+ self.hidden_size // self.num_attention_heads
876
+ )
877
+ # seq_len, b, nh, hidden_size
878
+ past_key_values = self.dropout(past_key_values)
879
+ past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2)
880
+ # past_key_values = [(v[0], v[1]) for v in past_key_values]
881
+ return past_key_values
882
+
883
+ @add_start_docstrings_to_model_forward(CHATGLM_6B_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
884
+ @add_code_sample_docstrings(
885
+ checkpoint=_CHECKPOINT_FOR_DOC,
886
+ output_type=BaseModelOutputWithPastAndCrossAttentions,
887
+ config_class=_CONFIG_FOR_DOC,
888
+ )
889
+ def forward(
890
+ self,
891
+ input_ids: Optional[torch.LongTensor] = None,
892
+ position_ids: Optional[torch.LongTensor] = None,
893
+ attention_mask: Optional[torch.Tensor] = None,
894
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
895
+ inputs_embeds: Optional[torch.LongTensor] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPast]:
901
+
902
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
903
+ output_hidden_states = (
904
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
905
+ )
906
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
907
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
908
+
909
+ if self.gradient_checkpointing and self.training:
910
+ if use_cache:
911
+ logger.warning_once(
912
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
913
+ )
914
+ use_cache = False
915
+
916
+ if input_ids is not None and inputs_embeds is not None:
917
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
918
+ elif input_ids is not None:
919
+ batch_size, seq_length = input_ids.shape[:2]
920
+ elif inputs_embeds is not None:
921
+ batch_size, seq_length = inputs_embeds.shape[:2]
922
+ else:
923
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
924
+
925
+ if inputs_embeds is None:
926
+ inputs_embeds = self.word_embeddings(input_ids)
927
+
928
+ if past_key_values is None:
929
+ if self.pre_seq_len is not None:
930
+ past_key_values = self.get_prompt(batch_size=input_ids.shape[0], device=input_ids.device,
931
+ dtype=inputs_embeds.dtype)
932
+ else:
933
+ past_key_values = tuple([None] * len(self.layers))
934
+
935
+ if attention_mask is None:
936
+ attention_mask = self.get_masks(
937
+ input_ids,
938
+ device=input_ids.device
939
+ )
940
+
941
+
942
+ if position_ids is None:
943
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
944
+ seqs = input_ids.tolist()
945
+
946
+ mask_positions, use_gmasks = [], []
947
+ for seq in seqs:
948
+ mask_token = gMASK if gMASK in seq else MASK
949
+ use_gmask = mask_token == gMASK
950
+ mask_positions.append(seq.index(mask_token))
951
+ use_gmasks.append(use_gmask)
952
+
953
+ position_ids = self.get_position_ids(
954
+ input_ids,
955
+ mask_positions=mask_positions,
956
+ device=input_ids.device,
957
+ use_gmasks=use_gmasks
958
+ )
959
+
960
+ if self.pre_seq_len is not None and attention_mask is not None:
961
+ prefix_attention_mask = torch.ones(batch_size, 1, input_ids.size(-1), self.pre_seq_len).to(
962
+ attention_mask.device)
963
+ prefix_attention_mask = (prefix_attention_mask < 0.5).bool()
964
+ attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=3)
965
+
966
+ # [seq_len, batch, hidden_size]
967
+ hidden_states = inputs_embeds.transpose(0, 1)
968
+
969
+ presents = () if use_cache else None
970
+ all_self_attentions = () if output_attentions else None
971
+ all_hidden_states = () if output_hidden_states else None
972
+
973
+ if attention_mask is None:
974
+ attention_mask = torch.zeros(1, 1, device=input_ids.device).bool()
975
+ else:
976
+ attention_mask = attention_mask.to(hidden_states.device)
977
+
978
+ for i, layer in enumerate(self.layers):
979
+
980
+ if output_hidden_states:
981
+ all_hidden_states = all_hidden_states + (hidden_states,)
982
+ layer_past = past_key_values[i]
983
+
984
+ if self.gradient_checkpointing and self.training:
985
+ layer_ret = torch.utils.checkpoint.checkpoint(
986
+ layer,
987
+ hidden_states,
988
+ position_ids,
989
+ attention_mask,
990
+ torch.tensor(i),
991
+ layer_past,
992
+ use_cache,
993
+ output_attentions
994
+ )
995
+ else:
996
+ layer_ret = layer(
997
+ hidden_states,
998
+ position_ids=position_ids,
999
+ attention_mask=attention_mask,
1000
+ layer_id=torch.tensor(i),
1001
+ layer_past=layer_past,
1002
+ use_cache=use_cache,
1003
+ output_attentions=output_attentions
1004
+ )
1005
+
1006
+ hidden_states = layer_ret[0]
1007
+
1008
+ if use_cache:
1009
+ presents = presents + (layer_ret[1],)
1010
+
1011
+ if output_attentions:
1012
+ all_self_attentions = all_self_attentions + (layer_ret[2 if use_cache else 1],)
1013
+
1014
+ # Final layer norm.
1015
+ hidden_states = self.final_layernorm(hidden_states)
1016
+
1017
+ if output_hidden_states:
1018
+ all_hidden_states = all_hidden_states + (hidden_states,)
1019
+
1020
+ if not return_dict:
1021
+ return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
1022
+
1023
+ return BaseModelOutputWithPast(
1024
+ last_hidden_state=hidden_states,
1025
+ past_key_values=presents,
1026
+ hidden_states=all_hidden_states,
1027
+ attentions=all_self_attentions,
1028
+ )
1029
+
1030
+
1031
+ class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
1032
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
1033
+ super().__init__(config)
1034
+ if empty_init:
1035
+ init_method = skip_init
1036
+ else:
1037
+ init_method = default_init
1038
+
1039
+ # self.hidden_size = config.hidden_size
1040
+ # self.params_dtype = torch.half
1041
+ # self.vocab_size = config.vocab_size
1042
+ self.max_sequence_length = config.max_sequence_length
1043
+
1044
+ self.position_encoding_2d = config.position_encoding_2d
1045
+
1046
+ self.transformer = ChatGLMModel(config, empty_init=empty_init)
1047
+
1048
+ self.lm_head = init_method(
1049
+ nn.Linear,
1050
+ config.hidden_size,
1051
+ config.vocab_size,
1052
+ bias=False,
1053
+ dtype=torch.half
1054
+ )
1055
+
1056
+ self.config = config
1057
+
1058
+ self.quantized = False
1059
+
1060
+ if self.config.quantization_bit:
1061
+ self.quantize(self.config.quantization_bit, empty_init=True)
1062
+
1063
+ def get_output_embeddings(self):
1064
+ return self.lm_head
1065
+
1066
+ def set_output_embeddings(self, new_embeddings):
1067
+ self.lm_head = new_embeddings
1068
+
1069
+ def _update_model_kwargs_for_generation(
1070
+ self,
1071
+ outputs: ModelOutput,
1072
+ model_kwargs: Dict[str, Any],
1073
+ is_encoder_decoder: bool = False,
1074
+ standardize_cache_format: bool = False,
1075
+ ) -> Dict[str, Any]:
1076
+ # update past_key_values
1077
+ model_kwargs["past_key_values"] = self._extract_past_from_model_output(
1078
+ outputs, standardize_cache_format=standardize_cache_format
1079
+ )
1080
+
1081
+ # update attention mask
1082
+ if "attention_mask" in model_kwargs:
1083
+ attention_mask = model_kwargs["attention_mask"]
1084
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1085
+ attention_mask = torch.cat(
1086
+ [attention_mask, attention_mask.new_ones((*attention_mask.shape[:3], 1))], dim=3)
1087
+ new_attention_mask = attention_mask[:, :, -1:].clone()
1088
+ new_attention_mask[..., -1] = False
1089
+ model_kwargs["attention_mask"] = torch.cat(
1090
+ [attention_mask, new_attention_mask], dim=2
1091
+ )
1092
+
1093
+ # update position ids
1094
+ if "position_ids" in model_kwargs:
1095
+ position_ids = model_kwargs["position_ids"]
1096
+ new_position_id = position_ids[..., -1:].clone()
1097
+ new_position_id[:, 1, :] += 1
1098
+ model_kwargs["position_ids"] = torch.cat(
1099
+ [position_ids, new_position_id], dim=-1
1100
+ )
1101
+
1102
+ return model_kwargs
1103
+
1104
+ def prepare_inputs_for_generation(
1105
+ self,
1106
+ input_ids: torch.LongTensor,
1107
+ past: Optional[torch.Tensor] = None,
1108
+ past_key_values: Optional[torch.Tensor] = None,
1109
+ attention_mask: Optional[torch.Tensor] = None,
1110
+ position_ids: Optional[torch.Tensor] = None,
1111
+ **kwargs
1112
+ ) -> dict:
1113
+ batch_size, seq_length = input_ids.shape
1114
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
1115
+ seqs = input_ids.tolist()
1116
+ mask_positions, use_gmasks = [], []
1117
+ for seq in seqs:
1118
+ mask_token = gMASK if gMASK in seq else MASK
1119
+ use_gmask = mask_token == gMASK
1120
+ mask_positions.append(seq.index(mask_token))
1121
+ use_gmasks.append(use_gmask)
1122
+
1123
+ # only last token for input_ids if past is not None
1124
+ if past is not None or past_key_values is not None:
1125
+ last_token = input_ids[:, -1].unsqueeze(-1)
1126
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1127
+ attention_mask = attention_mask[:, :, -1:]
1128
+ else:
1129
+ attention_mask = None
1130
+ if position_ids is not None:
1131
+ position_ids = position_ids[..., -1:]
1132
+ else:
1133
+ context_lengths = [seq.index(self.config.bos_token_id) for seq in seqs]
1134
+ if self.position_encoding_2d:
1135
+ position_ids = torch.tensor(
1136
+ [[mask_position, seq_length - context_length] for mask_position, context_length in
1137
+ zip(mask_positions, context_lengths)], dtype=torch.long, device=input_ids.device).unsqueeze(-1)
1138
+ else:
1139
+ position_ids = torch.tensor([mask_position for mask_position in mask_positions], dtype=torch.long,
1140
+ device=input_ids.device).unsqueeze(-1)
1141
+
1142
+ if past is None:
1143
+ past = past_key_values
1144
+ return {
1145
+ "input_ids": last_token,
1146
+ "past_key_values": past,
1147
+ "position_ids": position_ids,
1148
+ "attention_mask": attention_mask
1149
+ }
1150
+ else:
1151
+ if attention_mask is not None and attention_mask.dtype != torch.bool:
1152
+ logger.warning_once(f"The dtype of attention mask ({attention_mask.dtype}) is not bool")
1153
+ attention_mask = None
1154
+ if attention_mask is None:
1155
+ attention_mask = self.get_masks(
1156
+ input_ids,
1157
+ device=input_ids.device
1158
+ )
1159
+ if position_ids is None:
1160
+ position_ids = self.get_position_ids(
1161
+ input_ids,
1162
+ device=input_ids.device,
1163
+ mask_positions=mask_positions,
1164
+ use_gmasks=use_gmasks
1165
+ )
1166
+
1167
+ return {
1168
+ "input_ids": input_ids,
1169
+ "past_key_values": past,
1170
+ "position_ids": position_ids,
1171
+ "attention_mask": attention_mask
1172
+ }
1173
+
1174
+ def forward(
1175
+ self,
1176
+ input_ids: Optional[torch.Tensor] = None,
1177
+ position_ids: Optional[torch.Tensor] = None,
1178
+ attention_mask: Optional[torch.Tensor] = None,
1179
+ past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
1180
+ inputs_embeds: Optional[torch.Tensor] = None,
1181
+ labels: Optional[torch.Tensor] = None,
1182
+ use_cache: Optional[bool] = None,
1183
+ output_attentions: Optional[bool] = None,
1184
+ output_hidden_states: Optional[bool] = None,
1185
+ return_dict: Optional[bool] = None,
1186
+ ):
1187
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1188
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1189
+
1190
+ transformer_outputs = self.transformer(
1191
+ input_ids=input_ids,
1192
+ position_ids=position_ids,
1193
+ attention_mask=attention_mask,
1194
+ past_key_values=past_key_values,
1195
+ inputs_embeds=inputs_embeds,
1196
+ use_cache=use_cache,
1197
+ output_attentions=output_attentions,
1198
+ output_hidden_states=output_hidden_states,
1199
+ return_dict=return_dict,
1200
+ )
1201
+
1202
+ hidden_states = transformer_outputs[0]
1203
+
1204
+ lm_logits = self.lm_head(hidden_states).permute(1, 0, 2).contiguous()
1205
+
1206
+ loss = None
1207
+ if labels is not None:
1208
+ lm_logits = lm_logits.to(torch.float32)
1209
+
1210
+ # Shift so that tokens < n predict n
1211
+ shift_logits = lm_logits[..., :-1, :].contiguous()
1212
+ shift_labels = labels[..., 1:].contiguous()
1213
+ # Flatten the tokens
1214
+ loss_fct = CrossEntropyLoss(ignore_index=-100)
1215
+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
1216
+
1217
+ lm_logits = lm_logits.to(hidden_states.dtype)
1218
+ loss = loss.to(hidden_states.dtype)
1219
+
1220
+ if not return_dict:
1221
+ output = (lm_logits,) + transformer_outputs[1:]
1222
+ return ((loss,) + output) if loss is not None else output
1223
+
1224
+ return CausalLMOutputWithPast(
1225
+ loss=loss,
1226
+ logits=lm_logits,
1227
+ past_key_values=transformer_outputs.past_key_values,
1228
+ hidden_states=transformer_outputs.hidden_states,
1229
+ attentions=transformer_outputs.attentions,
1230
+ )
1231
+
1232
+ @staticmethod
1233
+ def _reorder_cache(
1234
+ past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor
1235
+ ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]:
1236
+ """
1237
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
1238
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
1239
+ beam_idx at every generation step.
1240
+
1241
+ Output shares the same memory storage as `past`.
1242
+ """
1243
+ return tuple(
1244
+ (
1245
+ layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)),
1246
+ layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)),
1247
+ )
1248
+ for layer_past in past
1249
+ )
1250
+
1251
+ def process_response(self, response):
1252
+ response = response.strip()
1253
+ response = response.replace("[[训练时间]]", "2023年")
1254
+ punkts = [
1255
+ [",", ","],
1256
+ ["!", "!"],
1257
+ [":", ":"],
1258
+ [";", ";"],
1259
+ ["\?", "?"],
1260
+ ]
1261
+ for item in punkts:
1262
+ response = re.sub(r"([\u4e00-\u9fff])%s" % item[0], r"\1%s" % item[1], response)
1263
+ response = re.sub(r"%s([\u4e00-\u9fff])" % item[0], r"%s\1" % item[1], response)
1264
+ return response
1265
+
1266
+ @torch.no_grad()
1267
+ def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048, num_beams=1,
1268
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1269
+ if history is None:
1270
+ history = []
1271
+ if logits_processor is None:
1272
+ logits_processor = LogitsProcessorList()
1273
+ logits_processor.append(InvalidScoreLogitsProcessor())
1274
+ gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p,
1275
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1276
+ if not history:
1277
+ prompt = query
1278
+ else:
1279
+ prompt = ""
1280
+ for i, (old_query, response) in enumerate(history):
1281
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1282
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1283
+ inputs = tokenizer([prompt], return_tensors="pt")
1284
+ inputs = inputs.to(self.device)
1285
+ outputs = self.generate(**inputs, **gen_kwargs)
1286
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1287
+ response = tokenizer.decode(outputs)
1288
+ response = self.process_response(response)
1289
+ history = history + [(query, response)]
1290
+ return response, history
1291
+
1292
+ @torch.no_grad()
1293
+ def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048,
1294
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1295
+ if history is None:
1296
+ history = []
1297
+ if logits_processor is None:
1298
+ logits_processor = LogitsProcessorList()
1299
+ logits_processor.append(InvalidScoreLogitsProcessor())
1300
+ gen_kwargs = {"max_length": max_length, "do_sample": do_sample, "top_p": top_p,
1301
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1302
+ if not history:
1303
+ prompt = query
1304
+ else:
1305
+ prompt = ""
1306
+ for i, (old_query, response) in enumerate(history):
1307
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1308
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1309
+ inputs = tokenizer([prompt], return_tensors="pt")
1310
+ inputs = inputs.to(self.device)
1311
+ for outputs in self.stream_generate(**inputs, **gen_kwargs):
1312
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1313
+ response = tokenizer.decode(outputs)
1314
+ response = self.process_response(response)
1315
+ new_history = history + [(query, response)]
1316
+ yield response, new_history
1317
+
1318
+ @torch.no_grad()
1319
+ def stream_generate(
1320
+ self,
1321
+ input_ids,
1322
+ generation_config: Optional[GenerationConfig] = None,
1323
+ logits_processor: Optional[LogitsProcessorList] = None,
1324
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1325
+ prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
1326
+ **kwargs,
1327
+ ):
1328
+ batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
1329
+
1330
+ if generation_config is None:
1331
+ generation_config = self.generation_config
1332
+ generation_config = copy.deepcopy(generation_config)
1333
+ model_kwargs = generation_config.update(**kwargs)
1334
+ bos_token_id, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id
1335
+
1336
+ if isinstance(eos_token_id, int):
1337
+ eos_token_id = [eos_token_id]
1338
+
1339
+ has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
1340
+ if has_default_max_length and generation_config.max_new_tokens is None:
1341
+ warnings.warn(
1342
+ f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
1343
+ "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
1344
+ " recommend using `max_new_tokens` to control the maximum length of the generation.",
1345
+ UserWarning,
1346
+ )
1347
+ elif generation_config.max_new_tokens is not None:
1348
+ generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
1349
+ if not has_default_max_length:
1350
+ logger.warn(
1351
+ f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
1352
+ f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
1353
+ "Please refer to the documentation for more information. "
1354
+ "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
1355
+ UserWarning,
1356
+ )
1357
+
1358
+ if input_ids_seq_length >= generation_config.max_length:
1359
+ input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
1360
+ logger.warning(
1361
+ f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to"
1362
+ f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
1363
+ " increasing `max_new_tokens`."
1364
+ )
1365
+
1366
+ # 2. Set generation parameters if not already defined
1367
+ logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
1368
+ stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
1369
+
1370
+ logits_processor = self._get_logits_processor(
1371
+ generation_config=generation_config,
1372
+ input_ids_seq_length=input_ids_seq_length,
1373
+ encoder_input_ids=input_ids,
1374
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1375
+ logits_processor=logits_processor,
1376
+ )
1377
+
1378
+ stopping_criteria = self._get_stopping_criteria(
1379
+ generation_config=generation_config, stopping_criteria=stopping_criteria
1380
+ )
1381
+ logits_warper = self._get_logits_warper(generation_config)
1382
+
1383
+ unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
1384
+ scores = None
1385
+ while True:
1386
+ model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
1387
+ # forward pass to get next token
1388
+ outputs = self(
1389
+ **model_inputs,
1390
+ return_dict=True,
1391
+ output_attentions=False,
1392
+ output_hidden_states=False,
1393
+ )
1394
+
1395
+ next_token_logits = outputs.logits[:, -1, :]
1396
+
1397
+ # pre-process distribution
1398
+ next_token_scores = logits_processor(input_ids, next_token_logits)
1399
+ next_token_scores = logits_warper(input_ids, next_token_scores)
1400
+
1401
+ # sample
1402
+ probs = nn.functional.softmax(next_token_scores, dim=-1)
1403
+ if generation_config.do_sample:
1404
+ next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
1405
+ else:
1406
+ next_tokens = torch.argmax(probs, dim=-1)
1407
+
1408
+ # update generated ids, model inputs, and length for next step
1409
+ input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
1410
+ model_kwargs = self._update_model_kwargs_for_generation(
1411
+ outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
1412
+ )
1413
+ unfinished_sequences = unfinished_sequences.mul((sum(next_tokens != i for i in eos_token_id)).long())
1414
+
1415
+ # stop when each sentence is finished, or if we exceed the maximum length
1416
+ if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
1417
+ break
1418
+ yield input_ids
1419
+
1420
+ def quantize(self, bits: int, empty_init=False, **kwargs):
1421
+ if bits == 0:
1422
+ return
1423
+
1424
+ from .quantization import quantize
1425
+
1426
+ if self.quantized:
1427
+ logger.info("Already quantized.")
1428
+ return self
1429
+
1430
+ self.quantized = True
1431
+
1432
+ self.config.quantization_bit = bits
1433
+
1434
+ self.transformer = quantize(self.transformer, bits, empty_init=empty_init, **kwargs)
1435
+ return self
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2040cd9904e7b657b554d0b4b9b65c4a604303e9e045948a559a71b03292ce9
3
+ size 234882351
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fd9079d9d67795180ab785a8d9be1931f209c43d811fe9a82d792150f83dddf
3
+ size 117441341
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/quantization.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.nn import Linear
2
+ from torch.nn.parameter import Parameter
3
+
4
+ import bz2
5
+ import torch
6
+ import base64
7
+ import ctypes
8
+ from transformers.utils import logging
9
+
10
+ from typing import List
11
+ from functools import partial
12
+
13
+ logger = logging.get_logger(__name__)
14
+
15
+ try:
16
+ from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up
17
+
18
+ class Kernel:
19
+ def __init__(self, code: bytes, function_names: List[str]):
20
+ self.code = code
21
+ self._function_names = function_names
22
+ self._cmodule = LazyKernelCModule(self.code)
23
+
24
+ for name in self._function_names:
25
+ setattr(self, name, KernelFunction(self._cmodule, name))
26
+
27
+ quantization_code = "$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ"
28
+
29
+ kernels = Kernel(
30
+ bz2.decompress(base64.b64decode(quantization_code)),
31
+ [
32
+ "int4WeightCompression",
33
+ "int4WeightExtractionFloat",
34
+ "int4WeightExtractionHalf",
35
+ "int8WeightExtractionFloat",
36
+ "int8WeightExtractionHalf",
37
+ ],
38
+ )
39
+ except Exception as exception:
40
+ kernels = None
41
+ logger.warning("Failed to load cpm_kernels:" + str(exception))
42
+
43
+
44
+ class W8A16Linear(torch.autograd.Function):
45
+ @staticmethod
46
+ def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, scale_w: torch.Tensor, weight_bit_width):
47
+ ctx.inp_shape = inp.size()
48
+ ctx.weight_bit_width = weight_bit_width
49
+ out_features = quant_w.size(0)
50
+ inp = inp.contiguous().view(-1, inp.size(-1))
51
+ weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width)
52
+ ctx.weight_shape = weight.size()
53
+ output = inp.mm(weight.t())
54
+ ctx.save_for_backward(inp, quant_w, scale_w)
55
+ return output.view(*(ctx.inp_shape[:-1] + (out_features,)))
56
+
57
+ @staticmethod
58
+ def backward(ctx, grad_output: torch.Tensor):
59
+ inp, quant_w, scale_w = ctx.saved_tensors
60
+ weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width)
61
+ grad_output = grad_output.contiguous().view(-1, weight.size(0))
62
+ grad_input = grad_output.mm(weight)
63
+ grad_weight = grad_output.t().mm(inp)
64
+ return grad_input.view(ctx.inp_shape), grad_weight.view(ctx.weight_shape), None, None
65
+
66
+
67
+ def compress_int4_weight(weight: torch.Tensor): # (n, m)
68
+ with torch.cuda.device(weight.device):
69
+ n, m = weight.size(0), weight.size(1)
70
+ assert m % 2 == 0
71
+ m = m // 2
72
+ out = torch.empty(n, m, dtype=torch.int8, device="cuda")
73
+ stream = torch.cuda.current_stream()
74
+
75
+ gridDim = (n, 1, 1)
76
+ blockDim = (min(round_up(m, 32), 1024), 1, 1)
77
+
78
+ kernels.int4WeightCompression(
79
+ gridDim,
80
+ blockDim,
81
+ 0,
82
+ stream,
83
+ [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)],
84
+ )
85
+ return out
86
+
87
+
88
+ def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int):
89
+ if source_bit_width == 8:
90
+ func = kernels.int8WeightExtractionHalf
91
+ elif source_bit_width == 4:
92
+ func = kernels.int4WeightExtractionHalf
93
+ else:
94
+ assert False, "Unsupported bit-width"
95
+
96
+ with torch.cuda.device(weight.device):
97
+ n, m = weight.size(0), weight.size(1)
98
+ out = torch.empty(n, m * (8 // source_bit_width), dtype=torch.half, device="cuda")
99
+ stream = torch.cuda.current_stream()
100
+
101
+ gridDim = (n, 1, 1)
102
+ blockDim = (min(round_up(m, 32), 1024), 1, 1)
103
+
104
+ func(
105
+ gridDim,
106
+ blockDim,
107
+ 0,
108
+ stream,
109
+ [
110
+ ctypes.c_void_p(weight.data_ptr()),
111
+ ctypes.c_void_p(scale_list.data_ptr()),
112
+ ctypes.c_void_p(out.data_ptr()),
113
+ ctypes.c_int32(n),
114
+ ctypes.c_int32(m),
115
+ ],
116
+ )
117
+ return out
118
+
119
+
120
+ class QuantizedLinear(Linear):
121
+ def __init__(self, weight_bit_width: int, weight_tensor=None, bias_tensor=None, empty_init=False, *args, **kwargs):
122
+ super(QuantizedLinear, self).__init__(*args, **kwargs)
123
+ self.weight_bit_width = weight_bit_width
124
+
125
+ shape = self.weight.shape
126
+ del self.weight
127
+
128
+ if weight_tensor is None or empty_init:
129
+ self.weight = torch.empty(
130
+ shape[0], shape[1] * weight_bit_width // 8, dtype=torch.int8, device=kwargs["device"]
131
+ )
132
+ self.weight_scale = torch.empty(shape[0], dtype=kwargs["dtype"], device=kwargs["device"])
133
+ else:
134
+ self.weight_scale = (weight_tensor.abs().max(dim=-1).values / ((2 ** (weight_bit_width - 1)) - 1)).half()
135
+ self.weight = torch.round(weight_tensor / self.weight_scale[:, None]).to(torch.int8)
136
+ if weight_bit_width == 4:
137
+ self.weight = compress_int4_weight(self.weight)
138
+
139
+ self.weight = Parameter(self.weight.to(kwargs["device"]), requires_grad=False)
140
+ self.weight_scale = Parameter(self.weight_scale.to(kwargs["device"]), requires_grad=False)
141
+ if bias_tensor is not None:
142
+ self.bias = Parameter(bias_tensor.to(kwargs["device"]), requires_grad=False)
143
+ else:
144
+ self.bias = None
145
+
146
+ def forward(self, input):
147
+ output = W8A16Linear.apply(input, self.weight, self.weight_scale, self.weight_bit_width)
148
+ if self.bias is not None:
149
+ output = output + self.bias
150
+ return output
151
+
152
+
153
+ def quantize(model, weight_bit_width, empty_init=False, **kwargs):
154
+ """Replace fp16 linear with quantized linear"""
155
+
156
+ for layer in model.layers:
157
+ layer.attention.query_key_value = QuantizedLinear(
158
+ weight_bit_width=weight_bit_width,
159
+ weight_tensor=layer.attention.query_key_value.weight.to(torch.cuda.current_device()),
160
+ bias_tensor=layer.attention.query_key_value.bias,
161
+ in_features=layer.attention.query_key_value.in_features,
162
+ out_features=layer.attention.query_key_value.out_features,
163
+ bias=True,
164
+ dtype=torch.half,
165
+ device=layer.attention.query_key_value.weight.device,
166
+ empty_init=empty_init
167
+ )
168
+ layer.attention.dense = QuantizedLinear(
169
+ weight_bit_width=weight_bit_width,
170
+ weight_tensor=layer.attention.dense.weight.to(torch.cuda.current_device()),
171
+ bias_tensor=layer.attention.dense.bias,
172
+ in_features=layer.attention.dense.in_features,
173
+ out_features=layer.attention.dense.out_features,
174
+ bias=True,
175
+ dtype=torch.half,
176
+ device=layer.attention.dense.weight.device,
177
+ empty_init=empty_init
178
+ )
179
+ layer.mlp.dense_h_to_4h = QuantizedLinear(
180
+ weight_bit_width=weight_bit_width,
181
+ weight_tensor=layer.mlp.dense_h_to_4h.weight.to(torch.cuda.current_device()),
182
+ bias_tensor=layer.mlp.dense_h_to_4h.bias,
183
+ in_features=layer.mlp.dense_h_to_4h.in_features,
184
+ out_features=layer.mlp.dense_h_to_4h.out_features,
185
+ bias=True,
186
+ dtype=torch.half,
187
+ device=layer.mlp.dense_h_to_4h.weight.device,
188
+ empty_init=empty_init
189
+ )
190
+ layer.mlp.dense_4h_to_h = QuantizedLinear(
191
+ weight_bit_width=weight_bit_width,
192
+ weight_tensor=layer.mlp.dense_4h_to_h.weight.to(torch.cuda.current_device()),
193
+ bias_tensor=layer.mlp.dense_4h_to_h.bias,
194
+ in_features=layer.mlp.dense_4h_to_h.in_features,
195
+ out_features=layer.mlp.dense_4h_to_h.out_features,
196
+ bias=True,
197
+ dtype=torch.half,
198
+ device=layer.mlp.dense_4h_to_h.weight.device,
199
+ empty_init=empty_init
200
+ )
201
+ return model
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0807d8b9b5da8a50ac37dc742c51f2fd14818229529350c25105e80232d0c12
3
+ size 14575
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8a7ee269617f79917695125e8a223db714e608f9db41b8e07a0b863380f2395
3
+ size 627
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<sop>",
3
+ "eos_token": "<eop>",
4
+ "mask_token": "[MASK]",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/tokenization_chatglm.py ADDED
@@ -0,0 +1,430 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tokenization classes for ChatGLM."""
2
+ from typing import List, Optional, Union
3
+ import os
4
+
5
+ from transformers.tokenization_utils import PreTrainedTokenizer
6
+ from transformers.utils import logging, PaddingStrategy
7
+ from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
8
+ from typing import Dict
9
+ import sentencepiece as spm
10
+ import numpy as np
11
+
12
+ logger = logging.get_logger(__name__)
13
+
14
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
15
+ "THUDM/chatglm-6b": 2048,
16
+ }
17
+
18
+
19
+ class TextTokenizer:
20
+ def __init__(self, model_path):
21
+ self.sp = spm.SentencePieceProcessor()
22
+ self.sp.Load(model_path)
23
+ self.num_tokens = self.sp.vocab_size()
24
+
25
+ def encode(self, text):
26
+ return self.sp.EncodeAsIds(text)
27
+
28
+ def decode(self, ids: List[int]):
29
+ return self.sp.DecodeIds(ids)
30
+
31
+ def tokenize(self, text):
32
+ return self.sp.EncodeAsPieces(text)
33
+
34
+ def convert_tokens_to_ids(self, tokens):
35
+ return [self.sp.PieceToId(token) for token in tokens]
36
+
37
+ def convert_token_to_id(self, token):
38
+ return self.sp.PieceToId(token)
39
+
40
+ def convert_id_to_token(self, idx):
41
+ return self.sp.IdToPiece(idx)
42
+
43
+ def __len__(self):
44
+ return self.num_tokens
45
+
46
+
47
+ class SPTokenizer:
48
+ def __init__(
49
+ self,
50
+ vocab_file,
51
+ num_image_tokens=20000,
52
+ max_blank_length=80,
53
+ byte_fallback=True,
54
+ ):
55
+ assert vocab_file is not None
56
+ self.vocab_file = vocab_file
57
+ self.num_image_tokens = num_image_tokens
58
+ self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
59
+ self.max_blank_length = max_blank_length
60
+ self.byte_fallback = byte_fallback
61
+ self.text_tokenizer = TextTokenizer(vocab_file)
62
+
63
+ def _get_text_tokenizer(self):
64
+ return self.text_tokenizer
65
+
66
+ @staticmethod
67
+ def get_blank_token(length: int):
68
+ assert length >= 2
69
+ return f"<|blank_{length}|>"
70
+
71
+ @staticmethod
72
+ def get_tab_token():
73
+ return f"<|tab|>"
74
+
75
+ @property
76
+ def num_text_tokens(self):
77
+ return self.text_tokenizer.num_tokens
78
+
79
+ @property
80
+ def num_tokens(self):
81
+ return self.num_image_tokens + self.num_text_tokens
82
+
83
+ @staticmethod
84
+ def _encode_whitespaces(text: str, max_len: int = 80):
85
+ text = text.replace("\t", SPTokenizer.get_tab_token())
86
+ for i in range(max_len, 1, -1):
87
+ text = text.replace(" " * i, SPTokenizer.get_blank_token(i))
88
+ return text
89
+
90
+ def _preprocess(self, text: str, linebreak=True, whitespaces=True):
91
+ if linebreak:
92
+ text = text.replace("\n", "<n>")
93
+ if whitespaces:
94
+ text = self._encode_whitespaces(text, max_len=self.max_blank_length)
95
+ return text
96
+
97
+ def encode(
98
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
99
+ ) -> List[int]:
100
+ """
101
+ @param text: Text to encode.
102
+ @param linebreak: Whether to encode newline (\n) in text.
103
+ @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
104
+ @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
105
+ @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
106
+ """
107
+ text = self._preprocess(text, linebreak, whitespaces)
108
+ if not add_dummy_prefix:
109
+ text = "<n>" + text
110
+ tmp = self._get_text_tokenizer().encode(text)
111
+ tokens = [x + self.num_image_tokens for x in tmp]
112
+ return tokens if add_dummy_prefix else tokens[2:]
113
+
114
+ def decode(self, text_ids: List[int]) -> str:
115
+ ids = [int(_id) - self.num_image_tokens for _id in text_ids]
116
+ ids = [_id for _id in ids if _id >= 0]
117
+ text = self._get_text_tokenizer().decode(ids)
118
+ text = text.replace("<n>", "\n")
119
+ text = text.replace(SPTokenizer.get_tab_token(), "\t")
120
+ for i in range(2, self.max_blank_length + 1):
121
+ text = text.replace(self.get_blank_token(i), " " * i)
122
+ return text
123
+
124
+ def tokenize(
125
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
126
+ ) -> List[str]:
127
+ """
128
+ @param text: Text to encode.
129
+ @param linebreak: Whether to encode newline (\n) in text.
130
+ @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
131
+ @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
132
+ @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
133
+ """
134
+ text = self._preprocess(text, linebreak, whitespaces)
135
+ if not add_dummy_prefix:
136
+ text = "<n>" + text
137
+ tokens = self._get_text_tokenizer().tokenize(text)
138
+ return tokens if add_dummy_prefix else tokens[2:]
139
+
140
+ def __getitem__(self, x: Union[int, str]):
141
+ if isinstance(x, int):
142
+ if x < self.num_image_tokens:
143
+ return "<image_{}>".format(x)
144
+ else:
145
+ return self.text_tokenizer.convert_id_to_token(x - self.num_image_tokens)
146
+ elif isinstance(x, str):
147
+ if x.startswith("<image_") and x.endswith(">") and x[7:-1].isdigit():
148
+ return int(x[7:-1])
149
+ else:
150
+ return self.text_tokenizer.convert_token_to_id(x) + self.num_image_tokens
151
+ else:
152
+ raise ValueError("The key should be str or int.")
153
+
154
+
155
+ class ChatGLMTokenizer(PreTrainedTokenizer):
156
+ """
157
+ Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding.
158
+
159
+ Args:
160
+ vocab_file (`str`):
161
+ Path to the vocabulary file.
162
+ """
163
+
164
+ vocab_files_names = {"vocab_file": "ice_text.model"}
165
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
166
+ model_input_names = ["input_ids", "attention_mask", "position_ids"]
167
+
168
+ def __init__(
169
+ self,
170
+ vocab_file,
171
+ do_lower_case=False,
172
+ remove_space=False,
173
+ bos_token='<sop>',
174
+ eos_token='<eop>',
175
+ end_token='</s>',
176
+ mask_token='[MASK]',
177
+ gmask_token='[gMASK]',
178
+ padding_side="left",
179
+ pad_token="<pad>",
180
+ unk_token="<unk>",
181
+ num_image_tokens=20000,
182
+ **kwargs
183
+ ) -> None:
184
+ super().__init__(
185
+ do_lower_case=do_lower_case,
186
+ remove_space=remove_space,
187
+ padding_side=padding_side,
188
+ bos_token=bos_token,
189
+ eos_token=eos_token,
190
+ end_token=end_token,
191
+ mask_token=mask_token,
192
+ gmask_token=gmask_token,
193
+ pad_token=pad_token,
194
+ unk_token=unk_token,
195
+ num_image_tokens=num_image_tokens,
196
+ **kwargs
197
+ )
198
+
199
+ self.do_lower_case = do_lower_case
200
+ self.remove_space = remove_space
201
+ self.vocab_file = vocab_file
202
+
203
+ self.bos_token = bos_token
204
+ self.eos_token = eos_token
205
+ self.end_token = end_token
206
+ self.mask_token = mask_token
207
+ self.gmask_token = gmask_token
208
+
209
+ self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
210
+
211
+ """ Initialisation """
212
+
213
+ @property
214
+ def gmask_token_id(self) -> Optional[int]:
215
+ if self.gmask_token is None:
216
+ return None
217
+ return self.convert_tokens_to_ids(self.gmask_token)
218
+
219
+ @property
220
+ def end_token_id(self) -> Optional[int]:
221
+ """
222
+ `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been
223
+ set.
224
+ """
225
+ if self.end_token is None:
226
+ return None
227
+ return self.convert_tokens_to_ids(self.end_token)
228
+
229
+ @property
230
+ def vocab_size(self):
231
+ """ Returns vocab size """
232
+ return self.sp_tokenizer.num_tokens
233
+
234
+ def get_vocab(self):
235
+ """ Returns vocab as a dict """
236
+ vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
237
+ vocab.update(self.added_tokens_encoder)
238
+ return vocab
239
+
240
+ def preprocess_text(self, inputs):
241
+ if self.remove_space:
242
+ outputs = " ".join(inputs.strip().split())
243
+ else:
244
+ outputs = inputs
245
+
246
+ if self.do_lower_case:
247
+ outputs = outputs.lower()
248
+
249
+ return outputs
250
+
251
+ def _tokenize(self, text, **kwargs):
252
+ """ Returns a tokenized string. """
253
+ text = self.preprocess_text(text)
254
+
255
+ seq = self.sp_tokenizer.tokenize(text)
256
+
257
+ return seq
258
+
259
+ def _decode(
260
+ self,
261
+ token_ids: Union[int, List[int]],
262
+ skip_special_tokens: bool = False,
263
+ clean_up_tokenization_spaces: bool = True,
264
+ **kwargs
265
+ ) -> str:
266
+ if isinstance(token_ids, int):
267
+ token_ids = [token_ids]
268
+ if len(token_ids) == 0:
269
+ return ""
270
+ if self.pad_token_id in token_ids: # remove pad
271
+ token_ids = list(filter((self.pad_token_id).__ne__, token_ids))
272
+ return self.sp_tokenizer.decode(token_ids)
273
+
274
+ def _convert_token_to_id(self, token):
275
+ """ Converts a token (str) in an id using the vocab. """
276
+ return self.sp_tokenizer[token]
277
+
278
+ def _convert_id_to_token(self, index):
279
+ """Converts an index (integer) in a token (str) using the vocab."""
280
+ return self.sp_tokenizer[index]
281
+
282
+ def save_vocabulary(self, save_directory, filename_prefix=None):
283
+ """
284
+ Save the vocabulary and special tokens file to a directory.
285
+
286
+ Args:
287
+ save_directory (`str`):
288
+ The directory in which to save the vocabulary.
289
+ filename_prefix (`str`, *optional*):
290
+ An optional prefix to add to the named of the saved files.
291
+
292
+ Returns:
293
+ `Tuple(str)`: Paths to the files saved.
294
+ """
295
+ if os.path.isdir(save_directory):
296
+ vocab_file = os.path.join(
297
+ save_directory, self.vocab_files_names["vocab_file"]
298
+ )
299
+ else:
300
+ vocab_file = save_directory
301
+
302
+ with open(self.vocab_file, 'rb') as fin:
303
+ proto_str = fin.read()
304
+
305
+ with open(vocab_file, "wb") as writer:
306
+ writer.write(proto_str)
307
+
308
+ return (vocab_file,)
309
+
310
+ def build_inputs_with_special_tokens(
311
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
312
+ ) -> List[int]:
313
+ """
314
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
315
+ adding special tokens. A BERT sequence has the following format:
316
+
317
+ - single sequence: `[CLS] X [SEP]`
318
+ - pair of sequences: `[CLS] A [SEP] B [SEP]`
319
+
320
+ Args:
321
+ token_ids_0 (`List[int]`):
322
+ List of IDs to which the special tokens will be added.
323
+ token_ids_1 (`List[int]`, *optional*):
324
+ Optional second list of IDs for sequence pairs.
325
+
326
+ Returns:
327
+ `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
328
+ """
329
+ gmask_id = self.sp_tokenizer[self.gmask_token]
330
+ eos_id = self.sp_tokenizer[self.eos_token]
331
+ token_ids_0 = token_ids_0 + [gmask_id, self.sp_tokenizer[self.bos_token]]
332
+ if token_ids_1 is not None:
333
+ token_ids_0 = token_ids_0 + token_ids_1 + [eos_id]
334
+ return token_ids_0
335
+
336
+ def _pad(
337
+ self,
338
+ encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
339
+ max_length: Optional[int] = None,
340
+ padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
341
+ pad_to_multiple_of: Optional[int] = None,
342
+ return_attention_mask: Optional[bool] = None,
343
+ ) -> dict:
344
+ """
345
+ Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
346
+
347
+ Args:
348
+ encoded_inputs:
349
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
350
+ max_length: maximum length of the returned list and optionally padding length (see below).
351
+ Will truncate by taking into account the special tokens.
352
+ padding_strategy: PaddingStrategy to use for padding.
353
+
354
+ - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
355
+ - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
356
+ - PaddingStrategy.DO_NOT_PAD: Do not pad
357
+ The tokenizer padding sides are defined in self.padding_side:
358
+
359
+ - 'left': pads on the left of the sequences
360
+ - 'right': pads on the right of the sequences
361
+ pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
362
+ This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
363
+ `>= 7.5` (Volta).
364
+ return_attention_mask:
365
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
366
+ """
367
+ # Load from model defaults
368
+ bos_token_id = self.sp_tokenizer[self.bos_token]
369
+ mask_token_id = self.sp_tokenizer[self.mask_token]
370
+ gmask_token_id = self.sp_tokenizer[self.gmask_token]
371
+ assert self.padding_side == "left"
372
+
373
+ required_input = encoded_inputs[self.model_input_names[0]]
374
+ seq_length = len(required_input)
375
+
376
+ if padding_strategy == PaddingStrategy.LONGEST:
377
+ max_length = len(required_input)
378
+
379
+ if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
380
+ max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
381
+
382
+ needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
383
+
384
+ # Initialize attention mask if not present.
385
+ if max_length is not None:
386
+ if "attention_mask" not in encoded_inputs:
387
+ if bos_token_id in required_input:
388
+ context_length = required_input.index(bos_token_id)
389
+ else:
390
+ context_length = seq_length
391
+ attention_mask = np.ones((1, seq_length, seq_length))
392
+ attention_mask = np.tril(attention_mask)
393
+ attention_mask[:, :, :context_length] = 1
394
+ attention_mask = np.bool_(attention_mask < 0.5)
395
+ encoded_inputs["attention_mask"] = attention_mask
396
+
397
+ if "position_ids" not in encoded_inputs:
398
+ if bos_token_id in required_input:
399
+ context_length = required_input.index(bos_token_id)
400
+ else:
401
+ context_length = seq_length
402
+ position_ids = np.arange(seq_length, dtype=np.int64)
403
+ mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id
404
+ if mask_token in required_input:
405
+ mask_position = required_input.index(mask_token)
406
+ position_ids[context_length:] = mask_position
407
+ block_position_ids = np.concatenate(
408
+ [np.zeros(context_length, dtype=np.int64),
409
+ np.arange(1, seq_length - context_length + 1, dtype=np.int64)])
410
+ encoded_inputs["position_ids"] = np.stack([position_ids, block_position_ids], axis=0)
411
+
412
+ if needs_to_be_padded:
413
+ difference = max_length - len(required_input)
414
+
415
+ if "attention_mask" in encoded_inputs:
416
+ encoded_inputs["attention_mask"] = np.pad(encoded_inputs["attention_mask"],
417
+ pad_width=[(0, 0), (difference, 0), (difference, 0)],
418
+ mode='constant', constant_values=True)
419
+ if "token_type_ids" in encoded_inputs:
420
+ encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
421
+ "token_type_ids"
422
+ ]
423
+ if "special_tokens_mask" in encoded_inputs:
424
+ encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
425
+ if "position_ids" in encoded_inputs:
426
+ encoded_inputs["position_ids"] = np.pad(encoded_inputs["position_ids"],
427
+ pad_width=[(0, 0), (difference, 0)])
428
+ encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
429
+
430
+ return encoded_inputs
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "tokenization_chatglm.ChatGLMTokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "bos_token": "<sop>",
9
+ "do_lower_case": false,
10
+ "end_token": "</s>",
11
+ "eos_token": "<eop>",
12
+ "gmask_token": "[gMASK]",
13
+ "mask_token": "[MASK]",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "num_image_tokens": 0,
16
+ "pad_token": "<pad>",
17
+ "padding_side": "left",
18
+ "remove_space": false,
19
+ "special_tokens_map_file": null,
20
+ "tokenizer_class": "ChatGLMTokenizer",
21
+ "unk_token": "<unk>"
22
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.013961727414724387,
5
+ "global_step": 100,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.0,
12
+ "learning_rate": 0.018000000000000002,
13
+ "loss": 5.1828,
14
+ "step": 10
15
+ },
16
+ {
17
+ "epoch": 0.0,
18
+ "learning_rate": 0.016,
19
+ "loss": 4.5979,
20
+ "step": 20
21
+ },
22
+ {
23
+ "epoch": 0.0,
24
+ "learning_rate": 0.013999999999999999,
25
+ "loss": 4.5608,
26
+ "step": 30
27
+ },
28
+ {
29
+ "epoch": 0.01,
30
+ "learning_rate": 0.012,
31
+ "loss": 4.375,
32
+ "step": 40
33
+ },
34
+ {
35
+ "epoch": 0.01,
36
+ "learning_rate": 0.01,
37
+ "loss": 4.4015,
38
+ "step": 50
39
+ },
40
+ {
41
+ "epoch": 0.01,
42
+ "learning_rate": 0.008,
43
+ "loss": 4.2929,
44
+ "step": 60
45
+ },
46
+ {
47
+ "epoch": 0.01,
48
+ "learning_rate": 0.006,
49
+ "loss": 4.3323,
50
+ "step": 70
51
+ },
52
+ {
53
+ "epoch": 0.01,
54
+ "learning_rate": 0.004,
55
+ "loss": 4.2657,
56
+ "step": 80
57
+ },
58
+ {
59
+ "epoch": 0.01,
60
+ "learning_rate": 0.002,
61
+ "loss": 4.2733,
62
+ "step": 90
63
+ },
64
+ {
65
+ "epoch": 0.01,
66
+ "learning_rate": 0.0,
67
+ "loss": 4.2842,
68
+ "step": 100
69
+ }
70
+ ],
71
+ "max_steps": 100,
72
+ "num_train_epochs": 1,
73
+ "total_flos": 3466572123340800.0,
74
+ "trial_name": null,
75
+ "trial_params": null
76
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69a6c8c7bd73e62303e866b1cd895083f806eb4826833c1bcdd3e9c460c635e1
3
+ size 3707
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/wangyan/project/hft/uptest",
3
+ "architectures": [
4
+ "ChatGLMForConditionalGeneration"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_chatglm.ChatGLMConfig",
8
+ "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
9
+ "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
10
+ },
11
+ "bos_token_id": 130004,
12
+ "eos_token_id": 130005,
13
+ "gmask_token_id": 130001,
14
+ "hidden_size": 4096,
15
+ "inner_hidden_size": 16384,
16
+ "layernorm_epsilon": 1e-05,
17
+ "mask_token_id": 130000,
18
+ "max_sequence_length": 2048,
19
+ "model_type": "chatglm",
20
+ "num_attention_heads": 32,
21
+ "num_layers": 28,
22
+ "pad_token_id": 3,
23
+ "position_encoding_2d": true,
24
+ "pre_seq_len": 128,
25
+ "prefix_projection": false,
26
+ "quantization_bit": 4,
27
+ "torch_dtype": "float16",
28
+ "transformers_version": "4.27.1",
29
+ "use_cache": true,
30
+ "vocab_size": 130528
31
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/configuration_chatglm.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ ChatGLM model configuration """
2
+
3
+ from transformers.configuration_utils import PretrainedConfig
4
+ from transformers.utils import logging
5
+
6
+ logger = logging.get_logger(__name__)
7
+
8
+
9
+ class ChatGLMConfig(PretrainedConfig):
10
+ r"""
11
+ This is the configuration class to store the configuration of a [`~ChatGLMModel`].
12
+ It is used to instantiate an ChatGLM model according to the specified arguments, defining the model
13
+ architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
14
+ the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture.
15
+
16
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
17
+ to control the model outputs. Read the documentation from [`PretrainedConfig`]
18
+ for more information.
19
+
20
+
21
+ Args:
22
+ vocab_size (`int`, *optional*, defaults to 150528):
23
+ Vocabulary size of the ChatGLM-6B model. Defines the number of different tokens that can be represented by the
24
+ `inputs_ids` passed when calling [`~ChatGLMModel`] or
25
+ [`~TFChatGLMModel`].
26
+ hidden_size (`int`, *optional*, defaults to 4096):
27
+ Dimension of the encoder layers and the pooler layer.
28
+ num_hidden_layers (`int`, *optional*, defaults to 28):
29
+ Number of hidden layers in the Transformer encoder.
30
+ num_attention_heads (`int`, *optional*, defaults to 32):
31
+ Number of attention heads for each attention layer in the Transformer encoder.
32
+ inner_hidden_size (`int`, *optional*, defaults to 16384):
33
+ Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
34
+ max_sequence_length (`int`, *optional*, defaults to 512):
35
+ The maximum sequence length that this model might ever be used with.
36
+ Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
37
+ layernorm_epsilon (`float`, *optional*, defaults to 1e-5):
38
+ The epsilon used by the layer normalization layers.
39
+ use_cache (`bool`, *optional*, defaults to `True`):
40
+ Whether the model should return the last key/values attentions (not used by all models).
41
+ Example:
42
+
43
+ ```python
44
+ >>> from configuration_chatglm import ChatGLMConfig
45
+ >>> from modeling_chatglm import ChatGLMModel
46
+
47
+ >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration
48
+ >>> configuration = ChatGLMConfig()
49
+
50
+ >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration
51
+ >>> model = ChatGLMModel(configuration)
52
+
53
+ >>> # Accessing the model configuration
54
+ >>> configuration = model.config
55
+ ```
56
+ """
57
+ model_type = "chatglm"
58
+
59
+ def __init__(
60
+ self,
61
+ vocab_size=150528,
62
+ hidden_size=4096,
63
+ num_layers=28,
64
+ num_attention_heads=32,
65
+ layernorm_epsilon=1e-5,
66
+ use_cache=False,
67
+ bos_token_id=150004,
68
+ eos_token_id=150005,
69
+ mask_token_id=150000,
70
+ gmask_token_id=150001,
71
+ pad_token_id=0,
72
+ max_sequence_length=2048,
73
+ inner_hidden_size=16384,
74
+ position_encoding_2d=True,
75
+ quantization_bit=0,
76
+ pre_seq_len=None,
77
+ prefix_projection=False,
78
+ **kwargs
79
+ ):
80
+ self.num_layers = num_layers
81
+ self.vocab_size = vocab_size
82
+ self.hidden_size = hidden_size
83
+ self.num_attention_heads = num_attention_heads
84
+ self.max_sequence_length = max_sequence_length
85
+ self.layernorm_epsilon = layernorm_epsilon
86
+ self.inner_hidden_size = inner_hidden_size
87
+ self.use_cache = use_cache
88
+ self.bos_token_id = bos_token_id
89
+ self.eos_token_id = eos_token_id
90
+ self.pad_token_id = pad_token_id
91
+ self.mask_token_id = mask_token_id
92
+ self.gmask_token_id = gmask_token_id
93
+ self.position_encoding_2d = position_encoding_2d
94
+ self.quantization_bit = quantization_bit
95
+ self.pre_seq_len = pre_seq_len
96
+ self.prefix_projection = prefix_projection
97
+
98
+ super().__init__(
99
+ pad_token_id=pad_token_id,
100
+ bos_token_id=bos_token_id,
101
+ eos_token_id=eos_token_id,
102
+ **kwargs
103
+ )
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 130004,
4
+ "eos_token_id": 130005,
5
+ "pad_token_id": 3,
6
+ "transformers_version": "4.27.1"
7
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/ice_text.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e974d9a69c242ce014c88c2b26089270f6198f3c0b700a887666cd3e816f17e
3
+ size 2706249
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/modeling_chatglm.py ADDED
@@ -0,0 +1,1435 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ PyTorch ChatGLM model. """
2
+
3
+ import math
4
+ import copy
5
+ import os
6
+ import warnings
7
+ import re
8
+ import sys
9
+
10
+ import torch
11
+ import torch.utils.checkpoint
12
+ import torch.nn.functional as F
13
+ from torch import nn
14
+ from torch.nn import CrossEntropyLoss, LayerNorm
15
+ from torch.nn.utils import skip_init
16
+ from typing import Optional, Tuple, Union, List, Callable, Dict, Any
17
+
18
+ from transformers.utils import (
19
+ add_code_sample_docstrings,
20
+ add_start_docstrings,
21
+ add_start_docstrings_to_model_forward,
22
+ )
23
+ from transformers.modeling_outputs import (
24
+ BaseModelOutputWithPast,
25
+ CausalLMOutputWithPast,
26
+ BaseModelOutputWithPastAndCrossAttentions,
27
+ )
28
+ from transformers.modeling_utils import PreTrainedModel
29
+ from transformers.utils import logging
30
+ from transformers.generation.logits_process import LogitsProcessor
31
+ from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationConfig, ModelOutput
32
+
33
+ from .configuration_chatglm import ChatGLMConfig
34
+
35
+ # flags required to enable jit fusion kernels
36
+
37
+ if sys.platform != 'darwin':
38
+ torch._C._jit_set_profiling_mode(False)
39
+ torch._C._jit_set_profiling_executor(False)
40
+ torch._C._jit_override_can_fuse_on_cpu(True)
41
+ torch._C._jit_override_can_fuse_on_gpu(True)
42
+
43
+ logger = logging.get_logger(__name__)
44
+
45
+ _CHECKPOINT_FOR_DOC = "THUDM/ChatGLM-6B"
46
+ _CONFIG_FOR_DOC = "ChatGLM6BConfig"
47
+
48
+ CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [
49
+ "THUDM/chatglm-6b",
50
+ # See all ChatGLM-6B models at https://huggingface.co/models?filter=chatglm
51
+ ]
52
+
53
+
54
+ class InvalidScoreLogitsProcessor(LogitsProcessor):
55
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
56
+ if torch.isnan(scores).any() or torch.isinf(scores).any():
57
+ scores.zero_()
58
+ scores[..., 5] = 5e4
59
+ return scores
60
+
61
+
62
+ def load_tf_weights_in_chatglm_6b(model, config, tf_checkpoint_path):
63
+ """Load tf checkpoints in a pytorch model."""
64
+ try:
65
+ import re
66
+
67
+ import numpy as np
68
+ import tensorflow as tf
69
+ except ImportError:
70
+ logger.error(
71
+ "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
72
+ "https://www.tensorflow.org/install/ for installation instructions."
73
+ )
74
+ raise
75
+ tf_path = os.path.abspath(tf_checkpoint_path)
76
+ logger.info(f"Converting TensorFlow checkpoint from {tf_path}")
77
+ # Load weights from TF model
78
+ init_vars = tf.train.list_variables(tf_path)
79
+ names = []
80
+ arrays = []
81
+ for name, shape in init_vars:
82
+ logger.info(f"Loading TF weight {name} with shape {shape}")
83
+ array = tf.train.load_variable(tf_path, name)
84
+ names.append(name)
85
+ arrays.append(array)
86
+
87
+ for name, array in zip(names, arrays):
88
+ name = name.split("/")
89
+ # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
90
+ # which are not required for using pretrained model
91
+ if any(
92
+ n in ["adam_v", "adam_m", "AdamWeightDecayOptimizer", "AdamWeightDecayOptimizer_1", "global_step"]
93
+ for n in name
94
+ ):
95
+ logger.info(f"Skipping {'/'.join(name)}")
96
+ continue
97
+ pointer = model
98
+ for m_name in name:
99
+ if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
100
+ scope_names = re.split(r"_(\d+)", m_name)
101
+ else:
102
+ scope_names = [m_name]
103
+ if scope_names[0] == "kernel" or scope_names[0] == "gamma":
104
+ pointer = getattr(pointer, "weight")
105
+ elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
106
+ pointer = getattr(pointer, "bias")
107
+ elif scope_names[0] == "output_weights":
108
+ pointer = getattr(pointer, "weight")
109
+ elif scope_names[0] == "squad":
110
+ pointer = getattr(pointer, "classifier")
111
+ else:
112
+ try:
113
+ pointer = getattr(pointer, scope_names[0])
114
+ except AttributeError:
115
+ logger.info(f"Skipping {'/'.join(name)}")
116
+ continue
117
+ if len(scope_names) >= 2:
118
+ num = int(scope_names[1])
119
+ pointer = pointer[num]
120
+ if m_name[-11:] == "_embeddings":
121
+ pointer = getattr(pointer, "weight")
122
+ elif m_name == "kernel":
123
+ array = np.transpose(array)
124
+ try:
125
+ assert (
126
+ pointer.shape == array.shape
127
+ ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"
128
+ except AssertionError as e:
129
+ e.args += (pointer.shape, array.shape)
130
+ raise
131
+ logger.info(f"Initialize PyTorch weight {name}")
132
+ pointer.data = torch.from_numpy(array)
133
+ return model
134
+
135
+
136
+ class PrefixEncoder(torch.nn.Module):
137
+ """
138
+ The torch.nn model to encode the prefix
139
+ Input shape: (batch-size, prefix-length)
140
+ Output shape: (batch-size, prefix-length, 2*layers*hidden)
141
+ """
142
+
143
+ def __init__(self, config):
144
+ super().__init__()
145
+ self.prefix_projection = config.prefix_projection
146
+ if self.prefix_projection:
147
+ # Use a two-layer MLP to encode the prefix
148
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.hidden_size)
149
+ self.trans = torch.nn.Sequential(
150
+ torch.nn.Linear(config.hidden_size, config.hidden_size),
151
+ torch.nn.Tanh(),
152
+ torch.nn.Linear(config.hidden_size, config.num_layers * config.hidden_size * 2)
153
+ )
154
+ else:
155
+ self.embedding = torch.nn.Embedding(config.pre_seq_len, config.num_layers * config.hidden_size * 2)
156
+
157
+ def forward(self, prefix: torch.Tensor):
158
+ if self.prefix_projection:
159
+ prefix_tokens = self.embedding(prefix)
160
+ past_key_values = self.trans(prefix_tokens)
161
+ else:
162
+ past_key_values = self.embedding(prefix)
163
+ return past_key_values
164
+
165
+
166
+ @torch.jit.script
167
+ def gelu_impl(x):
168
+ """OpenAI's gelu implementation."""
169
+ return 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x *
170
+ (1.0 + 0.044715 * x * x)))
171
+
172
+
173
+ def gelu(x):
174
+ return gelu_impl(x)
175
+
176
+
177
+ class RotaryEmbedding(torch.nn.Module):
178
+ def __init__(self, dim, base=10000, precision=torch.half, learnable=False):
179
+ super().__init__()
180
+ inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim))
181
+ inv_freq = inv_freq.half()
182
+ self.learnable = learnable
183
+ if learnable:
184
+ self.inv_freq = torch.nn.Parameter(inv_freq)
185
+ self.max_seq_len_cached = None
186
+ else:
187
+ self.register_buffer('inv_freq', inv_freq)
188
+ self.max_seq_len_cached = None
189
+ self.cos_cached = None
190
+ self.sin_cached = None
191
+ self.precision = precision
192
+
193
+ def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys,
194
+ error_msgs):
195
+ pass
196
+
197
+ def forward(self, x, seq_dim=1, seq_len=None):
198
+ if seq_len is None:
199
+ seq_len = x.shape[seq_dim]
200
+ if self.max_seq_len_cached is None or (seq_len > self.max_seq_len_cached):
201
+ self.max_seq_len_cached = None if self.learnable else seq_len
202
+ t = torch.arange(seq_len, device=x.device, dtype=self.inv_freq.dtype)
203
+ freqs = torch.einsum('i,j->ij', t, self.inv_freq)
204
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
205
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
206
+ if self.precision == torch.bfloat16:
207
+ emb = emb.float()
208
+
209
+ # [sx, 1 (b * np), hn]
210
+ cos_cached = emb.cos()[:, None, :]
211
+ sin_cached = emb.sin()[:, None, :]
212
+ if self.precision == torch.bfloat16:
213
+ cos_cached = cos_cached.bfloat16()
214
+ sin_cached = sin_cached.bfloat16()
215
+ if self.learnable:
216
+ return cos_cached, sin_cached
217
+ self.cos_cached, self.sin_cached = cos_cached, sin_cached
218
+ return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
219
+
220
+ def _apply(self, fn):
221
+ if self.cos_cached is not None:
222
+ self.cos_cached = fn(self.cos_cached)
223
+ if self.sin_cached is not None:
224
+ self.sin_cached = fn(self.sin_cached)
225
+ return super()._apply(fn)
226
+
227
+
228
+ def rotate_half(x):
229
+ x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
230
+ return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions
231
+
232
+
233
+ @torch.jit.script
234
+ def apply_rotary_pos_emb_index(q, k, cos, sin, position_id):
235
+ # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn]
236
+ cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \
237
+ F.embedding(position_id, sin.squeeze(1)).unsqueeze(2)
238
+ q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
239
+ return q, k
240
+
241
+
242
+ def attention_fn(
243
+ self,
244
+ query_layer,
245
+ key_layer,
246
+ value_layer,
247
+ attention_mask,
248
+ hidden_size_per_partition,
249
+ layer_id,
250
+ layer_past=None,
251
+ scaling_attention_score=True,
252
+ use_cache=False,
253
+ ):
254
+ if layer_past is not None:
255
+ past_key, past_value = layer_past[0], layer_past[1]
256
+ key_layer = torch.cat((past_key, key_layer), dim=0)
257
+ value_layer = torch.cat((past_value, value_layer), dim=0)
258
+
259
+ # seqlen, batch, num_attention_heads, hidden_size_per_attention_head
260
+ seq_len, b, nh, hidden_size = key_layer.shape
261
+
262
+ if use_cache:
263
+ present = (key_layer, value_layer)
264
+ else:
265
+ present = None
266
+
267
+ query_key_layer_scaling_coeff = float(layer_id + 1)
268
+ if scaling_attention_score:
269
+ query_layer = query_layer / (math.sqrt(hidden_size) * query_key_layer_scaling_coeff)
270
+
271
+ # ===================================
272
+ # Raw attention scores. [b, np, s, s]
273
+ # ===================================
274
+
275
+ # [b, np, sq, sk]
276
+ output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0))
277
+
278
+ # [sq, b, np, hn] -> [sq, b * np, hn]
279
+ query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
280
+ # [sk, b, np, hn] -> [sk, b * np, hn]
281
+ key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
282
+
283
+ matmul_result = torch.zeros(
284
+ 1, 1, 1,
285
+ dtype=query_layer.dtype,
286
+ device=query_layer.device,
287
+ )
288
+
289
+ matmul_result = torch.baddbmm(
290
+ matmul_result,
291
+ query_layer.transpose(0, 1), # [b * np, sq, hn]
292
+ key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk]
293
+ beta=0.0,
294
+ alpha=1.0,
295
+ )
296
+
297
+ # change view to [b, np, sq, sk]
298
+ attention_scores = matmul_result.view(*output_size)
299
+
300
+ if self.scale_mask_softmax:
301
+ self.scale_mask_softmax.scale = query_key_layer_scaling_coeff
302
+ attention_probs = self.scale_mask_softmax(attention_scores, attention_mask.contiguous())
303
+ else:
304
+ if not (attention_mask == 0).all():
305
+ # if auto-regressive, skip
306
+ attention_scores.masked_fill_(attention_mask, -10000.0)
307
+ dtype = attention_scores.dtype
308
+ attention_scores = attention_scores.float()
309
+ attention_scores = attention_scores * query_key_layer_scaling_coeff
310
+
311
+ attention_probs = F.softmax(attention_scores, dim=-1)
312
+
313
+ attention_probs = attention_probs.type(dtype)
314
+
315
+ # =========================
316
+ # Context layer. [sq, b, hp]
317
+ # =========================
318
+
319
+ # value_layer -> context layer.
320
+ # [sk, b, np, hn] --> [b, np, sq, hn]
321
+
322
+ # context layer shape: [b, np, sq, hn]
323
+ output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3))
324
+
325
+ # change view [sk, b * np, hn]
326
+ value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1)
327
+
328
+ # change view [b * np, sq, sk]
329
+ attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1)
330
+
331
+ # matmul: [b * np, sq, hn]
332
+ context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1))
333
+
334
+ # change view [b, np, sq, hn]
335
+ context_layer = context_layer.view(*output_size)
336
+
337
+ # [b, np, sq, hn] --> [sq, b, np, hn]
338
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
339
+
340
+ # [sq, b, np, hn] --> [sq, b, hp]
341
+ new_context_layer_shape = context_layer.size()[:-2] + (hidden_size_per_partition,)
342
+ context_layer = context_layer.view(*new_context_layer_shape)
343
+
344
+ outputs = (context_layer, present, attention_probs)
345
+
346
+ return outputs
347
+
348
+
349
+ def default_init(cls, *args, **kwargs):
350
+ return cls(*args, **kwargs)
351
+
352
+
353
+ class SelfAttention(torch.nn.Module):
354
+ def __init__(self, hidden_size, num_attention_heads,
355
+ layer_id, hidden_size_per_attention_head=None, bias=True,
356
+ params_dtype=torch.float, position_encoding_2d=True, empty_init=True):
357
+ if empty_init:
358
+ init_method = skip_init
359
+ else:
360
+ init_method = default_init
361
+ super(SelfAttention, self).__init__()
362
+
363
+ self.layer_id = layer_id
364
+ self.hidden_size = hidden_size
365
+ self.hidden_size_per_partition = hidden_size
366
+ self.num_attention_heads = num_attention_heads
367
+ self.num_attention_heads_per_partition = num_attention_heads
368
+ self.position_encoding_2d = position_encoding_2d
369
+ self.rotary_emb = RotaryEmbedding(
370
+ self.hidden_size // (self.num_attention_heads * 2)
371
+ if position_encoding_2d
372
+ else self.hidden_size // self.num_attention_heads,
373
+ base=10000,
374
+ precision=torch.half,
375
+ learnable=False,
376
+ )
377
+
378
+ self.scale_mask_softmax = None
379
+
380
+ if hidden_size_per_attention_head is None:
381
+ self.hidden_size_per_attention_head = hidden_size // num_attention_heads
382
+ else:
383
+ self.hidden_size_per_attention_head = hidden_size_per_attention_head
384
+
385
+ self.inner_hidden_size = num_attention_heads * self.hidden_size_per_attention_head
386
+
387
+ # Strided linear layer.
388
+ self.query_key_value = init_method(
389
+ torch.nn.Linear,
390
+ hidden_size,
391
+ 3 * self.inner_hidden_size,
392
+ bias=bias,
393
+ dtype=params_dtype,
394
+ )
395
+
396
+ self.dense = init_method(
397
+ torch.nn.Linear,
398
+ self.inner_hidden_size,
399
+ hidden_size,
400
+ bias=bias,
401
+ dtype=params_dtype,
402
+ )
403
+
404
+ @staticmethod
405
+ def attention_mask_func(attention_scores, attention_mask):
406
+ attention_scores.masked_fill_(attention_mask, -10000.0)
407
+ return attention_scores
408
+
409
+ def split_tensor_along_last_dim(self, tensor, num_partitions,
410
+ contiguous_split_chunks=False):
411
+ """Split a tensor along its last dimension.
412
+ Arguments:
413
+ tensor: input tensor.
414
+ num_partitions: number of partitions to split the tensor
415
+ contiguous_split_chunks: If True, make each chunk contiguous
416
+ in memory.
417
+ """
418
+ # Get the size and dimension.
419
+ last_dim = tensor.dim() - 1
420
+ last_dim_size = tensor.size()[last_dim] // num_partitions
421
+ # Split.
422
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
423
+ # Note: torch.split does not create contiguous tensors by default.
424
+ if contiguous_split_chunks:
425
+ return tuple(chunk.contiguous() for chunk in tensor_list)
426
+
427
+ return tensor_list
428
+
429
+ def forward(
430
+ self,
431
+ hidden_states: torch.Tensor,
432
+ position_ids,
433
+ attention_mask: torch.Tensor,
434
+ layer_id,
435
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
436
+ use_cache: bool = False,
437
+ output_attentions: bool = False,
438
+ ):
439
+ """
440
+ hidden_states: [seq_len, batch, hidden_size]
441
+ attention_mask: [(1, 1), seq_len, seq_len]
442
+ """
443
+
444
+ # [seq_len, batch, 3 * hidden_size]
445
+ mixed_raw_layer = self.query_key_value(hidden_states)
446
+
447
+ # [seq_len, batch, 3 * hidden_size] --> [seq_len, batch, num_attention_heads, 3 * hidden_size_per_attention_head]
448
+ new_tensor_shape = mixed_raw_layer.size()[:-1] + (
449
+ self.num_attention_heads_per_partition,
450
+ 3 * self.hidden_size_per_attention_head,
451
+ )
452
+ mixed_raw_layer = mixed_raw_layer.view(*new_tensor_shape)
453
+
454
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
455
+ (query_layer, key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_raw_layer, 3)
456
+
457
+ if self.position_encoding_2d:
458
+ q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1))
459
+ k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1))
460
+ cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1)
461
+ position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \
462
+ position_ids[:, 1, :].transpose(0, 1).contiguous()
463
+ q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids)
464
+ q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, block_position_ids)
465
+ query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1))
466
+ key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1))
467
+ else:
468
+ position_ids = position_ids.transpose(0, 1)
469
+ cos, sin = self.rotary_emb(value_layer, seq_len=position_ids.max() + 1)
470
+ # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
471
+ query_layer, key_layer = apply_rotary_pos_emb_index(query_layer, key_layer, cos, sin, position_ids)
472
+
473
+ # [seq_len, batch, hidden_size]
474
+ context_layer, present, attention_probs = attention_fn(
475
+ self=self,
476
+ query_layer=query_layer,
477
+ key_layer=key_layer,
478
+ value_layer=value_layer,
479
+ attention_mask=attention_mask,
480
+ hidden_size_per_partition=self.hidden_size_per_partition,
481
+ layer_id=layer_id,
482
+ layer_past=layer_past,
483
+ use_cache=use_cache
484
+ )
485
+
486
+ output = self.dense(context_layer)
487
+
488
+ outputs = (output, present)
489
+
490
+ if output_attentions:
491
+ outputs += (attention_probs,)
492
+
493
+ return outputs # output, present, attention_probs
494
+
495
+
496
+ class GEGLU(torch.nn.Module):
497
+ def __init__(self):
498
+ super().__init__()
499
+ self.activation_fn = F.gelu
500
+
501
+ def forward(self, x):
502
+ # dim=-1 breaks in jit for pt<1.10
503
+ x1, x2 = x.chunk(2, dim=(x.ndim - 1))
504
+ return x1 * self.activation_fn(x2)
505
+
506
+
507
+ class GLU(torch.nn.Module):
508
+ def __init__(self, hidden_size, inner_hidden_size=None,
509
+ layer_id=None, bias=True, activation_func=gelu, params_dtype=torch.float, empty_init=True):
510
+ super(GLU, self).__init__()
511
+ if empty_init:
512
+ init_method = skip_init
513
+ else:
514
+ init_method = default_init
515
+ self.layer_id = layer_id
516
+ self.activation_func = activation_func
517
+
518
+ # Project to 4h.
519
+ self.hidden_size = hidden_size
520
+ if inner_hidden_size is None:
521
+ inner_hidden_size = 4 * hidden_size
522
+ self.inner_hidden_size = inner_hidden_size
523
+ self.dense_h_to_4h = init_method(
524
+ torch.nn.Linear,
525
+ self.hidden_size,
526
+ self.inner_hidden_size,
527
+ bias=bias,
528
+ dtype=params_dtype,
529
+ )
530
+ # Project back to h.
531
+ self.dense_4h_to_h = init_method(
532
+ torch.nn.Linear,
533
+ self.inner_hidden_size,
534
+ self.hidden_size,
535
+ bias=bias,
536
+ dtype=params_dtype,
537
+ )
538
+
539
+ def forward(self, hidden_states):
540
+ """
541
+ hidden_states: [seq_len, batch, hidden_size]
542
+ """
543
+
544
+ # [seq_len, batch, inner_hidden_size]
545
+ intermediate_parallel = self.dense_h_to_4h(hidden_states)
546
+
547
+ intermediate_parallel = self.activation_func(intermediate_parallel)
548
+
549
+ output = self.dense_4h_to_h(intermediate_parallel)
550
+
551
+ return output
552
+
553
+
554
+ class GLMBlock(torch.nn.Module):
555
+ def __init__(
556
+ self,
557
+ hidden_size,
558
+ num_attention_heads,
559
+ layernorm_epsilon,
560
+ layer_id,
561
+ inner_hidden_size=None,
562
+ hidden_size_per_attention_head=None,
563
+ layernorm=LayerNorm,
564
+ use_bias=True,
565
+ params_dtype=torch.float,
566
+ num_layers=28,
567
+ position_encoding_2d=True,
568
+ empty_init=True
569
+ ):
570
+ super(GLMBlock, self).__init__()
571
+ # Set output layer initialization if not provided.
572
+
573
+ self.layer_id = layer_id
574
+
575
+ # Layernorm on the input data.
576
+ self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
577
+
578
+ self.position_encoding_2d = position_encoding_2d
579
+
580
+ # Self attention.
581
+ self.attention = SelfAttention(
582
+ hidden_size,
583
+ num_attention_heads,
584
+ layer_id,
585
+ hidden_size_per_attention_head=hidden_size_per_attention_head,
586
+ bias=use_bias,
587
+ params_dtype=params_dtype,
588
+ position_encoding_2d=self.position_encoding_2d,
589
+ empty_init=empty_init
590
+ )
591
+
592
+ # Layernorm on the input data.
593
+ self.post_attention_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
594
+
595
+ self.num_layers = num_layers
596
+
597
+ # GLU
598
+ self.mlp = GLU(
599
+ hidden_size,
600
+ inner_hidden_size=inner_hidden_size,
601
+ bias=use_bias,
602
+ layer_id=layer_id,
603
+ params_dtype=params_dtype,
604
+ empty_init=empty_init
605
+ )
606
+
607
+ def forward(
608
+ self,
609
+ hidden_states: torch.Tensor,
610
+ position_ids,
611
+ attention_mask: torch.Tensor,
612
+ layer_id,
613
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
614
+ use_cache: bool = False,
615
+ output_attentions: bool = False,
616
+ ):
617
+ """
618
+ hidden_states: [seq_len, batch, hidden_size]
619
+ attention_mask: [(1, 1), seq_len, seq_len]
620
+ """
621
+
622
+ # Layer norm at the begining of the transformer layer.
623
+ # [seq_len, batch, hidden_size]
624
+ attention_input = self.input_layernorm(hidden_states)
625
+
626
+ # Self attention.
627
+ attention_outputs = self.attention(
628
+ attention_input,
629
+ position_ids,
630
+ attention_mask=attention_mask,
631
+ layer_id=layer_id,
632
+ layer_past=layer_past,
633
+ use_cache=use_cache,
634
+ output_attentions=output_attentions
635
+ )
636
+
637
+ attention_output = attention_outputs[0]
638
+
639
+ outputs = attention_outputs[1:]
640
+
641
+ # Residual connection.
642
+ alpha = (2 * self.num_layers) ** 0.5
643
+ hidden_states = attention_input * alpha + attention_output
644
+
645
+ mlp_input = self.post_attention_layernorm(hidden_states)
646
+
647
+ # MLP.
648
+ mlp_output = self.mlp(mlp_input)
649
+
650
+ # Second residual connection.
651
+ output = mlp_input * alpha + mlp_output
652
+
653
+ if use_cache:
654
+ outputs = (output,) + outputs
655
+ else:
656
+ outputs = (output,) + outputs[1:]
657
+
658
+ return outputs # hidden_states, present, attentions
659
+
660
+
661
+ class ChatGLMPreTrainedModel(PreTrainedModel):
662
+ """
663
+ An abstract class to handle weights initialization and
664
+ a simple interface for downloading and loading pretrained models.
665
+ """
666
+
667
+ is_parallelizable = False
668
+ supports_gradient_checkpointing = True
669
+ config_class = ChatGLMConfig
670
+ base_model_prefix = "transformer"
671
+ _no_split_modules = ["GLMBlock"]
672
+
673
+ def __init__(self, *inputs, **kwargs):
674
+ super().__init__(*inputs, **kwargs)
675
+
676
+ def _init_weights(self, module: nn.Module):
677
+ """Initialize the weights."""
678
+ return
679
+
680
+ def get_masks(self, input_ids, device):
681
+ batch_size, seq_length = input_ids.shape
682
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
683
+ attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device)
684
+ attention_mask.tril_()
685
+ for i, context_length in enumerate(context_lengths):
686
+ attention_mask[i, :, :context_length] = 1
687
+ attention_mask.unsqueeze_(1)
688
+ attention_mask = (attention_mask < 0.5).bool()
689
+
690
+ return attention_mask
691
+
692
+ def get_position_ids(self, input_ids, mask_positions, device, use_gmasks=None):
693
+ batch_size, seq_length = input_ids.shape
694
+ if use_gmasks is None:
695
+ use_gmasks = [False] * batch_size
696
+ context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
697
+ if self.position_encoding_2d:
698
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
699
+ for i, context_length in enumerate(context_lengths):
700
+ position_ids[i, context_length:] = mask_positions[i]
701
+ block_position_ids = [torch.cat((
702
+ torch.zeros(context_length, dtype=torch.long, device=device),
703
+ torch.arange(seq_length - context_length, dtype=torch.long, device=device) + 1
704
+ )) for context_length in context_lengths]
705
+ block_position_ids = torch.stack(block_position_ids, dim=0)
706
+ position_ids = torch.stack((position_ids, block_position_ids), dim=1)
707
+ else:
708
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
709
+ for i, context_length in enumerate(context_lengths):
710
+ if not use_gmasks[i]:
711
+ position_ids[context_length:] = mask_positions[i]
712
+
713
+ return position_ids
714
+
715
+ def _set_gradient_checkpointing(self, module, value=False):
716
+ if isinstance(module, ChatGLMModel):
717
+ module.gradient_checkpointing = value
718
+
719
+
720
+ CHATGLM_6B_START_DOCSTRING = r"""
721
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
722
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
723
+ usage and behavior.
724
+
725
+ Parameters:
726
+ config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model.
727
+ Initializing with a config file does not load the weights associated with the model, only the configuration.
728
+ Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
729
+ """
730
+
731
+ CHATGLM_6B_INPUTS_DOCSTRING = r"""
732
+ Args:
733
+ input_ids (`torch.LongTensor` of shape `({0})`):
734
+ Indices of input sequence tokens in the vocabulary.
735
+
736
+ Indices can be obtained using [`ChatGLM6BTokenizer`].
737
+ See [`PreTrainedTokenizer.encode`] and
738
+ [`PreTrainedTokenizer.__call__`] for details.
739
+
740
+ [What are input IDs?](../glossary#input-ids)
741
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
742
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
743
+
744
+ - 1 for tokens that are **not masked**,
745
+ - 0 for tokens that are **masked**.
746
+
747
+ [What are attention masks?](../glossary#attention-mask)
748
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
749
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
750
+
751
+ - 0 corresponds to a *sentence A* token,
752
+ - 1 corresponds to a *sentence B* token.
753
+
754
+ [What are token type IDs?](../glossary#token-type-ids)
755
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
756
+ Indices of positions of each input sequence tokens in the position embeddings.
757
+ Selected in the range `[0, config.max_position_embeddings - 1]`.
758
+
759
+ [What are position IDs?](../glossary#position-ids)
760
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
761
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
762
+
763
+ - 1 indicates the head is **not masked**,
764
+ - 0 indicates the head is **masked**.
765
+
766
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
767
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
768
+ This is useful if you want more control over how to convert *input_ids* indices into associated vectors
769
+ than the model's internal embedding lookup matrix.
770
+ output_attentions (`bool`, *optional*):
771
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
772
+ tensors for more detail.
773
+ output_hidden_states (`bool`, *optional*):
774
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
775
+ more detail.
776
+ return_dict (`bool`, *optional*):
777
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
778
+ """
779
+
780
+
781
+ @add_start_docstrings(
782
+ "The bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.",
783
+ CHATGLM_6B_START_DOCSTRING,
784
+ )
785
+ class ChatGLMModel(ChatGLMPreTrainedModel):
786
+ """
787
+
788
+ The model can behave as an encoder (with only self-attention) as well
789
+ as a decoder, in which case a layer of cross-attention is added between
790
+ the self-attention layers, following the architecture described in [Attention is
791
+ all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
792
+ Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
793
+
794
+ To behave as an decoder the model needs to be initialized with the
795
+ `is_decoder` argument of the configuration set to `True`.
796
+ To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
797
+ argument and `add_cross_attention` set to `True`; an
798
+ `encoder_hidden_states` is then expected as an input to the forward pass.
799
+ """
800
+
801
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
802
+ super().__init__(config)
803
+ if empty_init:
804
+ init_method = skip_init
805
+ else:
806
+ init_method = default_init
807
+ # recording parameters
808
+ self.max_sequence_length = config.max_sequence_length
809
+ self.hidden_size = config.hidden_size
810
+ self.params_dtype = torch.half
811
+ self.num_attention_heads = config.num_attention_heads
812
+ self.vocab_size = config.vocab_size
813
+ self.num_layers = config.num_layers
814
+ self.layernorm_epsilon = config.layernorm_epsilon
815
+ self.inner_hidden_size = config.inner_hidden_size
816
+ self.hidden_size_per_attention_head = self.hidden_size // self.num_attention_heads
817
+ self.position_encoding_2d = config.position_encoding_2d
818
+ self.pre_seq_len = config.pre_seq_len
819
+ self.prefix_projection = config.prefix_projection
820
+
821
+ self.word_embeddings = init_method(
822
+ torch.nn.Embedding,
823
+ num_embeddings=self.vocab_size, embedding_dim=self.hidden_size,
824
+ dtype=self.params_dtype
825
+ )
826
+ self.gradient_checkpointing = False
827
+
828
+ def get_layer(layer_id):
829
+ return GLMBlock(
830
+ self.hidden_size,
831
+ self.num_attention_heads,
832
+ self.layernorm_epsilon,
833
+ layer_id,
834
+ inner_hidden_size=self.inner_hidden_size,
835
+ hidden_size_per_attention_head=self.hidden_size_per_attention_head,
836
+ layernorm=LayerNorm,
837
+ use_bias=True,
838
+ params_dtype=self.params_dtype,
839
+ position_encoding_2d=self.position_encoding_2d,
840
+ empty_init=empty_init
841
+ )
842
+
843
+ self.layers = torch.nn.ModuleList(
844
+ [get_layer(layer_id) for layer_id in range(self.num_layers)]
845
+ )
846
+
847
+ # Final layer norm before output.
848
+ self.final_layernorm = LayerNorm(self.hidden_size, eps=self.layernorm_epsilon)
849
+
850
+ if self.pre_seq_len is not None:
851
+ for param in self.parameters():
852
+ param.requires_grad = False
853
+ self.prefix_tokens = torch.arange(self.pre_seq_len).long()
854
+ self.prefix_encoder = PrefixEncoder(config)
855
+ self.dropout = torch.nn.Dropout(0.1)
856
+
857
+ # total_params = sum(p.numel() for p in self.parameters())
858
+ # trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad)
859
+ # print("Using p-tuning v2: # trainable_params = {} / {}".format(trainable_params, total_params))
860
+
861
+ def get_input_embeddings(self):
862
+ return self.word_embeddings
863
+
864
+ def set_input_embeddings(self, new_embeddings: torch.Tensor):
865
+ self.word_embeddings = new_embeddings
866
+
867
+ def get_prompt(self, batch_size, device, dtype=torch.half):
868
+ prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
869
+ past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)
870
+ past_key_values = past_key_values.view(
871
+ batch_size,
872
+ self.pre_seq_len,
873
+ self.num_layers * 2,
874
+ self.num_attention_heads,
875
+ self.hidden_size // self.num_attention_heads
876
+ )
877
+ # seq_len, b, nh, hidden_size
878
+ past_key_values = self.dropout(past_key_values)
879
+ past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2)
880
+ # past_key_values = [(v[0], v[1]) for v in past_key_values]
881
+ return past_key_values
882
+
883
+ @add_start_docstrings_to_model_forward(CHATGLM_6B_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
884
+ @add_code_sample_docstrings(
885
+ checkpoint=_CHECKPOINT_FOR_DOC,
886
+ output_type=BaseModelOutputWithPastAndCrossAttentions,
887
+ config_class=_CONFIG_FOR_DOC,
888
+ )
889
+ def forward(
890
+ self,
891
+ input_ids: Optional[torch.LongTensor] = None,
892
+ position_ids: Optional[torch.LongTensor] = None,
893
+ attention_mask: Optional[torch.Tensor] = None,
894
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
895
+ inputs_embeds: Optional[torch.LongTensor] = None,
896
+ use_cache: Optional[bool] = None,
897
+ output_attentions: Optional[bool] = None,
898
+ output_hidden_states: Optional[bool] = None,
899
+ return_dict: Optional[bool] = None,
900
+ ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPast]:
901
+
902
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
903
+ output_hidden_states = (
904
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
905
+ )
906
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
907
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
908
+
909
+ if self.gradient_checkpointing and self.training:
910
+ if use_cache:
911
+ logger.warning_once(
912
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
913
+ )
914
+ use_cache = False
915
+
916
+ if input_ids is not None and inputs_embeds is not None:
917
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
918
+ elif input_ids is not None:
919
+ batch_size, seq_length = input_ids.shape[:2]
920
+ elif inputs_embeds is not None:
921
+ batch_size, seq_length = inputs_embeds.shape[:2]
922
+ else:
923
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
924
+
925
+ if inputs_embeds is None:
926
+ inputs_embeds = self.word_embeddings(input_ids)
927
+
928
+ if past_key_values is None:
929
+ if self.pre_seq_len is not None:
930
+ past_key_values = self.get_prompt(batch_size=input_ids.shape[0], device=input_ids.device,
931
+ dtype=inputs_embeds.dtype)
932
+ else:
933
+ past_key_values = tuple([None] * len(self.layers))
934
+
935
+ if attention_mask is None:
936
+ attention_mask = self.get_masks(
937
+ input_ids,
938
+ device=input_ids.device
939
+ )
940
+
941
+
942
+ if position_ids is None:
943
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
944
+ seqs = input_ids.tolist()
945
+
946
+ mask_positions, use_gmasks = [], []
947
+ for seq in seqs:
948
+ mask_token = gMASK if gMASK in seq else MASK
949
+ use_gmask = mask_token == gMASK
950
+ mask_positions.append(seq.index(mask_token))
951
+ use_gmasks.append(use_gmask)
952
+
953
+ position_ids = self.get_position_ids(
954
+ input_ids,
955
+ mask_positions=mask_positions,
956
+ device=input_ids.device,
957
+ use_gmasks=use_gmasks
958
+ )
959
+
960
+ if self.pre_seq_len is not None and attention_mask is not None:
961
+ prefix_attention_mask = torch.ones(batch_size, 1, input_ids.size(-1), self.pre_seq_len).to(
962
+ attention_mask.device)
963
+ prefix_attention_mask = (prefix_attention_mask < 0.5).bool()
964
+ attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=3)
965
+
966
+ # [seq_len, batch, hidden_size]
967
+ hidden_states = inputs_embeds.transpose(0, 1)
968
+
969
+ presents = () if use_cache else None
970
+ all_self_attentions = () if output_attentions else None
971
+ all_hidden_states = () if output_hidden_states else None
972
+
973
+ if attention_mask is None:
974
+ attention_mask = torch.zeros(1, 1, device=input_ids.device).bool()
975
+ else:
976
+ attention_mask = attention_mask.to(hidden_states.device)
977
+
978
+ for i, layer in enumerate(self.layers):
979
+
980
+ if output_hidden_states:
981
+ all_hidden_states = all_hidden_states + (hidden_states,)
982
+ layer_past = past_key_values[i]
983
+
984
+ if self.gradient_checkpointing and self.training:
985
+ layer_ret = torch.utils.checkpoint.checkpoint(
986
+ layer,
987
+ hidden_states,
988
+ position_ids,
989
+ attention_mask,
990
+ torch.tensor(i),
991
+ layer_past,
992
+ use_cache,
993
+ output_attentions
994
+ )
995
+ else:
996
+ layer_ret = layer(
997
+ hidden_states,
998
+ position_ids=position_ids,
999
+ attention_mask=attention_mask,
1000
+ layer_id=torch.tensor(i),
1001
+ layer_past=layer_past,
1002
+ use_cache=use_cache,
1003
+ output_attentions=output_attentions
1004
+ )
1005
+
1006
+ hidden_states = layer_ret[0]
1007
+
1008
+ if use_cache:
1009
+ presents = presents + (layer_ret[1],)
1010
+
1011
+ if output_attentions:
1012
+ all_self_attentions = all_self_attentions + (layer_ret[2 if use_cache else 1],)
1013
+
1014
+ # Final layer norm.
1015
+ hidden_states = self.final_layernorm(hidden_states)
1016
+
1017
+ if output_hidden_states:
1018
+ all_hidden_states = all_hidden_states + (hidden_states,)
1019
+
1020
+ if not return_dict:
1021
+ return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
1022
+
1023
+ return BaseModelOutputWithPast(
1024
+ last_hidden_state=hidden_states,
1025
+ past_key_values=presents,
1026
+ hidden_states=all_hidden_states,
1027
+ attentions=all_self_attentions,
1028
+ )
1029
+
1030
+
1031
+ class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
1032
+ def __init__(self, config: ChatGLMConfig, empty_init=True):
1033
+ super().__init__(config)
1034
+ if empty_init:
1035
+ init_method = skip_init
1036
+ else:
1037
+ init_method = default_init
1038
+
1039
+ # self.hidden_size = config.hidden_size
1040
+ # self.params_dtype = torch.half
1041
+ # self.vocab_size = config.vocab_size
1042
+ self.max_sequence_length = config.max_sequence_length
1043
+
1044
+ self.position_encoding_2d = config.position_encoding_2d
1045
+
1046
+ self.transformer = ChatGLMModel(config, empty_init=empty_init)
1047
+
1048
+ self.lm_head = init_method(
1049
+ nn.Linear,
1050
+ config.hidden_size,
1051
+ config.vocab_size,
1052
+ bias=False,
1053
+ dtype=torch.half
1054
+ )
1055
+
1056
+ self.config = config
1057
+
1058
+ self.quantized = False
1059
+
1060
+ if self.config.quantization_bit:
1061
+ self.quantize(self.config.quantization_bit, empty_init=True)
1062
+
1063
+ def get_output_embeddings(self):
1064
+ return self.lm_head
1065
+
1066
+ def set_output_embeddings(self, new_embeddings):
1067
+ self.lm_head = new_embeddings
1068
+
1069
+ def _update_model_kwargs_for_generation(
1070
+ self,
1071
+ outputs: ModelOutput,
1072
+ model_kwargs: Dict[str, Any],
1073
+ is_encoder_decoder: bool = False,
1074
+ standardize_cache_format: bool = False,
1075
+ ) -> Dict[str, Any]:
1076
+ # update past_key_values
1077
+ model_kwargs["past_key_values"] = self._extract_past_from_model_output(
1078
+ outputs, standardize_cache_format=standardize_cache_format
1079
+ )
1080
+
1081
+ # update attention mask
1082
+ if "attention_mask" in model_kwargs:
1083
+ attention_mask = model_kwargs["attention_mask"]
1084
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1085
+ attention_mask = torch.cat(
1086
+ [attention_mask, attention_mask.new_ones((*attention_mask.shape[:3], 1))], dim=3)
1087
+ new_attention_mask = attention_mask[:, :, -1:].clone()
1088
+ new_attention_mask[..., -1] = False
1089
+ model_kwargs["attention_mask"] = torch.cat(
1090
+ [attention_mask, new_attention_mask], dim=2
1091
+ )
1092
+
1093
+ # update position ids
1094
+ if "position_ids" in model_kwargs:
1095
+ position_ids = model_kwargs["position_ids"]
1096
+ new_position_id = position_ids[..., -1:].clone()
1097
+ new_position_id[:, 1, :] += 1
1098
+ model_kwargs["position_ids"] = torch.cat(
1099
+ [position_ids, new_position_id], dim=-1
1100
+ )
1101
+
1102
+ return model_kwargs
1103
+
1104
+ def prepare_inputs_for_generation(
1105
+ self,
1106
+ input_ids: torch.LongTensor,
1107
+ past: Optional[torch.Tensor] = None,
1108
+ past_key_values: Optional[torch.Tensor] = None,
1109
+ attention_mask: Optional[torch.Tensor] = None,
1110
+ position_ids: Optional[torch.Tensor] = None,
1111
+ **kwargs
1112
+ ) -> dict:
1113
+ batch_size, seq_length = input_ids.shape
1114
+ MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
1115
+ seqs = input_ids.tolist()
1116
+ mask_positions, use_gmasks = [], []
1117
+ for seq in seqs:
1118
+ mask_token = gMASK if gMASK in seq else MASK
1119
+ use_gmask = mask_token == gMASK
1120
+ mask_positions.append(seq.index(mask_token))
1121
+ use_gmasks.append(use_gmask)
1122
+
1123
+ # only last token for input_ids if past is not None
1124
+ if past is not None or past_key_values is not None:
1125
+ last_token = input_ids[:, -1].unsqueeze(-1)
1126
+ if attention_mask is not None and attention_mask.dtype == torch.bool:
1127
+ attention_mask = attention_mask[:, :, -1:]
1128
+ else:
1129
+ attention_mask = None
1130
+ if position_ids is not None:
1131
+ position_ids = position_ids[..., -1:]
1132
+ else:
1133
+ context_lengths = [seq.index(self.config.bos_token_id) for seq in seqs]
1134
+ if self.position_encoding_2d:
1135
+ position_ids = torch.tensor(
1136
+ [[mask_position, seq_length - context_length] for mask_position, context_length in
1137
+ zip(mask_positions, context_lengths)], dtype=torch.long, device=input_ids.device).unsqueeze(-1)
1138
+ else:
1139
+ position_ids = torch.tensor([mask_position for mask_position in mask_positions], dtype=torch.long,
1140
+ device=input_ids.device).unsqueeze(-1)
1141
+
1142
+ if past is None:
1143
+ past = past_key_values
1144
+ return {
1145
+ "input_ids": last_token,
1146
+ "past_key_values": past,
1147
+ "position_ids": position_ids,
1148
+ "attention_mask": attention_mask
1149
+ }
1150
+ else:
1151
+ if attention_mask is not None and attention_mask.dtype != torch.bool:
1152
+ logger.warning_once(f"The dtype of attention mask ({attention_mask.dtype}) is not bool")
1153
+ attention_mask = None
1154
+ if attention_mask is None:
1155
+ attention_mask = self.get_masks(
1156
+ input_ids,
1157
+ device=input_ids.device
1158
+ )
1159
+ if position_ids is None:
1160
+ position_ids = self.get_position_ids(
1161
+ input_ids,
1162
+ device=input_ids.device,
1163
+ mask_positions=mask_positions,
1164
+ use_gmasks=use_gmasks
1165
+ )
1166
+
1167
+ return {
1168
+ "input_ids": input_ids,
1169
+ "past_key_values": past,
1170
+ "position_ids": position_ids,
1171
+ "attention_mask": attention_mask
1172
+ }
1173
+
1174
+ def forward(
1175
+ self,
1176
+ input_ids: Optional[torch.Tensor] = None,
1177
+ position_ids: Optional[torch.Tensor] = None,
1178
+ attention_mask: Optional[torch.Tensor] = None,
1179
+ past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
1180
+ inputs_embeds: Optional[torch.Tensor] = None,
1181
+ labels: Optional[torch.Tensor] = None,
1182
+ use_cache: Optional[bool] = None,
1183
+ output_attentions: Optional[bool] = None,
1184
+ output_hidden_states: Optional[bool] = None,
1185
+ return_dict: Optional[bool] = None,
1186
+ ):
1187
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1188
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1189
+
1190
+ transformer_outputs = self.transformer(
1191
+ input_ids=input_ids,
1192
+ position_ids=position_ids,
1193
+ attention_mask=attention_mask,
1194
+ past_key_values=past_key_values,
1195
+ inputs_embeds=inputs_embeds,
1196
+ use_cache=use_cache,
1197
+ output_attentions=output_attentions,
1198
+ output_hidden_states=output_hidden_states,
1199
+ return_dict=return_dict,
1200
+ )
1201
+
1202
+ hidden_states = transformer_outputs[0]
1203
+
1204
+ lm_logits = self.lm_head(hidden_states).permute(1, 0, 2).contiguous()
1205
+
1206
+ loss = None
1207
+ if labels is not None:
1208
+ lm_logits = lm_logits.to(torch.float32)
1209
+
1210
+ # Shift so that tokens < n predict n
1211
+ shift_logits = lm_logits[..., :-1, :].contiguous()
1212
+ shift_labels = labels[..., 1:].contiguous()
1213
+ # Flatten the tokens
1214
+ loss_fct = CrossEntropyLoss(ignore_index=-100)
1215
+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
1216
+
1217
+ lm_logits = lm_logits.to(hidden_states.dtype)
1218
+ loss = loss.to(hidden_states.dtype)
1219
+
1220
+ if not return_dict:
1221
+ output = (lm_logits,) + transformer_outputs[1:]
1222
+ return ((loss,) + output) if loss is not None else output
1223
+
1224
+ return CausalLMOutputWithPast(
1225
+ loss=loss,
1226
+ logits=lm_logits,
1227
+ past_key_values=transformer_outputs.past_key_values,
1228
+ hidden_states=transformer_outputs.hidden_states,
1229
+ attentions=transformer_outputs.attentions,
1230
+ )
1231
+
1232
+ @staticmethod
1233
+ def _reorder_cache(
1234
+ past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor
1235
+ ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]:
1236
+ """
1237
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
1238
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
1239
+ beam_idx at every generation step.
1240
+
1241
+ Output shares the same memory storage as `past`.
1242
+ """
1243
+ return tuple(
1244
+ (
1245
+ layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)),
1246
+ layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)),
1247
+ )
1248
+ for layer_past in past
1249
+ )
1250
+
1251
+ def process_response(self, response):
1252
+ response = response.strip()
1253
+ response = response.replace("[[训练时间]]", "2023年")
1254
+ punkts = [
1255
+ [",", ","],
1256
+ ["!", "!"],
1257
+ [":", ":"],
1258
+ [";", ";"],
1259
+ ["\?", "?"],
1260
+ ]
1261
+ for item in punkts:
1262
+ response = re.sub(r"([\u4e00-\u9fff])%s" % item[0], r"\1%s" % item[1], response)
1263
+ response = re.sub(r"%s([\u4e00-\u9fff])" % item[0], r"%s\1" % item[1], response)
1264
+ return response
1265
+
1266
+ @torch.no_grad()
1267
+ def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048, num_beams=1,
1268
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1269
+ if history is None:
1270
+ history = []
1271
+ if logits_processor is None:
1272
+ logits_processor = LogitsProcessorList()
1273
+ logits_processor.append(InvalidScoreLogitsProcessor())
1274
+ gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p,
1275
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1276
+ if not history:
1277
+ prompt = query
1278
+ else:
1279
+ prompt = ""
1280
+ for i, (old_query, response) in enumerate(history):
1281
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1282
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1283
+ inputs = tokenizer([prompt], return_tensors="pt")
1284
+ inputs = inputs.to(self.device)
1285
+ outputs = self.generate(**inputs, **gen_kwargs)
1286
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1287
+ response = tokenizer.decode(outputs)
1288
+ response = self.process_response(response)
1289
+ history = history + [(query, response)]
1290
+ return response, history
1291
+
1292
+ @torch.no_grad()
1293
+ def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048,
1294
+ do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
1295
+ if history is None:
1296
+ history = []
1297
+ if logits_processor is None:
1298
+ logits_processor = LogitsProcessorList()
1299
+ logits_processor.append(InvalidScoreLogitsProcessor())
1300
+ gen_kwargs = {"max_length": max_length, "do_sample": do_sample, "top_p": top_p,
1301
+ "temperature": temperature, "logits_processor": logits_processor, **kwargs}
1302
+ if not history:
1303
+ prompt = query
1304
+ else:
1305
+ prompt = ""
1306
+ for i, (old_query, response) in enumerate(history):
1307
+ prompt += "[Round {}]\n问:{}\n答:{}\n".format(i, old_query, response)
1308
+ prompt += "[Round {}]\n问:{}\n答:".format(len(history), query)
1309
+ inputs = tokenizer([prompt], return_tensors="pt")
1310
+ inputs = inputs.to(self.device)
1311
+ for outputs in self.stream_generate(**inputs, **gen_kwargs):
1312
+ outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
1313
+ response = tokenizer.decode(outputs)
1314
+ response = self.process_response(response)
1315
+ new_history = history + [(query, response)]
1316
+ yield response, new_history
1317
+
1318
+ @torch.no_grad()
1319
+ def stream_generate(
1320
+ self,
1321
+ input_ids,
1322
+ generation_config: Optional[GenerationConfig] = None,
1323
+ logits_processor: Optional[LogitsProcessorList] = None,
1324
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1325
+ prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
1326
+ **kwargs,
1327
+ ):
1328
+ batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
1329
+
1330
+ if generation_config is None:
1331
+ generation_config = self.generation_config
1332
+ generation_config = copy.deepcopy(generation_config)
1333
+ model_kwargs = generation_config.update(**kwargs)
1334
+ bos_token_id, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id
1335
+
1336
+ if isinstance(eos_token_id, int):
1337
+ eos_token_id = [eos_token_id]
1338
+
1339
+ has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
1340
+ if has_default_max_length and generation_config.max_new_tokens is None:
1341
+ warnings.warn(
1342
+ f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
1343
+ "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
1344
+ " recommend using `max_new_tokens` to control the maximum length of the generation.",
1345
+ UserWarning,
1346
+ )
1347
+ elif generation_config.max_new_tokens is not None:
1348
+ generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
1349
+ if not has_default_max_length:
1350
+ logger.warn(
1351
+ f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
1352
+ f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
1353
+ "Please refer to the documentation for more information. "
1354
+ "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
1355
+ UserWarning,
1356
+ )
1357
+
1358
+ if input_ids_seq_length >= generation_config.max_length:
1359
+ input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
1360
+ logger.warning(
1361
+ f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to"
1362
+ f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
1363
+ " increasing `max_new_tokens`."
1364
+ )
1365
+
1366
+ # 2. Set generation parameters if not already defined
1367
+ logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
1368
+ stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
1369
+
1370
+ logits_processor = self._get_logits_processor(
1371
+ generation_config=generation_config,
1372
+ input_ids_seq_length=input_ids_seq_length,
1373
+ encoder_input_ids=input_ids,
1374
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1375
+ logits_processor=logits_processor,
1376
+ )
1377
+
1378
+ stopping_criteria = self._get_stopping_criteria(
1379
+ generation_config=generation_config, stopping_criteria=stopping_criteria
1380
+ )
1381
+ logits_warper = self._get_logits_warper(generation_config)
1382
+
1383
+ unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
1384
+ scores = None
1385
+ while True:
1386
+ model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
1387
+ # forward pass to get next token
1388
+ outputs = self(
1389
+ **model_inputs,
1390
+ return_dict=True,
1391
+ output_attentions=False,
1392
+ output_hidden_states=False,
1393
+ )
1394
+
1395
+ next_token_logits = outputs.logits[:, -1, :]
1396
+
1397
+ # pre-process distribution
1398
+ next_token_scores = logits_processor(input_ids, next_token_logits)
1399
+ next_token_scores = logits_warper(input_ids, next_token_scores)
1400
+
1401
+ # sample
1402
+ probs = nn.functional.softmax(next_token_scores, dim=-1)
1403
+ if generation_config.do_sample:
1404
+ next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
1405
+ else:
1406
+ next_tokens = torch.argmax(probs, dim=-1)
1407
+
1408
+ # update generated ids, model inputs, and length for next step
1409
+ input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
1410
+ model_kwargs = self._update_model_kwargs_for_generation(
1411
+ outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
1412
+ )
1413
+ unfinished_sequences = unfinished_sequences.mul((sum(next_tokens != i for i in eos_token_id)).long())
1414
+
1415
+ # stop when each sentence is finished, or if we exceed the maximum length
1416
+ if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
1417
+ break
1418
+ yield input_ids
1419
+
1420
+ def quantize(self, bits: int, empty_init=False, **kwargs):
1421
+ if bits == 0:
1422
+ return
1423
+
1424
+ from .quantization import quantize
1425
+
1426
+ if self.quantized:
1427
+ logger.info("Already quantized.")
1428
+ return self
1429
+
1430
+ self.quantized = True
1431
+
1432
+ self.config.quantization_bit = bits
1433
+
1434
+ self.transformer = quantize(self.transformer, bits, empty_init=empty_init, **kwargs)
1435
+ return self
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e002d0c9702b39ed3641f99df3ed5f5112cd4b0c6cda46a5c44b813b11da8bf8
3
+ size 234882351
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1c10da4f2c7745bca3fd6175a5ad4de3a8a8bcc874b65c978682dd110d3dbf4
3
+ size 117441341
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/quantization.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.nn import Linear
2
+ from torch.nn.parameter import Parameter
3
+
4
+ import bz2
5
+ import torch
6
+ import base64
7
+ import ctypes
8
+ from transformers.utils import logging
9
+
10
+ from typing import List
11
+ from functools import partial
12
+
13
+ logger = logging.get_logger(__name__)
14
+
15
+ try:
16
+ from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up
17
+
18
+ class Kernel:
19
+ def __init__(self, code: bytes, function_names: List[str]):
20
+ self.code = code
21
+ self._function_names = function_names
22
+ self._cmodule = LazyKernelCModule(self.code)
23
+
24
+ for name in self._function_names:
25
+ setattr(self, name, KernelFunction(self._cmodule, name))
26
+
27
+ quantization_code = "$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ"
28
+
29
+ kernels = Kernel(
30
+ bz2.decompress(base64.b64decode(quantization_code)),
31
+ [
32
+ "int4WeightCompression",
33
+ "int4WeightExtractionFloat",
34
+ "int4WeightExtractionHalf",
35
+ "int8WeightExtractionFloat",
36
+ "int8WeightExtractionHalf",
37
+ ],
38
+ )
39
+ except Exception as exception:
40
+ kernels = None
41
+ logger.warning("Failed to load cpm_kernels:" + str(exception))
42
+
43
+
44
+ class W8A16Linear(torch.autograd.Function):
45
+ @staticmethod
46
+ def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, scale_w: torch.Tensor, weight_bit_width):
47
+ ctx.inp_shape = inp.size()
48
+ ctx.weight_bit_width = weight_bit_width
49
+ out_features = quant_w.size(0)
50
+ inp = inp.contiguous().view(-1, inp.size(-1))
51
+ weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width)
52
+ ctx.weight_shape = weight.size()
53
+ output = inp.mm(weight.t())
54
+ ctx.save_for_backward(inp, quant_w, scale_w)
55
+ return output.view(*(ctx.inp_shape[:-1] + (out_features,)))
56
+
57
+ @staticmethod
58
+ def backward(ctx, grad_output: torch.Tensor):
59
+ inp, quant_w, scale_w = ctx.saved_tensors
60
+ weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width)
61
+ grad_output = grad_output.contiguous().view(-1, weight.size(0))
62
+ grad_input = grad_output.mm(weight)
63
+ grad_weight = grad_output.t().mm(inp)
64
+ return grad_input.view(ctx.inp_shape), grad_weight.view(ctx.weight_shape), None, None
65
+
66
+
67
+ def compress_int4_weight(weight: torch.Tensor): # (n, m)
68
+ with torch.cuda.device(weight.device):
69
+ n, m = weight.size(0), weight.size(1)
70
+ assert m % 2 == 0
71
+ m = m // 2
72
+ out = torch.empty(n, m, dtype=torch.int8, device="cuda")
73
+ stream = torch.cuda.current_stream()
74
+
75
+ gridDim = (n, 1, 1)
76
+ blockDim = (min(round_up(m, 32), 1024), 1, 1)
77
+
78
+ kernels.int4WeightCompression(
79
+ gridDim,
80
+ blockDim,
81
+ 0,
82
+ stream,
83
+ [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)],
84
+ )
85
+ return out
86
+
87
+
88
+ def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int):
89
+ if source_bit_width == 8:
90
+ func = kernels.int8WeightExtractionHalf
91
+ elif source_bit_width == 4:
92
+ func = kernels.int4WeightExtractionHalf
93
+ else:
94
+ assert False, "Unsupported bit-width"
95
+
96
+ with torch.cuda.device(weight.device):
97
+ n, m = weight.size(0), weight.size(1)
98
+ out = torch.empty(n, m * (8 // source_bit_width), dtype=torch.half, device="cuda")
99
+ stream = torch.cuda.current_stream()
100
+
101
+ gridDim = (n, 1, 1)
102
+ blockDim = (min(round_up(m, 32), 1024), 1, 1)
103
+
104
+ func(
105
+ gridDim,
106
+ blockDim,
107
+ 0,
108
+ stream,
109
+ [
110
+ ctypes.c_void_p(weight.data_ptr()),
111
+ ctypes.c_void_p(scale_list.data_ptr()),
112
+ ctypes.c_void_p(out.data_ptr()),
113
+ ctypes.c_int32(n),
114
+ ctypes.c_int32(m),
115
+ ],
116
+ )
117
+ return out
118
+
119
+
120
+ class QuantizedLinear(Linear):
121
+ def __init__(self, weight_bit_width: int, weight_tensor=None, bias_tensor=None, empty_init=False, *args, **kwargs):
122
+ super(QuantizedLinear, self).__init__(*args, **kwargs)
123
+ self.weight_bit_width = weight_bit_width
124
+
125
+ shape = self.weight.shape
126
+ del self.weight
127
+
128
+ if weight_tensor is None or empty_init:
129
+ self.weight = torch.empty(
130
+ shape[0], shape[1] * weight_bit_width // 8, dtype=torch.int8, device=kwargs["device"]
131
+ )
132
+ self.weight_scale = torch.empty(shape[0], dtype=kwargs["dtype"], device=kwargs["device"])
133
+ else:
134
+ self.weight_scale = (weight_tensor.abs().max(dim=-1).values / ((2 ** (weight_bit_width - 1)) - 1)).half()
135
+ self.weight = torch.round(weight_tensor / self.weight_scale[:, None]).to(torch.int8)
136
+ if weight_bit_width == 4:
137
+ self.weight = compress_int4_weight(self.weight)
138
+
139
+ self.weight = Parameter(self.weight.to(kwargs["device"]), requires_grad=False)
140
+ self.weight_scale = Parameter(self.weight_scale.to(kwargs["device"]), requires_grad=False)
141
+ if bias_tensor is not None:
142
+ self.bias = Parameter(bias_tensor.to(kwargs["device"]), requires_grad=False)
143
+ else:
144
+ self.bias = None
145
+
146
+ def forward(self, input):
147
+ output = W8A16Linear.apply(input, self.weight, self.weight_scale, self.weight_bit_width)
148
+ if self.bias is not None:
149
+ output = output + self.bias
150
+ return output
151
+
152
+
153
+ def quantize(model, weight_bit_width, empty_init=False, **kwargs):
154
+ """Replace fp16 linear with quantized linear"""
155
+
156
+ for layer in model.layers:
157
+ layer.attention.query_key_value = QuantizedLinear(
158
+ weight_bit_width=weight_bit_width,
159
+ weight_tensor=layer.attention.query_key_value.weight.to(torch.cuda.current_device()),
160
+ bias_tensor=layer.attention.query_key_value.bias,
161
+ in_features=layer.attention.query_key_value.in_features,
162
+ out_features=layer.attention.query_key_value.out_features,
163
+ bias=True,
164
+ dtype=torch.half,
165
+ device=layer.attention.query_key_value.weight.device,
166
+ empty_init=empty_init
167
+ )
168
+ layer.attention.dense = QuantizedLinear(
169
+ weight_bit_width=weight_bit_width,
170
+ weight_tensor=layer.attention.dense.weight.to(torch.cuda.current_device()),
171
+ bias_tensor=layer.attention.dense.bias,
172
+ in_features=layer.attention.dense.in_features,
173
+ out_features=layer.attention.dense.out_features,
174
+ bias=True,
175
+ dtype=torch.half,
176
+ device=layer.attention.dense.weight.device,
177
+ empty_init=empty_init
178
+ )
179
+ layer.mlp.dense_h_to_4h = QuantizedLinear(
180
+ weight_bit_width=weight_bit_width,
181
+ weight_tensor=layer.mlp.dense_h_to_4h.weight.to(torch.cuda.current_device()),
182
+ bias_tensor=layer.mlp.dense_h_to_4h.bias,
183
+ in_features=layer.mlp.dense_h_to_4h.in_features,
184
+ out_features=layer.mlp.dense_h_to_4h.out_features,
185
+ bias=True,
186
+ dtype=torch.half,
187
+ device=layer.mlp.dense_h_to_4h.weight.device,
188
+ empty_init=empty_init
189
+ )
190
+ layer.mlp.dense_4h_to_h = QuantizedLinear(
191
+ weight_bit_width=weight_bit_width,
192
+ weight_tensor=layer.mlp.dense_4h_to_h.weight.to(torch.cuda.current_device()),
193
+ bias_tensor=layer.mlp.dense_4h_to_h.bias,
194
+ in_features=layer.mlp.dense_4h_to_h.in_features,
195
+ out_features=layer.mlp.dense_4h_to_h.out_features,
196
+ bias=True,
197
+ dtype=torch.half,
198
+ device=layer.mlp.dense_4h_to_h.weight.device,
199
+ empty_init=empty_init
200
+ )
201
+ return model
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21167843eea23c7bb13414b3a4505f43cf15f12d9e863a31b2ac887e3b2902d5
3
+ size 14575
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1b3555a90d88e426c39c52ddb55243e6c71e4454bd8b74c88bee0f30cea0d88
3
+ size 627
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<sop>",
3
+ "eos_token": "<eop>",
4
+ "mask_token": "[MASK]",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
ptuning/output/adgen-chatglm-6b-pt-128-2e-2/checkpoint-50/tokenization_chatglm.py ADDED
@@ -0,0 +1,430 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tokenization classes for ChatGLM."""
2
+ from typing import List, Optional, Union
3
+ import os
4
+
5
+ from transformers.tokenization_utils import PreTrainedTokenizer
6
+ from transformers.utils import logging, PaddingStrategy
7
+ from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
8
+ from typing import Dict
9
+ import sentencepiece as spm
10
+ import numpy as np
11
+
12
+ logger = logging.get_logger(__name__)
13
+
14
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
15
+ "THUDM/chatglm-6b": 2048,
16
+ }
17
+
18
+
19
+ class TextTokenizer:
20
+ def __init__(self, model_path):
21
+ self.sp = spm.SentencePieceProcessor()
22
+ self.sp.Load(model_path)
23
+ self.num_tokens = self.sp.vocab_size()
24
+
25
+ def encode(self, text):
26
+ return self.sp.EncodeAsIds(text)
27
+
28
+ def decode(self, ids: List[int]):
29
+ return self.sp.DecodeIds(ids)
30
+
31
+ def tokenize(self, text):
32
+ return self.sp.EncodeAsPieces(text)
33
+
34
+ def convert_tokens_to_ids(self, tokens):
35
+ return [self.sp.PieceToId(token) for token in tokens]
36
+
37
+ def convert_token_to_id(self, token):
38
+ return self.sp.PieceToId(token)
39
+
40
+ def convert_id_to_token(self, idx):
41
+ return self.sp.IdToPiece(idx)
42
+
43
+ def __len__(self):
44
+ return self.num_tokens
45
+
46
+
47
+ class SPTokenizer:
48
+ def __init__(
49
+ self,
50
+ vocab_file,
51
+ num_image_tokens=20000,
52
+ max_blank_length=80,
53
+ byte_fallback=True,
54
+ ):
55
+ assert vocab_file is not None
56
+ self.vocab_file = vocab_file
57
+ self.num_image_tokens = num_image_tokens
58
+ self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
59
+ self.max_blank_length = max_blank_length
60
+ self.byte_fallback = byte_fallback
61
+ self.text_tokenizer = TextTokenizer(vocab_file)
62
+
63
+ def _get_text_tokenizer(self):
64
+ return self.text_tokenizer
65
+
66
+ @staticmethod
67
+ def get_blank_token(length: int):
68
+ assert length >= 2
69
+ return f"<|blank_{length}|>"
70
+
71
+ @staticmethod
72
+ def get_tab_token():
73
+ return f"<|tab|>"
74
+
75
+ @property
76
+ def num_text_tokens(self):
77
+ return self.text_tokenizer.num_tokens
78
+
79
+ @property
80
+ def num_tokens(self):
81
+ return self.num_image_tokens + self.num_text_tokens
82
+
83
+ @staticmethod
84
+ def _encode_whitespaces(text: str, max_len: int = 80):
85
+ text = text.replace("\t", SPTokenizer.get_tab_token())
86
+ for i in range(max_len, 1, -1):
87
+ text = text.replace(" " * i, SPTokenizer.get_blank_token(i))
88
+ return text
89
+
90
+ def _preprocess(self, text: str, linebreak=True, whitespaces=True):
91
+ if linebreak:
92
+ text = text.replace("\n", "<n>")
93
+ if whitespaces:
94
+ text = self._encode_whitespaces(text, max_len=self.max_blank_length)
95
+ return text
96
+
97
+ def encode(
98
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
99
+ ) -> List[int]:
100
+ """
101
+ @param text: Text to encode.
102
+ @param linebreak: Whether to encode newline (\n) in text.
103
+ @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
104
+ @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
105
+ @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
106
+ """
107
+ text = self._preprocess(text, linebreak, whitespaces)
108
+ if not add_dummy_prefix:
109
+ text = "<n>" + text
110
+ tmp = self._get_text_tokenizer().encode(text)
111
+ tokens = [x + self.num_image_tokens for x in tmp]
112
+ return tokens if add_dummy_prefix else tokens[2:]
113
+
114
+ def decode(self, text_ids: List[int]) -> str:
115
+ ids = [int(_id) - self.num_image_tokens for _id in text_ids]
116
+ ids = [_id for _id in ids if _id >= 0]
117
+ text = self._get_text_tokenizer().decode(ids)
118
+ text = text.replace("<n>", "\n")
119
+ text = text.replace(SPTokenizer.get_tab_token(), "\t")
120
+ for i in range(2, self.max_blank_length + 1):
121
+ text = text.replace(self.get_blank_token(i), " " * i)
122
+ return text
123
+
124
+ def tokenize(
125
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
126
+ ) -> List[str]:
127
+ """
128
+ @param text: Text to encode.
129
+ @param linebreak: Whether to encode newline (\n) in text.
130
+ @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
131
+ @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
132
+ @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
133
+ """
134
+ text = self._preprocess(text, linebreak, whitespaces)
135
+ if not add_dummy_prefix:
136
+ text = "<n>" + text
137
+ tokens = self._get_text_tokenizer().tokenize(text)
138
+ return tokens if add_dummy_prefix else tokens[2:]
139
+
140
+ def __getitem__(self, x: Union[int, str]):
141
+ if isinstance(x, int):
142
+ if x < self.num_image_tokens:
143
+ return "<image_{}>".format(x)
144
+ else:
145
+ return self.text_tokenizer.convert_id_to_token(x - self.num_image_tokens)
146
+ elif isinstance(x, str):
147
+ if x.startswith("<image_") and x.endswith(">") and x[7:-1].isdigit():
148
+ return int(x[7:-1])
149
+ else:
150
+ return self.text_tokenizer.convert_token_to_id(x) + self.num_image_tokens
151
+ else:
152
+ raise ValueError("The key should be str or int.")
153
+
154
+
155
+ class ChatGLMTokenizer(PreTrainedTokenizer):
156
+ """
157
+ Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding.
158
+
159
+ Args:
160
+ vocab_file (`str`):
161
+ Path to the vocabulary file.
162
+ """
163
+
164
+ vocab_files_names = {"vocab_file": "ice_text.model"}
165
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
166
+ model_input_names = ["input_ids", "attention_mask", "position_ids"]
167
+
168
+ def __init__(
169
+ self,
170
+ vocab_file,
171
+ do_lower_case=False,
172
+ remove_space=False,
173
+ bos_token='<sop>',
174
+ eos_token='<eop>',
175
+ end_token='</s>',
176
+ mask_token='[MASK]',
177
+ gmask_token='[gMASK]',
178
+ padding_side="left",
179
+ pad_token="<pad>",
180
+ unk_token="<unk>",
181
+ num_image_tokens=20000,
182
+ **kwargs
183
+ ) -> None:
184
+ super().__init__(
185
+ do_lower_case=do_lower_case,
186
+ remove_space=remove_space,
187
+ padding_side=padding_side,
188
+ bos_token=bos_token,
189
+ eos_token=eos_token,
190
+ end_token=end_token,
191
+ mask_token=mask_token,
192
+ gmask_token=gmask_token,
193
+ pad_token=pad_token,
194
+ unk_token=unk_token,
195
+ num_image_tokens=num_image_tokens,
196
+ **kwargs
197
+ )
198
+
199
+ self.do_lower_case = do_lower_case
200
+ self.remove_space = remove_space
201
+ self.vocab_file = vocab_file
202
+
203
+ self.bos_token = bos_token
204
+ self.eos_token = eos_token
205
+ self.end_token = end_token
206
+ self.mask_token = mask_token
207
+ self.gmask_token = gmask_token
208
+
209
+ self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
210
+
211
+ """ Initialisation """
212
+
213
+ @property
214
+ def gmask_token_id(self) -> Optional[int]:
215
+ if self.gmask_token is None:
216
+ return None
217
+ return self.convert_tokens_to_ids(self.gmask_token)
218
+
219
+ @property
220
+ def end_token_id(self) -> Optional[int]:
221
+ """
222
+ `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been
223
+ set.
224
+ """
225
+ if self.end_token is None:
226
+ return None
227
+ return self.convert_tokens_to_ids(self.end_token)
228
+
229
+ @property
230
+ def vocab_size(self):
231
+ """ Returns vocab size """
232
+ return self.sp_tokenizer.num_tokens
233
+
234
+ def get_vocab(self):
235
+ """ Returns vocab as a dict """
236
+ vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
237
+ vocab.update(self.added_tokens_encoder)
238
+ return vocab
239
+
240
+ def preprocess_text(self, inputs):
241
+ if self.remove_space:
242
+ outputs = " ".join(inputs.strip().split())
243
+ else:
244
+ outputs = inputs
245
+
246
+ if self.do_lower_case:
247
+ outputs = outputs.lower()
248
+
249
+ return outputs
250
+
251
+ def _tokenize(self, text, **kwargs):
252
+ """ Returns a tokenized string. """
253
+ text = self.preprocess_text(text)
254
+
255
+ seq = self.sp_tokenizer.tokenize(text)
256
+
257
+ return seq
258
+
259
+ def _decode(
260
+ self,
261
+ token_ids: Union[int, List[int]],
262
+ skip_special_tokens: bool = False,
263
+ clean_up_tokenization_spaces: bool = True,
264
+ **kwargs
265
+ ) -> str:
266
+ if isinstance(token_ids, int):
267
+ token_ids = [token_ids]
268
+ if len(token_ids) == 0:
269
+ return ""
270
+ if self.pad_token_id in token_ids: # remove pad
271
+ token_ids = list(filter((self.pad_token_id).__ne__, token_ids))
272
+ return self.sp_tokenizer.decode(token_ids)
273
+
274
+ def _convert_token_to_id(self, token):
275
+ """ Converts a token (str) in an id using the vocab. """
276
+ return self.sp_tokenizer[token]
277
+
278
+ def _convert_id_to_token(self, index):
279
+ """Converts an index (integer) in a token (str) using the vocab."""
280
+ return self.sp_tokenizer[index]
281
+
282
+ def save_vocabulary(self, save_directory, filename_prefix=None):
283
+ """
284
+ Save the vocabulary and special tokens file to a directory.
285
+
286
+ Args:
287
+ save_directory (`str`):
288
+ The directory in which to save the vocabulary.
289
+ filename_prefix (`str`, *optional*):
290
+ An optional prefix to add to the named of the saved files.
291
+
292
+ Returns:
293
+ `Tuple(str)`: Paths to the files saved.
294
+ """
295
+ if os.path.isdir(save_directory):
296
+ vocab_file = os.path.join(
297
+ save_directory, self.vocab_files_names["vocab_file"]
298
+ )
299
+ else:
300
+ vocab_file = save_directory
301
+
302
+ with open(self.vocab_file, 'rb') as fin:
303
+ proto_str = fin.read()
304
+
305
+ with open(vocab_file, "wb") as writer:
306
+ writer.write(proto_str)
307
+
308
+ return (vocab_file,)
309
+
310
+ def build_inputs_with_special_tokens(
311
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
312
+ ) -> List[int]:
313
+ """
314
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
315
+ adding special tokens. A BERT sequence has the following format:
316
+
317
+ - single sequence: `[CLS] X [SEP]`
318
+ - pair of sequences: `[CLS] A [SEP] B [SEP]`
319
+
320
+ Args:
321
+ token_ids_0 (`List[int]`):
322
+ List of IDs to which the special tokens will be added.
323
+ token_ids_1 (`List[int]`, *optional*):
324
+ Optional second list of IDs for sequence pairs.
325
+
326
+ Returns:
327
+ `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
328
+ """
329
+ gmask_id = self.sp_tokenizer[self.gmask_token]
330
+ eos_id = self.sp_tokenizer[self.eos_token]
331
+ token_ids_0 = token_ids_0 + [gmask_id, self.sp_tokenizer[self.bos_token]]
332
+ if token_ids_1 is not None:
333
+ token_ids_0 = token_ids_0 + token_ids_1 + [eos_id]
334
+ return token_ids_0
335
+
336
+ def _pad(
337
+ self,
338
+ encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
339
+ max_length: Optional[int] = None,
340
+ padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
341
+ pad_to_multiple_of: Optional[int] = None,
342
+ return_attention_mask: Optional[bool] = None,
343
+ ) -> dict:
344
+ """
345
+ Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
346
+
347
+ Args:
348
+ encoded_inputs:
349
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
350
+ max_length: maximum length of the returned list and optionally padding length (see below).
351
+ Will truncate by taking into account the special tokens.
352
+ padding_strategy: PaddingStrategy to use for padding.
353
+
354
+ - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
355
+ - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
356
+ - PaddingStrategy.DO_NOT_PAD: Do not pad
357
+ The tokenizer padding sides are defined in self.padding_side:
358
+
359
+ - 'left': pads on the left of the sequences
360
+ - 'right': pads on the right of the sequences
361
+ pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
362
+ This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
363
+ `>= 7.5` (Volta).
364
+ return_attention_mask:
365
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
366
+ """
367
+ # Load from model defaults
368
+ bos_token_id = self.sp_tokenizer[self.bos_token]
369
+ mask_token_id = self.sp_tokenizer[self.mask_token]
370
+ gmask_token_id = self.sp_tokenizer[self.gmask_token]
371
+ assert self.padding_side == "left"
372
+
373
+ required_input = encoded_inputs[self.model_input_names[0]]
374
+ seq_length = len(required_input)
375
+
376
+ if padding_strategy == PaddingStrategy.LONGEST:
377
+ max_length = len(required_input)
378
+
379
+ if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
380
+ max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
381
+
382
+ needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
383
+
384
+ # Initialize attention mask if not present.
385
+ if max_length is not None:
386
+ if "attention_mask" not in encoded_inputs:
387
+ if bos_token_id in required_input:
388
+ context_length = required_input.index(bos_token_id)
389
+ else:
390
+ context_length = seq_length
391
+ attention_mask = np.ones((1, seq_length, seq_length))
392
+ attention_mask = np.tril(attention_mask)
393
+ attention_mask[:, :, :context_length] = 1
394
+ attention_mask = np.bool_(attention_mask < 0.5)
395
+ encoded_inputs["attention_mask"] = attention_mask
396
+
397
+ if "position_ids" not in encoded_inputs:
398
+ if bos_token_id in required_input:
399
+ context_length = required_input.index(bos_token_id)
400
+ else:
401
+ context_length = seq_length
402
+ position_ids = np.arange(seq_length, dtype=np.int64)
403
+ mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id
404
+ if mask_token in required_input:
405
+ mask_position = required_input.index(mask_token)
406
+ position_ids[context_length:] = mask_position
407
+ block_position_ids = np.concatenate(
408
+ [np.zeros(context_length, dtype=np.int64),
409
+ np.arange(1, seq_length - context_length + 1, dtype=np.int64)])
410
+ encoded_inputs["position_ids"] = np.stack([position_ids, block_position_ids], axis=0)
411
+
412
+ if needs_to_be_padded:
413
+ difference = max_length - len(required_input)
414
+
415
+ if "attention_mask" in encoded_inputs:
416
+ encoded_inputs["attention_mask"] = np.pad(encoded_inputs["attention_mask"],
417
+ pad_width=[(0, 0), (difference, 0), (difference, 0)],
418
+ mode='constant', constant_values=True)
419
+ if "token_type_ids" in encoded_inputs:
420
+ encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
421
+ "token_type_ids"
422
+ ]
423
+ if "special_tokens_mask" in encoded_inputs:
424
+ encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
425
+ if "position_ids" in encoded_inputs:
426
+ encoded_inputs["position_ids"] = np.pad(encoded_inputs["position_ids"],
427
+ pad_width=[(0, 0), (difference, 0)])
428
+ encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
429
+
430
+ return encoded_inputs