@inproceedings{bfe3bf595cc1492db4e9466f4d06f532,
title = "Harnessing the Power of Prompt Experts: Efficient Knowledge Distillation for Enhanced Language Understanding",
abstract = "Enhanced with machine learning, language understanding enables computers to not only comprehend but also learn from human language, thereby augmenting the capabilities of various NLP applications in AI. Multi-teacher distillation is a prominent method for knowledge transfer in language understanding, leveraging multiple teacher models to train a single student model. However, this approach incurs significant time and storage costs for training and inference with multiple teachers. To address these issues, we introduce PEE-KD, a simple yet effective framework that generates supervision for training a student model from a single language model. We implemented a language model with multiple prompts as the teacher model in multi-teacher distillation, achieving lightweight training and inference. Additionally, we propose an uncertainty-based method to enhance the robustness and accuracy of multiple prompts during training, along with a selector module to improve the inference speed of multi-teacher models. Experiments on NLU and NER tasks demonstrate that PEE-KD improves accuracy by up to 1.8\% and efficiency by up to 140\% compared to existing methods. Logit visualization comparisons between teacher and student models further validate the effectiveness of our approach. Our code and data are available at https://anonymous.4open.science/r/PEEKD-DF50/.",
keywords = "Deep learning, Multi-teacher knowledge distillation, Prompt tuning",
author = "Xv Meng and Jun Rao and Shuhan Qi and Lei Wang and Jing Xiao and Xuan Wang",
note = "Publisher Copyright: {\textcopyright} The Author(s), under exclusive license to Springer Nature Switzerland AG 2024.; European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases, ECML PKDD 2024 ; Conference date: 09-09-2024 Through 13-09-2024",
year = "2024",
doi = "10.1007/978-3-031-70371-3\_13",
language = "英语",
isbn = "9783031703706",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "218--234",
editor = "Albert Bifet and Povilas Daniu{\v s}is and Jesse Davis and Tomas Krilavi{\v c}ius and Meelis Kull and Eirini Ntoutsi and Kai Puolam{\"a}ki and Indrė {\v Z}liobaitė",
booktitle = "Machine Learning and Knowledge Discovery in Databases. Research Track and Demo Track - European Conference, ECML PKDD 2024, Proceedings",
address = "德国",
}