@inproceedings{d4e262b965424f01bb350f953a9baf82,
title = "JANUS: Joint Autoregressive and Non-autoregressive Training with Auxiliary Loss for Sequence Generation",
abstract = "Transformer-based autoregressive and non-autoregressive models have played an essential role in sequence generation tasks. The autoregressive model can obtain excellent performance, while the non-autoregressive model brings fast decoding speed for inference. In this paper, we propose JANUS, a Joint Autoregressive and Non-autoregressive training method using aUxiliary losS to enhance the model performance in both AR and NAR manner simultaneously and effectively alleviate the problem of distribution discrepancy. Further, we pre-train BART with JANUS on a large corpus with minimal cost (16 GPU days) and make the BART-JANUS capable of non-autoregressive generation, demonstrating that our approach can transfer the AR knowledge to NAR. Empirically, we show our approach and BART-JANUS can achieve significant improvement on multiple generation tasks, including machine translation and GLGE benchmarks. Our code is available at Github.",
author = "Xiaobo Liang and Juntao Li and Lijun Wu and Min Zhang",
note = "Publisher Copyright: {\textcopyright} 2022 Association for Computational Linguistics.; 2022 Conference on Empirical Methods in Natural Language Processing, EMNLP 2022 ; Conference date: 07-12-2022 Through 11-12-2022",
year = "2022",
doi = "10.18653/v1/2022.emnlp-main.550",
language = "英语",
series = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, EMNLP 2022",
publisher = "Association for Computational Linguistics (ACL)",
pages = "8050--8060",
editor = "Yoav Goldberg and Zornitsa Kozareva and Yue Zhang",
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, EMNLP 2022",
address = "澳大利亚",
}