@inproceedings{5e592a5c4b9d4363a70ed71de08f7ecd,
title = "Bilingual Alignment Pre-Training for Zero-Shot Cross-Lingual Transfer",
abstract = "Multilingual pre-trained models have achieved remarkable performance on cross-lingual transfer learning. Some multilingual models such as mBERT, have been pre-trained on unlabeled corpora, therefore the embeddings of different languages in the models may not be aligned very well. In this paper, we aim to improve the zero-shot cross-lingual transfer performance by proposing a pre-training task named Word-Exchange Aligning Model (WEAM), which uses the statistical alignment information as the prior knowledge to guide cross-lingual word prediction. We evaluate our model on multilingual machine reading comprehension task MLQA and natural language interface task XNLI. The results show that WEAM can significantly improve the zero-shot performance.",
author = "Ziqing Yang and Wentao Ma and Yiming Cui and Jiani Ye and Wanxiang Che and Shijin Wang",
note = "Publisher Copyright: {\textcopyright} 2021 Association for Computational Linguistics.; 3rd Workshop on Machine Reading for Question Answering, MRQA 2021 ; Conference date: 10-11-2021",
year = "2021",
language = "英语",
series = "Proceedings of the 3rd Workshop on Machine Reading for Question Answering, MRQA 2021",
publisher = "Association for Computational Linguistics (ACL)",
pages = "100--105",
editor = "Adam Fisch and Alon Talmor and Danqi Chen and Eunsol Choi and Minjoon Seo and Patrick Lewis and Robin Jia and Sewon Min",
booktitle = "Proceedings of the 3rd Workshop on Machine Reading for Question Answering, MRQA 2021",
address = "澳大利亚",
}