Adapter in Pfeiffer architecture trained on the binary SST task for 20 epochs with early stopping and a learning rate of 1e-4. See https://arxiv.org/pdf/2007.07779.pdf.
model = BertForSequenceClassification.from_pretrained("bert-base-uncased") config = AdapterConfig.load("pfeiffer") model.load_adapter("sentiment/sst-2@ukp", "text_task", config=config)
{ "ln_after": false, "ln_before": false, "mh_adapter": false, "output_adapter": true, "adapter_residual_before_ln": false, "non_linearity": "relu", "original_ln_after": true, "original_ln_before": true, "reduction_factor": 16, "residual_before_ln": true, "invertible_adapter": { "block_type": "nice", "non_linearity": "relu", "reduction_factor": 2 } }
Identifier | Comment | Score | Download |
---|---|---|---|
1 DEFAULT |
@article{pfeiffer2020AdapterHub, title={AdapterHub: A Framework for Adapting Transformers}, author={Jonas Pfeiffer and Andreas R\"uckl\'{e} and Clifton Poth and Aishwarya Kamath and Ivan Vuli\'{c} and Sebastian Ruder and Kyunghyun Cho and Iryna Gurevych}, journal={arXiv preprint}, year={2020}, url={https://arxiv.org/abs/2007.07779} }
@misc{pfeiffer2020adapterfusion, title={AdapterFusion: Non-Destructive Task Composition for Transfer Learning}, author={Jonas Pfeiffer and Aishwarya Kamath and Andreas Rücklé and Kyunghyun Cho and Iryna Gurevych}, year={2020}, eprint={2005.00247}, archivePrefix={arXiv}, primaryClass={cs.CL} }
@inproceedings{Socher2013RecursiveDM, title={Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank}, author={Richard Socher and Alex Perelygin and Jean Wu and Jason Chuang and Christopher D. Manning and Andrew Y. Ng and Christopher Potts}, booktitle={EMNLP}, year={2013} }