diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py
index 72550c12e2..59a82363c6 100644
--- a/composer/models/huggingface.py
+++ b/composer/models/huggingface.py
@@ -167,7 +167,7 @@ def _check_tokenizer_and_maybe_resize_embeddings(self, allow_embedding_resizing:
                     f' This would cause an error during training.'
                     f' Resizing the model embeddings to {len(self.tokenizer)} from {self.config.vocab_size}.',
                 )
-                self.model.resize_token_embeddings(len(self.tokenizer))
+                self.model.resize_token_embeddings(len(self.tokenizer), mean_resizing=False)
             else:
                 raise ValueError(
                     f'The number of tokens in the tokenizer is greater than the number of tokens in the model.'