diff --git a/dalle2_pytorch/trainer.py b/dalle2_pytorch/trainer.py
index 6c5d1f5..122da24 100644
--- a/dalle2_pytorch/trainer.py
+++ b/dalle2_pytorch/trainer.py
@@ -178,7 +178,7 @@ class EMA(nn.Module):
     def __init__(
         self,
         model,
-        beta = 0.9999,
+        beta = 0.99,
         update_after_step = 1000,
         update_every = 10,
     ):
diff --git a/dalle2_pytorch/version.py b/dalle2_pytorch/version.py
index 02f8497..e2f45ae 100644
--- a/dalle2_pytorch/version.py
+++ b/dalle2_pytorch/version.py
@@ -1 +1 @@
-__version__ = '0.6.4'
+__version__ = '0.6.5'