make sure vqgan-vae trainer supports mixed precision

dalle2_pytorch/dalle2_pytorch.py

@@ -264,7 +264,7 @@ class OpenAIClipAdapter(BaseClipAdapter):
         text_embed = self.clip.encode_text(text)
         text_encodings = self.text_encodings
         del self.text_encodings
-        return EmbeddedText(text_embed.float(), text_encodings.float(), text_mask)
+        return EmbeddedText(l2norm(text_embed.float()), text_encodings.float(), text_mask)
 
     @torch.no_grad()
     def embed_image(self, image):
@@ -272,7 +272,7 @@ class OpenAIClipAdapter(BaseClipAdapter):
         image = resize_image_to(image, self.image_size)
         image = self.clip_normalize(unnormalize_img(image))
         image_embed = self.clip.encode_image(image)
-        return EmbeddedImage(image_embed.float(), None)
+        return EmbeddedImage(l2norm(image_embed.float()), None)
 
 # classifier free guidance functions
 

dalle2_pytorch/train_vqgan_vae.py

@@ -3,14 +3,15 @@ import copy
 from random import choice
 from pathlib import Path
 from shutil import rmtree
+from PIL import Image
 
 import torch
 from torch import nn
-
+from torch.cuda.amp import autocast, GradScaler
-from PIL import Image
-from torchvision.datasets import ImageFolder
-import torchvision.transforms as T
 from torch.utils.data import Dataset, DataLoader, random_split
+
+import torchvision.transforms as T
+from torchvision.datasets import ImageFolder
 from torchvision.utils import make_grid, save_image
 
 from einops import rearrange
@@ -99,6 +100,7 @@ class VQGanVAETrainer(nn.Module):
         ema_update_after_step = 2000,
         ema_update_every = 10,
         apply_grad_penalty_every = 4,
+        amp = False
     ):
         super().__init__()
         assert isinstance(vae, VQGanVAE), 'vae must be instance of VQGanVAE'
@@ -120,6 +122,10 @@ class VQGanVAETrainer(nn.Module):
         self.optim = get_optimizer(vae_parameters, lr = lr, wd = wd)
         self.discr_optim = get_optimizer(discr_parameters, lr = lr, wd = wd)
 
+        self.amp = amp
+        self.scaler = GradScaler(enabled = amp)
+        self.discr_scaler = GradScaler(enabled = amp)
+
         # create dataset
 
         self.ds = ImageDataset(folder, image_size = image_size)
@@ -178,20 +184,22 @@ class VQGanVAETrainer(nn.Module):
             img = next(self.dl)
             img = img.to(device)
 
-            loss = self.vae(
+            with autocast(enabled = self.amp):
-                img,
+                loss = self.vae(
-                return_loss = True,
+                    img,
-                apply_grad_penalty = apply_grad_penalty
+                    return_loss = True,
-            )
+                    apply_grad_penalty = apply_grad_penalty
+                )
+
+
+                self.scaler.scale(loss / self.grad_accum_every).backward()
 
             accum_log(logs, {'loss': loss.item() / self.grad_accum_every})
 
-            (loss / self.grad_accum_every).backward()
+        self.scaler.step(self.optim)
-
+        self.scaler.update()
-        self.optim.step()
         self.optim.zero_grad()
 
-
         # update discriminator
 
         if exists(self.vae.discr):
@@ -200,12 +208,15 @@ class VQGanVAETrainer(nn.Module):
                 img = next(self.dl)
                 img = img.to(device)
 
-                loss = self.vae(img, return_discr_loss = True)
+                with autocast(enabled = self.amp):
+                    loss = self.vae(img, return_discr_loss = True)
+
+                    self.discr_scaler.scale(loss / self.grad_accum_every).backward()
+
                 accum_log(logs, {'discr_loss': loss.item() / self.grad_accum_every})
 
-                (loss / self.grad_accum_every).backward()
+            self.discr_scaler.step(self.discr_optim)
-
+            self.discr_scaler.update()
-            self.discr_optim.step()
             self.discr_optim.zero_grad()
 
             # log

setup.py

@@ -10,7 +10,7 @@ setup(
       'dream = dalle2_pytorch.cli:dream'
     ],
   },
-  version = '0.1.2',
+  version = '0.1.5',
   license='MIT',
   description = 'DALL-E 2',
   author = 'Phil Wang',