take care of gradient accumulation automatically for researchers, by passing in a max_batch_size on the decoder or diffusion prior trainer forward

dalle2_pytorch/train.py

@@ -68,23 +68,18 @@ def split(t, split_size = None):
 
 def split_args_and_kwargs(x, *args, split_size = None, **kwargs):
     batch_size = len(x)
-    split_size = default(split_size, batch_size)
-    chunk_size = ceil(batch_size / split_size)
+    chunk_size = ceil(batch_size / default(split_size, batch_size))
 
     dict_len = len(kwargs)
     dict_keys = kwargs.keys()
     all_args = (x, *args, *kwargs.values())
-    len_all_args = len(all_args)
-    split_kwargs_index = len_all_args - dict_len
-
     split_all_args = [split(arg, split_size = split_size) if exists(arg) and isinstance(arg, (torch.Tensor, Iterable)) else ((arg,) * chunk_size) for arg in all_args]
     chunk_sizes = tuple(map(len, split_all_args[0]))
 
     for (chunk_size, *chunked_all_args) in tuple(zip(chunk_sizes, *split_all_args)):
-        chunked_args, chunked_kwargs_values = chunked_all_args[:split_kwargs_index], chunked_all_args[split_kwargs_index:]
+        chunked_args, chunked_kwargs_values = chunked_all_args[:-dict_len], chunked_all_args[-dict_len:]
         chunked_kwargs = dict(tuple(zip(dict_keys, chunked_kwargs_values)))
-        chunk_size_frac = chunk_size / batch_size
-        yield chunk_size_frac, (chunked_args, chunked_kwargs)
+        yield chunk_size, (chunked_args, chunked_kwargs)
 
 # print helpers
 
@@ -254,22 +249,24 @@ class DiffusionPriorTrainer(nn.Module):
 
     def forward(
         self,
-        x,
         *args,
         max_batch_size = None,
         **kwargs
     ):
+        total_samples = 0
         total_loss = 0.
 
-        for chunk_size_frac, (chunked_args, chunked_kwargs) in split_args_and_kwargs(x, *args, split_size = max_batch_size, **kwargs):
+        for chunk_size, (chunked_args, chunked_kwargs) in split_args_and_kwargs(*args, split_size = max_batch_size, **kwargs):
             with autocast(enabled = self.amp):
-                loss = self.diffusion_prior(*chunked_args, **chunked_kwargs)
+                loss = self.diffusion_prior(*args, **kwargs)
 
-            loss = loss * chunk_size_frac
-            total_loss += loss.item()
-            self.scaler.scale(loss).backward()
+                total_loss += loss.item() * chunk_size
+                total_samples += chunk_size
 
-        return total_loss
+                scaled_loss = self.scaler.scale(loss)
+                scaled_loss.backward()
+
+        return total_loss / total_samples
 
 # decoder trainer
 
@@ -383,14 +380,17 @@ class DecoderTrainer(nn.Module):
         max_batch_size = None,
         **kwargs
     ):
+        total_samples = 0
         total_loss = 0.
 
-        for chunk_size_frac, (chunked_args, chunked_kwargs) in split_args_and_kwargs(x, split_size = max_batch_size, **kwargs):
+        for chunk_size, (chunked_args, chunked_kwargs) in split_args_and_kwargs(x, split_size = max_batch_size, **kwargs):
             with autocast(enabled = self.amp):
                 loss = self.decoder(*chunked_args, unet_number = unet_number, **chunked_kwargs)
 
-            loss = loss * chunk_size_frac
-            total_loss += loss.item()
-            self.scale(loss, unet_number = unet_number).backward()
+                total_loss += loss.item() * chunk_size
+                total_samples += chunk_size
 
-        return total_loss
+                scaled_loss = self.scale(loss, unet_number = unet_number)
+                scaled_loss.backward()
+
+        return total_loss / total_samples

setup.py

@@ -10,7 +10,7 @@ setup(
       'dream = dalle2_pytorch.cli:dream'
     ],
   },
-  version = '0.2.28',
+  version = '0.2.25',
   license='MIT',
   description = 'DALL-E 2',
   author = 'Phil Wang',