fairseq/optim/fp16_optimizer.py

Survived

Mutant 1894

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -14,7 +14,7 @@
 
     def __init__(
         self, init_scale=2.**15, scale_factor=2., scale_window=2000,
-        tolerance=0.05, threshold=None,
+        tolerance=1.05, threshold=None,
     ):
         self.loss_scale = init_scale
         self.scale_factor = scale_factor

Mutant 1895

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -46,7 +46,6 @@
         if self.threshold is not None:
             self.loss_scale = max(self.loss_scale, self.threshold)
 
-    @staticmethod
     def has_overflow(grad_norm):
         # detect inf and nan
         if grad_norm == float('inf') or grad_norm != grad_norm:

Mutant 1896

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -60,7 +60,6 @@
         # forward __init__ call to the next class in mro(method resolution order)
         super().__init__(*args, **kwargs)
 
-    @property
     def has_flat_params(self):
         return torch.is_tensor(self.fp32_params)
 

Mutant 1897

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -64,7 +64,6 @@
     def has_flat_params(self):
         return torch.is_tensor(self.fp32_params)
 
-    @classmethod
     def build_fp32_params(cls, params, flatten=True):
         # create FP32 copy of parameters and grads
         if flatten:

Mutant 1898

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -65,7 +65,7 @@
         return torch.is_tensor(self.fp32_params)
 
     @classmethod
-    def build_fp32_params(cls, params, flatten=True):
+    def build_fp32_params(cls, params, flatten=False):
         # create FP32 copy of parameters and grads
         if flatten:
             total_param_size = sum(p.data.numel() for p in params)

Mutant 1899

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -117,7 +117,7 @@
         loss.backward()
         self._needs_sync = True
 
-    def _sync_fp16_grads_to_fp32(self, multiply_grads=1.):
+    def _sync_fp16_grads_to_fp32(self, multiply_grads=2.0):
         if self._needs_sync:
             if self.scaler is not None:
                 # correct for dynamic loss scaler

Mutant 1900

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -246,7 +246,6 @@
             # disable loss scaling for bfloat16
             self.scaler = None
 
-    @classmethod
     def build_optimizer(cls, args, params):
         """
         Args:

Mutant 1901

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -268,7 +268,6 @@
             )
         return cls(args, params, fp32_optimizer, fp32_params)
 
-    @property
     def optimizer(self):
         return self.fp32_optimizer.optimizer
 

Mutant 1902

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -272,7 +272,6 @@
     def optimizer(self):
         return self.fp32_optimizer.optimizer
 
-    @property
     def optimizer_config(self):
         return self.fp32_optimizer.optimizer_config
 

Mutant 1903

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -289,7 +289,6 @@
         # forward __init__ call to the next class in MRO (method resolution order)
         super().__init__(*args, **kwargs)
 
-    @property
     def has_flat_params(self):
         return False
 

Mutant 1904

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -446,7 +446,6 @@
             # disable loss scaling for bfloat16
             self.scaler = None
 
-    @classmethod
     def build_optimizer(cls, args, params):
         """
         Args:

Mutant 1905

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -456,7 +456,6 @@
         fp16_optimizer = optim.build_optimizer(args, params)
         return cls(args, params, fp16_optimizer)
 
-    @property
     def optimizer(self):
         return self.wrapped_optimizer.optimizer
 

Mutant 1906

--- fairseq/optim/fp16_optimizer.py
+++ fairseq/optim/fp16_optimizer.py
@@ -460,7 +460,6 @@
     def optimizer(self):
         return self.wrapped_optimizer.optimizer
 
-    @property
     def optimizer_config(self):
         return self.wrapped_optimizer.optimizer_config