cleverhans/future/torch/attacks/spsa.py
Killed 82 out of 114 mutantsSurvived
Survived mutation testing. These mutants show holes in your test suite.Mutant 22
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -5,7 +5,7 @@
from cleverhans.future.torch.utils import clip_eta
-def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
+def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=+np.inf, clip_max=np.inf, y=None,
targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
"""
Mutant 24
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -6,7 +6,7 @@
def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
- targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
+ targeted=False, early_stop_loss_threshold=None, learning_rate=1.01, delta=0.01,
spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
"""
This implements the SPSA adversary, as in https://arxiv.org/abs/1802.05666
Mutant 25
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -6,7 +6,7 @@
def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
- targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
+ targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=1.01,
spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
"""
This implements the SPSA adversary, as in https://arxiv.org/abs/1802.05666
Mutant 26
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -7,7 +7,7 @@
def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
- spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
+ spsa_samples=129, spsa_iters=1, is_debug=False, sanity_checks=True):
"""
This implements the SPSA adversary, as in https://arxiv.org/abs/1802.05666
(Uesato et al. 2018). SPSA is a gradient-free optimization method, which is useful when
Mutant 28
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -7,7 +7,7 @@
def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
- spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
+ spsa_samples=128, spsa_iters=1, is_debug=True, sanity_checks=True):
"""
This implements the SPSA adversary, as in https://arxiv.org/abs/1802.05666
(Uesato et al. 2018). SPSA is a gradient-free optimization method, which is useful when
Mutant 29
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -7,7 +7,7 @@
def spsa(model_fn, x, eps, nb_iter, norm=np.inf, clip_min=-np.inf, clip_max=np.inf, y=None,
targeted=False, early_stop_loss_threshold=None, learning_rate=0.01, delta=0.01,
- spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=True):
+ spsa_samples=128, spsa_iters=1, is_debug=False, sanity_checks=False):
"""
This implements the SPSA adversary, as in https://arxiv.org/abs/1802.05666
(Uesato et al. 2018). SPSA is a gradient-free optimization method, which is useful when
Mutant 42
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -65,7 +65,7 @@
adv_x.append(adv_x_single)
return torch.cat(adv_x)
- if eps < 0:
+ if eps <= 0:
raise ValueError(
"eps must be greater than or equal to 0, got {} instead".format(eps))
if eps == 0:
Mutant 46
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -71,7 +71,7 @@
if eps == 0:
return x
- if clip_min is not None and clip_max is not None:
+ if clip_min is None and clip_max is not None:
if clip_min > clip_max:
raise ValueError(
"clip_min must be less than or equal to clip_max, got clip_min={} and clip_max={}"
Mutant 47
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -71,7 +71,7 @@
if eps == 0:
return x
- if clip_min is not None and clip_max is not None:
+ if clip_min is not None and clip_max is None:
if clip_min > clip_max:
raise ValueError(
"clip_min must be less than or equal to clip_max, got clip_min={} and clip_max={}"
Mutant 48
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -71,7 +71,7 @@
if eps == 0:
return x
- if clip_min is not None and clip_max is not None:
+ if clip_min is not None or clip_max is not None:
if clip_min > clip_max:
raise ValueError(
"clip_min must be less than or equal to clip_max, got clip_min={} and clip_max={}"
Mutant 49
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -72,7 +72,7 @@
return x
if clip_min is not None and clip_max is not None:
- if clip_min > clip_max:
+ if clip_min >= clip_max:
raise ValueError(
"clip_min must be less than or equal to clip_max, got clip_min={} and clip_max={}"
.format(clip_min, clip_max))
Mutant 51
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -80,7 +80,7 @@
asserts = []
# If a data range was specified, check that the input was in that range
- asserts.append(torch.all(x >= clip_min))
+ asserts.append(torch.all(x > clip_min))
asserts.append(torch.all(x <= clip_max))
if is_debug:
Mutant 52
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -81,7 +81,7 @@
# If a data range was specified, check that the input was in that range
asserts.append(torch.all(x >= clip_min))
- asserts.append(torch.all(x <= clip_max))
+ asserts.append(torch.all(x < clip_max))
if is_debug:
print("Starting SPSA attack with eps = {}".format(eps))
Mutant 53
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -86,7 +86,7 @@
if is_debug:
print("Starting SPSA attack with eps = {}".format(eps))
- perturbation = (torch.rand_like(x) * 2 - 1) * eps
+ perturbation = (torch.rand_like(x) / 2 - 1) * eps
_project_perturbation(perturbation, norm, eps, x, clip_min, clip_max)
optimizer = optim.Adam([perturbation], lr=learning_rate)
Mutant 54
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -86,7 +86,7 @@
if is_debug:
print("Starting SPSA attack with eps = {}".format(eps))
- perturbation = (torch.rand_like(x) * 2 - 1) * eps
+ perturbation = (torch.rand_like(x) * 3 - 1) * eps
_project_perturbation(perturbation, norm, eps, x, clip_min, clip_max)
optimizer = optim.Adam([perturbation], lr=learning_rate)
Mutant 62
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -96,7 +96,7 @@
Margin logit loss, with correct sign for targeted vs untargeted loss.
"""
logits = model_fn(x + pert)
- loss_multiplier = 1 if targeted else -1
+ loss_multiplier = 2 if targeted else -1
return loss_multiplier * _margin_logit_loss(logits, y.expand(len(pert)))
spsa_grad = _compute_spsa_gradient(loss_fn, x, delta=delta,
Mutant 64
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -96,7 +96,7 @@
Margin logit loss, with correct sign for targeted vs untargeted loss.
"""
logits = model_fn(x + pert)
- loss_multiplier = 1 if targeted else -1
+ loss_multiplier = 1 if targeted else -2
return loss_multiplier * _margin_logit_loss(logits, y.expand(len(pert)))
spsa_grad = _compute_spsa_gradient(loss_fn, x, delta=delta,
Mutant 69
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -106,7 +106,7 @@
_project_perturbation(perturbation, norm, eps, x, clip_min, clip_max)
- loss = loss_fn(perturbation).item()
+ loss = None
if is_debug:
print('Iteration {}: loss = {}'.format(i, loss))
if early_stop_loss_threshold is not None and loss < early_stop_loss_threshold:
Mutant 71
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -109,7 +109,7 @@
loss = loss_fn(perturbation).item()
if is_debug:
print('Iteration {}: loss = {}'.format(i, loss))
- if early_stop_loss_threshold is not None and loss < early_stop_loss_threshold:
+ if early_stop_loss_threshold is not None and loss <= early_stop_loss_threshold:
break
adv_x = torch.clamp((x + perturbation).detach(), clip_min, clip_max)
Mutant 75
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -114,7 +114,7 @@
adv_x = torch.clamp((x + perturbation).detach(), clip_min, clip_max)
- if norm == np.inf:
+ if norm != np.inf:
asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
else:
asserts.append(torch.all(torch.abs(
Mutant 77
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -115,7 +115,7 @@
adv_x = torch.clamp((x + perturbation).detach(), clip_min, clip_max)
if norm == np.inf:
- asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
+ asserts.append(torch.all(torch.abs(adv_x - x) < eps + 1e-6))
else:
asserts.append(torch.all(torch.abs(
torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
Mutant 79
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -115,7 +115,7 @@
adv_x = torch.clamp((x + perturbation).detach(), clip_min, clip_max)
if norm == np.inf:
- asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
+ asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1.000001))
else:
asserts.append(torch.all(torch.abs(
torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
Mutant 81
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -118,7 +118,7 @@
asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
else:
asserts.append(torch.all(torch.abs(
- torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
+ torch.renorm(adv_x - x, p=norm, dim=1, maxnorm=eps) - (adv_x - x)) < 1e-6))
asserts.append(torch.all(adv_x >= clip_min))
asserts.append(torch.all(adv_x <= clip_max))
Mutant 84
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -118,7 +118,7 @@
asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
else:
asserts.append(torch.all(torch.abs(
- torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
+ torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) <= 1e-6))
asserts.append(torch.all(adv_x >= clip_min))
asserts.append(torch.all(adv_x <= clip_max))
Mutant 85
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -118,7 +118,7 @@
asserts.append(torch.all(torch.abs(adv_x - x) <= eps + 1e-6))
else:
asserts.append(torch.all(torch.abs(
- torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
+ torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1.000001))
asserts.append(torch.all(adv_x >= clip_min))
asserts.append(torch.all(adv_x <= clip_max))
Mutant 86
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -119,7 +119,7 @@
else:
asserts.append(torch.all(torch.abs(
torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
- asserts.append(torch.all(adv_x >= clip_min))
+ asserts.append(torch.all(adv_x > clip_min))
asserts.append(torch.all(adv_x <= clip_max))
if sanity_checks:
Mutant 87
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -120,7 +120,7 @@
asserts.append(torch.all(torch.abs(
torch.renorm(adv_x - x, p=norm, dim=0, maxnorm=eps) - (adv_x - x)) < 1e-6))
asserts.append(torch.all(adv_x >= clip_min))
- asserts.append(torch.all(adv_x <= clip_max))
+ asserts.append(torch.all(adv_x < clip_max))
if sanity_checks:
assert np.all(asserts)
Mutant 88
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -128,7 +128,7 @@
return adv_x
-def _project_perturbation(perturbation, norm, epsilon, input_image, clip_min=-np.inf,
+def _project_perturbation(perturbation, norm, epsilon, input_image, clip_min=+np.inf,
clip_max=np.inf):
"""
Project `perturbation` onto L-infinity ball of radius `epsilon`. Also project into
Mutant 104
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -157,7 +157,7 @@
grad_list = []
for i in range(iters):
- delta_x = delta * torch.sign(torch.rand_like(x_batch) - 0.5)
+ delta_x = delta / torch.sign(torch.rand_like(x_batch) - 0.5)
delta_x = torch.cat([delta_x, -delta_x])
with torch.no_grad():
loss_vals = loss_fn(x + delta_x)
Mutant 113
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -162,7 +162,7 @@
with torch.no_grad():
loss_vals = loss_fn(x + delta_x)
while len(loss_vals.size()) < num_dims:
- loss_vals = loss_vals.unsqueeze(-1)
+ loss_vals = loss_vals.unsqueeze(+1)
avg_grad = torch.mean(loss_vals * torch.sign(delta_x), dim=0, keepdim=True) / delta
grad_list.append(avg_grad)
Mutant 116
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -163,7 +163,7 @@
loss_vals = loss_fn(x + delta_x)
while len(loss_vals.size()) < num_dims:
loss_vals = loss_vals.unsqueeze(-1)
- avg_grad = torch.mean(loss_vals * torch.sign(delta_x), dim=0, keepdim=True) / delta
+ avg_grad = torch.mean(loss_vals / torch.sign(delta_x), dim=0, keepdim=True) / delta
grad_list.append(avg_grad)
return torch.mean(torch.cat(grad_list), dim=0, keepdim=True)
Mutant 119
--- cleverhans/future/torch/attacks/spsa.py
+++ cleverhans/future/torch/attacks/spsa.py
@@ -163,7 +163,7 @@
loss_vals = loss_fn(x + delta_x)
while len(loss_vals.size()) < num_dims:
loss_vals = loss_vals.unsqueeze(-1)
- avg_grad = torch.mean(loss_vals * torch.sign(delta_x), dim=0, keepdim=True) / delta
+ avg_grad = torch.mean(loss_vals * torch.sign(delta_x), dim=0, keepdim=True) * delta
grad_list.append(avg_grad)
return torch.mean(torch.cat(grad_list), dim=0, keepdim=True)