diff:
@@ -62,6 +62,7 @@ public class Applications {
private final String metricsUri;
private final ApplicationService api;
private final Spaces spaces;
+ private final Integer resultsPerPage;
private final LoadingCache<String, CloudFoundryServerGroup> serverGroupCache =
CacheBuilder.newBuilder()
@@ -95,7 +96,7 @@ public class Applications {
public List<CloudFoundryApplication> all() {
List<Application> newCloudFoundryAppList =
- collectPages("applications", page -> api.all(page, 500, null, null));
+ collectPages("applications", page -> api.all(page, resultsPerPage, null, null));
// Evict Records from `cache` that are no longer in the foundation
List<String> availableAppIds =

diff:
@@ -180,7 +180,8 @@ public class HttpCloudFoundryClient implements CloudFoundryClient {
String apiHost,
String user,
String password,
- boolean skipSslValidation) {
+ boolean skipSslValidation,
+ Integer resultsPerPage) {
this.apiHost = apiHost;
this.user = user;
this.password = password;
@@ -209,7 +210,12 @@ public class HttpCloudFoundryClient implements CloudFoundryClient {
this.spaces = new Spaces(createService(SpaceService.class), organizations);
this.applications =
new Applications(
- account, appsManagerUri, metricsUri, createService(ApplicationService.class), spaces);
+ account,
+ appsManagerUri,
+ metricsUri,
+ createService(ApplicationService.class),
+ spaces,
+ resultsPerPage);
this.domains = new Domains(createService(DomainService.class), organizations);
this.serviceInstances =
new ServiceInstances(
@@ -218,7 +224,13 @@ public class HttpCloudFoundryClient implements CloudFoundryClient {
organizations,
spaces);
this.routes =
- new Routes(account, createService(RouteService.class), applications, domains, spaces);
+ new Routes(
+ account,
+ createService(RouteService.class),
+ applications,
+ domains,
+ spaces,
+ resultsPerPage);
this.serviceKeys = new ServiceKeys(createService(ServiceKeyService.class), spaces);
this.tasks = new Tasks(createService(TaskService.class));

diff:
@@ -56,6 +56,7 @@ public class Routes {
private final Applications applications;
private final Domains domains;
private final Spaces spaces;
+ private final Integer resultsPerPage;
private LoadingCache<String, List<RouteMapping>> routeMappings =
CacheBuilder.newBuilder()
@@ -143,7 +144,7 @@ public class Routes {
public List<CloudFoundryLoadBalancer> all() throws CloudFoundryApiException {
List<Resource<Route>> routeResources =
- collectPageResources("routes", pg -> api.all(pg, 500, null));
+ collectPageResources("routes", pg -> api.all(pg, resultsPerPage, null));
List<CloudFoundryLoadBalancer> loadBalancers = new ArrayList<>(routeResources.size());
for (Resource<Route> routeResource : routeResources) {
loadBalancers.add(map(routeResource));

diff:
@@ -60,5 +60,6 @@ public class CloudFoundryConfigurationProperties implements DisposableBean {
private String password;
private String environment;
private boolean skipSslValidation;
+ private Integer resultsPerPage;
}
}

diff:
@@ -51,6 +51,8 @@ public class CloudFoundryCredentials implements AccountCredentials<CloudFoundryC
@Deprecated private final List<String> requiredGroupMembership = Collections.emptyList();
private final boolean skipSslValidation;
+ @Nullable private final Integer resultsPerPage;
+
private CloudFoundryClient credentials;
public CloudFoundryCredentials(
@@ -61,7 +63,8 @@ public class CloudFoundryCredentials implements AccountCredentials<CloudFoundryC
String userName,
String password,
String environment,
- boolean skipSslValidation) {
+ boolean skipSslValidation,
+ Integer resultsPerPage) {
this.name = name;
this.appsManagerUri = appsManagerUri;
this.metricsUri = metricsUri;
@@ -70,13 +73,21 @@ public class CloudFoundryCredentials implements AccountCredentials<CloudFoundryC
this.password = password;
this.environment = Optional.ofNullable(environment).orElse("dev");
this.skipSslValidation = skipSslValidation;
+ this.resultsPerPage = Optional.ofNullable(resultsPerPage).orElse(500);
}
public CloudFoundryClient getCredentials() {
if (this.credentials == null) {
this.credentials =
new HttpCloudFoundryClient(
- name, appsManagerUri, metricsUri, apiHost, userName, password, skipSslValidation);
+ name,
+ appsManagerUri,
+ metricsUri,
+ apiHost,
+ userName,
+ password,
+ skipSslValidation,
+ resultsPerPage);
}
return credentials;
}
@@ -109,12 +120,20 @@ public class CloudFoundryCredentials implements AccountCredentials<CloudFoundryC
&& Objects.equals(userName, that.userName)
&& Objects.equals(password, that.password)
&& Objects.equals(environment, that.environment)
- && Objects.equals(skipSslValidation, that.skipSslValidation);
+ && Objects.equals(skipSslValidation, that.skipSslValidation)
+ && Objects.equals(resultsPerPage, that.resultsPerPage);
}
@Override
public int hashCode() {
return Objects.hash(
- name, appsManagerUri, metricsUri, userName, password, environment, skipSslValidation);
+ name,
+ appsManagerUri,
+ metricsUri,
+ userName,
+ password,
+ environment,
+ skipSslValidation,
+ resultsPerPage);
}
}

diff:
@@ -86,7 +86,8 @@ public class CloudFoundryCredentialsSynchronizer implements CredentialsInitializ
managedAccount.getUser(),
managedAccount.getPassword(),
managedAccount.getEnvironment(),
- managedAccount.isSkipSslValidation());
+ managedAccount.isSkipSslValidation(),
+ managedAccount.getResultsPerPage());
AccountCredentials existingCredentials =
accountCredentialsRepository.getOne(credentials.getName());

diff:
@@ -46,7 +46,8 @@ class ApplicationsTest {
private ApplicationService applicationService = mock(ApplicationService.class);
private Spaces spaces = mock(Spaces.class);
private Applications apps =
- new Applications("pws", "some-apps-man-uri", "some-metrics-uri", applicationService, spaces);
+ new Applications(
+ "pws", "some-apps-man-uri", "some-metrics-uri", applicationService, spaces, 500);
private String spaceId = "space-guid";
private CloudFoundrySpace cloudFoundrySpace =
CloudFoundrySpace.builder()
@@ -65,7 +66,8 @@ class ApplicationsTest {
"api.run.pivotal.io",
"baduser",
"badpassword",
- false);
+ false,
+ 500);
assertThatThrownBy(() -> client.getApplications().all())
.isInstanceOf(CloudFoundryApiException.class);

diff:
@@ -68,7 +68,7 @@ class HttpCloudFoundryClientTest {
HttpCloudFoundryClient cloudFoundryClient =
new HttpCloudFoundryClient(
- "account", "appsManUri", "metricsUri", "host", "user", "password", false);
+ "account", "appsManUri", "metricsUri", "host", "user", "password", false, 500);
Response response = cloudFoundryClient.createRetryInterceptor(chain);
try {
@@ -97,7 +97,7 @@ class HttpCloudFoundryClientTest {
HttpCloudFoundryClient cloudFoundryClient =
new HttpCloudFoundryClient(
- "account", "appsManUri", "metricsUri", "host", "user", "password", false);
+ "account", "appsManUri", "metricsUri", "host", "user", "password", false, 500);
Response response = cloudFoundryClient.createRetryInterceptor(chain);
try {
@@ -126,7 +126,7 @@ class HttpCloudFoundryClientTest {
HttpCloudFoundryClient cloudFoundryClient =
new HttpCloudFoundryClient(
- "account", "appsManUri", "metricsUri", "host", "user", "password", false);
+ "account", "appsManUri", "metricsUri", "host", "user", "password", false, 500);
Response response = cloudFoundryClient.createRetryInterceptor(chain);
try {

diff:
@@ -57,7 +57,7 @@ class RoutesTest {
when(routeService.all(any(), any(), any())).thenReturn(Page.singleton(route, "abc123"));
when(routeService.routeMappings(any(), any())).thenReturn(new Page<>());
- Routes routes = new Routes("pws", routeService, null, domains, spaces);
+ Routes routes = new Routes("pws", routeService, null, domains, spaces, 500);
RouteId routeId = routes.toRouteId("demo1-prod.apps.calabasas.cf-app.com/path");
assertThat(routeId).isNotNull();
assertThat(routeId.getHost()).isEqualTo("demo1-prod");
@@ -67,7 +67,7 @@ class RoutesTest {
@Test
void toRouteIdReturnsNullForInvalidRoute() {
- Routes routes = new Routes(null, null, null, null, null);
+ Routes routes = new Routes(null, null, null, null, null, 500);
assertNull(routes.toRouteId("demo1-pro cf-app.com/path"));
}
@@ -127,7 +127,7 @@ class RoutesTest {
when(routeService.all(any(), any(), any())).thenReturn(routePage);
when(routeService.routeMappings(any(), any())).thenReturn(routeMappingPage);
- Routes routes = new Routes("pws", routeService, null, domains, spaces);
+ Routes routes = new Routes("pws", routeService, null, domains, spaces, 500);
CloudFoundryLoadBalancer loadBalancer =
routes.find(new RouteId().setHost("somehost").setDomainGuid("domain-guid"), "space-guid");

diff:
@@ -85,7 +85,7 @@ class AbstractLoadBalancersAtomicOperationConverterTest {
}
private final CloudFoundryCredentials cloudFoundryCredentials =
- new CloudFoundryCredentials("test", "", "", "", "", "", "", false) {
+ new CloudFoundryCredentials("test", "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -52,7 +52,7 @@ class CreateCloudFoundryServiceKeyAtomicOperationConverterTest {
}
private final CloudFoundryCredentials cloudFoundryCredentials =
- new CloudFoundryCredentials("my-account", "", "", "", "", "", "", false) {
+ new CloudFoundryCredentials("my-account", "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -57,7 +57,7 @@ class DeleteCloudFoundryServiceKeyAtomicOperationConverterTest {
}
private final CloudFoundryCredentials cloudFoundryCredentials =
- new CloudFoundryCredentials("my-account", "", "", "", "", "", "", false) {
+ new CloudFoundryCredentials("my-account", "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -69,7 +69,7 @@ class DeployCloudFoundryServerGroupAtomicOperationConverterTest {
.build()));
}
- return new CloudFoundryCredentials(name, "", "", "", "", "", "", false) {
+ return new CloudFoundryCredentials(name, "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -71,7 +71,7 @@ class DeployCloudFoundryServiceAtomicOperationConverterTest {
}
private final CloudFoundryCredentials cloudFoundryCredentials =
- new CloudFoundryCredentials("test", "", "", "", "", "", "", false) {
+ new CloudFoundryCredentials("test", "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -60,7 +60,7 @@ class ScaleCloudFoundryServerGroupAtomicOperationConverterTest {
}
private final CloudFoundryCredentials cloudFoundryCredentials =
- new CloudFoundryCredentials("test", "", "", "", "", "", "", false) {
+ new CloudFoundryCredentials("test", "", "", "", "", "", "", false, 500) {
public CloudFoundryClient getClient() {
return cloudFoundryClient;
}

diff:
@@ -149,7 +149,7 @@ class CloudFoundryCredentialsSynchronizerTest {
private CloudFoundryCredentials createCredentials(String name) {
return new CloudFoundryCredentials(
- name, null, null, "api." + name, "user-" + name, "pwd-" + name, null, false);
+ name, null, null, "api." + name, "user-" + name, "pwd-" + name, null, false, 500);
}
private void loadProviderFromRepository() {

original_message: fix(cf): allow results per page to be configurable (#4289)


change_type: MODIFY

diff:
@@ -111,6 +111,7 @@ public class CloudFoundryServerGroupCachingAgent extends AbstractCloudFoundryCac
SERVER_GROUPS.getNs(),
serverGroups.stream()
.map(sg -> setServerGroupCacheData(toKeep, sg, loadDataStart))
+ .filter(c -> c != null && c.getId() != null)
.collect(Collectors.toSet()));
results.put(
INSTANCES.getNs(),

license: Apache License 2.0

repo: spinnaker/clouddriver

original_message: fix(cf): remove the empty cache results for SG (#4310)


diff:
@@ -103,6 +103,15 @@ public class EcsCreateServerGroupDescriptionValidator extends CommonValidator {
rejectValue(errors, "ecsClusterName", "not.nullable");
}
+ if (createServerGroupDescription.getServiceDiscoveryAssociations() != null) {
+ for (CreateServerGroupDescription.ServiceDiscoveryAssociation association :
+ createServerGroupDescription.getServiceDiscoveryAssociations()) {
+ if (association.getRegistry() == null) {
+ rejectValue(errors, "serviceDiscoveryAssociations", "item.invalid");
+ }
+ }
+ }
+
boolean hasTargetGroup = StringUtils.isNotBlank(createServerGroupDescription.getTargetGroup());
if (!createServerGroupDescription.isUseTaskDefinitionArtifact()) {

original_message: fix(ecs): Add validation for empty serviceDiscoveryAssociations (#4337)


diff:
@@ -165,7 +165,7 @@ class AwsProviderConfig {
} else {
// This caching agent runs across all accounts in one iteration (to maintain consistency).
newlyAddedAgents << new ReservationReportCachingAgent(
- registry, amazonClientProvider, amazonS3DataProvider, allAccounts, objectMapper, reservationReportPool, ctx
+ registry, amazonClientProvider, amazonS3DataProvider, allAccounts, objectMapper, reservationReportPool.get(), ctx
)
}
}

original_message: fix(aws): add .get() to optional known to be present (#4355)


message:
fix(aws): fix `AmazonInstance#getZone`
`any` was renamed to `getExtraAttributes`, but this call site was
missed.

diff:
@@ -85,7 +85,7 @@ class AmazonInstance implements Instance, Serializable {
@Override
String getZone() {
- any().get("placement")?.availabilityZone
+ getExtraAttributes().get("placement")?.availabilityZone
}
@Override

original_message:
fix(aws): fix `AmazonInstance#getZone` (#4385)

`any` was renamed to `getExtraAttributes`, but this call site was
missed.


diff:
@@ -18,9 +18,7 @@ package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH;
-import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS;
-import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.ecs.AmazonECS;
@@ -65,8 +63,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
private final Logger log = LoggerFactory.getLogger(getClass());
private Collection<String> taskEvictions;
- private Collection<String> serviceEvictions;
- private Collection<String> taskDefEvictions;
private ObjectMapper objectMapper;
public TaskHealthCachingAgent(
@@ -105,8 +101,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
List<TaskHealth> taskHealthList = new LinkedList<>();
taskEvictions = new LinkedList<>();
- serviceEvictions = new LinkedList<>();
- taskDefEvictions = new LinkedList<>();
Collection<Task> tasks = taskCacheClient.getAll(accountName, region);
if (tasks != null) {
@@ -222,18 +216,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
return overallTaskHealth;
}
- private void evictStaleData(Task task, Service loadBalancerService) {
- String serviceEvictionKey =
- Keys.getTaskDefinitionKey(accountName, region, loadBalancerService.getServiceName());
- serviceEvictions.add(serviceEvictionKey);
- String taskEvictionKey = Keys.getTaskKey(accountName, region, task.getTaskId());
- taskEvictions.add(taskEvictionKey);
-
- String taskDefArn = loadBalancerService.getTaskDefinition();
- String taskDefKey = Keys.getTaskDefinitionKey(accountName, region, taskDefArn);
- taskDefEvictions.add(taskDefKey);
- }
-
private TaskHealth makeTaskHealth(
Task task, String serviceName, TargetHealthDescription healthDescription) {
String targetHealth = STATUS_UNKNOWN;
@@ -333,7 +315,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
if (targetHealth == null) {
log.debug("Cached EcsTargetHealth is empty for targetGroup {}", targetGroupArn);
- evictStaleData(task, loadBalancerService);
return makeTaskHealth(task, serviceName, null);
}
TargetHealthDescription targetHealthDescription =
@@ -345,7 +326,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
targetGroupArn,
targetId,
targetPort);
- evictStaleData(task, loadBalancerService);
return makeTaskHealth(task, serviceName, null);
}
@@ -451,20 +431,6 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
evictions.put(TASKS.toString(), taskEvictions);
}
}
- if (!serviceEvictions.isEmpty()) {
- if (evictions.containsKey(SERVICES.toString())) {
- evictions.get(SERVICES.toString()).addAll(serviceEvictions);
- } else {
- evictions.put(SERVICES.toString(), serviceEvictions);
- }
- }
- if (!taskDefEvictions.isEmpty()) {
- if (evictions.containsKey(TASK_DEFINITIONS.toString())) {
- evictions.get(TASK_DEFINITIONS.toString()).addAll(taskDefEvictions);
- } else {
- evictions.put(TASK_DEFINITIONS.toString(), taskDefEvictions);
- }
- }
return evictions;
}

original_message: fix(ecs): Remove extra evictions from TaskHealthCachingAgent (#4417)


diff:
@@ -72,6 +72,7 @@ public class TargetHealthCachingAgent extends AbstractEcsAwsAwareCachingAgent<Ec
EcsTargetGroupCacheClient targetGroupCacheClient =
new EcsTargetGroupCacheClient(awsProviderCache, objectMapper);
Set<String> targetGroups = fetchTargetGroups(targetGroupCacheClient);
+ log.debug("Fetched {} target groups for which to get target healths", targetGroups.size());
AmazonElasticLoadBalancing amazonLoadBalancing =
amazonClientProvider.getAmazonElasticLoadBalancingV2(account, region, false);
@@ -114,7 +115,15 @@ public class TargetHealthCachingAgent extends AbstractEcsAwsAwareCachingAgent<Ec
Collection<String> targetGroupKeys =
awsProviderCache.filterIdentifiers(TARGET_GROUPS.getNs(), searchKey);
- List<EcsTargetGroup> tgList = cacheClient.find(targetGroupKeys);
+ if (targetGroupKeys.size() > 0) {
+ log.debug("Found " + targetGroupKeys.size() + " target group keys in " + getAgentType());
+ }
+
+ // TODO: refine search key instead of filtering these cache results
+ List<EcsTargetGroup> tgList =
+ cacheClient.find(targetGroupKeys).stream()
+ .filter(t -> t.getTargetGroupArn().contains(account.getAccountId()))
+ .collect(Collectors.toList());
return tgList.stream().map(EcsTargetGroup::getTargetGroupArn).collect(Collectors.toSet());
}

diff:
@@ -41,7 +41,7 @@ class TargetHealthCachingAgentSpec extends Specification {
def awsProviderCache = Mock(ProviderCache)
def credentialsProvider = Mock(AWSCredentialsProvider)
def amazonloadBalancing = Mock(AmazonElasticLoadBalancing)
- def targetGroupArn = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':123456789012:targetgroup/test-tg/9e8997b7cff00c62'
+ def targetGroupArn = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':' + CommonCachingAgent.ACCOUNT_ID + ':targetgroup/test-tg/9e8997b7cff00c62'
ObjectMapper mapper = new ObjectMapper()
@Subject
@@ -140,4 +140,61 @@ class TargetHealthCachingAgentSpec extends Specification {
targetHealthList.size() == 0
}
+
+ def 'should describe and return target groups for matching account only'() {
+ given:
+ // set up correct account cache data
+ def targetGroupAttributes = [
+ loadBalancerNames: ['loadBalancerName'],
+ targetGroupArn: targetGroupArn,
+ targetGroupName: 'test-tg',
+ vpcId: 'vpc-id',
+ ]
+ def targetGroupKey =
+ Keys.getTargetGroupKey('test-tg', CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, 'ip', 'vpc-id')
+ def loadbalancerKey =
+ Keys.getLoadBalancerKey('loadBalancerName', CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, 'vpc-id', 'ip')
+ def relations = [loadBalancers: [loadbalancerKey]]
+ def targetGroupCacheData =
+ new DefaultCacheData(targetGroupKey, targetGroupAttributes, relations)
+
+ // set up other account cache data
+ def targetGroupArn2 = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':210987654321:targetgroup/other-tg/7uf491b7cff00c62'
+ def otherAccountName = "other-account"
+ def targetGroupAttributes2 = [
+ loadBalancerNames: ['loadBalancerName'],
+ targetGroupArn: targetGroupArn2,
+ targetGroupName: 'other-tg',
+ vpcId: 'vpc-id',
+ ]
+ def targetGroupKey2 =
+ Keys.getTargetGroupKey('test-tg', otherAccountName, CommonCachingAgent.REGION, 'ip', 'vpc-id')
+ def loadbalancerKey2 =
+ Keys.getLoadBalancerKey('loadBalancerName', otherAccountName, CommonCachingAgent.REGION, 'vpc-id', 'ip')
+ def relations2 = [loadBalancers: [loadbalancerKey2]]
+ def targetGroupCacheData2 =
+ new DefaultCacheData(targetGroupKey2, targetGroupAttributes2, relations2)
+
+ TargetHealthDescription targetHealth =
+ new TargetHealthDescription().withTarget(
+ new TargetDescription().withId('10.0.0.3').withPort(80))
+ .withTargetHealth(new TargetHealth().withState(TargetHealthStateEnum.Healthy))
+
+ // return cache data for both target groups
+ awsProviderCache.getAll(TARGET_GROUPS.getNs(), _, _) >> [targetGroupCacheData, targetGroupCacheData2]
+
+ when:
+ agent.setAwsCache(awsProviderCache)
+ def targetHealthList = agent.getItems(ecs, Mock(ProviderCache))
+
+ then:
+ // expect one describe call, with correct target group
+ 1 * amazonloadBalancing.describeTargetHealth({ DescribeTargetHealthRequest request ->
+ request.targetGroupArn == targetGroupArn
+ }) >> new DescribeTargetHealthResult().withTargetHealthDescriptions(targetHealth)
+
+ targetHealthList.size() == 1
+ EcsTargetHealth targetHealthDescription = targetHealthList.get(0)
+ targetHealthDescription.getTargetGroupArn() == targetGroupArn
+ }
}

diff:
@@ -53,7 +53,7 @@ class TaskHealthCachingAgentSpec extends Specification {
def clientProvider = Mock(AmazonClientProvider)
def providerCache = Mock(ProviderCache)
def credentialsProvider = Mock(AWSCredentialsProvider)
- def targetGroupArn = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':769716316905:targetgroup/test-target-group/9e8997b7cff00c62'
+ def targetGroupArn = 'arn:aws:elasticloadbalancing:' + CommonCachingAgent.REGION + ':' + CommonCachingAgent.ACCOUNT_ID + ':targetgroup/test-target-group/9e8997b7cff00c62'
ObjectMapper mapper = new ObjectMapper()

diff:
@@ -32,7 +32,8 @@ import org.junit.BeforeClass;
public class CommonCachingAgent {
static final String REGION = "us-west-2";
- private static final String ECS_SERIVCE = "arn:aws:ecs:" + REGION + ":012345678910:";
+ static final String ACCOUNT_ID = "012345678910";
+ private static final String ECS_SERIVCE = "arn:aws:ecs:" + REGION + ":" + ACCOUNT_ID + ":";
static final String ACCOUNT = "test-account";
static final String APP_NAME = "testapp";
static final String ROLE_ARN = ECS_SERIVCE + "service/test-role";
@@ -77,6 +78,7 @@ public class CommonCachingAgent {
static {
netflixAmazonCredentials = mock(NetflixAmazonCredentials.class);
when(netflixAmazonCredentials.getName()).thenReturn(ACCOUNT);
+ when(netflixAmazonCredentials.getAccountId()).thenReturn(ACCOUNT_ID);
}
@BeforeClass

original_message: fix(ecs): Correctly scope TargetHealthCachingAgent describe calls (#4429)


diff:
@@ -104,6 +104,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
Collection<Task> tasks = taskCacheClient.getAll(accountName, region);
if (tasks != null) {
+ log.debug("Found {} tasks to retrieve health for.", tasks.size());
for (Task task : tasks) {
String containerInstanceCacheKey =
Keys.getContainerInstanceKey(accountName, region, task.getContainerInstanceArn());
@@ -117,6 +118,10 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
if (service == null) {
String taskEvictionKey = Keys.getTaskKey(accountName, region, task.getTaskId());
taskEvictions.add(taskEvictionKey);
+ log.debug(
+ "Service '{}' for task '{}' is null. Will not retrieve health.",
+ serviceName,
+ task.getTaskArn());
continue;
}
@@ -125,6 +130,8 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
TaskDefinition taskDefinition = taskDefinitionCacheClient.get(taskDefinitionCacheKey);
if (isContainerMissingNetworking(task)) {
+ log.debug(
+ "Task '{}' is missing networking. Will not retrieve health.", task.getTaskArn());
continue;
}
@@ -157,6 +164,8 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
}
log.debug("TaskHealthList contains the following elements: {}", taskHealthList);
}
+ } else {
+ log.debug("Task list is null. No healths to describe.");
}
return taskHealthList;
@@ -170,7 +179,7 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
TaskDefinition taskDefinition) {
if (taskDefinition == null) {
- log.debug("Provided task definition is null.");
+ log.debug("Provided task definition is null for task '{}'.", task.getTaskArn());
return null;
}

original_message: fix(ecs): Improve debug messaging for task health (#4431)


diff:
package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.model;
+import com.google.common.collect.ImmutableList;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesCacheDataConverter;
@@ -25,7 +26,6 @@ import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.Kube
import com.netflix.spinnaker.clouddriver.model.LoadBalancer;
import com.netflix.spinnaker.clouddriver.model.LoadBalancerProvider;
import com.netflix.spinnaker.clouddriver.model.LoadBalancerServerGroup;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -69,14 +69,13 @@ public class KubernetesV2LoadBalancer extends ManifestBasedModel
Set<LoadBalancerServerGroup> serverGroups =
serverGroupData.stream()
- // ignoring load balancers here since they are discarded by ::toLoadBalancerServerGroup
.map(
d ->
KubernetesV2ServerGroup.fromCacheData(
KubernetesV2ServerGroupCacheData.builder()
.serverGroupData(d)
.instanceData(serverGroupToInstanceData.get(d.getId()))
- .loadBalancerData(new ArrayList<>())
+ .loadBalancerData(ImmutableList.of(cd))
.build()))
.filter(Objects::nonNull)
.map(KubernetesV2ServerGroup::toLoadBalancerServerGroup)

original_message: fix(kubernetes): properly set isDisabled on server groups (#4494)


message: chore(bumpdeps): enable autobumps

change_type: ADD

diff:
+name: Bump Dependencies
+
+on:
+ repository_dispatch:
+ types: [bump-dependencies]
+
+jobs:
+ bump-dependencies:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: spinnaker/bumpdeps@master
+ with:
+ ref: ${{ github.event.client_payload.ref }}
+ key: clouddriverVersion
+ repositories: front50,halyard
+ env:
+ GITHUB_OAUTH: ${{ secrets.SPINNAKER_GITHUB_TOKEN }}

diff:
@@ -20,7 +20,7 @@ jobs:
- uses: actions/setup-java@v1
with:
java-version: 11
- - uses: actions/cache@v1.1.2
+ - uses: actions/cache@v1
with:
path: ~/.gradle
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
@@ -63,3 +63,11 @@ jobs:
${{ steps.release_info.outputs.CHANGELOG }}
draft: false
prerelease: ${{ steps.release_info.outputs.IS_CANDIDATE }}
+ - name: Pause before dependency bump
+ run: sleep 300
+ - name: Trigger dependency bump workflow
+ uses: peter-evans/repository-dispatch@v1
+ with:
+ token: ${{ secrets.SPINNAKER_GITHUB_TOKEN }}
+ event-type: bump-dependencies
+ client-payload: '{"ref": "${{ github.ref }}"}'

original_message: chore(bumpdeps): enable autobumps (#4506)


diff:
@@ -25,12 +25,8 @@ import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.cache.RelationshipCacheFilter;
import com.netflix.spinnaker.clouddriver.aws.data.Keys;
import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsLoadBalancerCache;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.netflix.spinnaker.clouddriver.ecs.provider.view.EcsAccountMapper;
+import java.util.*;
import java.util.stream.Collectors;
import org.springframework.stereotype.Component;
@@ -39,10 +35,13 @@ public class EcsLoadbalancerCacheClient {
private final Cache cacheView;
private final ObjectMapper objectMapper;
+ private final EcsAccountMapper ecsAccountMapper;
- public EcsLoadbalancerCacheClient(Cache cacheView, ObjectMapper objectMapper) {
+ public EcsLoadbalancerCacheClient(
+ Cache cacheView, ObjectMapper objectMapper, EcsAccountMapper ecsAccountMapper) {
this.cacheView = cacheView;
this.objectMapper = objectMapper;
+ this.ecsAccountMapper = ecsAccountMapper;
}
public List<EcsLoadBalancerCache> find(String account, String region) {
@@ -56,6 +55,12 @@ public class EcsLoadbalancerCacheClient {
private Set<Map<String, Object>> fetchFromCache(String account, String region) {
String accountFilter = account != null ? account : "*";
+ if (!"*".equals(accountFilter)) {
+ String awsAccountName = ecsAccountMapper.fromEcsAccountNameToAwsAccountName(accountFilter);
+ if (awsAccountName != null) {
+ accountFilter = awsAccountName;
+ }
+ }
String regionFilter = region != null ? region : "*";
String searchKey = Keys.getLoadBalancerKey("*", accountFilter, regionFilter, "*", "*") + "*";
@@ -66,12 +71,16 @@ public class EcsLoadbalancerCacheClient {
return fetchLoadBalancerAttributes(loadbalancerKeys);
}
- public Set<EcsLoadBalancerCache> findWithTargetGroups(Set<String> targetGroups) {
- return findAll().stream()
- .filter(
- ecsLoadBalancerCache ->
- targetGroups.containsAll(ecsLoadBalancerCache.getTargetGroups()))
- .collect(Collectors.toSet());
+ public List<EcsLoadBalancerCache> findWithTargetGroups(Set<String> targetGroupKeys) {
+ Set<CacheData> targetGroupCacheData =
+ new HashSet<>(
+ cacheView.getAll(
+ TARGET_GROUPS.getNs(),
+ targetGroupKeys,
+ RelationshipCacheFilter.include(LOAD_BALANCERS.getNs())));
+ Set<String> lbKeys = inferAssociatedLoadBalancers(targetGroupCacheData);
+ Set<Map<String, Object>> loadbalancerAttributes = fetchLoadBalancerAttributes(lbKeys);
+ return convertToLoadbalancer(loadbalancerAttributes);
}
private EcsLoadBalancerCache convertToLoadBalancer(Map<String, Object> targetGroupAttributes) {
@@ -89,8 +98,9 @@ public class EcsLoadbalancerCacheClient {
return ecsTargetGroups;
}
- private Set<Map<String, Object>> fetchLoadBalancerAttributes(Collection<String> targetGroupKeys) {
- Set<CacheData> loadBalancerCache = fetchLoadBalancers(targetGroupKeys);
+ private Set<Map<String, Object>> fetchLoadBalancerAttributes(
+ Collection<String> loadBalancerKeys) {
+ Set<CacheData> loadBalancerCache = fetchLoadBalancers(loadBalancerKeys);
return loadBalancerCache.stream()
.filter(this::hashTargetGroups)
@@ -108,7 +118,8 @@ public class EcsLoadbalancerCacheClient {
Map<String, String> parts = Keys.parse(loadbalancerCache.getId());
attributes.put("region", parts.get("region"));
- attributes.put("account", parts.get("account"));
+ String ecsAccount = ecsAccountMapper.fromAwsAccountNameToEcsAccountName(parts.get("account"));
+ attributes.put("account", ecsAccount);
attributes.put("loadBalancerType", parts.get("loadBalancerType"));
attributes.put(
"targetGroups",
@@ -119,18 +130,12 @@ public class EcsLoadbalancerCacheClient {
return attributes;
}
- private Set<Map<String, Object>> retrieveLoadbalancers(
- Set<String> loadbalancersAssociatedWithTargetGroups) {
- Collection<CacheData> loadbalancers =
- cacheView.getAll(LOAD_BALANCERS.getNs(), loadbalancersAssociatedWithTargetGroups);
- return loadbalancers.stream().map(CacheData::getAttributes).collect(Collectors.toSet());
- }
-
private Set<String> inferAssociatedLoadBalancers(Set<CacheData> targetGroups) {
Set<String> loadbalancersAssociatedWithTargetGroups = new HashSet<>();
for (CacheData targetGroup : targetGroups) {
- Collection<String> relatedLoadbalancer = targetGroup.getRelationships().get("loadbalancer");
+ Collection<String> relatedLoadbalancer =
+ targetGroup.getRelationships().get(LOAD_BALANCERS.ns);
if (relatedLoadbalancer != null && relatedLoadbalancer.size() > 0) {
loadbalancersAssociatedWithTargetGroups.addAll(relatedLoadbalancer);
}

diff:
@@ -61,6 +61,10 @@ public class EcsTargetGroupCacheClient {
return targetGroups;
}
+ public Collection<String> getAllKeys() {
+ return cacheView.getIdentifiers(TARGET_GROUPS.ns);
+ }
+
private EcsTargetGroup convertToTargetGroup(Map<String, Object> targetGroupAttributes) {
EcsTargetGroup ecsTargetGroup =
objectMapper.convertValue(targetGroupAttributes, EcsTargetGroup.class);

diff:
@@ -54,6 +54,6 @@ public class EcsLoadBalancerCache implements LoadBalancer {
@Override
public String getType() {
- return loadBalancerType;
+ return cloudProvider;
}
}

diff:
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer;
+
+import com.amazonaws.services.elasticloadbalancingv2.model.Listener;
+import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider;
+import com.netflix.spinnaker.clouddriver.model.LoadBalancer;
+import com.netflix.spinnaker.clouddriver.model.LoadBalancerServerGroup;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import lombok.Data;
+
+@Data
+public class EcsLoadBalancer implements LoadBalancer {
+ // TODO: refactor EcsLoadBalancerCache so can be extended here?
+
+ private String account;
+ private String region;
+ private String loadBalancerArn;
+ private String loadBalancerType;
+ private String cloudProvider = EcsCloudProvider.ID;
+ private List<Listener> listeners;
+ private List<String> availabilityZones;
+ private String ipAddressType;
+ private String loadBalancerName;
+ private String canonicalHostedZoneId;
+ private String vpcId;
+ private String dnsname;
+ private Long createdTime;
+ private List<String> subnets;
+ private List<String> securityGroups;
+ private List<EcsTargetGroup> targetGroups;
+ private Set<LoadBalancerServerGroup> serverGroups;
+ private Map<String, Set<String>> targetGroupServices;
+
+ @Override
+ public String getName() {
+ return loadBalancerName;
+ }
+
+ @Override
+ public String getType() {
+ return cloudProvider;
+ }
+}

diff:
@@ -28,7 +28,7 @@ public class EcsLoadBalancerDetail implements Details {
String region;
String name;
String vpcId;
- String type = "aws";
+ String type = "ecs";
String loadBalancerType;
List<String> securityGroups = new LinkedList<>();
List<String> targetGroups = new LinkedList<>();

diff:
@@ -27,6 +27,7 @@ public class EcsTargetGroup implements LoadBalancerProvider.Details {
List<String> loadBalancerNames;
List<String> instances;
+ String targetType;
Integer healthCheckTimeoutSeconds;
String targetGroupArn;
String healthCheckPort;

diff:
package com.netflix.spinnaker.clouddriver.ecs.provider.view;
-import com.netflix.spinnaker.clouddriver.aws.model.AmazonLoadBalancer;
+import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.TARGET_GROUPS;
+
+import com.amazonaws.services.ecs.model.LoadBalancer;
+import com.netflix.spinnaker.clouddriver.aws.AmazonCloudProvider;
+import com.netflix.spinnaker.clouddriver.aws.data.ArnUtils;
import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider;
import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsLoadbalancerCacheClient;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsTargetGroupCacheClient;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient;
import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsLoadBalancerCache;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
+import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsLoadBalancer;
import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsLoadBalancerDetail;
import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsLoadBalancerSummary;
-import com.netflix.spinnaker.clouddriver.ecs.security.ECSCredentialsConfig;
+import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsTargetGroup;
import com.netflix.spinnaker.clouddriver.model.LoadBalancerProvider;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
-public class EcsLoadBalancerProvider implements LoadBalancerProvider<AmazonLoadBalancer> {
+public class EcsLoadBalancerProvider implements LoadBalancerProvider<EcsLoadBalancer> {
private final EcsLoadbalancerCacheClient ecsLoadbalancerCacheClient;
- private final ECSCredentialsConfig ecsCredentialsConfig;
+ private final EcsAccountMapper ecsAccountMapper;
+ private final ServiceCacheClient ecsServiceCacheClient;
+ private final EcsTargetGroupCacheClient ecsTargetGroupCacheClient;
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
@Autowired
public EcsLoadBalancerProvider(
EcsLoadbalancerCacheClient ecsLoadbalancerCacheClient,
- ECSCredentialsConfig ecsCredentialsConfig) {
+ EcsAccountMapper ecsAccountMapper,
+ ServiceCacheClient ecsServiceCacheClient,
+ EcsTargetGroupCacheClient ecsTargetGroupCacheClient) {
this.ecsLoadbalancerCacheClient = ecsLoadbalancerCacheClient;
- this.ecsCredentialsConfig = ecsCredentialsConfig;
+ this.ecsAccountMapper = ecsAccountMapper;
+ this.ecsServiceCacheClient = ecsServiceCacheClient;
+ this.ecsTargetGroupCacheClient = ecsTargetGroupCacheClient;
}
@Override
@@ -57,8 +72,7 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<AmazonLoadB
List<EcsLoadBalancerCache> loadBalancers = ecsLoadbalancerCacheClient.findAll();
for (EcsLoadBalancerCache lb : loadBalancers) {
- String account = getEcsAccountName(lb.getAccount());
- if (account == null) {
+ if (lb.getAccount() == null) {
continue;
}
@@ -72,7 +86,7 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<AmazonLoadB
}
EcsLoadBalancerDetail loadBalancer = new EcsLoadBalancerDetail();
- loadBalancer.setAccount(account);
+ loadBalancer.setAccount(lb.getAccount());
loadBalancer.setRegion(region);
loadBalancer.setName(name);
loadBalancer.setVpcId(lb.getVpcId());
@@ -81,7 +95,7 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<AmazonLoadB
loadBalancer.setTargetGroups(lb.getTargetGroups());
summary
- .getOrCreateAccount(account)
+ .getOrCreateAccount(lb.getAccount())
.getOrCreateRegion(region)
.getLoadBalancers()
.add(loadBalancer);
@@ -92,27 +106,113 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<AmazonLoadB
@Override
public Item get(String name) {
- return null; // TODO - Implement this.
+ return null; // intentionally null, implement if/when needed in Deck.
}
@Override
public List<Details> byAccountAndRegionAndName(String account, String region, String name) {
- return null; // TODO - Implement this. This is used to show the details view of a load balancer
- // which is not even implemented yet
+ return null; // intentionally null, implement if/when needed in Deck.
}
@Override
- public Set<AmazonLoadBalancer> getApplicationLoadBalancers(String application) {
- return null; // TODO - Implement this. This is used to show load balancers and reveals other
- // buttons
+ public Set<EcsLoadBalancer> getApplicationLoadBalancers(String application) {
+ // Find the load balancers currently in use by ECS services in this application
+ Set<Service> services =
+ ecsServiceCacheClient.getAll().stream()
+ .filter(service -> service.getApplicationName().equals(application))
+ .collect(Collectors.toSet());
+ log.debug("Retrieved {} services for application '{}'", services.size(), application);
+
+ Collection<String> allTargetGroupKeys = ecsTargetGroupCacheClient.getAllKeys();
+ log.debug(
+ "Retrieved {} target group keys for application '{}'",
+ allTargetGroupKeys.size(),
+ application);
+
+ Map<String, Set<String>> targetGroupToServicesMap = new HashMap<>();
+ Set<String> targetGroupKeys = new HashSet<>();
+
+ // find all the target group cache keys
+ for (Service service : services) {
+ String awsAccountName =
+ ecsAccountMapper.fromEcsAccountNameToAwsAccountName(service.getAccount());
+ for (LoadBalancer loadBalancer : service.getLoadBalancers()) {
+ if (loadBalancer.getTargetGroupArn() != null) {
+ String tgArn = loadBalancer.getTargetGroupArn();
+ String keyPrefix =
+ String.format(
+ "%s:%s:%s:%s:%s:",
+ AmazonCloudProvider.ID,
+ TARGET_GROUPS.getNs(),
+ awsAccountName,
+ service.getRegion(),
+ ArnUtils.extractTargetGroupName(tgArn).get());
+ Set<String> matchingKeys =
+ allTargetGroupKeys.stream()
+ .filter(key -> key.startsWith(keyPrefix))
+ .collect(Collectors.toSet());
+ targetGroupKeys.addAll(matchingKeys);
+ // associate target groups with services it contains targets for
+ if (targetGroupToServicesMap.containsKey(tgArn)) {
+ Set<String> serviceList = targetGroupToServicesMap.get(tgArn);
+ serviceList.add(service.getServiceName());
+ targetGroupToServicesMap.put(tgArn, serviceList);
+ } else {
+ Set<String> srcServices = Collections.singleton(service.getServiceName());
+ targetGroupToServicesMap.put(tgArn, srcServices);
+ }
+ }
+ }
}
- private String getEcsAccountName(String awsAccountName) {
- for (ECSCredentialsConfig.Account ecsAccount : ecsCredentialsConfig.getAccounts()) {
- if (ecsAccount.getAwsAccount().equals(awsAccountName)) {
- return ecsAccount.getName();
+ // retrieve matching target groups
+ List<EcsTargetGroup> tgs = ecsTargetGroupCacheClient.find(targetGroupKeys);
+
+ // find the load balancers for those target groups
+ List<EcsLoadBalancerCache> tgLBs =
+ ecsLoadbalancerCacheClient.findWithTargetGroups(targetGroupKeys);
+ log.debug(
+ "Retrieved {} load balancers for {} target group keys.",
+ tgLBs.size(),
+ targetGroupKeys.size());
+
+ Set<EcsLoadBalancer> ecsLoadBalancers = new HashSet<>();
+ for (EcsLoadBalancerCache loadBalancerCache : tgLBs) {
+ List<EcsTargetGroup> matchingTGs =
+ tgs.stream()
+ .filter(tg -> loadBalancerCache.getTargetGroups().contains(tg.getTargetGroupName()))
+ .collect(Collectors.toList());
+ EcsLoadBalancer ecsLB =
+ makeEcsLoadBalancer(loadBalancerCache, matchingTGs, targetGroupToServicesMap);
+ ecsLoadBalancers.add(ecsLB);
}
+
+ return ecsLoadBalancers;
}
- return null;
+
+ private EcsLoadBalancer makeEcsLoadBalancer(
+ EcsLoadBalancerCache elbCacheData,
+ List<EcsTargetGroup> tgCacheData,
+ Map<String, Set<String>> tgToServiceMap) {
+ EcsLoadBalancer ecsLoadBalancer = new EcsLoadBalancer();
+ ecsLoadBalancer.setAccount(elbCacheData.getAccount());
+ ecsLoadBalancer.setRegion(elbCacheData.getRegion());
+ ecsLoadBalancer.setLoadBalancerArn(elbCacheData.getLoadBalancerArn());
+ ecsLoadBalancer.setLoadBalancerName(elbCacheData.getLoadBalancerName());
+ ecsLoadBalancer.setLoadBalancerType(elbCacheData.getLoadBalancerType());
+ ecsLoadBalancer.setCloudProvider(elbCacheData.getCloudProvider());
+ ecsLoadBalancer.setListeners(elbCacheData.getListeners());
+ ecsLoadBalancer.setAvailabilityZones(elbCacheData.getAvailabilityZones());
+ ecsLoadBalancer.setIpAddressType(elbCacheData.getIpAddressType());
+ ecsLoadBalancer.setDnsname(elbCacheData.getDnsname());
+ ecsLoadBalancer.setVpcId(elbCacheData.getVpcId());
+ ecsLoadBalancer.setCreatedTime(elbCacheData.getCreatedTime());
+ ecsLoadBalancer.setSecurityGroups(elbCacheData.getSecurityGroups());
+ ecsLoadBalancer.setSubnets(elbCacheData.getSubnets());
+ ecsLoadBalancer.setTargetGroups(tgCacheData);
+ ecsLoadBalancer.setTargetGroupServices(tgToServiceMap);
+ // TODO: get, add target healths per service/tg
+
+ return ecsLoadBalancer;
}
}

diff:
@@ -25,6 +25,7 @@ import com.netflix.spinnaker.clouddriver.aws.data.Keys
import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider
import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsLoadbalancerCacheClient
import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsLoadBalancerCache
+import com.netflix.spinnaker.clouddriver.ecs.provider.view.EcsAccountMapper
import spock.lang.Specification
import spock.lang.Subject
@@ -32,24 +33,29 @@ class EcsLoadbalancerCacheClientSpec extends Specification {
def cacheView = Mock(Cache)
def objectMapper = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ def accountMapper = Mock(EcsAccountMapper)
@Subject
- EcsLoadbalancerCacheClient client = new EcsLoadbalancerCacheClient(cacheView, objectMapper)
+ def client = new EcsLoadbalancerCacheClient(cacheView, objectMapper, accountMapper)
def 'should convert cache data into object'() {
given:
def loadBalancerName = 'test-name'
- def account = 'test-account'
+ def ecsAccount = 'ecsAccount'
+ def awsAccount = 'awsAccount'
def region = 'us-west-1'
def vpcId = 'vpc-id'
def loadBalancerType = 'classic'
def targetGroupName = 'test-target-group'
- def loadbalancerKey = Keys.getLoadBalancerKey(loadBalancerName, account, region, vpcId, loadBalancerType)
- def targetGroupKey = Keys.getTargetGroupKey(targetGroupName, account, region, TargetTypeEnum.Instance.toString(), vpcId)
+ accountMapper.fromEcsAccountNameToAwsAccountName(ecsAccount) >> awsAccount
+ accountMapper.fromAwsAccountNameToEcsAccountName(awsAccount) >> ecsAccount
+
+ def loadbalancerKey = Keys.getLoadBalancerKey(loadBalancerName, awsAccount, region, vpcId, loadBalancerType)
+ def targetGroupKey = Keys.getTargetGroupKey(targetGroupName, awsAccount, region, TargetTypeEnum.Instance.toString(), vpcId)
def givenEcsLoadbalancer = new EcsLoadBalancerCache(
- account: account,
+ account: ecsAccount,
region: region,
loadBalancerArn: 'arn',
loadBalancerType: loadBalancerType,

diff:
@@ -52,6 +52,7 @@ class EcsTargetGroupCacheClientSpec extends Specification {
instances: [],
healthCheckTimeoutSeconds: 30,
targetGroupArn: 'arn',
+ targetType: 'ip',
healthCheckPort: 1337,
matcher: [:],
healthCheckProtocol: 'http',

diff:
package com.netflix.spinnaker.clouddriver.ecs.provider.view
+import com.amazonaws.services.ecs.model.LoadBalancer
import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider
import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsLoadbalancerCacheClient
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsTargetGroupCacheClient
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient
import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsLoadBalancerCache
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service
+import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsLoadBalancer
+import com.netflix.spinnaker.clouddriver.ecs.model.loadbalancer.EcsTargetGroup
import com.netflix.spinnaker.clouddriver.ecs.security.ECSCredentialsConfig
+import com.netflix.spinnaker.clouddriver.ecs.security.NetflixECSCredentials
import spock.lang.Specification
import spock.lang.Subject
class EcsLoadBalancerProviderSpec extends Specification {
- def client = Mock(EcsLoadbalancerCacheClient)
- def ecsCredentialsConfig = Mock(ECSCredentialsConfig)
+ def ECS_ACCOUNT = 'ecsAccount'
+ def AWS_ACCOUNT = 'awsAccount'
+
+ def mockLBCache = Mock(EcsLoadbalancerCacheClient)
+ def mockServiceCache = Mock(ServiceCacheClient)
+ def mockTargetGroupCache = Mock(EcsTargetGroupCacheClient)
+ def accountMapper = Mock(EcsAccountMapper)
+
@Subject
- def provider = new EcsLoadBalancerProvider(client, ecsCredentialsConfig)
+ def provider = new EcsLoadBalancerProvider(
+ mockLBCache,
+ accountMapper,
+ mockServiceCache,
+ mockTargetGroupCache)
+
+ def setup() {
+ accountMapper.fromEcsAccountNameToAwsAccountName(ECS_ACCOUNT) >> AWS_ACCOUNT
+ }
def 'should retrieve an empty list'() {
when:
def retrievedList = provider.list()
then:
- client.findAll() >> Collections.emptyList()
+ mockLBCache.findAll() >> Collections.emptyList()
retrievedList.size() == 0
}
@@ -42,7 +63,6 @@ class EcsLoadBalancerProviderSpec extends Specification {
given:
def expectedNumberOfLoadbalancers = 2
def givenList = []
- def accounts = []
(1..expectedNumberOfLoadbalancers).forEach() {
givenList << new EcsLoadBalancerCache(
account: 'test-account-' + it,
@@ -64,20 +84,108 @@ class EcsLoadBalancerProviderSpec extends Specification {
targetGroups: ['target-group-' + it],
serverGroups: []
)
-
- accounts << new ECSCredentialsConfig.Account(
- name: 'test-account-' + it,
- awsAccount: 'test-account-' + it
- )
}
- ecsCredentialsConfig.getAccounts() >> accounts
when:
def retrievedList = provider.list()
then:
- client.findAll() >> givenList
+ mockLBCache.findAll() >> givenList
retrievedList.size() == expectedNumberOfLoadbalancers
retrievedList*.getName().containsAll(givenList*.loadBalancerName)
}
+
+ def 'should retrieve application load balancers'() {
+ given:
+ def applicationName = 'myEcsApp'
+ def tgArn1 = 'arn:aws:elasticloadbalancing:us-west-1:1234567890:targetgroup/test-tg-1/2136bac'
+ def tgArn2 = 'arn:aws:elasticloadbalancing:us-west-1:1234567890:targetgroup/test-tg-2/2136bac'
+
+ // define 2 ports to expose on our task
+ LoadBalancer ecsLb1 = new LoadBalancer()
+ ecsLb1.setContainerName("container-name")
+ ecsLb1.setContainerPort(8080)
+ ecsLb1.setTargetGroupArn(tgArn1)
+
+ LoadBalancer ecsLb2 = new LoadBalancer()
+ ecsLb2.setContainerName("container-name")
+ ecsLb2.setContainerPort(443)
+ ecsLb2.setTargetGroupArn(tgArn2)
+
+ // add these to our service
+ Service ecsService = new Service()
+ ecsService.setServiceName('ecs-test-detail-000v')
+ ecsService.setServiceArn('arn:aws:ecs:service/ecs-test-detail-000v')
+ ecsService.setLoadBalancers([ecsLb1, ecsLb2])
+ ecsService.setAccount('test-account')
+ ecsService.setApplicationName(applicationName)
+
+ // mock the cache entries for the TGs and their associated LBs
+ EcsTargetGroup ecsTg1 = new EcsTargetGroup()
+ ecsTg1.setTargetGroupArn(tgArn1)
+ ecsTg1.setTargetGroupName('test-tg-1')
+
+ EcsTargetGroup ecsTg2 = new EcsTargetGroup()
+ ecsTg2.setTargetGroupArn(tgArn2)
+ ecsTg2.setTargetGroupName('test-tg-2')
+
+ EcsLoadBalancerCache ecsLoadBalancerCache1 = new EcsLoadBalancerCache(
+ account: 'test-account',
+ region: 'us-west-2',
+ loadBalancerArn: 'arn:1',
+ loadBalancerType: 'application',
+ cloudProvider: EcsCloudProvider.ID,
+ listeners: [],
+ scheme: 'scheme',
+ availabilityZones: [],
+ ipAddressType: 'ipv4',
+ loadBalancerName: 'load-balancer-name1',
+ canonicalHostedZoneId: 'zone-id',
+ vpcId: 'vpc-id',
+ dnsname: 'dns-name',
+ createdTime: System.currentTimeMillis(),
+ subnets: [],
+ securityGroups: [],
+ targetGroups: [ecsTg1.getTargetGroupName()],
+ serverGroups: []
+ )
+ EcsLoadBalancerCache ecsLoadBalancerCache2 = new EcsLoadBalancerCache(
+ account: 'test-account',
+ region: 'us-west-2',
+ loadBalancerArn: 'arn:2',
+ loadBalancerType: 'application',
+ cloudProvider: EcsCloudProvider.ID,
+ listeners: [],
+ scheme: 'scheme',
+ availabilityZones: [],
+ ipAddressType: 'ipv4',
+ loadBalancerName: 'load-balancer-name2',
+ canonicalHostedZoneId: 'zone-id',
+ vpcId: 'vpc-id',
+ dnsname: 'dns-name',
+ createdTime: System.currentTimeMillis(),
+ subnets: [],
+ securityGroups: [],
+ targetGroups: [ecsTg2.getTargetGroupName()],
+ serverGroups: []
+ )
+
+ when:
+ def loadBalancerList = provider.getApplicationLoadBalancers(applicationName)
+
+ then:
+ mockServiceCache.getAll() >> Collections.singletonList(ecsService)
+ mockTargetGroupCache.getAllKeys() >> ['fake-tg-key-1', 'fake-tg-key-2']
+ mockTargetGroupCache.find(_) >> [ecsTg1, ecsTg2]
+ mockLBCache.findWithTargetGroups(_) >> [ecsLoadBalancerCache1, ecsLoadBalancerCache2]
+
+ loadBalancerList.size() == 2
+ for (EcsLoadBalancer lb : loadBalancerList) {
+ lb.targetGroupServices.size() == 1
+ lb.targetGroups.size() == 1
+
+ def tgArn = lb.targetGroups[0].getTargetGroupArn()
+ lb.targetGroupServices[tgArn][0] == ecsService.getServiceName()
+ }
+ }
}

original_message:
feat(ecs): Return application load balancer info from LB provider (#4526)

Co-authored-by: Clare Liguori <liguori@amazon.com>


diff:
@@ -19,6 +19,7 @@ package com.netflix.spinnaker.clouddriver.ecs.provider.view;
import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.TARGET_GROUPS;
import com.amazonaws.services.ecs.model.LoadBalancer;
+import com.google.common.collect.Sets;
import com.netflix.spinnaker.clouddriver.aws.AmazonCloudProvider;
import com.netflix.spinnaker.clouddriver.aws.data.ArnUtils;
import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider;
@@ -158,7 +159,7 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<EcsLoadBala
serviceList.add(service.getServiceName());
targetGroupToServicesMap.put(tgArn, serviceList);
} else {
- Set<String> srcServices = Collections.singleton(service.getServiceName());
+ Set<String> srcServices = Sets.newHashSet(service.getServiceName());
targetGroupToServicesMap.put(tgArn, srcServices);
}
}

original_message:
fix(ecs): make targetgroup service set mutable (#4542)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


diff:
@@ -153,12 +153,14 @@ public class EcsLoadBalancerProvider implements LoadBalancerProvider<EcsLoadBala
.filter(key -> key.startsWith(keyPrefix))
.collect(Collectors.toSet());
targetGroupKeys.addAll(matchingKeys);
- // associate target groups with services it contains targets for
+ // associate target group with services it contains targets for
if (targetGroupToServicesMap.containsKey(tgArn)) {
+ log.debug("Mapping additional service '{}' to '{}'", service.getServiceName(), tgArn);
Set<String> serviceList = targetGroupToServicesMap.get(tgArn);
serviceList.add(service.getServiceName());
targetGroupToServicesMap.put(tgArn, serviceList);
} else {
+ log.debug("Mapping service '{}' to '{}'", service.getServiceName(), tgArn);
Set<String> srcServices = Sets.newHashSet(service.getServiceName());
targetGroupToServicesMap.put(tgArn, srcServices);
}

diff:
@@ -188,4 +188,78 @@ class EcsLoadBalancerProviderSpec extends Specification {
lb.targetGroupServices[tgArn][0] == ecsService.getServiceName()
}
}
+
+ def 'should retrieve load balancers mapped to multiple services'() {
+ given:
+ def applicationName = 'myEcsApp'
+ def tgArn = 'arn:aws:elasticloadbalancing:us-west-1:1234567890:targetgroup/test-tg-1/2136bac'
+
+ // define 2 ecs services load balanced behind the same target group
+ LoadBalancer service1lb = new LoadBalancer()
+ service1lb.setContainerName("container-name")
+ service1lb.setContainerPort(8080)
+ service1lb.setTargetGroupArn(tgArn)
+
+ Service ecsService1 = new Service()
+ ecsService1.setServiceName('ecs-test-one-000v')
+ ecsService1.setServiceArn('arn:aws:ecs:service/ecs-test-one-000v')
+ ecsService1.setLoadBalancers([service1lb])
+ ecsService1.setAccount('test-account')
+ ecsService1.setApplicationName(applicationName)
+
+ LoadBalancer service2lb = new LoadBalancer()
+ service2lb.setContainerName("container-name")
+ service2lb.setContainerPort(8080)
+ service2lb.setTargetGroupArn(tgArn)
+
+ Service ecsService2 = new Service()
+ ecsService2.setServiceName('ecs-test-two-000v')
+ ecsService2.setServiceArn('arn:aws:ecs:service/ecs-test-two-000v')
+ ecsService2.setLoadBalancers([service2lb])
+ ecsService2.setAccount('test-account')
+ ecsService2.setApplicationName(applicationName)
+
+ // mock the cache entries for the TG and associated LB
+ EcsTargetGroup ecsTg = new EcsTargetGroup()
+ ecsTg.setTargetGroupArn(tgArn)
+ ecsTg.setTargetGroupName('test-tg-1')
+
+ EcsLoadBalancerCache ecsLoadBalancerCache = new EcsLoadBalancerCache(
+ account: 'test-account',
+ region: 'us-west-2',
+ loadBalancerArn: 'arn:1',
+ loadBalancerType: 'application',
+ cloudProvider: EcsCloudProvider.ID,
+ listeners: [],
+ scheme: 'scheme',
+ availabilityZones: [],
+ ipAddressType: 'ipv4',
+ loadBalancerName: 'load-balancer-name1',
+ canonicalHostedZoneId: 'zone-id',
+ vpcId: 'vpc-id',
+ dnsname: 'dns-name',
+ createdTime: System.currentTimeMillis(),
+ subnets: [],
+ securityGroups: [],
+ targetGroups: [ecsTg.getTargetGroupName()],
+ serverGroups: []
+ )
+
+ when:
+ def loadBalancerList = provider.getApplicationLoadBalancers(applicationName)
+
+ then:
+ mockServiceCache.getAll() >> [ecsService1, ecsService2]
+ mockTargetGroupCache.getAllKeys() >> ['fake-tg-key-1']
+ mockTargetGroupCache.find(_) >> [ecsTg]
+ mockLBCache.findWithTargetGroups(_) >> Collections.singletonList(ecsLoadBalancerCache)
+
+ loadBalancerList.size() == 1
+
+ def lb = loadBalancerList[0]
+ lb.targetGroupServices.size() == 1
+ lb.targetGroups.size() == 1
+ def services = lb.targetGroupServices[tgArn]
+ services.size() == 2
+ }
}

original_message:
chore(ecs): add unit test, debug msgs to application load balancers (#4546)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


message: refactor(core): remove skins

diff:
@@ -46,7 +46,6 @@ public class KubernetesConfigurationProperties {
private boolean configureImagePullSecrets = true;
private List<String> namespaces = new ArrayList<>();
private List<String> omitNamespaces = new ArrayList<>();
- private String skin;
private int cacheThreads = DEFAULT_CACHE_THREADS;
private List<LinkedDockerRegistryConfiguration> dockerRegistries;
private List<String> requiredGroupMembership = new ArrayList<>();

diff:
@@ -44,8 +44,6 @@ public class KubernetesNamedAccountCredentials<C extends KubernetesCredentials>
@Include private final String accountType;
- @Include private final String skin;
-
@Include private final int cacheThreads;
@Include private final C credentials;
@@ -64,9 +62,6 @@ public class KubernetesNamedAccountCredentials<C extends KubernetesCredentials>
Optional.ofNullable(managedAccount.getEnvironment()).orElse(managedAccount.getName());
this.accountType =
Optional.ofNullable(managedAccount.getAccountType()).orElse(managedAccount.getName());
- this.skin =
- Optional.ofNullable(managedAccount.getSkin())
- .orElse(managedAccount.getProviderVersion().toString());
this.cacheThreads = managedAccount.getCacheThreads();
this.cacheIntervalSeconds = managedAccount.getCacheIntervalSeconds();

diff:
@@ -105,7 +105,6 @@ class KubernetesV2ProviderSynchronizableSpec extends Specification {
credentials.getName() == "test-account"
credentials.getEnvironment() == "test-account"
credentials.getAccountType() == "test-account"
- credentials.getSkin() == "v2"
credentials.getCacheIntervalSeconds() == null
credentials.getCacheThreads() == 1

diff:
@@ -77,17 +77,6 @@ public interface AccountCredentials<T> {
return ProviderVersion.v1;
}
- /**
- * Provides a named "skin" as a signal for Spinnaker API clients, e.g. Deck, to alter their
- * behavior. By default, returns an account's provider version, but does not need to be coupled to
- * a provider version.
- *
- * @return the account's skin.
- */
- default String getSkin() {
- return getProviderVersion().toString();
- }
-
/** @return the id for the account (may be null if not supported by underlying cloud provider) */
default String getAccountId() {
return null;

diff:
@@ -73,8 +73,7 @@ class CredentialsController {
'requiredGroupMembership',
'permissions',
'providerVersion',
- 'accountId',
- 'skin'])
+ 'accountId'])
}
cred.type = accountCredentials.cloudProvider

diff:
@@ -53,7 +53,7 @@ class CredentialsControllerSpec extends Specification {
List<Map> parsedResponse = new JsonSlurper().parseText(result.response.contentAsString) as List
- parsedResponse == [[name: "test", environment: "env", accountType: "acctType", cloudProvider: "testProvider", type: "testProvider", requiredGroupMembership: ["test"], permissions: [READ:["test"], WRITE:["test"]], challengeDestructiveActions: false, primaryAccount: false, providerVersion: "v1", skin: "v1"]]
+ parsedResponse == [[name: "test", environment: "env", accountType: "acctType", cloudProvider: "testProvider", type: "testProvider", requiredGroupMembership: ["test"], permissions: [READ:["test"], WRITE:["test"]], challengeDestructiveActions: false, primaryAccount: false, providerVersion: "v1"]]
}
static class TestNamedAccountCredentials implements AccountCredentials<Map> {

original_message:
refactor(core): remove skins (#4576)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


diff:
@@ -38,13 +38,13 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/applications/{application}/serverGroupManagers")
public class ServerGroupManagerController {
- final List<ServerGroupManagerProvider<ServerGroupManager>> serverGroupManagerProviders;
+ final List<ServerGroupManagerProvider<? extends ServerGroupManager>> serverGroupManagerProviders;
final RequestQueue requestQueue;
@Autowired
public ServerGroupManagerController(
- List<ServerGroupManagerProvider<ServerGroupManager>> serverGroupManagerProviders,
+ List<ServerGroupManagerProvider<? extends ServerGroupManager>> serverGroupManagerProviders,
RequestQueue requestQueue) {
this.serverGroupManagerProviders = serverGroupManagerProviders;
this.requestQueue = requestQueue;

original_message: fix(kubernetes): return kubernetes server group managers (#4583)


diff:
@@ -122,7 +122,10 @@ public class TargetHealthCachingAgent extends AbstractEcsAwsAwareCachingAgent<Ec
// TODO: refine search key instead of filtering these cache results
List<EcsTargetGroup> tgList =
cacheClient.find(targetGroupKeys).stream()
- .filter(t -> t.getTargetGroupArn().contains(account.getAccountId()))
+ .filter(
+ t ->
+ t.getTargetGroupArn() != null
+ && t.getTargetGroupArn().contains(account.getAccountId()))
.collect(Collectors.toList());
return tgList.stream().map(EcsTargetGroup::getTargetGroupArn).collect(Collectors.toSet());

diff:
@@ -197,4 +197,47 @@ class TargetHealthCachingAgentSpec extends Specification {
EcsTargetHealth targetHealthDescription = targetHealthList.get(0)
targetHealthDescription.getTargetGroupArn() == targetGroupArn
}
+
+ def 'should handle null targetGroupArn in cache data'() {
+ given:
+ def targetGroupAttributes = [
+ loadBalancerNames: ['loadBalancerName'],
+ targetGroupArn: null,
+ targetGroupName: null,
+ vpcId: 'vpc-id',
+ ]
+ def targetGroupKey =
+ Keys.getTargetGroupKey('test-tg', CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, 'ip', 'vpc-id')
+ def loadbalancerKey =
+ Keys.getLoadBalancerKey('loadBalancerName', CommonCachingAgent.ACCOUNT, CommonCachingAgent.REGION, 'vpc-id', 'ip')
+ def relations = [loadBalancers: [loadbalancerKey]]
+ def targetGroupCacheData =
+ new DefaultCacheData(targetGroupKey, targetGroupAttributes, relations)
+
+
+ def badAwsProviderCache = Mock(ProviderCache)
+ badAwsProviderCache.filterIdentifiers(_, _) >> []
+ badAwsProviderCache.getAll(TARGET_GROUPS.getNs(), _, _) >> [targetGroupCacheData]
+
+ when:
+ agent.setAwsCache(badAwsProviderCache)
+ def targetHealthList = agent.getItems(ecs, Mock(ProviderCache))
+
+ then:
+ targetHealthList.size() == 0
+ }
+
+ def 'should return empty set when no target group cache data available'() {
+ given:
+ def emptyAwsProviderCache = Mock(ProviderCache)
+ emptyAwsProviderCache.filterIdentifiers(_, _) >> []
+ emptyAwsProviderCache.getAll(TARGET_GROUPS.getNs(), _, _) >> []
+
+ when:
+ agent.setAwsCache(emptyAwsProviderCache)
+ def targetHealthList = agent.getItems(ecs, Mock(ProviderCache))
+
+ then:
+ targetHealthList.size() == 0
+ }
}

original_message: fix(ecs): add null check for target group cache data (#4665)


message:
fix(ecs): set IAMRoleCachingAgent AWS region
* fix(ecs): set IAMRoleCachingAgent AWS region
Previously, DEFAULT_REGION was always used, breaking for users in China or GovCloud.
This change uses a configured region, if present, and logs the choice.
* use default unless sample shows otherwise

diff:
@@ -46,6 +46,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,8 +84,7 @@ public class IamRoleCachingAgent implements CachingAgent {
@Override
public CacheResult loadData(ProviderCache providerCache) {
- AmazonIdentityManagement iam =
- amazonClientProvider.getIam(account, Regions.DEFAULT_REGION.getName(), false);
+ AmazonIdentityManagement iam = amazonClientProvider.getIam(account, getIamRegion(), false);
Set<IamRole> cacheableRoles = fetchIamRoles(iam, accountName);
Map<String, Collection<CacheData>> newDataMap = generateFreshData(cacheableRoles);
@@ -134,6 +134,23 @@ public class IamRoleCachingAgent implements CachingAgent {
return evictionsByKey;
}
+ protected String getIamRegion() {
+ // sample a region from the account in case the default won't work
+ String testRegion =
+ !account.getRegions().isEmpty() && account.getRegions().get(0) != null
+ ? account.getRegions().get(0).getName()
+ : "";
+
+ if (StringUtils.isNotBlank(testRegion)
+ && (testRegion.startsWith("cn-") || testRegion.startsWith("us-gov-"))) {
+ log.debug("retrieving IAM Roles from given region: {}", testRegion);
+ return testRegion;
+ }
+
+ log.debug("retrieving IAM Roles from default region: {}", Regions.DEFAULT_REGION.getName());
+ return Regions.DEFAULT_REGION.getName();
+ }
+
Map<String, Collection<CacheData>> generateFreshData(Set<IamRole> cacheableRoles) {
Collection<CacheData> dataPoints = new HashSet<>();
Map<String, Collection<CacheData>> newDataMap = new HashMap<>();

diff:
@@ -23,11 +23,13 @@ import static org.mockito.Matchers.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import com.amazonaws.regions.Regions;
import com.amazonaws.services.identitymanagement.AmazonIdentityManagement;
import com.amazonaws.services.identitymanagement.model.ListRolesRequest;
import com.amazonaws.services.identitymanagement.model.ListRolesResult;
import com.amazonaws.services.identitymanagement.model.Role;
import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonCredentials;
import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
import com.netflix.spinnaker.clouddriver.ecs.cache.model.IamRole;
@@ -152,4 +154,39 @@ public class IamRoleCachingAgentTest extends CommonCachingAgent {
keys.contains(cacheData.getId()));
}
}
+
+ @Test
+ public void shouldGetDefaultRegion() {
+ // given
+ String defaultRegionName = Regions.DEFAULT_REGION.getName();
+ when(netflixAmazonCredentials.getRegions())
+ .thenReturn(Collections.singletonList(new AmazonCredentials.AWSRegion("us-east-1", null)));
+
+ // when
+ String actualRegionName = agent.getIamRegion();
+
+ // then
+ assertEquals(
+ "Expected region to equal " + defaultRegionName + ", but got " + actualRegionName,
+ defaultRegionName,
+ actualRegionName);
+ }
+
+ @Test
+ public void shouldGetConfiguredIamRegion() {
+ // given
+ String expectedRegionName = "cn-north-1";
+ when(netflixAmazonCredentials.getRegions())
+ .thenReturn(
+ Collections.singletonList(new AmazonCredentials.AWSRegion(expectedRegionName, null)));
+
+ // when
+ String actualRegionName = agent.getIamRegion();
+
+ // then
+ assertEquals(
+ "Expected region to equal " + expectedRegionName + ", but got " + actualRegionName,
+ expectedRegionName,
+ actualRegionName);
+ }
}

original_message:
fix(ecs): set IAMRoleCachingAgent AWS region (#4674)

* fix(ecs): set IAMRoleCachingAgent AWS region

Previously, DEFAULT_REGION was always used, breaking for users in China or GovCloud.
This change uses a configured region, if present, and logs the choice.

* use default unless sample shows otherwise

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


message:
fix(gce): use the new ScaleInControl autoscaler API
I also changed the clouddriver API for this functionality to match the new terminology. Since the only client of this API (deck) has it hidden behind a feature flag, I felt okay about breaking the API to make the code less confusing.

diff:
@@ -847,18 +847,19 @@ class GCEUtil {
}
}
- if (scaleDownControl) {
- def scaledDownReplicasInput = scaleDownControl.maxScaledDownReplicas
- def maxScaledDownReplicas = null
- if (scaledDownReplicasInput != null) {
- maxScaledDownReplicas = FixedOrPercent.newInstance()
- .setFixed(scaledDownReplicasInput.fixed)
- .setPercent(scaledDownReplicasInput.percent)
- }
- gceAutoscalingPolicy.scaleDownControl =
- new AutoscalingPolicyScaleDownControl(
- maxScaledDownReplicas: maxScaledDownReplicas,
- timeWindowSec: scaleDownControl.timeWindowSec)
+ if (scaleInControl) {
+ def scaledInReplicasInput = scaleInControl.maxScaledInReplicas
+ FixedOrPercent maxScaledInReplicas = null
+ if (scaledInReplicasInput != null) {
+ maxScaledInReplicas = new FixedOrPercent(
+ fixed: scaledInReplicasInput.fixed,
+ percent: scaledInReplicasInput.percent
+ )
+ }
+ gceAutoscalingPolicy.scaleInControl =
+ new AutoscalingPolicyScaleInControl(
+ maxScaledInReplicas: maxScaledInReplicas,
+ timeWindowSec: scaleInControl.timeWindowSec)
}
new Autoscaler(name: serverGroupName,

diff:
@@ -229,14 +229,14 @@ class UpsertGoogleAutoscalingPolicyAtomicOperation extends GoogleAtomicOperation
}
}
- // If scaleDownControl is completely absent, we leave the previous value.
+ // If scaleInControl is completely absent, we leave the previous value.
// To remove it, set it to an empty object.
- if (update.scaleDownControl != null) {
- def scaleDownControl = update.scaleDownControl
- if (scaleDownControl.timeWindowSec != null && scaleDownControl.maxScaledDownReplicas != null) {
- newDescription.scaleDownControl = scaleDownControl
+ if (update.scaleInControl != null) {
+ def scaleInControl = update.scaleInControl
+ if (scaleInControl.timeWindowSec != null && scaleInControl.maxScaledInReplicas != null) {
+ newDescription.scaleInControl = scaleInControl
} else {
- newDescription.scaleDownControl = null
+ newDescription.scaleInControl = null
}
}

diff:
@@ -18,11 +18,13 @@ package com.netflix.spinnaker.clouddriver.google.model
import groovy.transform.AutoClone
import groovy.transform.Canonical
+import groovy.transform.CompileStatic
import groovy.transform.ToString
@AutoClone
@Canonical
@ToString(includeNames = true)
+@CompileStatic
class GoogleAutoscalingPolicy {
Integer minNumReplicas
Integer maxNumReplicas
@@ -31,7 +33,7 @@ class GoogleAutoscalingPolicy {
CpuUtilization cpuUtilization
LoadBalancingUtilization loadBalancingUtilization
List<CustomMetricUtilization> customMetricUtilizations
- ScaleDownControl scaleDownControl
+ ScaleInControl scaleInControl
AutoscalingMode mode
@ToString(includeNames = true)
@@ -57,8 +59,8 @@ class GoogleAutoscalingPolicy {
}
}
- static class ScaleDownControl {
- FixedOrPercent maxScaledDownReplicas
+ static class ScaleInControl {
+ FixedOrPercent maxScaledInReplicas
Integer timeWindowSec
}

diff:
@@ -43,7 +43,7 @@ import com.google.api.services.compute.model.AutoscalingPolicy;
import com.google.api.services.compute.model.AutoscalingPolicyCpuUtilization;
import com.google.api.services.compute.model.AutoscalingPolicyCustomMetricUtilization;
import com.google.api.services.compute.model.AutoscalingPolicyLoadBalancingUtilization;
-import com.google.api.services.compute.model.AutoscalingPolicyScaleDownControl;
+import com.google.api.services.compute.model.AutoscalingPolicyScaleInControl;
import com.google.api.services.compute.model.DistributionPolicy;
import com.google.api.services.compute.model.Instance;
import com.google.api.services.compute.model.InstanceGroupManager;
@@ -85,7 +85,7 @@ import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.Cu
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.CustomMetricUtilization.UtilizationTargetType;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.FixedOrPercent;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.LoadBalancingUtilization;
-import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.ScaleDownControl;
+import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.ScaleInControl;
import com.netflix.spinnaker.clouddriver.google.model.GoogleDistributionPolicy;
import com.netflix.spinnaker.clouddriver.google.model.GoogleInstance;
import com.netflix.spinnaker.clouddriver.google.model.GoogleInstances;
@@ -922,7 +922,7 @@ abstract class AbstractGoogleServerGroupCachingAgent
output.setMaxNumReplicas(input.getMaxNumReplicas());
output.setMinNumReplicas(input.getMinNumReplicas());
output.setMode(valueOf(AutoscalingMode.class, input.getMode()));
- output.setScaleDownControl(convertScaleDownControl(input.getScaleDownControl()));
+ output.setScaleInControl(convertScaleInControl(input.getScaleInControl()));
return output;
}
@@ -969,20 +969,20 @@ abstract class AbstractGoogleServerGroupCachingAgent
return output;
}
- private static ScaleDownControl convertScaleDownControl(
- @Nullable AutoscalingPolicyScaleDownControl input) {
+ private static ScaleInControl convertScaleInControl(
+ @Nullable AutoscalingPolicyScaleInControl input) {
if (input == null) {
return null;
}
- FixedOrPercent maxScaledDownReplicas = null;
- if (input.getMaxScaledDownReplicas() != null) {
- maxScaledDownReplicas = new FixedOrPercent();
- maxScaledDownReplicas.setFixed(input.getMaxScaledDownReplicas().getFixed());
- maxScaledDownReplicas.setPercent(input.getMaxScaledDownReplicas().getPercent());
+ FixedOrPercent maxScaledInReplicas = null;
+ if (input.getMaxScaledInReplicas() != null) {
+ maxScaledInReplicas = new FixedOrPercent();
+ maxScaledInReplicas.setFixed(input.getMaxScaledInReplicas().getFixed());
+ maxScaledInReplicas.setPercent(input.getMaxScaledInReplicas().getPercent());
}
- ScaleDownControl output = new ScaleDownControl();
+ ScaleInControl output = new ScaleInControl();
output.setTimeWindowSec(input.getTimeWindowSec());
- output.setMaxScaledDownReplicas(maxScaledDownReplicas);
+ output.setMaxScaledInReplicas(maxScaledInReplicas);
return output;
}

diff:
@@ -33,7 +33,7 @@ import com.google.api.services.compute.model.AutoscalingPolicy;
import com.google.api.services.compute.model.AutoscalingPolicyCpuUtilization;
import com.google.api.services.compute.model.AutoscalingPolicyCustomMetricUtilization;
import com.google.api.services.compute.model.AutoscalingPolicyLoadBalancingUtilization;
-import com.google.api.services.compute.model.AutoscalingPolicyScaleDownControl;
+import com.google.api.services.compute.model.AutoscalingPolicyScaleInControl;
import com.google.api.services.compute.model.DistributionPolicy;
import com.google.api.services.compute.model.DistributionPolicyZoneConfiguration;
import com.google.api.services.compute.model.FixedOrPercent;
@@ -738,10 +738,10 @@ class AbstractGoogleServerGroupCachingAgentTest {
.setUtilizationTarget(911.23)
.setUtilizationTargetType("GAUGE"),
new AutoscalingPolicyCustomMetricUtilization()))
- .setScaleDownControl(
- new AutoscalingPolicyScaleDownControl()
+ .setScaleInControl(
+ new AutoscalingPolicyScaleInControl()
.setTimeWindowSec(10111)
- .setMaxScaledDownReplicas(new FixedOrPercent().setFixed(123).setPercent(456)));
+ .setMaxScaledInReplicas(new FixedOrPercent().setFixed(123).setPercent(456)));
InstanceGroupManager instanceGroupManager =
new InstanceGroupManager().setName("myServerGroup").setZone(ZONE_URL);
@@ -769,11 +769,9 @@ class AbstractGoogleServerGroupCachingAgentTest {
assertThat(converted.getMaxNumReplicas()).isEqualTo(input.getMaxNumReplicas());
assertThat(converted.getMinNumReplicas()).isEqualTo(input.getMinNumReplicas());
assertThat(converted.getMode().toString()).isEqualTo(input.getMode());
- assertThat(converted.getScaleDownControl().getTimeWindowSec()).isEqualTo(10111);
- assertThat(converted.getScaleDownControl().getMaxScaledDownReplicas().getFixed())
- .isEqualTo(123);
- assertThat(converted.getScaleDownControl().getMaxScaledDownReplicas().getPercent())
- .isEqualTo(456);
+ assertThat(converted.getScaleInControl().getTimeWindowSec()).isEqualTo(10111);
+ assertThat(converted.getScaleInControl().getMaxScaledInReplicas().getFixed()).isEqualTo(123);
+ assertThat(converted.getScaleInControl().getMaxScaledInReplicas().getPercent()).isEqualTo(456);
assertThat(converted.getCustomMetricUtilizations())
.hasSize(input.getCustomMetricUtilizations().size());
@@ -823,7 +821,7 @@ class AbstractGoogleServerGroupCachingAgentTest {
assertThat(converted.getMaxNumReplicas()).isNull();
assertThat(converted.getMinNumReplicas()).isNull();
assertThat(converted.getMode()).isNull();
- assertThat(converted.getScaleDownControl()).isNull();
+ assertThat(converted.getScaleInControl()).isNull();
}
public static AbstractGoogleServerGroupCachingAgent createCachingAgent(

original_message:
fix(gce): use the new ScaleInControl autoscaler API (#4705)

I also changed the clouddriver API for this functionality to match the new terminology. Since the only client of this API (deck) has it hidden behind a feature flag, I felt okay about breaking the API to make the code less confusing.

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


diff:
@@ -129,30 +129,28 @@ public class TaskHealthCachingAgent extends AbstractEcsCachingAgent<TaskHealth>
Keys.getTaskDefinitionKey(accountName, region, service.getTaskDefinition());
TaskDefinition taskDefinition = taskDefinitionCacheClient.get(taskDefinitionCacheKey);
- boolean lacksNetworkBindings = isTaskMissingNetworkBindings(task);
+ boolean lacksNetworkInterfaces = isTaskMissingNetworkInterfaces(task);
if (task.getContainers().isEmpty()
- || (lacksNetworkBindings && isTaskMissingNetworkInterfaces(task))) {
+ || (isTaskMissingNetworkBindings(task) && lacksNetworkInterfaces)) {
log.debug(
"Task '{}' is missing networking. Will not retrieve load balancer health.",
task.getTaskArn());
continue;
}
- TaskHealth taskHealth = null;
+ TaskHealth taskHealth;
// ideally, could determine health check method by looking at taskDef.networkMode,
// however this isn't reliably cached yet, so reusing network binding check.
- if (!lacksNetworkBindings) {
- taskHealth =
- inferHealthNetworkBindedContainer(
- targetHealthCacheClient, task, containerInstance, serviceName, service);
- }
-
- if (taskHealth == null) {
+ if (!lacksNetworkInterfaces) {
+ // if network interfaces are present, assume awsvpc mode
taskHealth =
inferHealthNetworkInterfacedContainer(
targetHealthCacheClient, task, serviceName, service, taskDefinition);
+ } else {
+ taskHealth =
+ inferHealthNetworkBindedContainer(
+ targetHealthCacheClient, task, containerInstance, serviceName, service);
}
-
log.debug("Task Health contains the following elements: {}", taskHealth);
if (taskHealth != null) {

original_message: chore(ecs): Clarify network check in task health agent (#4714)


message: chore(ecs): Update role info in README

diff:
-## AWS ECS Clouddriver
+## Amazon ECS Clouddriver
-The clouddriver-ecs module allows for ECS deployments of dockerized applications. **You need to enable the AWS cloud provider in order for the ECS cloud provider to work**.
-
-It is a work in progress
+The clouddriver-ecs module allows for Amazon ECS deployments of dockerized applications. **You need to enable the AWS cloud provider in order for the ECS cloud provider to work**.
## Clouddriver configuration
-In order for the ECS cloud provider to work, a corresponding AWS account must be configured and enabled. An ECS account will be tied to a given AWS account by its name. Below is an example snippet of `clouddriver.yml`:
+In order for the Amazon ECS cloud provider to work, a corresponding AWS account must be configured and enabled. An ECS account will be tied to a given AWS account by its name. Below is an example snippet of `clouddriver.yml`:
```
aws:
@@ -27,29 +25,13 @@ ecs:
```
+## Spinnaker role
+In Spinnaker 1.19 and later, the Amazon ECS cloud provider requires [service-linked roles](https://docs.aws.amazon.com/AmazonECS/latest/userguide/using-service-linked-roles.html) for Amazon ECS and Application Auto Scaling. To create the required service-linked roles, run the following `aws-cli` commands:
-## Spinnaker role
-Make sure that you allow the `application-autoscaling.amazonaws.com` and `ecs.amazonaws.com` principals to assume the SpinnakerManaged role by adding it as a principal. See example code below. Failure to do so will prevent you from deploying ECS server groups:
```
-{
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Principal": {
- "Service": [
- "ecs.amazonaws.com",
- "application-autoscaling.amazonaws.com"
- ],
- },
- "Action": "sts:AssumeRole"
- }
- ]
-}
+aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com
+aws iam create-service-linked-role --aws-service-name ecs.application-autoscaling.amazonaws.com
```
-##
-
-TODO Wishlist:
-1. Perhaps clouddriver should try to add the 2 required trust relationships on startup if they are detected as not being present
+See the official Spinnaker [Amazon ECS provider setup docs](https://spinnaker.io/setup/install/providers/aws/aws-ecs/#service-linked-iam-roles) for more information.

original_message: chore(ecs): Update role info in README (#4736)


message:
feat(kubernetes): add apiVersion, displayName, and namespace to ManifestBasedModel
Deck expects instances, serverGroups, serverGroupManagers, securityGroups, and loadBalancers to each include these four properties. Currently, Deck is combining data returned from each view with data returned from each manifest. Instead, let's just include these properties in each view.

diff:
@@ -36,6 +36,18 @@ public abstract class ManifestBasedModel {
return getManifest().getFullResourceName();
}
+ public String getDisplayName() {
+ return getManifest().getName();
+ }
+
+ public String getApiVersion() {
+ return getManifest().getApiVersion().toString();
+ }
+
+ public String getNamespace() {
+ return getManifest().getNamespace();
+ }
+
// Spinnaker namespace hacks
public String getZone() {
return getManifest().getNamespace();

original_message:
feat(kubernetes): add apiVersion, displayName, and namespace to ManifestBasedModel (#4765)

Deck expects instances, serverGroups, serverGroupManagers, securityGroups, and loadBalancers to each include these four properties. Currently, Deck is combining data returned from each view with data returned from each manifest. Instead, let's just include these properties in each view.

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


message:
fix(kubernetes): replace getCreatedTime in KubernetesResource interface
We [removed](https://github.com/spinnaker/clouddriver/pull/4769) getCreatedTime from the shared interface extended by Kubernetes resource types and pushed it down to KubernetesServerGroup only, but it turns out that Deck expects all resources to have this field.

diff:
@@ -32,4 +32,6 @@ public interface KubernetesResource {
KubernetesApiVersion getApiVersion();
String getNamespace();
+
+ Long getCreatedTime();
}

diff:
@@ -56,6 +56,7 @@ public final class KubernetesV2Instance implements Instance, KubernetesResource
private final KubernetesKind kind;
private final Map<String, String> labels;
private final Moniker moniker;
+ private final Long createdTime;
@Null
@Override
@@ -73,6 +74,7 @@ public final class KubernetesV2Instance implements Instance, KubernetesResource
this.kind = manifest.getKind();
this.labels = ImmutableMap.copyOf(manifest.getLabels());
this.moniker = moniker;
+ this.createdTime = manifest.getFormattedCreationTimestamp();
this.health = new ArrayList<>();
V1PodStatus status =

diff:
@@ -49,6 +49,7 @@ public final class KubernetesV2LoadBalancer
private final KubernetesKind kind;
private final Map<String, String> labels;
private final Moniker moniker;
+ private final Long createdTime;
private KubernetesV2LoadBalancer(
KubernetesManifest manifest,
@@ -64,6 +65,7 @@ public final class KubernetesV2LoadBalancer
this.labels = ImmutableMap.copyOf(manifest.getLabels());
this.moniker = moniker;
this.serverGroups = serverGroups;
+ this.createdTime = manifest.getFormattedCreationTimestamp();
}
@Nullable

diff:
@@ -71,6 +71,7 @@ public final class KubernetesV2SecurityGroup implements KubernetesResource, Secu
private final KubernetesKind kind;
private final Map<String, String> labels;
private final Moniker moniker;
+ private final Long createdTime;
private final Set<Rule> inboundRules;
private final Set<Rule> outboundRules;
@@ -104,6 +105,7 @@ public final class KubernetesV2SecurityGroup implements KubernetesResource, Secu
this.namespace = manifest.getNamespace();
this.labels = ImmutableMap.copyOf(manifest.getLabels());
this.moniker = moniker;
+ this.createdTime = manifest.getFormattedCreationTimestamp();
this.inboundRules = inboundRules;
this.outboundRules = outboundRules;

diff:
@@ -20,7 +20,6 @@ package com.netflix.spinnaker.clouddriver.kubernetes.caching.view.model;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -43,8 +42,6 @@ import com.netflix.spinnaker.clouddriver.model.ServerGroup;
import com.netflix.spinnaker.clouddriver.model.ServerGroupManager.ServerGroupManagerSummary;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.moniker.Moniker;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -135,7 +132,7 @@ public final class KubernetesV2ServerGroup implements KubernetesResource, Server
this.displayName = manifest.getName();
this.labels = ImmutableMap.copyOf(manifest.getLabels());
this.moniker = moniker;
- this.createdTime = getCreatedTime(manifest);
+ this.createdTime = manifest.getFormattedCreationTimestamp();
this.buildInfo =
ImmutableMap.of(
"images",
@@ -269,21 +266,6 @@ public final class KubernetesV2ServerGroup implements KubernetesResource, Server
.build());
}
- private static Long getCreatedTime(KubernetesManifest manifest) {
- Map<String, String> metadata =
- (Map<String, String>) manifest.getOrDefault("metadata", new HashMap<>());
- String timestamp = metadata.get("creationTimestamp");
- try {
- if (!Strings.isNullOrEmpty(timestamp)) {
- return (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX").parse(timestamp)).getTime();
- }
- } catch (ParseException e) {
- log.warn("Failed to parse timestamp: ", e);
- }
-
- return null;
- }
-
@Override
public String getType() {
return KubernetesCloudProvider.ID;

diff:
@@ -49,6 +49,7 @@ public final class KubernetesV2ServerGroupManager
private final KubernetesKind kind;
private final Map<String, String> labels;
private final Moniker moniker;
+ private final Long createdTime;
private KubernetesV2ServerGroupManager(
KubernetesManifest manifest,
@@ -64,6 +65,7 @@ public final class KubernetesV2ServerGroupManager
this.labels = ImmutableMap.copyOf(manifest.getLabels());
this.moniker = moniker;
this.serverGroups = serverGroups;
+ this.createdTime = manifest.getFormattedCreationTimestamp();
}
public static KubernetesV2ServerGroupManager fromCacheData(

diff:
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,9 +31,11 @@ import java.util.Optional;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+@Slf4j
public class KubernetesManifest extends HashMap<String, Object> {
private static final ObjectMapper mapper = new ObjectMapper();
@@ -151,6 +155,21 @@ public class KubernetesManifest extends HashMap<String, Object> {
: "";
}
+ @JsonIgnore
+ @Nullable
+ public Long getFormattedCreationTimestamp() {
+ String timestamp = getCreationTimestamp();
+ try {
+ if (!Strings.isNullOrEmpty(timestamp)) {
+ return (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX").parse(timestamp)).getTime();
+ }
+ } catch (ParseException e) {
+ log.warn("Failed to parse timestamp: ", e);
+ }
+
+ return null;
+ }
+
@JsonIgnore
@Nonnull
public List<OwnerReference> getOwnerReferences() {

original_message:
fix(kubernetes): replace getCreatedTime in KubernetesResource interface (#4791)

We [removed](https://github.com/spinnaker/clouddriver/pull/4769) getCreatedTime from the shared interface extended by Kubernetes resource types and pushed it down to KubernetesServerGroup only, but it turns out that Deck expects all resources to have this field.

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


diff:
@@ -840,7 +840,8 @@ class GCEUtil {
if (cpuUtilization) {
gceAutoscalingPolicy.cpuUtilization =
- new AutoscalingPolicyCpuUtilization(utilizationTarget: cpuUtilization.utilizationTarget)
+ new AutoscalingPolicyCpuUtilization(utilizationTarget: cpuUtilization.utilizationTarget,
+ predictiveMethod: cpuUtilization.predictiveMethod)
}
if (loadBalancingUtilization) {

diff:
@@ -39,6 +39,12 @@ class GoogleAutoscalingPolicy {
@ToString(includeNames = true)
static class CpuUtilization {
Double utilizationTarget
+ PredictiveMethod predictiveMethod
+
+ enum PredictiveMethod {
+ NONE,
+ STANDARD
+ }
}
@ToString(includeNames = true)

diff:
@@ -83,6 +83,7 @@ import com.netflix.spinnaker.clouddriver.google.compute.Instances;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.AutoscalingMode;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.CpuUtilization;
+import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.CpuUtilization.PredictiveMethod;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.CustomMetricUtilization;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.CustomMetricUtilization.UtilizationTargetType;
import com.netflix.spinnaker.clouddriver.google.model.GoogleAutoscalingPolicy.FixedOrPercent;
@@ -942,6 +943,7 @@ abstract class AbstractGoogleServerGroupCachingAgent
}
CpuUtilization output = new CpuUtilization();
output.setUtilizationTarget(input.getUtilizationTarget());
+ output.setPredictiveMethod(valueOf(PredictiveMethod.class, input.getPredictiveMethod()));
return output;
}

diff:
@@ -725,7 +725,10 @@ class AbstractGoogleServerGroupCachingAgentTest {
AutoscalingPolicy input =
new AutoscalingPolicy()
.setCoolDownPeriodSec(123)
- .setCpuUtilization(new AutoscalingPolicyCpuUtilization().setUtilizationTarget(9.87))
+ .setCpuUtilization(
+ new AutoscalingPolicyCpuUtilization()
+ .setUtilizationTarget(9.87)
+ .setPredictiveMethod("STANDARD"))
.setLoadBalancingUtilization(
new AutoscalingPolicyLoadBalancingUtilization().setUtilizationTarget(6.54))
.setMaxNumReplicas(99)
@@ -764,6 +767,8 @@ class AbstractGoogleServerGroupCachingAgentTest {
assertThat(converted.getCoolDownPeriodSec()).isEqualTo(input.getCoolDownPeriodSec());
assertThat(converted.getCpuUtilization().getUtilizationTarget())
.isEqualTo(input.getCpuUtilization().getUtilizationTarget());
+ assertThat(converted.getCpuUtilization().getPredictiveMethod().toString())
+ .isEqualTo(input.getCpuUtilization().getPredictiveMethod());
assertThat(converted.getLoadBalancingUtilization().getUtilizationTarget())
.isEqualTo(input.getLoadBalancingUtilization().getUtilizationTarget());
assertThat(converted.getMaxNumReplicas()).isEqualTo(input.getMaxNumReplicas());
@@ -824,6 +829,65 @@ class AbstractGoogleServerGroupCachingAgentTest {
assertThat(converted.getScaleInControl()).isNull();
}
+ @Test
+ void serverGroupAutoscalingPolicy_unknownPredictiveAutoscalerMethod() {
+
+ InstanceGroupManager instanceGroupManager =
+ new InstanceGroupManager().setName("myServerGroup").setZone(ZONE_URL);
+ Autoscaler autoscaler =
+ new Autoscaler()
+ .setZone(ZONE_URL)
+ .setTarget("myServerGroup")
+ .setAutoscalingPolicy(
+ new AutoscalingPolicy()
+ .setCpuUtilization(
+ new AutoscalingPolicyCpuUtilization()
+ .setPredictiveMethod("SOME THING THAT DOESN'T REALLY EXIST")));
+
+ Compute compute =
+ new StubComputeFactory()
+ .setInstanceGroupManagers(instanceGroupManager)
+ .setAutoscalers(autoscaler)
+ .create();
+ AbstractGoogleServerGroupCachingAgent cachingAgent =
+ createCachingAgent(
+ compute, ImmutableList.of(instanceGroupManager), ImmutableList.of(autoscaler));
+
+ CacheResult cacheResult = cachingAgent.loadData(inMemoryProviderCache());
+ GoogleServerGroup serverGroup = getOnlyServerGroup(cacheResult);
+ GoogleAutoscalingPolicy converted = serverGroup.getAutoscalingPolicy();
+
+ assertThat(converted.getCpuUtilization().getPredictiveMethod()).isNull();
+ }
+
+ @Test
+ void serverGroupAutoscalingPolicy_emptyPredictiveAutoscalerMethod() {
+
+ InstanceGroupManager instanceGroupManager =
+ new InstanceGroupManager().setName("myServerGroup").setZone(ZONE_URL);
+ Autoscaler autoscaler =
+ new Autoscaler()
+ .setZone(ZONE_URL)
+ .setTarget("myServerGroup")
+ .setAutoscalingPolicy(
+ new AutoscalingPolicy().setCpuUtilization(new AutoscalingPolicyCpuUtilization()));
+
+ Compute compute =
+ new StubComputeFactory()
+ .setInstanceGroupManagers(instanceGroupManager)
+ .setAutoscalers(autoscaler)
+ .create();
+ AbstractGoogleServerGroupCachingAgent cachingAgent =
+ createCachingAgent(
+ compute, ImmutableList.of(instanceGroupManager), ImmutableList.of(autoscaler));
+
+ CacheResult cacheResult = cachingAgent.loadData(inMemoryProviderCache());
+ GoogleServerGroup serverGroup = getOnlyServerGroup(cacheResult);
+ GoogleAutoscalingPolicy converted = serverGroup.getAutoscalingPolicy();
+
+ assertThat(converted.getCpuUtilization().getPredictiveMethod()).isNull();
+ }
+
public static AbstractGoogleServerGroupCachingAgent createCachingAgent(
Compute compute, Collection<InstanceGroupManager> instanceGroupManagers) {
return createCachingAgent(

original_message:
feat(gce): add support for the predictive autoscaler (#4794)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


message:
refactor(kubernetes): add missing to non-generated code
* refactor(kubernetes): add missing to non-generated code
We cannot yet enable the error-prone warnings as there are a few dozen classes with Lombok-generated getter overrides that I'll handle in a followup PR.
* fix(kubernetes): run spotlessApply

diff:
@@ -41,6 +41,7 @@ public class KubernetesCoreCachingAgent extends KubernetesV2OnDemandCachingAgent
super(namedAccountCredentials, objectMapper, registry, agentIndex, agentCount, agentInterval);
}
+ @Override
public Collection<AgentDataType> getProvidedDataTypes() {
// The ARTIFACT kind is deprecated; no new entries of this type will be created. We are leaving
// it in the authoritative types for now so that existing entries get evicted.
@@ -72,6 +73,7 @@ public class KubernetesCoreCachingAgent extends KubernetesV2OnDemandCachingAgent
* these requests, and have it return all pending on-demand refresh requests (not just ones
* related to its slice of namespaces).
*/
+ @Override
protected boolean handlePendingOnDemandRequests() {
return agentIndex == 0;
}

diff:
@@ -40,6 +40,7 @@ public class KubernetesUnregisteredCustomResourceCachingAgent
super(namedAccountCredentials, objectMapper, registry, agentIndex, agentCount, agentInterval);
}
+ @Override
public ImmutableSet<AgentDataType> getProvidedDataTypes() {
return primaryKinds().stream()
.map(k -> AUTHORITATIVE.forType(k.toString()))

diff:
@@ -28,6 +28,7 @@ public final class KubernetesV2Application implements Application {
private final String name;
private final Map<String, Set<String>> clusterNames;
+ @Override
public Map<String, String> getAttributes() {
return new ImmutableMap.Builder<String, String>().put("name", name).build();
}

diff:
@@ -57,6 +57,7 @@ public class KubernetesV2JobStatus implements JobStatus {
this.createdTime = job.getMetadata().getCreationTimestamp().getMillis();
}
+ @Override
public Map<String, String> getCompletionDetails() {
Map<String, String> details = new HashMap<>();
details.put("exitCode", this.exitCode != null ? this.exitCode.toString() : "");
@@ -66,6 +67,7 @@ public class KubernetesV2JobStatus implements JobStatus {
return details;
}
+ @Override
public JobState getJobState() {
V1JobStatus status = job.getStatus();
if (status == null) {

diff:
@@ -21,6 +21,7 @@ import com.netflix.spinnaker.clouddriver.kubernetes.description.manifest.Kuberne
import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesCredentials;
public interface CanUndoRollout extends CanRollout {
+ @Override
KubernetesKind kind();
default void undoRollout(

diff:
@@ -51,6 +51,7 @@ public abstract class KubernetesHandler implements CanDeploy, CanDelete, CanPatc
public abstract int deployPriority();
+ @Override
@Nonnull
public abstract KubernetesKind kind();

diff:
@@ -55,6 +55,7 @@ public class KubernetesRunJobOperation
return TaskRepository.threadLocalTask.get();
}
+ @Override
public KubernetesRunJobDeploymentResult operate(List<KubernetesRunJobDeploymentResult> _unused) {
getTask().updateStatus(OP_NAME, "Running Kubernetes job...");
KubernetesManifest jobSpec = this.description.getManifest();

diff:
@@ -54,6 +54,7 @@ public class KubernetesV2JobProvider implements JobProvider<KubernetesV2JobStatu
this.manifestProvider = manifestProvider;
}
+ @Override
@Nullable
public KubernetesV2JobStatus collectJob(String account, String location, String id) {
Optional<V1Job> optionalJob = getKubernetesJob(account, location, id);
@@ -84,6 +85,7 @@ public class KubernetesV2JobProvider implements JobProvider<KubernetesV2JobStatu
return jobStatus;
}
+ @Override
@Nullable
public Map<String, Object> getFileContents(
String account, String location, String id, String containerName) {
@@ -104,6 +106,7 @@ public class KubernetesV2JobProvider implements JobProvider<KubernetesV2JobStatu
.orElse(null);
}
+ @Override
public void cancelJob(String account, String location, String id) {
throw new NotImplementedException(
"cancelJob is not implemented for the V2 Kubernetes provider");

diff:
@@ -212,6 +212,7 @@ public class KubernetesCredentials {
.build(key -> supplier.get());
}
+ @Override
public T get() {
return cache.get(CACHE_KEY);
}

original_message:
refactor(kubernetes): add missing @Overrides to non-generated code (#4852)

* refactor(kubernetes): add missing @Overrides to non-generated code

We cannot yet enable the error-prone warnings as there are a few dozen classes with Lombok-generated getter overrides that I'll handle in a followup PR.

* fix(kubernetes): run spotlessApply

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>


diff:
@@ -23,5 +23,6 @@ import lombok.EqualsAndHashCode;
@EqualsAndHashCode(callSuper = false)
public class DeleteLambdaFunctionDescription extends AbstractLambdaFunctionDescription {
String functionName;
+ String qualifier;
// TODO : Stub, implement later
}

diff:
@@ -50,6 +50,8 @@ public class DeleteLambdaAtomicOperation
DeleteFunctionRequest request =
new DeleteFunctionRequest().withFunctionName(cache.getFunctionArn());
+ request.withQualifier(description.getQualifier());
+
DeleteFunctionResult result = client.deleteFunction(request);
updateTaskStatus("Finished deletion of AWS Lambda Function Operation...");

original_message:
fix(aws/lambda): Add qualifier flag to support deleting specific version of lambda (#4873)

Co-authored-by: smaniyedath <shyam_maniyedath@intuit.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>