From b2a161b9b78de5addecfde14304678d7cc3a4088 Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Wed, 15 Dec 2021 16:00:46 -0800 Subject: [PATCH] Use IMDSV2 to get instance metadata (#977) Co-authored-by: ayushis --- build.gradle | 1 + .../netflix/priam/aws/S3FileSystemBase.java | 14 ++-- .../identity/config/AWSInstanceInfo.java | 76 +++++-------------- .../identity/config/TestAWSInstanceInfo.java | 34 ++++----- 4 files changed, 47 insertions(+), 78 deletions(-) diff --git a/build.gradle b/build.gradle index db5eaa911..5ae51364f 100644 --- a/build.gradle +++ b/build.gradle @@ -40,6 +40,7 @@ allprojects { compile 'com.google.code.findbugs:jsr305:3.0.2' // AWS Services + compile 'com.amazonaws:aws-java-sdk-core:latest.release' compile 'com.amazonaws:aws-java-sdk-s3:latest.release' compile 'com.amazonaws:aws-java-sdk-sns:latest.release' compile 'com.amazonaws:aws-java-sdk-ec2:latest.release' diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 3fe07736a..4ff3d10df 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -19,11 +19,7 @@ import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.lifecycle.LifecycleAndOperator; -import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter; -import com.amazonaws.services.s3.model.lifecycle.LifecyclePredicateVisitor; -import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate; -import com.amazonaws.services.s3.model.lifecycle.LifecycleTagPredicate; +import com.amazonaws.services.s3.model.lifecycle.*; import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; @@ -142,6 +138,14 @@ public void visit(LifecycleTagPredicate lifecycleTagPredicate) {} @Override public void visit(LifecycleAndOperator lifecycleAndOperator) {} + + @Override + public void visit( + LifecycleObjectSizeGreaterThanPredicate lifecycleObjectSizeGreaterThanPredicate) {} + + @Override + public void visit( + LifecycleObjectSizeLessThanPredicate lifecycleObjectSizeLessThanPredicate) {} } private Optional getBucketLifecycleRule(List rules, String prefix) { diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java index 6102c9daa..78a9f0455 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -16,17 +16,15 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; +import com.amazonaws.util.EC2MetadataUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.cred.ICredential; import com.netflix.priam.utils.RetryableCallable; -import com.netflix.priam.utils.SystemUtils; import java.util.List; -import java.util.Optional; import org.apache.commons.lang3.StringUtils; -import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,12 +32,12 @@ @Singleton public class AWSInstanceInfo implements InstanceInfo { private static final Logger logger = LoggerFactory.getLogger(AWSInstanceInfo.class); - static final String PUBLIC_HOSTNAME_URL = - "http://169.254.169.254/latest/meta-data/public-hostname"; - static final String LOCAL_HOSTNAME_URL = - "http://169.254.169.254/latest/meta-data/local-hostname"; - static final String PUBLIC_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/public-ipv4"; - static final String LOCAL_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/local-ipv4"; + + static final String PUBLIC_HOSTNAME_URL = "/latest/meta-data/public-hostname"; + static final String LOCAL_HOSTNAME_URL = "/latest/meta-data/local-hostname"; + static final String PUBLIC_HOSTIP_URL = "/latest/meta-data/public-ipv4"; + static final String LOCAL_HOSTIP_URL = "/latest/meta-data/local-ipv4"; + private JSONObject identityDocument = null; private String privateIp; private String hostIP; @@ -61,9 +59,7 @@ public AWSInstanceInfo(ICredential credential) { @Override public String getPrivateIP() { if (privateIp == null) { - privateIp = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/local-ipv4"); + privateIp = EC2MetadataUtils.getPrivateIpAddress(); } return privateIp; } @@ -71,9 +67,7 @@ public String getPrivateIP() { @Override public String getRac() { if (rac == null) { - rac = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/placement/availability-zone"); + rac = EC2MetadataUtils.getAvailabilityZone(); } return rac; } @@ -98,9 +92,7 @@ public List getDefaultRacks() { @Override public String getInstanceId() { if (instanceId == null) { - instanceId = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/instance-id"); + instanceId = EC2MetadataUtils.getInstanceId(); } return instanceId; } @@ -108,41 +100,21 @@ public String getInstanceId() { @Override public String getInstanceType() { if (instanceType == null) { - instanceType = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/instance-type"); + instanceType = EC2MetadataUtils.getInstanceType(); } return instanceType; } private String getMac() { if (mac == null) { - mac = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/") - .trim(); + mac = EC2MetadataUtils.getNetworkInterfaces().get(0).getMacAddress(); } return mac; } @Override public String getRegion() { - try { - getIdentityDocument(); - return this.identityDocument.getString("region"); - } catch (JSONException e) { - // If there is any issue in getting region, use AZ as backup. - return getRac().substring(0, getRac().length() - 1); - } - } - - private void getIdentityDocument() throws JSONException { - if (this.identityDocument == null) { - String jsonStr = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/dynamic/instance-identity/document"); - this.identityDocument = new JSONObject(jsonStr); - } + return EC2MetadataUtils.getEC2InstanceRegion(); } @Override @@ -152,12 +124,7 @@ public String getVpcId() { if (vpcId == null) try { - vpcId = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" - + nacId - + "vpc-id") - .trim(); + vpcId = EC2MetadataUtils.getNetworkInterfaces().get(0).getVpcId(); } catch (Exception e) { logger.info( "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", @@ -211,9 +178,9 @@ public InstanceEnvironment getInstanceEnvironment() { @Override public String getHostname() { if (hostName == null) { + String publicHostName = tryGetDataFromUrl(PUBLIC_HOSTNAME_URL); hostName = - tryGetDataFromUrl(PUBLIC_HOSTNAME_URL) - .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTNAME_URL)); + publicHostName == null ? tryGetDataFromUrl(LOCAL_HOSTNAME_URL) : publicHostName; } return hostName; } @@ -221,18 +188,17 @@ public String getHostname() { @Override public String getHostIP() { if (hostIP == null) { - hostIP = - tryGetDataFromUrl(PUBLIC_HOSTIP_URL) - .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTIP_URL)); + String publicHostIP = tryGetDataFromUrl(PUBLIC_HOSTIP_URL); + hostIP = publicHostIP == null ? tryGetDataFromUrl(LOCAL_HOSTIP_URL) : publicHostIP; } return hostIP; } - Optional tryGetDataFromUrl(String url) { + String tryGetDataFromUrl(String url) { try { - return Optional.of(SystemUtils.getDataFromUrl(url)); + return EC2MetadataUtils.getData(url); } catch (Exception e) { - return Optional.empty(); + return null; } } } diff --git a/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java index c71fa663c..ffc3fd5cf 100644 --- a/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java +++ b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java @@ -1,9 +1,7 @@ package com.netflix.priam.identity.config; import com.google.common.truth.Truth; -import com.netflix.priam.utils.SystemUtils; import mockit.Expectations; -import mockit.Mocked; import org.junit.Before; import org.junit.Test; @@ -21,10 +19,10 @@ public void setUp() { } @Test - public void testPublicHostIP(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testPublicHostIP() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); result = "1.2.3.4"; } }; @@ -32,12 +30,12 @@ public void testPublicHostIP(@Mocked SystemUtils systemUtils) { } @Test - public void testMissingPublicHostIP(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testMissingPublicHostIP() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); - result = new RuntimeException(); - SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTIP_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + result = null; + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.LOCAL_HOSTIP_URL); result = "1.2.3.4"; } }; @@ -45,10 +43,10 @@ public void testMissingPublicHostIP(@Mocked SystemUtils systemUtils) { } @Test - public void testPublicHostname(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testPublicHostname() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); result = "hostname"; } }; @@ -56,12 +54,12 @@ public void testPublicHostname(@Mocked SystemUtils systemUtils) { } @Test - public void testMissingPublicHostname(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testMissingPublicHostname() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); - result = new RuntimeException(); - SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTNAME_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + result = null; + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.LOCAL_HOSTNAME_URL); result = "hostname"; } };