Skip to content

Commit 7579622

Browse files
author
buishglc
committed
About a year and a half ago, we delivered this automated installer which can install open-source ranger, as emr-native ranger feature release, we deeply refactored it, combined open-source and emr-native ranger together, make it support 4 high applicability scenarios ( No.1, 2, 3, 4) altogether. for this commit, Scenario 1: "OpenLDAP + EMR-Native Ranger" is fully tested, works very well.
1 parent 76d4ea9 commit 7579622

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+7241
-1152
lines changed

‎bin/ad.ps1

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# 安装Active Directory服务
2+
Install-windowsfeature -name AD-Domain-Services -IncludeManagementTools
3+
4+
# 创建Forest, 命令执行成功之后会自动重启服务器，虽然使用-NoRebootOnCompletion:$true可以规避自动重启，但是如果要正常使用AD服务，还是要重启服务器。
5+
Install-ADDSForest -DomainName example.com -SafeModeAdministratorPassword (ConvertTo-SecureString 'Admin1234!' -AsPlainText -Force) -DomainMode WinThreshold -DomainNetbiosName ABC -ForestMode WinThreshold -DatabasePath "C:\Windows\NTDS" -LogPath "C:\Windows\NTDS" -SysvolPath "C:\Windows\SYSVOL" -CreateDnsDelegation:$false -InstallDns:$true -NoRebootOnCompletion:$false -Force:$true
6+
7+
ksetup /addkdc CN-NORTH-1.COMPUTE.INTERNAL
8+
9+
netdom trust CN-NORTH-1.COMPUTE.INTERNAL /Domain:EXAMPLE.COM /add /realm /passwordt:Admin1234!
10+
11+
# 执行该命令前，需在AD服务上手都配置安全组策略, 确保“Kerberos允许的加密类型”这一配置项包含如下两种加密类型中的一种或全部！
12+
ksetup /SetEncTypeAttr CN-NORTH-1.COMPUTE.INTERNAL AES256-CTS-HMAC-SHA1-96 AES128-CTS-HMAC-SHA1-96
13+
14+
# ------------------------------------------ ou: Service Accounts ------------------------------------------- #
15+
16+
# Remove OU "Service Accounts" recursively if exists
17+
# Get-ADOrganizationalUnit -Identity "OU=Service Accounts,DC=EXAMPLE,DC=COM" |
18+
# Set-ADObject -ProtectedFromAccidentalDeletion:$false -PassThru |
19+
# Remove-ADOrganizationalUnit -Confirm:$false -Recursive
20+
21+
# Add OU "Service Accounts"
22+
New-ADOrganizationalUnit -Name "Service Accounts" -Path "DC=EXAMPLE,DC=COM"
23+
24+
# ------------------------------------------- user: ranger-binder ------------------------------------------- #
25+
26+
# Remove service account "ranger-binder" if exists
27+
Remove-ADUser -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Confirm:$false
28+
29+
# Create service account "ranger-binder"
30+
New-ADUser -Name "ranger-binder" -OtherAttributes @{'title'="ranger-binder"} -path "OU=Service Accounts,DC=EXAMPLE,DC=COM"
31+
Set-ADAccountPassword -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
32+
Set-ADUser -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
33+
Enable-ADAccount -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM"
34+
35+
# ---------------------------------------- user: emr-ad-domain-joiner --------------------------------------- #
36+
37+
# Remove service account "emr-ad-domain-joiner" if exists
38+
Remove-ADUser -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Confirm:$false
39+
40+
# Create service account "emr-ad-domain-joiner"
41+
New-ADUser -Name "emr-ad-domain-joiner" -OtherAttributes @{'title'="emr-ad-domain-joiner"} -path "OU=Service Accounts,DC=EXAMPLE,DC=COM"
42+
Set-ADAccountPassword -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
43+
Set-ADUser -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
44+
Enable-ADAccount -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
45+
46+
# Add service account "emr-ad-domain-joiner" to "Domain Admins", "Schema Admins", "Enterprise Admins" groups,
47+
# so as it has enough privileges to add EMR cluster nodes into AD domain.
48+
Add-ADGroupMember -Identity "Domain Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
49+
Add-ADGroupMember -Identity "Schema Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
50+
Add-ADGroupMember -Identity "Enterprise Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
51+
52+
# --------------------------------------------- user: ad-user-1 --------------------------------------------- #
53+
54+
# Remove service account "emr-ad-domain-joiner" if exists
55+
Remove-ADUser -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -Confirm:$false
56+
57+
# Create normal domain user "ad-user-1"
58+
New-ADUser -Name "ad-user-1" -OtherAttributes @{'title'="ad-user-1"} -path "CN=Users,DC=EXAMPLE,DC=COM"
59+
Set-ADAccountPassword -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
60+
Set-ADUser -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
61+
Enable-ADAccount -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM"

‎bin/crt.sh

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# This script apply follow naming role:
5+
# for a principal, i.e. ranger-admin, its:
6+
# 1. private key: ranger-admin.key
7+
# 2. certificate: ranger-admin.crt
8+
# 3. pcks12 file: ranger-admin.p12
9+
# 4. keystore: ranger-admin-keystore.jks
10+
# 5. truststore: ranger-admin-truststore.jks
11+
#
12+
13+
createRangerSecrets() {
14+
printHeading "CREATE KEY, CRT, KEYSTORE & TRUSTSTORE FOR RANGER"
15+
16+
rm -rf $RANGER_SECRETS_DIR
17+
mkdir -p $RANGER_SECRETS_DIR
18+
19+
# ranger-admin
20+
printHeading "CREATE KEY, CRT, KEYSTORE FOR [ ranger-admin ]"
21+
createKeyCertPair "ranger-admin"
22+
createKeystore "ranger-admin"
23+
24+
# ranger-plugin
25+
# it is NOT possible to use ranger-plugin certs, because they will be generated & managed by EMR
26+
# but for other open-source plugins, i.e. HBase, Presto, these certs will be required if ranger admin https enabled!
27+
printHeading "CREATE KEY, CRT, KEYSTORE FOR [ ranger-plugin ]"
28+
createKeyCertPair "ranger-plugin"
29+
createKeystore "ranger-plugin"
30+
31+
# build truststore file used by ranger admin, be careful, this file NOT USED!
32+
# instead, plugins crts will install to JVM cacerts file.
33+
createOrImportToTruststore "ranger-admin" "ranger-admin"# trust self
34+
createOrImportToTruststore "ranger-admin" "ranger-plugin" # trust plugin
35+
36+
# build truststore file used by ranger open-source plugins.
37+
# for emr-native plugins, EMR will regenerate truststore with key & certs from secrets manager.
38+
createOrImportToTruststore "ranger-plugin" "ranger-plugin" # trust self
39+
createOrImportToTruststore "ranger-plugin" "ranger-admin" # trust admin
40+
41+
# NOTE: because there is explicit truststore config items in ranger admin,
42+
# there is no way to set the path of 'ranger-admin-truststore.jks',
43+
# so, we HAVE TO add all trusted crts to JAVA default crts store: $JAVA_HOME/jre/lib/security/cacerts
44+
removeFromJavaDefaultTruststore "ranger-admin"
45+
importToJavaDefaultTruststore "ranger-admin" # trust self
46+
removeFromJavaDefaultTruststore "ranger-plugin"
47+
importToJavaDefaultTruststore "ranger-plugin" # trust self
48+
49+
listCerts "ranger-admin"
50+
listCerts "ranger-plugin"
51+
52+
removeSecretsOnSecretsManager
53+
uploadRangerSecretsToSecretsManager
54+
}
55+
56+
removeRangerSecrets() {
57+
removeSecretsOnSecretsManager
58+
removeFromJavaDefaultTruststore "ranger-admin"
59+
removeFromJavaDefaultTruststore "ranger-plugin"
60+
rm -rf $RANGER_SECRETS_DIR
61+
}
62+
63+
createKeyCertPair() {
64+
principal="$1"
65+
openssl req -x509 -newkey rsa:2048 -nodes -keyout $RANGER_SECRETS_DIR/${principal}.key -out $RANGER_SECRETS_DIR/${principal}.crt -days 3650 -subj "/C=CN/ST=SHANGHAI/L=SHANGHAI/O=EXAMPLE/OU=IT/CN=$CERTIFICATE_CN"
66+
}
67+
68+
createKeystore() {
69+
principal="$1"
70+
openssl pkcs12 -export -in $RANGER_SECRETS_DIR/${principal}.crt -inkey $RANGER_SECRETS_DIR/${principal}.key -chain -CAfile $RANGER_SECRETS_DIR/${principal}.crt -name ${principal} -out $RANGER_SECRETS_DIR/${principal}.p12 -password pass:changeit
71+
keytool -importkeystore -deststorepass changeit -destkeystore $RANGER_SECRETS_DIR/${principal}-keystore.jks -srckeystore $RANGER_SECRETS_DIR/${principal}.p12 -srcstoretype PKCS12 -srcstorepass changeit
72+
}
73+
74+
createOrImportToTruststore() {
75+
trustingPrincipal="$1"
76+
trustedPrincipal="$2"
77+
keytool -import -file $RANGER_SECRETS_DIR/${trustedPrincipal}.crt -alias $trustedPrincipal -keystore $RANGER_SECRETS_DIR/${trustingPrincipal}-truststore.jks -storepass changeit -noprompt
78+
}
79+
80+
removeFromJavaDefaultTruststore() {
81+
trustedPrincipal="$1"
82+
keytool -delete -alias $trustedPrincipal -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit
83+
}
84+
85+
importToJavaDefaultTruststore() {
86+
trustedPrincipal="$1"
87+
keytool -import -file $RANGER_SECRETS_DIR/${trustedPrincipal}.crt -alias $trustedPrincipal -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit -noprompt
88+
}
89+
90+
listCerts() {
91+
principal="$1"
92+
echo "${principal}-keystore.jks contains following certificates:"
93+
keytool -list -v -keystore $RANGER_SECRETS_DIR/${principal}-keystore.jks -storepass changeit|grep Alias.*
94+
echo "${principal}-truststore.jks contains following certificates:"
95+
keytool -list -v -keystore $RANGER_SECRETS_DIR/${principal}-truststore.jks -storepass changeit|grep Alias.*
96+
}
97+
98+
uploadRangerSecretsToSecretsManager() {
99+
printHeading "UPLOAD SECRETS TO SECRETSMANAGER"
100+
# append postfix for secret id in case there are multiple clusters to be created!
101+
aws secretsmanager create-secret --name "ranger-admin@${RANGER_HOST}" --description "Ranger Admin Certificate" --secret-string file://$RANGER_SECRETS_DIR/ranger-admin.crt --region $REGION
102+
# EMR Security Configuration need a secret file contains plugin key and cert both.
103+
cat $RANGER_SECRETS_DIR/ranger-plugin.key $RANGER_SECRETS_DIR/ranger-plugin.crt > $RANGER_SECRETS_DIR/ranger-plugin.key+crt
104+
aws secretsmanager create-secret --name "ranger-plugin@${RANGER_HOST}" --description "Ranger Admin Private Key & Certificate" --secret-string file://$RANGER_SECRETS_DIR/ranger-plugin.key+crt --region $REGION
105+
}
106+
107+
removeSecretsOnSecretsManager() {
108+
printHeading "REMOVE SECRETS FROM SECRETSMANAGER"
109+
aws secretsmanager delete-secret --secret-id "ranger-admin@${RANGER_HOST}" --force-delete-without-recovery --region $REGION --cli-read-timeout 10 --cli-connect-timeout 10
110+
aws secretsmanager delete-secret --secret-id "ranger-plugin@${RANGER_HOST}" --force-delete-without-recovery --region $REGION --cli-read-timeout 10 --cli-connect-timeout 10
111+
112+
# deleting secrets is async job, have to wait for a while...
113+
sleep 30
114+
}
115+

‎bin/ec2.sh

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
3+
initEc2() {
4+
# init ec2 is a one-time job, no need to run duplicatly,
5+
# especially when re-run all-in-one install
6+
if [ -f $INIT_EC2_FLAG_FILE ]; then
7+
echo "This ec2 instance has been initialized, nothing to do!"
8+
echo "If you want to force re-init this ec2, please execute force-init-ec2 command"
9+
else
10+
if [[ "$REGION" = "" || "$ACCESS_KEY_ID" = "" || "$SECRET_ACCESS_KEY" = "" ]]; then
11+
echo "ERROR! --region or --access-key-id or --secret-access-key is not provided!"
12+
exit 1
13+
fi
14+
installTools
15+
configSsh
16+
installAwsCli
17+
installJdk8IfNotExists
18+
touch $INIT_EC2_FLAG_FILE
19+
fi
20+
}
21+
22+
forceInitEc2() {
23+
rm -f $INIT_EC2_FLAG_FILE
24+
initEc2
25+
}
26+
27+
installTools() {
28+
printHeading "INSTALL COMMON TOOLS ON EC2"
29+
yum -y update
30+
# install common tools
31+
yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E '%{rhel}').noarch.rpm
32+
yum -y install lrzsz vim wget zip unzip expect tree htop iotop nc telnet jq
33+
34+
# change timezone
35+
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
36+
}
37+
38+
configSsh() {
39+
printHeading "CONFIG SSH"
40+
# enable ssh login with password
41+
sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/g" /etc/ssh/sshd_config
42+
echo "PermitRootLogin yes" | tee -a /etc/ssh/sshd_config
43+
echo "RSAAuthentication yes" | tee -a /etc/ssh/sshd_config
44+
systemctl restart sshd
45+
46+
# overwrite authorized_keys of root, because it blocks root login with: "no-port-forwarding, ... echo;sleep 10""
47+
# if user: hadoop exists, this is a node of EMR, otherwise, this is a normal EC2 instance!
48+
egrep "^hadoop\:" /etc/passwd >& /dev/null
49+
if [ "$?" == "0" ]; then
50+
cat /home/hadoop/.ssh/authorized_keys > /root/.ssh/authorized_keys
51+
else
52+
cat /home/ec2-user/.ssh/authorized_keys > /root/.ssh/authorized_keys
53+
fi
54+
}
55+
56+
installAwsCli() {
57+
printHeading "INSTALL AWS CLI V2"
58+
# aws cli is very stupid!
59+
# for v1, it is installed via rpm/yum, so with 'yum list installed awscli', we can't get version
60+
# for v2, it is installed via zip Packages, it does NOT work with 'yum list installed awscli', only 'aws --version' works
61+
# but for v1, 'aws --version' does not work, it DO print version message, but does NOT return string value!
62+
# if let message=$(aws --version), it prints message on console, but $message is empty!
63+
# so, it is REQUIRED to append '2>&1', the following is right way to get version:
64+
# awscliVer=$(aws --version 2>&1 | grep -o '[0-9]*\.[0-9]*\.[0-9]*' | head -n1)
65+
rm /tmp/awscli -rf
66+
echo "Remove awscli v1 if exists ..."
67+
yum -y remove awscli
68+
69+
echo "Remove awscli v2 if exists in case not latest version ..."
70+
rm /usr/bin/aws
71+
rm /usr/local/bin/aws
72+
rm /usr/bin/aws_completer
73+
rm /usr/local/bin/aws_completer
74+
rm -rf /usr/local/aws-cli
75+
76+
echo "Install latest awscli v2 ..."
77+
wget "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -P "/tmp/awscli/" &> /dev/null
78+
unzip /tmp/awscli/awscli-exe-linux-x86_64.zip -d /tmp/awscli/ &> /dev/null
79+
/tmp/awscli/aws/install
80+
ln -s /usr/local/bin/aws /usr/bin/aws
81+
82+
echo "Create credentials for awscli ..."
83+
tee /tmp/config <<EOF
84+
[default]
85+
region = $REGION
86+
EOF
87+
tee /tmp/credentials <<EOF
88+
[default]
89+
aws_access_key_id = $ACCESS_KEY_ID
90+
aws_secret_access_key = $SECRET_ACCESS_KEY
91+
EOF
92+
# for non-root users
93+
users=(ec2-user hadoop)
94+
for user in "${users[@]}"; do
95+
# if user exists, add awscli credentials
96+
egrep "^$user\:" /etc/passwd >& /dev/null
97+
if [ "$?" == "0" ]; then
98+
rm -rf /home/$user/.aws
99+
mkdir /home/$user/.aws
100+
cp /tmp/config /home/$user/.aws/
101+
cp /tmp/credentials /home/$user/.aws/
102+
chown -R $user:$user /home/$user/.aws
103+
fi
104+
done
105+
# for root user
106+
rm -rf /root/.aws
107+
mkdir /root/.aws
108+
cp /tmp/config /root/.aws/
109+
cp /tmp/credentials /root/.aws/
110+
chown -R root:root /root/.aws
111+
112+
rm -f /tmp/config /tmp/credentials
113+
}
114+
115+
installJdk8IfNotExists() {
116+
rpm -q java-1.8.0-openjdk-devel &>/dev/null
117+
if [ ! "$?" = "0" ]; then
118+
printHeading "INSTALL OPEN JDK8"
119+
yum -y install java-1.8.0-openjdk-devel
120+
printHeading "MAKE AND EXPORT JAVA ENV VARS"
121+
echo "export JAVA_HOME=$JAVA_HOME;export PATH=$JAVA_HOME/bin:$PATH" > /etc/profile.d/java.sh
122+
source /etc/profile.d/java.sh
123+
# make sure ec2-user also set JAVA_HOME, only works after re-login as ec2-user
124+
# not work for current ssh session!
125+
sudo -i -u ec2-user source /etc/profile.d/java.sh
126+
fi
127+
}

0 commit comments

Comments
 (0)