Skip to content

Commit 2ff4155

Browse files
Zhongxiang Zhengsmiklosovicbeobal
committed
Enable JMX server configuration to be in cassandra.yaml
patch by Zhongxiang Zheng; reviewed by Stefan Miklosovic, Maulin Vasavada, Cheng Wang, Jordan West for CASSANDRA-11695 Co-authored-by: Stefan Miklosovic <[email protected]> Co-authored-by: Sam Tunnicliffe <[email protected]>
1 parent 54e4688 commit 2ff4155

38 files changed

+1042
-574
lines changed

CHANGES.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
5.1
2+
* Enable JMX server configuration to be in cassandra.yaml (CASSANDRA-11695)
23
* Parallelized UCS compactions (CASSANDRA-18802)
34
* Avoid prepared statement invalidation race when committing schema changes (CASSANDRA-20116)
45
* Restore optimization in MultiCBuilder around building one clustering (CASSANDRA-20129)

NEWS.txt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ New features
8888
generate a password of configured password strength policy upon role creation or alteration
8989
when 'GENERATED PASSWORD' clause is used. Character sets supported are: English, Cyrillic, modern Cyrillic,
9090
German, Polish and Czech.
91-
- JMX SSL configuration can be now done in cassandra.yaml via jmx_encryption_options section instead of cassandra-env.sh
9291
- There is new MBean of name org.apache.cassandra.service.snapshot:type=SnapshotManager which exposes user-facing
9392
snapshot operations. Snapshot-related methods on StorageServiceMBean are still present and functional
9493
but marked as deprecated.
@@ -102,6 +101,12 @@ New features
102101
compactions. To avoid the possibility of starving background compactions from resources, the number of threads
103102
used for major compactions is limited to half the available compaction threads by default, and can be controlled
104103
by a new --jobs / -j option of nodetool compact.
104+
- It is possible to configure JMX server in cassandra.yaml in jmx_server_options configuration section.
105+
JMX SSL configuration can be configured via jmx_encryption_options in jmx_server_options. The old way of
106+
configuring JMX is still present and default, but new way is preferable as it will e.g. not leak credentials for
107+
JMX to JVM parameters. You have to opt-in to use the configuration via cassandra.yaml by uncommenting
108+
the respective configuration sections and by commenting out `configure_jmx` function call in cassandra-env.sh.
109+
Enabling both ways of configuring JMX will result in a node failing to start.
105110

106111

107112
Upgrading

bin/nodetool

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,19 @@ if [ -f "$CASSANDRA_CONF/cassandra-env.sh" ]; then
5252
JVM_OPTS="$JVM_OPTS_SAVE"
5353
fi
5454

55+
# In case JMX_PORT is not set (when configure_jmx in cassandra-env.sh is commented out),
56+
# try to parse it from cassandra.yaml.
57+
if [ "x$JMX_PORT" = "x" ]; then
58+
if [ -f "$CASSANDRA_CONF/cassandra.yaml" ]; then
59+
JMX_PORT=`grep jmx_port $CASSANDRA_CONF/cassandra.yaml | cut -d ':' -f 2 | tr -d '[[:space:]]'`
60+
fi
61+
fi
62+
63+
# If, by any chance, it is not there either, set it to default.
64+
if [ "x$JMX_PORT" = "x" ]; then
65+
JMX_PORT=7199
66+
fi
67+
5568
# JMX Port passed via cmd line args (-p 9999 / --port 9999 / --port=9999)
5669
# should override the value from cassandra-env.sh
5770
ARGS=""

conf/cassandra-env.sh

Lines changed: 59 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -218,55 +218,66 @@ if [ "x$LOCAL_JMX" = "x" ]; then
218218
LOCAL_JMX=yes
219219
fi
220220

221-
# Specifies the default port over which Cassandra will be available for
222-
# JMX connections.
223-
# For security reasons, you should not expose this port to the internet. Firewall it if needed.
224-
JMX_PORT="7199"
225-
226-
if [ "$LOCAL_JMX" = "yes" ]; then
227-
JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT"
228-
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
229-
else
230-
JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.port=$JMX_PORT"
231-
# if ssl is enabled the same port cannot be used for both jmx and rmi so either
232-
# pick another value for this property or comment out to use a random port (though see CASSANDRA-7087 for origins)
233-
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
234-
235-
# turn on JMX authentication. See below for further options
236-
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"
237-
238-
# jmx ssl options
239-
# Consider using the jmx_encryption_options section of cassandra.yaml instead
240-
# to prevent sensitive information being exposed.
241-
# In case jmx ssl options are configured in both the places - this file and cassandra.yaml, and
242-
# if com.sun.management.jmxremote.ssl is configured to be true here and encryption_options are marked enabled in
243-
# cassandra.yaml, then we will get exception at the startup
244-
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=true"
245-
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.need.client.auth=true"
246-
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.protocols=<enabled-protocols>"
247-
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.cipher.suites=<enabled-cipher-suites>"
248-
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStore=/path/to/keystore"
249-
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStorePassword=<keystore-password>"
250-
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStore=/path/to/truststore"
251-
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStorePassword=<truststore-password>"
252-
fi
221+
configure_jmx()
222+
{
223+
JMX_PORT=$1
224+
225+
if [ "$LOCAL_JMX" = "yes" ]; then
226+
JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT"
227+
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
228+
else
229+
JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.port=$JMX_PORT"
230+
# if ssl is enabled the same port cannot be used for both jmx and rmi so either
231+
# pick another value for this property or comment out to use a random port (though see CASSANDRA-7087 for origins)
232+
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
233+
234+
# turn on JMX authentication. See below for further options
235+
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"
236+
237+
# jmx ssl options
238+
# Consider using the jmx_encryption_options section of jmx_server_options in cassandra.yaml instead
239+
# to prevent sensitive information being exposed.
240+
# In case jmx ssl options are configured in both the places - this file and cassandra.yaml, and
241+
# if com.sun.management.jmxremote.ssl is configured to be true here and encryption_options are marked enabled in
242+
# cassandra.yaml, then we will get exception at the startup
243+
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=true"
244+
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.need.client.auth=true"
245+
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.protocols=<enabled-protocols>"
246+
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.cipher.suites=<enabled-cipher-suites>"
247+
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStore=/path/to/keystore"
248+
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStorePassword=<keystore-password>"
249+
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStore=/path/to/truststore"
250+
#JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStorePassword=<truststore-password>"
251+
fi
252+
253+
# jmx authentication and authorization options. By default, auth is only
254+
# activated for remote connections but they can also be enabled for local only JMX
255+
## Basic file based authn & authz
256+
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"
257+
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.access.file=/etc/cassandra/jmxremote.access"
258+
## Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
259+
## JAAS login modules can be used for authentication by uncommenting these two properties.
260+
## Cassandra ships with a LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule -
261+
## which delegates to the IAuthenticator configured in cassandra.yaml. See the sample JAAS configuration
262+
## file cassandra-jaas.config
263+
#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"
264+
#JVM_OPTS="$JVM_OPTS -Djava.security.auth.login.config=$CASSANDRA_CONF/cassandra-jaas.config"
265+
266+
## Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
267+
## uncomment this to use it. Requires one of the two authentication options to be enabled
268+
#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"
269+
}
253270

254-
# jmx authentication and authorization options. By default, auth is only
255-
# activated for remote connections but they can also be enabled for local only JMX
256-
## Basic file based authn & authz
257-
JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"
258-
#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.access.file=/etc/cassandra/jmxremote.access"
259-
## Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
260-
## JAAS login modules can be used for authentication by uncommenting these two properties.
261-
## Cassandra ships with a LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule -
262-
## which delegates to the IAuthenticator configured in cassandra.yaml. See the sample JAAS configuration
263-
## file cassandra-jaas.config
264-
#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"
265-
#JVM_OPTS="$JVM_OPTS -Djava.security.auth.login.config=$CASSANDRA_CONF/cassandra-jaas.config"
266-
267-
## Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
268-
## uncomment this to use it. Requires one of the two authentication options to be enabled
269-
#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"
271+
# If this function call is commented out, then Cassandra will start with no system properties for JMX set whatsoever.
272+
# We will be expecting the settings in jmx_server_options and jmx_encryption_options respectively instead.
273+
# The argument specifies the default port over which Cassandra will be available for JMX connections.
274+
#
275+
# If you comment out configure_jmx method call, then JMX_PORT variable will not be set, which means
276+
# nodetool which sources this file will not see it either and port from cassandra.yaml will be parsed instead,
277+
# if not found there either, it defaults to 7199.
278+
#
279+
# For security reasons, you should not expose this port to the internet. Firewall it if needed.
280+
configure_jmx 7199
270281

271282
# To use mx4j, an HTML interface for JMX, add mx4j-tools.jar to the lib/
272283
# directory.

conf/cassandra.yaml

Lines changed: 53 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1721,14 +1721,59 @@ client_encryption_options:
17211721
# JMX SSL.
17221722
# Similar to `client/server_encryption_options`, you can specify PEM-based
17231723
# key material or customize the SSL configuration using `ssl_context_factory` in `jmx_encryption_options`.
1724-
#jmx_encryption_options:
1725-
# enabled: true
1726-
# cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
1727-
# accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
1728-
# keystore: conf/cassandra_ssl.keystore
1729-
# keystore_password: cassandra
1730-
# truststore: conf/cassandra_ssl.truststore
1731-
# truststore_password: cassandra
1724+
# If you uncomment this section, please be sure that you comment out configure_jmx function call in cassandra-env.sh
1725+
# as it is errorneous to have JMX set by two ways, both in cassandra-env.sh and in this yaml.
1726+
#jmx_server_options:
1727+
# enabled: true
1728+
# remote: false
1729+
# jmx_port: 7199
1730+
#
1731+
# Port used by the RMI registry when remote connections are enabled.
1732+
# To simplify firewall configs, this can be set to the same as the JMX server port (port). See CASSANDRA-7087.
1733+
# However, if ssl is enabled the same port cannot be used for both jmx and rmi so either
1734+
# pick another value for this property. Alternatively, comment out or set to 0 to use a random
1735+
# port (pre-CASSANDRA-7087 behaviour)
1736+
# rmi_port: 7199
1737+
#
1738+
# jmx ssl options - only apply when remote connections are enabled
1739+
#
1740+
# jmx_encryption_options:
1741+
# enabled: true
1742+
# cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
1743+
# accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
1744+
# keystore: conf/cassandra_ssl.keystore
1745+
# keystore_password: cassandra
1746+
# truststore: conf/cassandra_ssl.truststore
1747+
# truststore_password: cassandra
1748+
#
1749+
# jmx authentication and authorization options.
1750+
# authenticate: false
1751+
#
1752+
# Options for basic file based authentication & authorization
1753+
# password_file: /etc/cassandra/jmxremote.password
1754+
# access_file: /etc/cassandra/jmxremote.access
1755+
#
1756+
# Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
1757+
# JAAS login modules can be used for authentication using this property.Cassandra ships with a
1758+
# LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule - which delegates
1759+
# to the IAuthenticator configured in cassandra.yaml.
1760+
#
1761+
# login_config_name refers to the Application Name in the JAAS configuration under which the
1762+
# desired LoginModule(s) are configured.
1763+
# The location of the JAAS config file may be set using the standard JVM mechanism, by setting
1764+
# the system property "java.security.auth.login.config". If this property is set, it's value
1765+
# will be used to locate the config file. For convenience, if the property is not already set
1766+
# at startup, a value can be supplied here via the login_config_file setting.
1767+
#
1768+
# The Application Name specified must be present in the JAAS config or an error will be thrown
1769+
# when authentication is attempted.
1770+
# See the sample JAAS configuration file conf/cassandra-jaas.config
1771+
# login_config_name: CassandraLogin
1772+
# login_config_file: conf/cassandra-jaas.config
1773+
#
1774+
# Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
1775+
# uncomment this to use it. Requires one of the two authentication options to be enabled
1776+
# authorizer: org.apache.cassandra.auth.jmx.AuthorizationProxy
17321777

17331778
# internode_compression controls whether traffic between nodes is
17341779
# compressed.

conf/cassandra_latest.yaml

Lines changed: 53 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1685,14 +1685,59 @@ client_encryption_options:
16851685
# JMX SSL.
16861686
# Similar to `client/server_encryption_options`, you can specify PEM-based
16871687
# key material or customize the SSL configuration using `ssl_context_factory` in `jmx_encryption_options`.
1688-
#jmx_encryption_options:
1689-
# enabled: true
1690-
# cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
1691-
# accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
1692-
# keystore: conf/cassandra_ssl.keystore
1693-
# keystore_password: cassandra
1694-
# truststore: conf/cassandra_ssl.truststore
1695-
# truststore_password: cassandra
1688+
# If you uncomment this section, please be sure that you comment out configure_jmx function call in cassandra-env.sh
1689+
# as it is errorneous to have JMX set by two ways, both in cassandra-env.sh and in this yaml.
1690+
#jmx_server_options:
1691+
# enabled: true
1692+
# remote: false
1693+
# jmx_port: 7199
1694+
#
1695+
# Port used by the RMI registry when remote connections are enabled.
1696+
# To simplify firewall configs, this can be set to the same as the JMX server port (port). See CASSANDRA-7087.
1697+
# However, if ssl is enabled the same port cannot be used for both jmx and rmi so either
1698+
# pick another value for this property. Alternatively, comment out or set to 0 to use a random
1699+
# port (pre-CASSANDRA-7087 behaviour)
1700+
# rmi_port: 7199
1701+
#
1702+
# jmx ssl options - only apply when remote connections are enabled
1703+
#
1704+
# jmx_encryption_options:
1705+
# enabled: true
1706+
# cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
1707+
# accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
1708+
# keystore: conf/cassandra_ssl.keystore
1709+
# keystore_password: cassandra
1710+
# truststore: conf/cassandra_ssl.truststore
1711+
# truststore_password: cassandra
1712+
#
1713+
# jmx authentication and authorization options.
1714+
# authenticate: false
1715+
#
1716+
# Options for basic file based authentication & authorization
1717+
# password_file: /etc/cassandra/jmxremote.password
1718+
# access_file: /etc/cassandra/jmxremote.access
1719+
#
1720+
# Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
1721+
# JAAS login modules can be used for authentication using this property.Cassandra ships with a
1722+
# LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule - which delegates
1723+
# to the IAuthenticator configured in cassandra.yaml.
1724+
#
1725+
# login_config_name refers to the Application Name in the JAAS configuration under which the
1726+
# desired LoginModule(s) are configured.
1727+
# The location of the JAAS config file may be set using the standard JVM mechanism, by setting
1728+
# the system property "java.security.auth.login.config". If this property is set, it's value
1729+
# will be used to locate the config file. For convenience, if the property is not already set
1730+
# at startup, a value can be supplied here via the login_config_file setting.
1731+
#
1732+
# The Application Name specified must be present in the JAAS config or an error will be thrown
1733+
# when authentication is attempted.
1734+
# See the sample JAAS configuration file conf/cassandra-jaas.config
1735+
# login_config_name: CassandraLogin
1736+
# login_config_file: conf/cassandra-jaas.config
1737+
#
1738+
# Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
1739+
# uncomment this to use it. Requires one of the two authentication options to be enabled
1740+
# authorizer: org.apache.cassandra.auth.jmx.AuthorizationProxy
16961741

16971742
# internode_compression controls whether traffic between nodes is
16981743
# compressed.

doc/scripts/convert_yaml_to_adoc.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
'hints_compression',
5050
'server_encryption_options',
5151
'client_encryption_options',
52-
'jmx_encryption_options',
52+
'jmx_server_options',
5353
'transparent_data_encryption_options',
5454
'hinted_handoff_disabled_datacenters',
5555
'startup_checks',

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ public enum CassandraRelevantProperties
146146
COM_SUN_MANAGEMENT_JMXREMOTE_PASSWORD_FILE("com.sun.management.jmxremote.password.file"),
147147
/** Port number to enable JMX RMI connections - com.sun.management.jmxremote.port */
148148
COM_SUN_MANAGEMENT_JMXREMOTE_PORT("com.sun.management.jmxremote.port"),
149+
/** Enables SSL sockets for the RMI registry from which clients obtain the JMX connector stub */
150+
COM_SUN_MANAGEMENT_JMXREMOTE_REGISTRY_SSL("com.sun.management.jmxremote.registry.ssl"),
149151
/**
150152
* The port number to which the RMI connector will be bound - com.sun.management.jmxremote.rmi.port.
151153
* An Integer object that represents the value of the second argument is returned
@@ -287,6 +289,10 @@ public enum CassandraRelevantProperties
287289
IO_NETTY_EVENTLOOP_THREADS("io.netty.eventLoopThreads"),
288290
IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT("io.netty.transport.estimateSizeOnSubmit"),
289291
IO_NETTY_TRANSPORT_NONATIVE("io.netty.transport.noNative"),
292+
JAVAX_NET_SSL_KEYSTORE("javax.net.ssl.keyStore"),
293+
JAVAX_NET_SSL_KEYSTOREPASSWORD("javax.net.ssl.keyStorePassword"),
294+
JAVAX_NET_SSL_TRUSTSTORE("javax.net.ssl.trustStore"),
295+
JAVAX_NET_SSL_TRUSTSTOREPASSWORD("javax.net.ssl.trustStorePassword"),
290296
JAVAX_RMI_SSL_CLIENT_ENABLED_CIPHER_SUITES("javax.rmi.ssl.client.enabledCipherSuites"),
291297
JAVAX_RMI_SSL_CLIENT_ENABLED_PROTOCOLS("javax.rmi.ssl.client.enabledProtocols"),
292298
/** Java class path. */

src/java/org/apache/cassandra/config/Config.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,8 @@ public static class SSTableConfig
434434

435435
public EncryptionOptions.ServerEncryptionOptions server_encryption_options = new EncryptionOptions.ServerEncryptionOptions();
436436
public EncryptionOptions client_encryption_options = new EncryptionOptions();
437-
public EncryptionOptions jmx_encryption_options = new EncryptionOptions();
437+
438+
public JMXServerOptions jmx_server_options;
438439

439440
public InternodeCompression internode_compression = InternodeCompression.none;
440441

@@ -1322,7 +1323,8 @@ public enum BatchlogEndpointStrategy
13221323
private static final Set<String> SENSITIVE_KEYS = new HashSet<String>() {{
13231324
add("client_encryption_options");
13241325
add("server_encryption_options");
1325-
add("jmx_encryption_options");
1326+
// jmx_server_options output (JMXServerOptions.toString()) doesn't
1327+
// include sensitive encryption config so no need to blocklist here
13261328
}};
13271329

13281330
public static void log(Config config)

0 commit comments

Comments
 (0)