STAR-508: Cursor-based memtable walks

CHANGES.txt

@@ -1,3 +1,6 @@
+Future version (tbd)
+ * Require only MODIFY permission on base when updating table with MV (STAR-564)
+
 4.0.1
  * Cleanup dependency scopes (CASSANDRA-16704)
  * Make JmxHistogram#getRecentValues() and JmxTimer#getRecentValues() thread-safe (CASSANDRA-16707)

README.asc

@@ -22,8 +22,8 @@ and running, and demonstrate some simple reads and writes. For a more-complete g
 
 First, we'll unpack our archive:
 
-  $ tar -zxvf apache-cassandra-$VERSION.tar.gz
-  $ cd apache-cassandra-$VERSION
+  $ tar -zxvf dse-db-$VERSION.tar.gz
+  $ cd dse-db-$VERSION
 
 After that we start the server. Running the startup script with the -f argument will cause
 Cassandra to remain in the foreground and log to standard out; it can be stopped with ctrl-C.

bin/cassandra.in.sh

@@ -30,10 +30,10 @@ CLASSPATH="$CASSANDRA_CONF"
 # compiled classes. NOTE: This isn't needed by the startup script,
 # it's just used here in constructing the classpath.
 if [ -d $CASSANDRA_HOME/build ] ; then
-    #cassandra_bin="$CASSANDRA_HOME/build/classes/main"
-    cassandra_bin=`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
+    #dse_db_bin="$CASSANDRA_HOME/build/classes/main"
+    dse_db_bin=`ls -1 $CASSANDRA_HOME/build/dse-db*.jar`
 
-    CLASSPATH="$CLASSPATH:$cassandra_bin"
+    CLASSPATH="$CLASSPATH:$dse_db_bin"
 fi
 
 # the default location for commitlogs, sstables, and saved caches
@@ -112,16 +112,16 @@ JAVA_VERSION=11
 if [ "$JVM_VERSION" = "1.8.0" ]  ; then
     JVM_PATCH_VERSION=${jvmver#*_}
     if [ "$JVM_VERSION" \< "1.8" ] || [ "$JVM_VERSION" \> "1.8.2" ] ; then
-        echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $JVM_VERSION is not supported."
+        echo "DSE DB 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java $JVM_VERSION is not supported."
         exit 1;
     fi
     if [ "$JVM_PATCH_VERSION" -lt 151 ] ; then
-        echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java 8 update $JVM_PATCH_VERSION is not supported."
+        echo "DSE DB 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer). Java 8 update $JVM_PATCH_VERSION is not supported."
         exit 1;
     fi
     JAVA_VERSION=8
 elif [ "$JVM_VERSION" \< "11" ] ; then
-    echo "Cassandra 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
+    echo "DSE DB 4.0 requires either Java 8 (update 151 or newer) or Java 11 (or newer)."
     exit 1;
 fi
 

conf/cassandra-env.sh

@@ -99,7 +99,7 @@ if [ $JAVA_VERSION -ge 11 ] ; then
     if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
         # only add -Xlog:gc if it's not mentioned in jvm-server.options file
         mkdir -p ${CASSANDRA_LOG_DIR}
-        JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+        JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
     fi
 else
     # Java 8

conf/cassandra.yaml

@@ -546,7 +546,7 @@ concurrent_materialized_view_writes: 32
 #
 # offheap_objects
 #    off heap objects
-memtable_allocation_type: heap_buffers
+memtable_allocation_type: offheap_objects
 
 # Limit memory usage for Merkle tree calculations during repairs. The default
 # is 1/16th of the available heap. The main tradeoff is that smaller trees
@@ -633,7 +633,7 @@ index_summary_resize_interval_in_minutes: 60
 # buffers. Enable this to avoid sudden dirty buffer flushing from
 # impacting read latencies. Almost always a good idea on SSDs; not
 # necessarily on platters.
-trickle_fsync: false
+trickle_fsync: true
 trickle_fsync_interval_in_kb: 10240
 
 # TCP port, for commands and data
@@ -1243,18 +1243,6 @@ transparent_data_encryption_options:
 # SAFETY THRESHOLDS #
 #####################
 
-# When executing a scan, within or across a partition, we need to keep the
-# tombstones seen in memory so we can return them to the coordinator, which
-# will use them to make sure other replicas also know about the deleted rows.
-# With workloads that generate a lot of tombstones, this can cause performance
-# problems and even exaust the server heap.
-# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
-# Adjust the thresholds here if you understand the dangers and want to
-# scan more tombstones anyway.  These thresholds may also be adjusted at runtime
-# using the StorageService mbean.
-tombstone_warn_threshold: 1000
-tombstone_failure_threshold: 100000
-
 # Filtering and secondary index queries at read consistency levels above ONE/LOCAL_ONE use a
 # mechanism called replica filtering protection to ensure that results from stale replicas do
 # not violate consistency. (See CASSANDRA-8272 and CASSANDRA-15907 for more details.) This
@@ -1275,19 +1263,6 @@ replica_filtering_protection:
     cached_rows_warn_threshold: 2000
     cached_rows_fail_threshold: 32000
 
-# Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default.
-# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
-batch_size_warn_threshold_in_kb: 5
-
-# Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default.
-batch_size_fail_threshold_in_kb: 50
-
-# Log WARN on any batches not of type LOGGED than span across more partitions than this limit
-unlogged_batch_across_partitions_warn_threshold: 10
-
-# Log a warning when compacting partitions larger than this value
-compaction_large_partition_warning_threshold_mb: 100
-
 # GC Pauses greater than 200 ms will be logged at INFO level
 # This threshold can be adjusted to minimize logging if necessary
 # gc_log_threshold_in_ms: 200
@@ -1419,6 +1394,10 @@ report_unconfirmed_repaired_data_mismatches: false
 # table_count_warn_threshold: 150
 # keyspace_count_warn_threshold: 40
 
+# This is the page size used internally by aggregation queries. It aims to limit the memory used by aggregation
+# queries when there is a lot of data to aggregate.
+# aggregation_subpage_size_in_kb: 2048
+
 #########################
 # EXPERIMENTAL FEATURES #
 #########################
@@ -1434,3 +1413,138 @@ enable_sasi_indexes: false
 # Enables creation of transiently replicated keyspaces on this node.
 # Transient replication is experimental and is not recommended for production use.
 enable_transient_replication: false
+
+  # Emulates DataStax Constellation database-as-a-service defaults.
+  #
+  # When enabled, some defaults are modified to match those used by DataStax Constellation (DataStax cloud data
+  # platform). This includes (but is not limited to) stricter guardrails defaults.
+  #
+  # This can be used as an convenience to develop and test applications meant to run on DataStax Constellation.
+  #
+  # Warning: when enabled, the updated defaults reflect those of DataStax Constellation _at the time_ of the currently
+  #                 used DSE release. This is a best-effort emulation of said defaults. Further, all nodes must use the same
+  #                 config value.
+  # emulate_dbaas_defaults: false
+
+  # Guardrails settings.
+  # guardrails:
+  # When executing a scan, within or across a partition, we need to keep the
+  # tombstones seen in memory so we can return them to the coordinator, which
+  # will use them to make sure other replicas also know about the deleted rows.
+  # With workloads that generate a lot of tombstones, this can cause performance
+  # problems and even exhaust the server heap.
+  # (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
+  # Adjust the thresholds here if you understand the dangers and want to
+  # scan more tombstones anyway.  These thresholds may also be adjusted at runtime
+  # using the StorageService mbean.
+  #
+  # Default tombstone_warn_threshold is 1000, may differ if emulate_dbaas_defaults is enabled
+  # Default tombstone_failure_threshold is 100000, may differ if emulate_dbaas_defaults is enabled
+  # tombstone_warn_threshold: 1000
+  # tombstone_failure_threshold: 100000
+
+  # Log a warning when compacting partitions larger than this value.
+  # Default value is 100mb, may differ if emulate_dbaas_defaults is enabled
+  # partition_size_warn_threshold_in_mb: 100
+
+  # Log WARN on any multiple-partition batch size that exceeds this value. 64kb per batch by default.
+  # Use caution when increasing the size of this threshold as it can lead to node instability.
+  # Default value is 64kb, may differ if emulate_dbaas_defaults is enabled
+  # batch_size_warn_threshold_in_kb: 64
+
+  # Fail any multiple-partition batch that exceeds this value. The calculated default is 640kb (10x warn threshold).
+  # Default value is 640kb, may differ if emulate_dbaas_defaults is enabled
+  # batch_size_fail_threshold_in_kb: 640
+
+  # Log WARN on any batches not of type LOGGED than span across more partitions than this limit.
+  # Default value is 10, may differ if emulate_dbaas_defaults is enabled
+  # unlogged_batch_across_partitions_warn_threshold: 10
+
+  # Failure threshold to prevent writing large column value into Cassandra.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # column_value_size_failure_threshold_in_kb: -1
+
+  # Failure threshold to prevent creating more columns per table than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # columns_per_table_failure_threshold: -1
+
+  # Failure threshold to prevent creating more fields in user-defined-type than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # fields_per_udt_failure_threshold: -1
+
+  # Warning threshold to warn when encountering larger size of collection data than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # collection_size_warn_threshold_in_kb: -1
+
+  # Warning threshold to warn when encountering more elements in collection than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # items_per_collection_warn_threshold: -1
+
+  # Whether read-before-write operation is allowed, eg. setting list element by index, removing list element
+  # by index. Note: LWT is always allowed.
+  # Default true to allow read before write operation, may differ if emulate_dbaas_defaults is enabled
+  # read_before_write_list_operations_enabled: true
+
+  # Failure threshold to prevent creating more secondary index per table than threshold (does not apply to CUSTOM INDEX StorageAttachedIndex)
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # secondary_index_per_table_failure_threshold: -1
+
+  # Failure threshold for number of StorageAttachedIndex per table (only applies to CUSTOM INDEX StorageAttachedIndex)
+  # Default is 10 (same when emulate_dbaas_defaults is enabled)
+  # sai_indexes_per_table_failure_threshold: 10
+  #
+  # Failure threshold for total number of StorageAttachedIndex across all keyspaces (only applies to CUSTOM INDEX StorageAttachedIndex)
+  # Default is 10 (same when emulate_dbaas_defaults is enabled)
+  # sai_indexes_total_failure_threshold: 100
+
+  # Failure threshold to prevent creating more materialized views per table than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # materialized_view_per_table_failure_threshold: -1
+
+  # Warn threshold to warn creating more tables than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # tables_warn_threshold: -1
+
+  # Failure threshold to prevent creating more tables than threshold.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # tables_failure_threshold: -1
+
+  # Preventing creating tables with provided configurations.
+  # Default all properties are allowed, may differ if emulate_dbaas_defaults is enabled
+  # table_properties_disallowed:
+
+  # Whether to allow user-provided timestamp in write request
+  # Default true to allow user-provided timestamp, may differ if emulate_dbaas_defaults is enabled
+  # user_timestamps_enabled: true
+
+  # Preventing query with provided consistency levels
+  # Default all consistency levels are allowed.
+  # write_consistency_levels_disallowed:
+
+  # Failure threshold to prevent providing larger paging by bytes than threshold, also served as a hard paging limit
+  # when paging by rows is used.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # page_size_failure_threshold_in_kb: -1
+
+  # Failure threshold to prevent IN query creating size of cartesian product exceeding threshold, eg.
+  # "a in (1,2,...10) and b in (1,2...10)" results in cartesian product of 100.
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # in_select_cartesian_product_failure_threshold: -1
+
+  # Failure threshold to prevent IN query containing more partition keys than threshold
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # partition_keys_in_select_failure_threshold: -1
+
+  # Warning threshold to warn when local disk usage exceeding threshold. Valid values: (1, 100]
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # disk_usage_percentage_warn_threshold: -1
+
+  # Failure threshold to reject write requests if replica disk usage exceeding threshold. Valid values: (1, 100]
+  # Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+  # disk_usage_percentage_failure_threshold: -1
+
+  # Allows configuring max disk size of data directories when calculating thresholds for disk_usage_percentage_warn_threshold
+  # and disk_usage_percentage_failure_threshold. Valid values: (1, max available disk size of all data directories]
+  # Default -1 to disable and use the physically available disk size of data directories during calculations.
+  # may differ if emulate_dbaas_defaults is enabled
+  # disk_usage_max_disk_size_in_gb: -1

conf/cqlshrc.sample.cloud

@@ -0,0 +1,17 @@
+; Copyright DataStax, Inc.
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+; Sample ~/.cqlshrc file with cloud configuration.
+[connection]
+secure_connect_bundle = /path/to/creds.zip

conf/jvm11-server.options

@@ -11,34 +11,34 @@
 
 
 ### CMS Settings
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
+#-XX:+UseConcMarkSweepGC
+#-XX:+CMSParallelRemarkEnabled
+#-XX:SurvivorRatio=8
+#-XX:MaxTenuringThreshold=1
+#-XX:CMSInitiatingOccupancyFraction=75
+#-XX:+UseCMSInitiatingOccupancyOnly
+#-XX:CMSWaitDuration=10000
+#-XX:+CMSParallelInitialMarkEnabled
+#-XX:+CMSEdenChunksRecordAlways
 ## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
+#-XX:+CMSClassUnloadingEnabled
 
 
 
 ### G1 Settings
 ## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#-XX:+ParallelRefProcEnabled
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
 
 #
 ## Have the JVM do less remembered set work during STW, instead
 ## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
+-XX:G1RSetUpdatingPauseTimePercent=5
 #
 ## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
 ## 200ms is the JVM default and lowest viable setting
 ## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=500
+-XX:MaxGCPauseMillis=500
 
 ## Optional G1 Settings
 # Save CPU time on large (>= 16GB) heaps by delaying region scanning
@@ -80,7 +80,7 @@
 # Java 11 (and newer) GC logging options:
 # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
 # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
-#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+-Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
 
 # Notes for Java 8 migration:
 #

conf/jvm8-server.options

@@ -18,33 +18,33 @@
 #################
 
 ### CMS Settings
--XX:+UseParNewGC
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
+#-XX:+UseParNewGC
+#-XX:+UseConcMarkSweepGC
+#-XX:+CMSParallelRemarkEnabled
+#-XX:SurvivorRatio=8
+#-XX:MaxTenuringThreshold=1
+#-XX:CMSInitiatingOccupancyFraction=75
+#-XX:+UseCMSInitiatingOccupancyOnly
+#-XX:CMSWaitDuration=10000
+#-XX:+CMSParallelInitialMarkEnabled
+#-XX:+CMSEdenChunksRecordAlways
 ## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
+#-XX:+CMSClassUnloadingEnabled
 
 ### G1 Settings
 ## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#-XX:+ParallelRefProcEnabled
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
 
 #
 ## Have the JVM do less remembered set work during STW, instead
 ## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
+-XX:G1RSetUpdatingPauseTimePercent=5
 #
 ## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
 ## 200ms is the JVM default and lowest viable setting
 ## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=500
+-XX:MaxGCPauseMillis=500
 
 ## Optional G1 Settings
 # Save CPU time on large (>= 16GB) heaps by delaying region scanning