Add Redis OSS standalone benchmark setup for c8g.16xlarge -ARM

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/README.md

@@ -0,0 +1,46 @@
+# oss-redis-standalone-arm64-ubuntu24.04-c8g.16xlarge
+
+Deploy Multi-VM benchmark sceneario, including 2 clients and 1 DB machine.
+
+- Cloud provider: AWS
+- OS: Ubuntu 24.04
+- Client machine: c8g.16xlarge
+- Benchmark machine: c8g.16xlarge
+
+---
+
+#### Tested scenarios
+
+- TBD
+
+#### Deployment
+
+##### Required env variables
+
+The terraform and ansible scripts expect the following env variables to be filled:
+
+```
+export EC2_REGION={ ## INSERT REGION ## }
+export EC2_ACCESS_KEY={ ## INSERT EC2 ACCESS KEY ## }
+export EC2_SECRET_KEY={ ## INSERT EC2 SECRET KEY ## }
+export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+```
+
+##### Required pub/private keys
+
+The terraform script expects the following public private keys to be present on ~/.ssh/ dir:
+
+```
+~/.ssh/perf-ci.pem
+~/.ssh/perf-ci.pub
+```
+
+##### Deployment steps
+
+within project repo
+
+```bash
+cd terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge
+terraform plan
+terraform apply
+```

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/bench-client-resources.tf

@@ -0,0 +1,59 @@
+
+resource "aws_instance" "client" {
+  count                  = var.client_instance_count
+  ami                    = var.instance_ami
+  instance_type          = var.instance_type
+  subnet_id              = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id
+  vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name               = var.key_name
+  placement_group        = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b
+  availability_zone      = "us-east-2b"
+
+  # Cloud-init configuration from external file to install and build memtier
+  user_data = file("${path.module}/cloud-init-client.yaml")
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+  }
+
+  volume_tags = {
+    Environment  = "${var.environment}"
+    Name         = "ebs_block_device-${var.setup_name}-CLIENT-${count.index + 1}"
+    setup        = "${var.setup_name}"
+    redis_module = "${var.redis_module}"
+    github_actor = "${var.github_actor}"
+    github_repo  = "${var.github_repo}"
+    github_sha   = "${var.github_sha}"
+    timeout_secs = "${var.timeout_secs}"
+  }
+
+  tags = {
+    Environment  = "${var.environment}"
+    Name         = "${var.setup_name}-CLIENT-${count.index + 1}"
+    setup        = "${var.setup_name}"
+    redis_module = "${var.redis_module}"
+    github_actor = "${var.github_actor}"
+    github_repo  = "${var.github_repo}"
+    github_sha   = "${var.github_sha}"
+    timeout_secs = "${var.timeout_secs}"
+  }
+
+  ################################################################################
+  # This will ensure we wait here until the instance is ready to receive the ssh connection 
+  ################################################################################
+  provisioner "remote-exec" {
+    script = "./../../scripts/wait_for_instance.sh"
+    connection {
+      host        = self.public_ip # The `self` variable is like `this` in many programming languages
+      type        = "ssh"          # in this case, `self` is the resource (the server).
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      #need to increase timeout to larger then 5m for metal instances
+      timeout = "15m"
+      agent   = "false"
+    }
+  }
+}

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/cloud-init-client.yaml

@@ -0,0 +1,40 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+packages:
+  - git
+  - build-essential
+  - autoconf
+  - automake
+  - libpcre3-dev
+  - libevent-dev
+  - pkg-config
+  - zlib1g-dev
+  - libssl-dev
+  - libtool
+  - ca-certificates
+  - wget
+
+# Create the memtier installation script
+write_files:
+  - path: /tmp/install_memtier.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash
+      set -e # exit immediately on error
+      # Clone memtier benchmark
+      cd /tmp
+      rm -rf memtier_benchmark
+      git clone https://github.com/RedisLabs/memtier_benchmark.git
+      cd memtier_benchmark
+      # Build and install
+      autoreconf -ivf
+      ./configure
+      make -j
+      sudo make install
+      echo "Memtier benchmark installed successfully"
+# Run the installation script
+runcmd:
+  - bash /tmp/install_memtier.sh
+  - echo "Cloud-init installation completed"

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/cloud-init-db.yaml

@@ -0,0 +1,79 @@
+#cloud-config
+
+# Update and upgrade packages
+package_update: true
+package_upgrade: true
+
+# Install required packages
+packages:
+  ### DragonflyDB dependencies
+  - ninja-build
+  - libunwind-dev
+  - libboost-context-dev
+  - libssl-dev
+  - autoconf-archive
+  - libtool
+  - cmake
+  - g++
+  - bison
+  - zlib1g-dev
+  - git
+  - make
+  - pkg-config
+
+  # Redis dependencies
+  - git
+  - dpkg-dev
+  - gcc
+  - g++
+  - libc6-dev
+  - libssl-dev
+  - make
+  - cmake
+  - clang
+  - automake
+  - autoconf
+  - libtool
+  - ca-certificates
+  - wget
+
+# Create directories
+runcmd:
+  ### DragonflyDB
+  # Log start time
+  - echo "Starting DragonflyDB installation at $(date)"
+
+  # Clone DragonflyDB repository
+  - echo "Cloning DragonflyDB repository..."
+  - cd /home/ubuntu
+  - git clone --recursive https://github.com/dragonflydb/dragonfly
+
+  # Build DragonflyDB
+  - echo "Building DragonflyDB (this may take a while)..."
+  - cd /home/ubuntu/dragonfly
+  - ./helio/blaze.sh -release
+  - cd build-opt && ninja dragonfly
+
+  # Log completion
+  - echo "DragonflyDB installation completed at $(date)"
+
+  ### Redis
+  # Log start time
+  - echo "Start Redis installation at $(date)"
+
+  # Clone Redis repository
+  - echo "Cloning Redis repository..."
+  - cd /home/ubuntu
+  - git clone https://github.com/redis/redis
+
+  # Build Redis
+  - echo "Building Redis (this may take a while)..."
+  - cd /home/ubuntu/redis
+  - make distclean
+  - make BUILD_TLS=yes -j
+
+  # Fix ownership of /home/ubuntu files to ubuntu
+  - chown -R ubuntu:ubuntu /home/ubuntu/
+
+  # Log completion
+  - echo "Redis installation completed at $(date)"

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/common.tf

@@ -0,0 +1,12 @@
+
+################################################################################
+# This is the bucket holding this specific setup tfstate
+################################################################################
+terraform {
+  backend "s3" {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge.tfstate"
+    region = "us-east-1"
+  }
+}
+

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/db-resources.tf

@@ -0,0 +1,93 @@
+
+resource "aws_instance" "server" {
+  count                  = var.server_instance_count
+  ami                    = var.instance_ami
+  instance_type          = var.instance_type
+  subnet_id              = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id
+  vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name               = var.key_name
+  placement_group        = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b
+  availability_zone      = "us-east-2b"
+
+  # Cloud-init configuration from external file to install and build DragonflyDB from source
+  user_data = file("${path.module}/cloud-init-db.yaml")
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+  }
+
+  volume_tags = {
+    Environment  = "${var.environment}"
+    Name         = "ebs_block_device-${var.setup_name}-DB-${count.index + 1}"
+    setup        = "${var.setup_name}"
+    redis_module = "${var.redis_module}"
+    github_actor = "${var.github_actor}"
+    github_repo  = "${var.github_repo}"
+    github_sha   = "${var.github_sha}"
+    timeout_secs = "${var.timeout_secs}"
+  }
+
+  tags = {
+    Environment  = "${var.environment}"
+    Name         = "${var.setup_name}-DB-${count.index + 1}"
+    setup        = "${var.setup_name}"
+    redis_module = "${var.redis_module}"
+    github_actor = "${var.github_actor}"
+    github_repo  = "${var.github_repo}"
+    github_sha   = "${var.github_sha}"
+    timeout_secs = "${var.timeout_secs}"
+  }
+
+  ################################################################################
+  # This will ensure we wait here until the instance is ready to receive the ssh connection
+  ################################################################################
+  provisioner "remote-exec" {
+    script = "./../../scripts/wait_for_instance.sh"
+    connection {
+      host        = self.public_ip # The `self` variable is like `this` in many programming languages
+      type        = "ssh"          # in this case, `self` is the resource (the server).
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      #need to increase timeout to larger then 5m for metal instances
+      timeout = "15m"
+      agent   = "false"
+    }
+  }
+
+  ################################################################################
+  # Copy create-cluster script to the server
+  ################################################################################
+  provisioner "file" {
+    source      = "./../../scripts/create-cluster"
+    destination = "/home/${var.ssh_user}/create-cluster"
+    connection {
+      host        = self.public_ip
+      type        = "ssh"
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      timeout     = "1m"
+      agent       = "false"
+    }
+  }
+
+  ################################################################################
+  # Make create-cluster script executable
+  ################################################################################
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /home/${var.ssh_user}/create-cluster",
+      "echo 'create-cluster script copied and made executable'"
+    ]
+    connection {
+      host        = self.public_ip
+      type        = "ssh"
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      timeout     = "1m"
+      agent       = "false"
+    }
+  }
+}

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/output.tf

@@ -0,0 +1,15 @@
+output "server_public_ip" {
+  value = ["${aws_instance.server[*].public_ip}"]
+}
+
+output "server_private_ip" {
+  value = ["${aws_instance.server[*].private_ip}"]
+}
+
+output "client_public_ip" {
+  value = ["${aws_instance.client[*].public_ip}"]
+}
+
+output "client_private_ip" {
+  value = ["${aws_instance.client[*].private_ip}"]
+}

terraform/oss-standalone-arm64-ubuntu24.04-c8g.16xlarge/shared_resources.tf

@@ -0,0 +1,17 @@
+# provider
+provider "aws" {
+  region = var.region
+}
+
+################################################################################
+# This is the shared resources bucket key -- you will need it across environments like security rules,etc...
+# !! do not change this !!
+################################################################################
+data "terraform_remote_state" "shared_resources" {
+  backend = "s3"
+  config = {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/shared_resources.tfstate"
+    region = "us-east-1"
+  }
+}