Add stats-based random alloc/dealloc generation and simulation.

random_alloc_test/generateAllocs.cpp

@@ -0,0 +1,107 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <vector>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <algorithm>
+
+#define MAX_SIZE 1000000000
+void no_opt_ptr(void* ptr) {
+    // Dummy function to prevent optimization
+    asm volatile("" : : "g"(ptr) : "memory");
+}
+
+struct AllocationRange {
+    size_t minSize;
+    size_t maxSize;
+    double cumulativeProbability;
+};
+
+std::vector<AllocationRange> readAllocationRanges(const char* inputFilename) {
+    std::vector<AllocationRange> ranges;
+    std::ifstream infile(inputFilename);
+    std::string line;
+    double cumulativeProbability = 0;
+    while (std::getline(infile, line)) {
+        std::istringstream iss(line);
+        double minSize, probability;
+        if (!(iss >> minSize >> probability)) {
+            std::cerr << "Error reading line: " << line << std::endl;
+            continue;
+        }
+        // The minSize in this line is the maxSize of the last line
+        if (ranges.size() > 0) {
+            ranges[ranges.size()-1].maxSize = minSize;
+        }
+	cumulativeProbability += probability;
+        ranges.push_back({static_cast<size_t>(minSize), 0, cumulativeProbability});
+    }
+    ranges[ranges.size()-1].maxSize = MAX_SIZE;
+    return ranges;
+}
+
+size_t getRandomSize(const std::vector<AllocationRange>& ranges) {
+    double randValue = static_cast<double>(rand()) / RAND_MAX;
+    auto it = std::lower_bound(ranges.begin(), ranges.end(), randValue, 
+        [](const AllocationRange& range, double value) {
+            return range.cumulativeProbability < value;
+        });
+    if (it != ranges.end()) {
+        return it->minSize + (rand() % (it->maxSize - it->minSize));
+    }
+    std::cout << "Should be unreachable!" << std::endl;
+    return ranges.back().minSize; // Fallback in case of rounding errors
+}
+
+int main(int argc, char* argv[]) {
+    if (argc != 4) {
+        fprintf(stderr, "Usage: %s <numOperations> <outputFilename> <inputFilename>\n", argv[0]);
+        return 1;
+    }
+
+    int numOperations = atoi(argv[1]);
+    const char* outputFilename = argv[2];
+    const char* inputFilename = argv[3];
+
+    FILE* file = fopen(outputFilename, "w");
+    if (!file) {
+        perror("Failed to open output file");
+        return 1;
+    }
+
+    std::vector<AllocationRange> allocationRanges = readAllocationRanges(inputFilename);
+    if (allocationRanges.empty()) {
+        fprintf(stderr, "No valid allocation ranges found in input file\n");
+        return 1;
+    }
+
+    std::vector<size_t> allocations;
+    srand((unsigned int)time(NULL));
+
+    for (int i = 0; i < numOperations; ++i) {
+        if (rand() % 2) {
+            size_t size = getRandomSize(allocationRanges);
+            allocations.push_back(size);
+            fprintf(file, "alloc %zu\n", size);
+        } else {
+            if (!allocations.empty()) {
+                size_t index = rand() % allocations.size();
+                allocations.erase(allocations.begin() + index);
+                fprintf(file, "dalloc %zu\n", index);
+            }
+        }
+    }
+
+/*
+    for (int i=allocations.size()-1; i >= 0; i--) {
+        fprintf(file, "dalloc %zu\n", i);
+    }
+*/
+
+    fclose(file);
+
+    return 0;
+}

random_alloc_test/generateAllocs.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+g++ -o generateAllocs generateAllocs.cpp
+#sample usage
+./generateAllocs 2000000 booting_sim.log booting_stats.log

random_alloc_test/multiThreadRandomTest.cpp

@@ -0,0 +1,132 @@
+#include <iostream>
+#include <thread>
+#include <vector>
+#include <chrono>
+#include <cstdlib>
+#include <ctime>
+#include <mutex>
+#include <fstream>
+#include <jemalloc/jemalloc.h>
+
+std::mutex mtx;
+
+void dumpMallocStats() {
+    const char* opts = "g";  // General information
+    malloc_stats_print(NULL, NULL, opts);
+}
+
+void probeMemoryStats(size_t time) {
+    uint64_t epoch = 1;
+    size_t sz = sizeof(epoch);
+    mallctl("epoch", &epoch, &sz, &epoch, sz);
+
+    size_t allocated, active, resident, pdirty, purged, metadata;
+    sz = sizeof(size_t);
+    mallctl("stats.allocated", &allocated, &sz, NULL, 0);
+    mallctl("stats.active", &active, &sz, NULL, 0);
+    mallctl("stats.resident", &resident, &sz, NULL, 0);
+    mallctl("stats.arenas.4096.pdirty", &pdirty, &sz, NULL, 0);
+    mallctl("stats.arenas.4096.dirty_purged", &purged, &sz, NULL, 0);
+    mallctl("stats.metadata", &metadata, &sz, NULL, 0);
+    std::cout << allocated << ";" << active << ";" << resident << ";"<< pdirty << ";"<<purged<<";"<<metadata<<";" << time << std::endl;
+}
+
+static inline void *no_opt_ptr(void *ptr) {
+    asm volatile("" : "+r"(ptr));
+    return ptr;
+}
+
+void performOperations(int id, int runTimeSeconds, std::string file_name) {
+    auto startTime = std::chrono::steady_clock::now();
+    std::vector<void*> allocations;
+    std::ifstream inputFile;
+    long long numOps = 0;
+
+    inputFile.open(file_name);
+    if (!inputFile.is_open()) {
+        std::cerr << "Failed to open file: " << file_name << std::endl;
+        return;
+    }
+
+    while (true) {
+        auto currentTime = std::chrono::steady_clock::now();
+        auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(currentTime - startTime).count();
+
+        if (elapsed >= runTimeSeconds) {
+            break;
+        }
+        std::string operation;
+        size_t size_or_index;
+        if (inputFile >> operation >> size_or_index) {
+            if (operation == "alloc") {
+                void* ptr = malloc(size_or_index);
+                if (ptr) {
+                    no_opt_ptr(ptr);
+                    allocations.push_back(ptr);
+                }
+            } else if (operation == "dalloc") {
+                if (size_or_index < allocations.size()) {
+                    free(allocations[size_or_index]);
+                    allocations.erase(allocations.begin() + size_or_index);
+                }
+            }
+            numOps ++;
+        } else {
+            // End of file or invalid input
+            break;
+        }
+    }
+
+    for (void* ptr : allocations) {
+        free(ptr);
+    }
+    allocations.clear();
+    std::cout << "Total numOps: " << numOps <<std::endl;
+}
+
+int main(int argc, char* argv[]) {
+    if (argc < 5) {
+        std::cerr << "Usage: " << argv[0] << " <num_threads> <runTimeSeconds> <probeInterval> <file>" << std::endl;
+        return 1;
+    }
+
+    int num_threads = std::stoi(argv[1]);
+    int runTimeSeconds = std::stoi(argv[2]);
+    int probeInterval = std::stoi(argv[3]);
+    std::string file_name = argv[4];
+
+    std::vector<std::thread> threads;
+
+    // Launch threads
+    for (int i = 0; i < num_threads; ++i) {
+        threads.emplace_back(performOperations, i, runTimeSeconds, file_name);
+    }
+
+    // Main thread probes memory stats
+    auto startTime = std::chrono::steady_clock::now();
+    while (true) {
+        auto currentTime = std::chrono::steady_clock::now();
+        auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(currentTime - startTime).count();
+
+        if (elapsed >= runTimeSeconds) {
+            break;
+        }
+
+        probeMemoryStats(elapsed);
+        std::this_thread::sleep_for(std::chrono::seconds(probeInterval));
+    }
+
+    // Join all threads
+    for (auto& th : threads) {
+        th.join();
+    }
+
+    // Final 30 seconds wait with probing every 10 seconds
+    for (int i = 0; i < 3; ++i) {
+        std::this_thread::sleep_for(std::chrono::seconds(10));
+        probeMemoryStats(1000000);
+    }
+
+    std::cout << "Program ended." << std::endl;
+    return 0;
+}

random_alloc_test/randomTest.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Turn off turbos and chef
+sudo dynamo "$(hostname)" turnOffTurbo
+sudo dynamo "$(hostname)" checkTurboStatus
+sudo /usr/facebook/ops/scripts/chef/stop_chef_temporarily -r "Performance testing with turbo disabled" -t 12
+
+INPUT_FILE="ART_full_sim.log"
+RUNNING_COMMAND="
+MALLOC_CONF=\"stats_print:true,dirty_decay_ms:5000,muzzy_decay_ms:0,background_thread:true\" ./multiThreadRandom 10 200 5 ${INPUT_FILE}"
+
+echo "Testing on dev"
+cd ~/gdai-jemalloc/jemalloc
+git checkout dev
+./autogen.sh --enable-prof && make -j64 && sudo make install
+
+cd ~/gdai-jemalloc/jemalloc-experiments/random_alloc_test
+g++ multiThreadRandomTest.cpp -o multiThreadRandom `jemalloc-config --libdir`/libjemalloc.a `jemalloc-config --libs` -O3
+export LD_PRELOAD=/usr/local/lib/libjemalloc.so
+eval ${RUNNING_COMMAND}
+
+echo "Testing on the changes"
+cd ~/gdai-jemalloc/jemalloc
+#git checkout hpa_debug
+git checkout hpa_test_2
+./autogen.sh --enable-prof --enable-limit-usize-gap && make -j64 && sudo make install
+
+cd ~/gdai-jemalloc/jemalloc-experiments/random_alloc_test
+g++ multiThreadRandomTest.cpp -o multiThreadRandom `jemalloc-config --libdir`/libjemalloc.a `jemalloc-config --libs` -O3
+export LD_PRELOAD=/usr/local/lib/libjemalloc.so
+eval ${RUNNING_COMMAND}
+