tweaks and fixes

Author: michaelhollman

city.c

@@ -65,7 +65,6 @@ city_t *read_city(int fileNum)
 void write_city(int fileNum, city_t *city)
 {
     // set up file
-    mkdir("cities", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "cities/city_%08d.dat", fileNum);

datestamp.c

@@ -64,7 +64,6 @@ datestamp_t *read_datestamp(int fileNum)
 void write_datestamp(int fileNum, datestamp_t *datestamp)
 {
     // set up file
-    mkdir("datestamps", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "datestamps/datestamp_%08d.dat", fileNum);

int_node.c

@@ -91,7 +91,6 @@ int_node_t *read_node(int fileNum, int tableType)
 
 void write_node(int fileNum, int_node_t *node) {
     // set up file
-    mkdir("bplus", 0777);
     FILE *fp;
     char filename[1024];
     filename_for_node(filename, node->tableType, fileNum);

int_node.h

@@ -6,7 +6,7 @@
 #include <stdbool.h>
 
 // Change this value to adjust the fan-out of the b+ trees
-#define FAN_OUT         2
+#define FAN_OUT         200
 
 #define TEXT_SHORT      64
 #define TEXT_LONG       1024

make_bplus_trees.c

@@ -1,5 +1,7 @@
 #include <stdio.h>
 #include <sys/time.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
 #include "record.h"
 #include "user.h"
@@ -18,6 +20,8 @@ int main(int argc, char **argv)
     struct timeval sysTimeStart, sysTimeEnd;
     gettimeofday(&sysTimeStart, NULL);
 
+    mkdir("bplus", 0777);
+    
     int i;
     // open file count information
     file_count_t *fc = read_file_count();

make_tables.c

@@ -3,6 +3,8 @@
 #include <sys/time.h>
 #include <string.h>
 #include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
 #include "record.h"
 #include "user.h"
@@ -49,6 +51,15 @@ int main(int argc, char **argv)
     struct timeval sysTimeStart, sysTimeEnd;
     gettimeofday(&sysTimeStart, NULL);
 
+    // set up directories;
+    mkdir("cities", 0777);
+    mkdir("datestamps", 0777);
+    mkdir("messages", 0777);
+    mkdir("states", 0777);
+    mkdir("timestamps", 0777);
+    mkdir("users", 0777);
+    mkdir("bplus", 0777);
+
     // set up some counters, etc.
     unsigned int recordCount = 0,
     userCount = 0,

message.c

@@ -62,7 +62,6 @@ message_t *read_message(int fileNum)
 void write_message(int fileNum, message_t *message)
 {
     // set up file
-    mkdir("messages", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "messages/message_%08d.dat", fileNum);
@@ -94,56 +93,8 @@ void free_message(message_t *message)
     free(message);
 }
 
-int compare_message_sorts(const void *a, const void *b)
+int compare_messages(const void *a, const void *b)
 {
     // compare by timestampId
-    return (int) ((message_sort_t*)a)->timestampId - ((message_sort_t*)b)->timestampId;
+    return (int) ((message_t*)a)->timestampId - ((message_t*)b)->timestampId;
 }
-
-void presort_message(int fileNum)
-{
-    char old[1024];
-    sprintf(old, "messages/message_%08d.dat", fileNum);
-    char new[1024];
-    sprintf(new, "messages/presort_message_%08d.dat", fileNum);
-    rename(old, new);
-}
-
-message_t *read_presorted_message(int fileNum)
-{
-    // set up file
-    FILE *fp;
-    char filename[1024];
-    sprintf(filename, "messages/presort_message_%08d.dat", fileNum);
-    
-    // open file
-    fp = fopen(filename, "rb");
-    
-    if (!fp) {
-        fprintf(stderr, "Cannot open %s\n", filename);
-        exit(0);
-    }
-    
-    message_t *message = (message_t *)malloc(sizeof(message_t));
-    
-    // read message
-    fread(&(message->messageId), sizeof(int), 1, fp);
-    fread(&(message->userId), sizeof(int), 1, fp);
-    fread(&(message->timestampId), sizeof(int), 1, fp);
-    fread(&(message->datestampId), sizeof(int), 1, fp);
-    fread(&(message->text[0]), sizeof(char), TEXT_LONG, fp);
-    
-    fclose(fp);
-    
-    return message;
-}
-
-void unpresort_message(int presortFileNum, int newFileNum)
-{
-    char old[1024];
-    sprintf(old, "messages/presort_message_%08d.dat", presortFileNum);
-    char new[1024];
-    sprintf(new, "messages/message_%08d.dat", newFileNum);
-    rename(old, new);
-}
-

message.h

@@ -25,16 +25,7 @@ void write_message(int fileNum, message_t *message);
 
 void free_message(message_t *message);
 
-// sorting wrapper
-typedef struct {
-    int fileNum;
-    int timestampId;
-} message_sort_t;
-
-int compare_message_sorts(const void *a, const void *b);
-
-void presort_message(int fileNum);
+int compare_messages(const void *a, const void *b);
 
-void unpresort_message(int presortFileNum, int newFileNum);
 
 #endif

sort_tables.c

@@ -16,7 +16,7 @@
 int main(int argc, char **argv)
 {
     printf("Sorting tables starting...\n");
-
+    
     // time the program
     struct timeval sysTimeStart, sysTimeEnd;
     gettimeofday(&sysTimeStart, NULL);
@@ -140,48 +140,46 @@ int main(int argc, char **argv)
     }
 
     // messages
-    // note: this sort is quite a bit different since we cant dump all 1.9 million messages into memory.
+    // note: this sort uses the heap
     {
         printf("Sorting %d messages\n", fc->messages);
 
+        
         // time this section
         struct timeval startSysTimeSub, endSysTimeSub;
         gettimeofday(&startSysTimeSub, NULL);
 
         // get count
         int count = fc->messages;
-        message_sort_t message_sorts[count];
+        message_t *messages = malloc(sizeof(message_t) * count);
 
-        // read into temp struct/array, rename file
+        // read
         for (i = 0; i < count; i++)
         {
-            message_t *msg = read_message(i);
-            presort_message(i);
-
-            message_sort_t tmp;
-            tmp.fileNum = i;
-            tmp.timestampId = msg->timestampId;
-            
-            message_sorts[i] = tmp;
-            free_message(msg);
+            message_t *tmp = read_message(i);
+            messages[i] = *tmp;
+            free_message(tmp);
         }
 
         // sort
-        qsort(message_sorts, count, sizeof(message_sorts[0]), compare_message_sorts);
+        qsort(messages, count, sizeof(messages[0]), compare_messages);
 
-        // rename back in sorted order
+        // write
         for (i = 0; i < count; i++)
         {
-            unpresort_message(message_sorts[i].fileNum, i);
+            write_message(i, &messages[i]);
+            //print_city(&cities[i]);
         }
 
+        free(messages);
+        
         // end timing this section
         gettimeofday(&endSysTimeSub, NULL);
         float totalTime = (endSysTimeSub.tv_sec - startSysTimeSub.tv_sec)
         + (endSysTimeSub.tv_usec - startSysTimeSub.tv_usec) / 1000000.0f;
         printf("Table process time %f seconds\n", totalTime);
     }
-
+    
     // timestamps
     {
         printf("Sorting %d timestamps\n", fc->timestamps);
@@ -256,7 +254,7 @@ int main(int argc, char **argv)
         + (endSysTimeSub.tv_usec - startSysTimeSub.tv_usec) / 1000000.0f;
         printf("Table process time %f seconds\n", totalTime);
     }
-         
+    
     free_file_count(fc);
 
     // end timing the program

state.c

@@ -63,7 +63,6 @@ state_t *read_state(int fileNum)
 void write_state(int fileNum, state_t *state)
 {
     // set up file
-    mkdir("states", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "states/state_%08d.dat", fileNum);

timestamp.c

@@ -61,7 +61,6 @@ timestamp_t *read_timestamp(int fileNum)
 void write_timestamp(int fileNum, timestamp_t *timestamp)
 {
     // set up file
-    mkdir("timestamps", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "timestamps/timestamp_%08d.dat", fileNum);

user.c

@@ -65,7 +65,6 @@ user_t *read_user(int fileNum)
 void write_user(int fileNum, user_t *user)
 {
     // set up file
-    mkdir("users", 0777);
     FILE *fp;
     char filename[1024];
     sprintf(filename, "users/user_%08d.dat", fileNum);