Skip to content

Commit eeff224

Browse files
szarnyasgszarnyasg
committedApr 29, 2021
Add flush log flag
·
v2.0.00.3.4
1 parent 720328a commit eeff224

15 files changed

+89
-28
lines changed
 

‎src/main/java/com/ldbc/driver/client/CreateValidationParamsMode.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,8 @@ public Object startExecutionAndAwaitCompletion() throws ClientException
133133
int rowsWrittenSoFar = 0;
134134
try ( SimpleCsvFileWriter simpleCsvFileWriter = new SimpleCsvFileWriter(
135135
validationFileToGenerate,
136-
SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR ) )
136+
SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR,
137+
controlService.configuration().flushLog() ) )
137138
{
138139
DecimalFormat decimalFormat = new DecimalFormat( "###,###,##0" );
139140
while ( csvRows.hasNext() )

‎src/main/java/com/ldbc/driver/client/ExecuteWorkloadMode.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,10 @@ private void doInit( boolean warmup ) throws ClientException
160160
{
161161
resultsLogWriter = (null == resultsLog)
162162
? new NullResultsLogWriter()
163-
: new SimpleResultsLogWriter( resultsLog, controlService.configuration().timeUnit() );
163+
: new SimpleResultsLogWriter(
164+
resultsLog,
165+
controlService.configuration().timeUnit(),
166+
controlService.configuration().flushLog() );
164167
}
165168
catch ( IOException e )
166169
{

‎src/main/java/com/ldbc/driver/control/ConsoleAndFileDriverConfiguration.java

+26-3
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration
7777
public static final String HELP_DEFAULT_STRING = Boolean.toString( HELP_DEFAULT );
7878
private static final String HELP_DESCRIPTION = "print usage instruction";
7979

80+
public static final String FLUSH_LOG_ARG = "flush_log";
81+
public static final boolean FLUSH_LOG_DEFAULT = false;
82+
public static final String FLUSH_LOG_DEFAULT_STRING = Boolean.toString(FLUSH_LOG_DEFAULT);
83+
private static final String FLUSH_LOG_DESCRIPTION = "flush log to disk after each operation";
84+
8085
public static final String NAME_ARG = "nm";
8186
private static final String NAME_ARG_LONG = "name";
8287
public static final String NAME_DEFAULT = "LDBC";
@@ -185,6 +190,7 @@ public static Map<String,String> defaultsAsMap() throws DriverConfigurationExcep
185190
Map<String,String> defaultParamsMap = new HashMap<>();
186191
defaultParamsMap.put( IGNORE_SCHEDULED_START_TIMES_ARG, IGNORE_SCHEDULED_START_TIMES_DEFAULT_STRING );
187192
defaultParamsMap.put( HELP_ARG, HELP_DEFAULT_STRING );
193+
defaultParamsMap.put( FLUSH_LOG_ARG, FLUSH_LOG_DEFAULT_STRING );
188194
defaultParamsMap.put( OPERATION_COUNT_ARG, OPERATION_COUNT_DEFAULT_STRING );
189195
defaultParamsMap.put( WORKLOAD_ARG, WORKLOAD_DEFAULT_STRING );
190196
defaultParamsMap.put( NAME_ARG, NAME_DEFAULT_STRING );
@@ -280,6 +286,7 @@ public static ConsoleAndFileDriverConfiguration fromParamsMap( Map<String,String
280286
boolean printHelp = Boolean.parseBoolean( paramsMap.get( HELP_ARG ) );
281287
boolean ignoreScheduledStartTimes =
282288
Boolean.parseBoolean( paramsMap.get( IGNORE_SCHEDULED_START_TIMES_ARG ) );
289+
boolean flushLog = Boolean.parseBoolean( paramsMap.get( FLUSH_LOG_ARG ) );
283290
return new ConsoleAndFileDriverConfiguration(
284291
paramsMap,
285292
name,
@@ -298,7 +305,8 @@ public static ConsoleAndFileDriverConfiguration fromParamsMap( Map<String,String
298305
printHelp,
299306
ignoreScheduledStartTimes,
300307
warmupCount,
301-
skipCount
308+
skipCount,
309+
flushLog
302310
);
303311
}
304312
catch ( DriverConfigurationException e )
@@ -605,6 +613,10 @@ private static Options buildOptions()
605613
Option printHelpOption = OptionBuilder.withDescription( HELP_DESCRIPTION ).create( HELP_ARG );
606614
options.addOption( printHelpOption );
607615

616+
617+
Option flushOption = OptionBuilder.withDescription(FLUSH_LOG_DESCRIPTION).create(FLUSH_LOG_ARG);
618+
options.addOption( flushOption );
619+
608620
Option ignoreScheduledStartTimesOption =
609621
OptionBuilder.withDescription( IGNORE_SCHEDULED_START_TIMES_DESCRIPTION )
610622
.create( IGNORE_SCHEDULED_START_TIMES_ARG );
@@ -684,6 +696,7 @@ public static String commandlineHelpString()
684696
private final boolean ignoreScheduledStartTimes;
685697
private final long warmupCount;
686698
private final long skipCount;
699+
private final boolean flushLog;
687700

688701
public ConsoleAndFileDriverConfiguration( Map<String,String> paramsMap,
689702
String name,
@@ -702,7 +715,8 @@ public ConsoleAndFileDriverConfiguration( Map<String,String> paramsMap,
702715
boolean printHelp,
703716
boolean ignoreScheduledStartTimes,
704717
long warmupCount,
705-
long skipCount )
718+
long skipCount,
719+
boolean flushLog )
706720
{
707721
if ( null == paramsMap )
708722
{
@@ -726,6 +740,7 @@ public ConsoleAndFileDriverConfiguration( Map<String,String> paramsMap,
726740
this.ignoreScheduledStartTimes = ignoreScheduledStartTimes;
727741
this.warmupCount = warmupCount;
728742
this.skipCount = skipCount;
743+
this.flushLog = flushLog;
729744

730745
if ( null != name )
731746
{
@@ -762,6 +777,7 @@ public ConsoleAndFileDriverConfiguration( Map<String,String> paramsMap,
762777
paramsMap.put( IGNORE_SCHEDULED_START_TIMES_ARG, Boolean.toString( ignoreScheduledStartTimes ) );
763778
paramsMap.put( WARMUP_COUNT_ARG, Long.toString( warmupCount ) );
764779
paramsMap.put( SKIP_COUNT_ARG, Long.toString( skipCount ) );
780+
paramsMap.put( FLUSH_LOG_ARG, Boolean.toString( flushLog ) );
765781
}
766782

767783
@Override
@@ -879,6 +895,9 @@ public long skipCount()
879895
return skipCount;
880896
}
881897

898+
@Override
899+
public boolean flushLog() { return flushLog; }
900+
882901
@Override
883902
public Map<String,String> asMap()
884903
{
@@ -995,6 +1014,9 @@ public DriverConfiguration applyArgs( Map<String,String> newParamsMap ) throws D
9951014
long newSkipCount = (newParamsMapWithShortKeys.containsKey( SKIP_COUNT_ARG )) ?
9961015
Long.parseLong( newParamsMapWithShortKeys.get( SKIP_COUNT_ARG ) ) :
9971016
skipCount;
1017+
boolean newFlushLog = (newParamsMapWithShortKeys.containsKey( FLUSH_LOG_ARG )) ?
1018+
Boolean.parseBoolean( newParamsMapWithShortKeys.get( FLUSH_LOG_ARG ) ) :
1019+
flushLog;
9981020

9991021
return new ConsoleAndFileDriverConfiguration(
10001022
newOtherParams,
@@ -1014,7 +1036,8 @@ public DriverConfiguration applyArgs( Map<String,String> newParamsMap ) throws D
10141036
newPrintHelp,
10151037
newIgnoreScheduledStartTimes,
10161038
newWarmupCount,
1017-
newSkipCount
1039+
newSkipCount,
1040+
newFlushLog
10181041
);
10191042
}
10201043

‎src/main/java/com/ldbc/driver/control/DriverConfiguration.java

+2
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ public interface DriverConfiguration
4141

4242
long skipCount();
4343

44+
boolean flushLog();
45+
4446
String toPropertiesString() throws DriverConfigurationException;
4547

4648
Map<String,String> asMap();

‎src/main/java/com/ldbc/driver/csv/simple/SimpleCsvFileWriter.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,15 @@ public class SimpleCsvFileWriter implements Closeable
1616

1717
private final BufferedWriter bufferedWriter;
1818
private final String columnSeparator;
19+
private final boolean flushLog;
1920

20-
public SimpleCsvFileWriter( File file, String columnSeparator ) throws IOException
21+
public SimpleCsvFileWriter( File file, String columnSeparator, boolean flushLog ) throws IOException
2122
{
2223
this.bufferedWriter =
2324
new BufferedWriter( new OutputStreamWriter( new FileOutputStream( file ), Charsets.UTF_8 ) );
2425

2526
this.columnSeparator = columnSeparator;
27+
this.flushLog = flushLog;
2628
}
2729

2830
public void writeRows( Iterator<String[]> csvRows ) throws IOException
@@ -40,7 +42,9 @@ public void writeRow( String... columns ) throws IOException
4042
}
4143
bufferedWriter.write( columns[columns.length - 1] );
4244
bufferedWriter.newLine();
43-
bufferedWriter.flush();
45+
if (flushLog) {
46+
bufferedWriter.flush();
47+
}
4448
}
4549

4650
@Override

‎src/main/java/com/ldbc/driver/runtime/metrics/SimpleResultsLogWriter.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ public class SimpleResultsLogWriter implements ResultsLogWriter
1111
private final SimpleCsvFileWriter writer;
1212
private final TimeUnit unit;
1313

14-
public SimpleResultsLogWriter( File resultsLog, TimeUnit unit ) throws IOException
14+
public SimpleResultsLogWriter( File resultsLog, TimeUnit unit, boolean flushLog ) throws IOException
1515
{
16-
this.writer = new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR );
16+
this.writer = new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR, flushLog );
1717
this.unit = unit;
1818
resultsLog.createNewFile();
1919
writer.writeRow(

‎src/main/resources/configuration/ldbc_driver_default.properties

+6
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ help=false
8181
# COMMAND: -ignore_scheduled_start_times
8282
ignore_scheduled_start_times=false
8383

84+
# flush log to disk after each operation
85+
# useful for durability tests
86+
# BOOLEAN
87+
# COMMAND: -flush_log
88+
flush_log=false
89+
8490
# ***************************************************************
8591
# *** the following should be set by workload implementations ***
8692
# ***************************************************************

‎src/test/java/com/ldbc/driver/control/ConsoleAndFileDriverConfigurationTest.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ public void toMapThenFromMapShouldReturnSameResultWhenAllParamsAreInitiallySetVi
107107
long warmupCount = 5;
108108
long skipCount = 6;
109109
Map<String,String> paramsMap = new HashMap<>();
110+
boolean flushLog = false;
110111

111112
ConsoleAndFileDriverConfiguration configurationBefore = new ConsoleAndFileDriverConfiguration(
112113
paramsMap,
@@ -126,7 +127,8 @@ public void toMapThenFromMapShouldReturnSameResultWhenAllParamsAreInitiallySetVi
126127
printHelp,
127128
ignoreScheduledStartTimes,
128129
warmupCount,
129-
skipCount
130+
skipCount,
131+
flushLog
130132
);
131133

132134
DriverConfiguration configurationAfter =
@@ -565,6 +567,7 @@ public void shouldReturnSameAsConstructedWith()
565567
boolean ignoreScheduledStartTimes = false;
566568
long warmupCount = 10;
567569
long skipCount = 100;
570+
boolean flushLog = false;
568571

569572
ConsoleAndFileDriverConfiguration params = new ConsoleAndFileDriverConfiguration(
570573
paramsMap,
@@ -584,7 +587,8 @@ public void shouldReturnSameAsConstructedWith()
584587
printHelp,
585588
ignoreScheduledStartTimes,
586589
warmupCount,
587-
skipCount
590+
skipCount,
591+
flushLog
588592
);
589593

590594
assertThat( params.asMap(), equalTo( paramsMap ) );

‎src/test/java/com/ldbc/driver/generator/TimeMappingOperationGeneratorTest.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,7 @@ public void shouldNotBreakTheMonotonicallyIncreasingScheduledStartTimesOfOperati
362362
boolean ignoreScheduledStartTimes = false;
363363
long warmupCount = 0;
364364
long skipCount = 0;
365+
boolean flushLog = false;
365366

366367
ConsoleAndFileDriverConfiguration configuration = new ConsoleAndFileDriverConfiguration(
367368
paramsMap,
@@ -381,7 +382,8 @@ public void shouldNotBreakTheMonotonicallyIncreasingScheduledStartTimesOfOperati
381382
printHelp,
382383
ignoreScheduledStartTimes,
383384
warmupCount,
384-
skipCount
385+
skipCount,
386+
flushLog
385387
);
386388

387389
Map<String,String> updateStreamParams = MapUtils.loadPropertiesToMap(

‎src/test/java/com/ldbc/driver/runtime/QueuePerformanceTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ public void operationQueuePerformanceTest()
8686
boolean ignoreScheduledStartTimes = false;
8787
long warmupCount = 0;
8888
long skipCount = 0;
89+
boolean flushLog = false;
8990

9091
DriverConfiguration config = new ConsoleAndFileDriverConfiguration(
9192
paramsMap,
@@ -105,7 +106,8 @@ public void operationQueuePerformanceTest()
105106
printHelp,
106107
ignoreScheduledStartTimes,
107108
warmupCount,
108-
skipCount
109+
skipCount,
110+
flushLog
109111
);
110112

111113
GeneratorFactory gf = new GeneratorFactory( new RandomDataGeneratorFactory( 42L ) );

‎src/test/java/com/ldbc/driver/runtime/WorkloadRunnerTest.java

+16-8
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ public void shouldRunReadOnlyLdbcWorkloadWithNothingDbAndCrashInSaneManner()
114114
boolean ignoreScheduledStartTimes = false;
115115
long warmupCount = 100;
116116
long skipCount = 10;
117+
boolean flushLog = false;
117118

118119
ConsoleAndFileDriverConfiguration configuration = new ConsoleAndFileDriverConfiguration(
119120
paramsMap,
@@ -133,7 +134,8 @@ public void shouldRunReadOnlyLdbcWorkloadWithNothingDbAndCrashInSaneManner()
133134
printHelp,
134135
ignoreScheduledStartTimes,
135136
warmupCount,
136-
skipCount
137+
skipCount,
138+
flushLog
137139
);
138140

139141
configuration = (ConsoleAndFileDriverConfiguration) configuration
@@ -171,7 +173,7 @@ public void shouldRunReadOnlyLdbcWorkloadWithNothingDbAndCrashInSaneManner()
171173

172174
File resultsLog = temporaryFolder.newFile();
173175
SimpleCsvFileWriter csvResultsLogWriter =
174-
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR );
176+
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR, flushLog );
175177
metricsService = ThreadedQueuedMetricsService.newInstanceUsingBlockingBoundedQueue(
176178
timeSource,
177179
errorReporter,
@@ -315,6 +317,7 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbAndReturnExpectedMetric
315317
boolean ignoreScheduledStartTimes = false;
316318
long warmupCount = 100;
317319
long skipCount = 10;
320+
boolean flushLog = false;
318321

319322
ConsoleAndFileDriverConfiguration configuration = new ConsoleAndFileDriverConfiguration(
320323
paramsMap,
@@ -334,7 +337,8 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbAndReturnExpectedMetric
334337
printHelp,
335338
ignoreScheduledStartTimes,
336339
warmupCount,
337-
skipCount
340+
skipCount,
341+
flushLog
338342
);
339343

340344
configuration = (ConsoleAndFileDriverConfiguration) configuration
@@ -372,7 +376,7 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbAndReturnExpectedMetric
372376

373377
File resultsLog = temporaryFolder.newFile();
374378
SimpleCsvFileWriter csvResultsLogWriter =
375-
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR );
379+
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR, flushLog );
376380
metricsService = ThreadedQueuedMetricsService.newInstanceUsingBlockingBoundedQueue(
377381
timeSource,
378382
errorReporter,
@@ -534,6 +538,7 @@ public void doShouldRunReadWriteLdbcWorkloadWithNothingDbAndReturnExpectedMetric
534538
boolean ignoreScheduledStartTimes = false;
535539
long warmupCount = 100;
536540
long skipCount = 10;
541+
boolean flushLog = false;
537542

538543
ConsoleAndFileDriverConfiguration configuration = new ConsoleAndFileDriverConfiguration(
539544
paramsMap,
@@ -553,7 +558,8 @@ public void doShouldRunReadWriteLdbcWorkloadWithNothingDbAndReturnExpectedMetric
553558
printHelp,
554559
ignoreScheduledStartTimes,
555560
warmupCount,
556-
skipCount
561+
skipCount,
562+
flushLog
557563
);
558564

559565
configuration = (ConsoleAndFileDriverConfiguration) configuration
@@ -591,7 +597,7 @@ public void doShouldRunReadWriteLdbcWorkloadWithNothingDbAndReturnExpectedMetric
591597

592598
File resultsLog = temporaryFolder.newFile();
593599
SimpleCsvFileWriter csvResultsLogWriter =
594-
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR );
600+
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR, flushLog );
595601
metricsService = ThreadedQueuedMetricsService.newInstanceUsingBlockingBoundedQueue(
596602
timeSource,
597603
errorReporter,
@@ -799,6 +805,7 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbWhileIgnoringScheduledS
799805
boolean ignoreScheduledStartTimes = true;
800806
long warmupCount = 100;
801807
long skipCount = 10;
808+
boolean flushLog = false;
802809

803810
ConsoleAndFileDriverConfiguration configuration = new ConsoleAndFileDriverConfiguration(
804811
paramsMap,
@@ -818,7 +825,8 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbWhileIgnoringScheduledS
818825
printHelp,
819826
ignoreScheduledStartTimes,
820827
warmupCount,
821-
skipCount
828+
skipCount,
829+
flushLog
822830
);
823831

824832
configuration = (ConsoleAndFileDriverConfiguration) configuration
@@ -859,7 +867,7 @@ private void doShouldRunReadOnlyLdbcWorkloadWithNothingDbWhileIgnoringScheduledS
859867

860868
File resultsLog = temporaryFolder.newFile();
861869
SimpleCsvFileWriter csvResultsLogWriter =
862-
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR );
870+
new SimpleCsvFileWriter( resultsLog, SimpleCsvFileWriter.DEFAULT_COLUMN_SEPARATOR, flushLog );
863871
metricsService = ThreadedQueuedMetricsService.newInstanceUsingBlockingBoundedQueue(
864872
timeSource,
865873
errorReporter,

‎src/test/java/com/ldbc/driver/runtime/metrics/ResultsLogReaderWriterTest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ public void shouldWriteAndReadSameValues() throws Exception
2424
{
2525
File resultsLog = temporaryFolder.newFile();
2626
TimeUnit unit = NANOSECONDS;
27+
boolean flushLog = false;
2728

28-
try ( ResultsLogWriter writer = new SimpleResultsLogWriter( resultsLog, unit ) )
29+
try ( ResultsLogWriter writer = new SimpleResultsLogWriter( resultsLog, unit, flushLog ) )
2930
{
3031
writer.write(
3132
"a",
@@ -70,8 +71,9 @@ public void shouldConvertUnitsCorrectly() throws Exception
7071
{
7172
File resultsLog = temporaryFolder.newFile();
7273
TimeUnit unit = MILLISECONDS;
74+
boolean flushLog = false;
7375

74-
try ( ResultsLogWriter writer = new SimpleResultsLogWriter( resultsLog, unit ) )
76+
try ( ResultsLogWriter writer = new SimpleResultsLogWriter( resultsLog, unit, flushLog ) )
7577
{
7678
writer.write(
7779
"a",

0 commit comments

Comments
 (0)
Please sign in to comment.