JAPI-498 DFSClient: Reading long varstrings corrupted (hpcc-systems#597)

* JAPI-498 DFSClient: Reading long varstrings corrupted

- Modified BinaryRecordReader to correctly handle read ahead in varstr
- Added a long string test

Signed-off-by: James McMullan [email protected]

* Code review changes

Signed-off-by: James McMullan [email protected]

Author: jpmcmu

dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java

@@ -959,7 +959,7 @@ private String getNullTerminatedString(HpccSrcType stype) throws IOException
                     throw new IOException("Error, unexpected EOS while constructing UTF16 string.");
                 }
 
-                readSize = (readSize / 2) * 2;
+                readSize = ((readSize + 1) / 2) * 2;
                 if (readSize > OPTIMIZED_STRING_READ_AHEAD)
                 {
                     readSize = OPTIMIZED_STRING_READ_AHEAD;
@@ -970,7 +970,7 @@ private String getNullTerminatedString(HpccSrcType stype) throws IOException
 
                 for (int j = 0; j < readSize-1; j += 2)
                 {
-                    if (scratchBuffer[j] == '\0' && scratchBuffer[j + 1] == '\0')
+                    if (scratchBuffer[strByteLen + j] == '\0' && scratchBuffer[strByteLen + j + 1] == '\0')
                     {
                         eosLocation = j;
                         break;
@@ -1018,7 +1018,7 @@ private String getNullTerminatedString(HpccSrcType stype) throws IOException
 
                 for (int j = 0; j < readSize; j++)
                 {
-                    if (scratchBuffer[j] == '\0')
+                    if (scratchBuffer[strByteLen + j] == '\0')
                     {
                         eosLocation = j;
                         break;

dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java

@@ -276,7 +276,6 @@ public void getNullMetadataTest() throws Exception
         assertNull("Meta should be null for nonexistent file",meta);
     }
 
-
     private static final String       ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_";
     private static final SecureRandom RANDOM   = new SecureRandom();
 
@@ -376,6 +375,47 @@ public void unsigned8ToDecimalTest() throws Exception
         }
     }
 
+    @Test
+    public void longStringTest() throws Exception
+    {
+        // Create a large record dataset
+        FieldDef[] fieldDefs = new FieldDef[4];
+        fieldDefs[0] = new FieldDef("LongVarUnicode", FieldType.VAR_STRING, "", 4, false, false, HpccSrcType.UTF16LE, new FieldDef[0]);
+        fieldDefs[1] = new FieldDef("LongUnicode", FieldType.STRING, "", 64, true, false, HpccSrcType.UTF16LE, new FieldDef[0]);
+        fieldDefs[2] = new FieldDef("LongVarString", FieldType.VAR_STRING, "", 4, false, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]);
+        fieldDefs[3] = new FieldDef("LongString", FieldType.STRING, "", 64, true, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]);
+        FieldDef recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs);
+
+        List<HPCCRecord> originalRecords = new ArrayList<HPCCRecord>();
+        for (int i = 0; i < 10; i++)
+        {
+            Object[] fields = new Object[4];
+            fields[0] = generateRandomString(1024);
+            fields[1] = generateRandomString(64);
+            fields[2] = generateRandomString(1024);
+            fields[3] = generateRandomString(64);
+            HPCCRecord record = new HPCCRecord(fields, recordDef);
+            originalRecords.add(record);
+        }
+
+        String datasetName = "benchmark::long_string::10rows"; 
+        writeFile(originalRecords, datasetName, recordDef,connTO);
+
+        HPCCFile file = new HPCCFile(datasetName, connString , hpccUser, hpccPass);
+        List<HPCCRecord> readRecords = readFile(file, connTO, false);
+        if (readRecords.size() < 10)
+        {
+            Assert.fail("Failed to read " + datasetName + " dataset");
+        }
+
+        for (int i = 0; i < 10; i++)
+        {
+            HPCCRecord originalRecord = originalRecords.get(i);
+            HPCCRecord readRecord = originalRecords.get(i);
+            Assert.assertEquals(originalRecord, readRecord);
+        }
+    }
+
     @Test
     public void numericOverflowTest() throws Exception
     {