Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions ext/java/org/jruby/ext/stringio/StringIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -898,42 +898,46 @@ private IRubyObject getline(ThreadContext context, final IRubyObject rs, int lim
int p;
int w = 0;

if (limit > 0 && s + limit < e) {
// a fixed-width encoding may step past a truncated last character
e = Math.min(e, getEncoding().rightAdjustCharHead(stringBytes, s, s + limit, e));
final int end = e;
if (rs == context.nil || ((RubyString) rs).size() != 0) {
e = getlineLimitEnd(stringBytes, s, e, limit);
}
if (rs == context.nil) {
if (chomp) {
w = chompNewlineWidth(stringBytes, s, e);
}
str = strioSubstr(runtime, pos, e - s - w, enc);
} else if ((n = ((RubyString) rs).size()) == 0) {
int paragraph_end = 0;
int paragraph_end = -1;
int line_end;
p = s;
while (stringBytes[p] == '\n') {
if (++p == e) {
ptr.pos = e - begin;
return context.nil;
}
}
s = p;
// like IO, the limit counts from the first non-newline character
e = getlineLimitEnd(stringBytes, s, e, limit);
while ((p = Helpers.memchr(stringBytes, p, '\n', e - p)) != -1 && (p != e)) {
p++;
if (!((p < e && stringBytes[p] == '\n') ||
(p + 1 < e && stringBytes[p] == '\r' && stringBytes[p+1] == '\n'))) {
continue;
}
paragraph_end = p - ((stringBytes[p-2] == '\r') ? 2 : 1);
while ((p < e && stringBytes[p] == '\n') ||
(p + 1 < e && stringBytes[p] == '\r' && stringBytes[p+1] == '\n')) {
p += (stringBytes[p] == '\r') ? 2 : 1;
}
e = p;
// like IO, return the paragraph with only one empty line
e = p + ((stringBytes[p] == '\r') ? 2 : 1);
break;
}
if (chomp && paragraph_end != 0) {
w = e - paragraph_end;
line_end = (chomp && paragraph_end != -1) ? paragraph_end : e;
// like IO, skip the following empty lines, even past the limit
while ((e < end && stringBytes[e] == '\n') ||
(e + 1 < end && stringBytes[e] == '\r' && stringBytes[e+1] == '\n')) {
e += (stringBytes[e] == '\r') ? 2 : 1;
}
str = strioSubstr(runtime, s - begin, e - s - w, enc);
str = strioSubstr(runtime, s - begin, line_end - s, enc);
} else if (n == 1) {
RubyString strStr = (RubyString) rs;
ByteList strByteList = strStr.getByteList();
Expand Down Expand Up @@ -979,6 +983,14 @@ private IRubyObject getline(ThreadContext context, final IRubyObject rs, int lim
return str;
}

private int getlineLimitEnd(byte[] bytes, int s, int e, int limit) {
if (limit > 0 && s + limit < e) {
// a fixed-width encoding may step past a truncated last character
e = Math.min(e, getEncoding().rightAdjustCharHead(bytes, s, s + limit, e));
}
return e;
}

private static int chompNewlineWidth(byte[] bytes, int s, int e) {
if (e > s && bytes[--e] == '\n') {
if (e > s && bytes[--e] == '\r') return 2;
Expand Down
44 changes: 28 additions & 16 deletions ext/stringio/stringio.c
Original file line number Diff line number Diff line change
Expand Up @@ -1552,10 +1552,22 @@ chomp_newline_width(const char *s, const char *e)
return 0;
}

static inline const char *
getline_limit_end(struct StringIO *ptr, const char *s, const char *e, rb_len_t limit)
{
if (limit > 0 && (size_t)limit < (size_t)(e - s)) {
const char *const end = e;
e = rb_enc_right_char_head(s, s + limit, e, get_enc(ptr));
/* a fixed-width encoding may step past a truncated last character */
if (e > end) e = end;
}
return e;
}

static VALUE
strio_getline(struct getline_arg *arg, struct StringIO *ptr)
{
const char *s, *e, *p;
const char *s, *e, *p, *end;
rb_len_t n, limit = arg->limit;
VALUE str = arg->rs;
rb_len_t w = 0;
Expand All @@ -1565,13 +1577,10 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr)
return Qnil;
}
s = RSTRING_PTR(ptr->string);
e = s + RSTRING_LEN(ptr->string);
end = e = s + RSTRING_LEN(ptr->string);
s += ptr->pos;
if (limit > 0 && (size_t)limit < (size_t)(e - s)) {
const char *const end = e;
e = rb_enc_right_char_head(s, s + limit, e, get_enc(ptr));
/* a fixed-width encoding may step past a truncated last character */
if (e > end) e = end;
if (NIL_P(str) || RSTRING_LEN(str) != 0) {
e = getline_limit_end(ptr, s, e, limit);
}
if (NIL_P(str)) {
if (arg->chomp) {
Expand All @@ -1580,33 +1589,36 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr)
str = strio_substr(ptr, ptr->pos, e - s - w, enc);
}
else if ((n = RSTRING_LEN(str)) == 0) {
const char *paragraph_end = NULL;
const char *paragraph_end = NULL, *line_end;
p = s;
while (p[(p + 1 < e) && (*p == '\r') && 0] == '\n') {
p += *p == '\r';
if (++p == e) {
ptr->pos = e - RSTRING_PTR(ptr->string);
return Qnil;
}
}
s = p;
/* like IO, the limit counts from the first non-newline character */
e = getline_limit_end(ptr, s, e, limit);
while ((p = memchr(p, '\n', e - p)) && (p != e)) {
p++;
if (!((p < e && *p == '\n') ||
(p + 1 < e && *p == '\r' && *(p+1) == '\n'))) {
continue;
}
paragraph_end = p - ((*(p-2) == '\r') ? 2 : 1);
while ((p < e && *p == '\n') ||
(p + 1 < e && *p == '\r' && *(p+1) == '\n')) {
p += (*p == '\r') ? 2 : 1;
}
e = p;
/* like IO, return the paragraph with only one empty line */
e = p + ((*p == '\r') ? 2 : 1);
break;
}
if (arg->chomp && paragraph_end) {
w = e - paragraph_end;
line_end = (arg->chomp && paragraph_end) ? paragraph_end : e;
/* like IO, skip the following empty lines, even past the limit */
while ((e < end && *e == '\n') ||
(e + 1 < end && *e == '\r' && *(e+1) == '\n')) {
e += (*e == '\r') ? 2 : 1;
}
str = strio_substr(ptr, s - RSTRING_PTR(ptr->string), e - s - w, enc);
str = strio_substr(ptr, s - RSTRING_PTR(ptr->string), line_end - s, enc);
}
else if (n == 1) {
if ((p = memchr(s, RSTRING_PTR(str)[0], e - s)) != 0) {
Expand Down
38 changes: 36 additions & 2 deletions test/stringio/test_stringio.rb
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,39 @@ def test_gets
assert_string("", Encoding::UTF_8, StringIO.new("foo").gets(0))
end

def test_gets_paragraph_extra_newlines
# Like IO, return the paragraph with only one empty line and skip
# the rest.
stringio = StringIO.new("abc\n\n\n\ndef\n\n\n")
assert_equal("abc\n\n", stringio.gets(""))
assert_equal(7, stringio.pos)
assert_equal("def\n\n", stringio.gets(""))
assert_equal(13, stringio.pos)
assert_nil(stringio.gets(""))
assert_equal(["abc\n\n", "def\n\n"], StringIO.new("abc\n\n\n\ndef\n\n\n").readlines(""))
assert_equal(["abc", "def"], StringIO.new("abc\n\n\n\ndef\n\n\n").readlines("", chomp: true))
end

def test_gets_paragraph_only_newlines
stringio = StringIO.new("\n\n\n")
assert_nil(stringio.gets(""))
assert_equal(3, stringio.pos)
end

def test_gets_paragraph_limit
# Like IO, the limit starts after the leading newlines.
stringio = StringIO.new("\n\nabc\n\ndef")
assert_equal("a", stringio.gets("", 1))
assert_equal("bc", stringio.gets("", 2))
# and the empty lines after a line cut by the limit are skipped.
assert_equal(7, stringio.pos)
assert_equal("def", stringio.gets("", 5))
assert_nil(stringio.gets("", 1))

assert_equal("\u3042", StringIO.new("\n\n\u3042\u3044").gets("", 1))
assert_equal("abc\n\n", StringIO.new("abc\n\n\n\nd").gets("", 5))
end

def test_gets_utf_16
stringio = StringIO.new("line1\nline2\nline3\n".encode("utf-16le"))
assert_equal("line1\n".encode("utf-16le"), stringio.gets)
Expand Down Expand Up @@ -198,6 +231,7 @@ def test_gets_chomp_eol
stringio = StringIO.new("abc\r\n\r\ndef\r\n")
assert_equal("abc", stringio.gets("", chomp: true))
assert_equal("def\r\n", stringio.gets("", chomp: true))
assert_equal(["", "abc"], StringIO.new("\r\n\r\nabc").each_line("", chomp: true).to_a)
end

def test_readlines
Expand Down Expand Up @@ -695,12 +729,12 @@ def test_each
f.rewind
assert_equal(["foo", "bar", "baz"], f.each(chomp: true).to_a)
f = StringIO.new("foo\nbar\n\n\nbaz\n")
assert_equal(["foo\nbar\n\n\n", "baz\n"], f.each("").to_a)
assert_equal(["foo\nbar\n\n", "baz\n"], f.each("").to_a)
f.rewind
assert_equal(["foo\nbar", "baz\n"], f.each("", chomp: true).to_a)

f = StringIO.new("foo\r\nbar\r\n\r\n\r\nbaz\r\n")
assert_equal(["foo\r\nbar\r\n\r\n\r\n", "baz\r\n"], f.each("").to_a)
assert_equal(["foo\r\nbar\r\n\r\n", "baz\r\n"], f.each("").to_a)
f.rewind
assert_equal(["foo\r\nbar", "baz\r\n"], f.each("", chomp: true).to_a)

Expand Down