Skip to content

Commit 950a47c

Browse files
committed
added options for ignoring blank lines and corrupt data
1 parent 6cfa463 commit 950a47c

File tree

6 files changed

+103
-11
lines changed

6 files changed

+103
-11
lines changed

README.rst

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ Random Access File Reader
33

44
| This is a small library that allows for reading any given line in a file without having to read all the lines before it
55
or load the entire file into memory. Only the line indexes and lengths are held in memory, which enables random
6-
access even on very large files at a very minuscule memory cost.
6+
access even on very large files for a very minuscule memory cost.
77
88
Installation
99
============
@@ -33,6 +33,10 @@ Usage
3333
for l in lines:
3434
print l
3535

36+
| Optional arguments in the constructor:
37+
- ``endline_character`` - self-explanatory (default is endline character ``\\n``)
38+
- ``ignore_blank_lines`` - if set to ``True``, blank lines in the file will not be read or indexed (default is ``False``)
39+
3640
|
3741
| Csv example:
3842
@@ -51,3 +55,10 @@ Usage
5155
for l in lines:
5256
for x in l:
5357
print x + " = " l[x]
58+
59+
| Optional arguments in the constructor:
60+
- ``endline_character`` - self-explanatory (default is endline character ``\\n``)
61+
- ``ignore_blank_lines`` - if set to ``True``, blank lines in the file will not be read or indexed (default is ``True``)
62+
- ``values_delimiter`` - character used by the csv to separate values within a line (default is ``,``)
63+
- ``quotechar`` - character used by the csv to surround values that contain the value delimiting character (default is ``"``)
64+
- ``ignore_corrupt`` - if set to ``True``, lines with an invalid length will return blank instead of raising an exception (default is ``False``)

randomAccessReader/__init__.py

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@
1919

2020
class RandomAccessReader(object):
2121

22-
def __init__(self, filepath, endline_character='\n'):
22+
def __init__(self, filepath, endline_character='\n', ignore_blank_lines=False):
2323
"""
2424
:param filepath: Absolute path to file
2525
:param endline_character: Delimiter for lines. Defaults to newline character (\n)
2626
"""
2727
self._filepath = filepath
2828
self._endline = endline_character
29+
self._ignore_blanks = ignore_blank_lines
2930
self._lines = self._get_line_data()
3031

3132
@property
@@ -46,7 +47,8 @@ def _get_line_data(self):
4647

4748
if current == self._endline or current == '':
4849
# we've reached the end of the current line
49-
lines.append({"position": start_position, "length": current_line})
50+
if not self._ignore_blanks or current_line > 0:
51+
lines.append({"position": start_position, "length": current_line})
5052
start_position += current_line + 1
5153
current_line = 0
5254
if current == '':
@@ -75,11 +77,18 @@ def get_lines(self, line_number, amount=1):
7577

7678
class CsvRandomAccessReader(RandomAccessReader):
7779

78-
def __init__(self, filepath, has_header=True, endline_character='\n', values_delimiter=',', quotechar='"'):
79-
super(CsvRandomAccessReader, self).__init__(filepath, endline_character)
80+
def __init__(self, filepath, has_header=True, **kwargs):
81+
"""
82+
83+
:param filepath:
84+
:param has_header:
85+
:param kwargs: endline_character='\n', values_delimiter=',', quotechar='"', ignore_corrupt=False, ignore_blank_lines=True
86+
"""
87+
super(CsvRandomAccessReader, self).__init__(filepath, kwargs.get('endline_character','\n'), kwargs.get('ignore_blank_lines', True))
8088
self._headers = None
81-
self._delimiter = values_delimiter
82-
self._quotechar = quotechar
89+
self._delimiter = kwargs.get('values_delimiter', ',')
90+
self._quotechar = kwargs.get('quotechar', '"')
91+
self._ignore_bad_lines = kwargs.get('ignore_corrupt', False)
8392
self.has_header = has_header
8493
if has_header:
8594
dialect = self.MyDialect(self._endline, self._quotechar, self._delimiter)
@@ -108,7 +117,9 @@ def _get_line_values(self, line):
108117
r = csv.reader(b, dialect)
109118
values = tuple(r.next())
110119
if len(self._headers) != len(values):
111-
raise ValueError("Corrupt csv - header and row have different lengths")
120+
if not self._ignore_bad_lines:
121+
raise ValueError("Corrupt csv - header and row have different lengths")
122+
return None
112123
return values
113124

114125
def get_line_dicts(self, line_number, amount=1):
@@ -125,7 +136,11 @@ def get_line_dicts(self, line_number, amount=1):
125136
lines = []
126137
text_lines = self.get_lines(line_number, amount)
127138
for x in xrange(amount):
128-
lines.append(dict(zip(self._headers, self._get_line_values(text_lines[x]))))
139+
vals = self._get_line_values(text_lines[x])
140+
if vals is None:
141+
lines.append(dict(zip(self._headers, range(len(self._headers)))))
142+
else:
143+
lines.append(dict(zip(self._headers, vals)))
129144
return lines
130145

131146
class MyDialect(csv.Dialect):

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# Versions should comply with PEP440. For a discussion on single-sourcing
1616
# the version across setup.py and the project code, see
1717
# https://packaging.python.org/en/latest/single_source_version.html
18-
version='0.2.5',
18+
version='0.3.0',
1919

2020
description='A python random access file reader',
2121
long_description=long_description,

tests/blank_line.csv

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
Ad,a,Description line 1,b,Description line 2,c,Headline 1,d,Headline 2,e,Description,f,Path 1,g,Path 2,dd,Display URL,ee,Destination URL,cc,Final URL
2+
How to Start Blogging,,A Dummy Proof Step-By-Step Guide.,,"Get Your Own Blog Today, Live Help!",,Guide: How to Create a Blog,,Dummy Proof Step-By-Step Guide,,"Follow Our Free Step-By-Step Guide. Learn How to Create a Blog, Live Chat Help!",,Wordpress,,Create-a-Blog,,BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r4/,,http://www.buildyourownblog.net
3+
How to Start a Blog Today,,Follow My Free Step-By-Step Guide.,,"Create Any Type of Blog, Live Help!",,How to Make an Online Blog,,A Step-By-Step Guide,,A Step-By-Step Guide For Beginners. Follow Me & Start Your Own Blog.,,Money-From-Blog,,78,,Blogspot.BuildYourOwnBlog.net,,http://www.buildyourownblog.net,,http://www.buildyourownblog.net/r5/
4+
{KeyWord:Blogging For Dummies},,Free Setup Guide and Blog Tutorials,,"Make a Passive Income, Live Help!",,Learn How to Run a Blog,,Learn How to Start a Blog,,A Dummy Proof Step-By-Step Guide. Learn Tons of Blogging Tips &Tricks,,Passive-Income,,,,Wordpress.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/welcome-mobile/,,http://www.buildyourownblog.net/
5+
Make Money With a Blog,,Dummy Proof Step-By-Step Guide.,,"Start Any Type of Blog, Live Help!",,How To Make Money With a Blog,,Read Our Guide to Start a Blog,,"Follow Our Free Step-By-Step Guide. Make a Passive Income, Live Chat Support!",,Money,,,,Money.BuildYourOwnBlog.net,,http://Buildyourownblog.net,,http://www.buildyourownblog.net/r1/
6+
How to Build a Blog Today,,A Step-By-Step Guide For Beginners.,,Learn Tons of Blogging Tips &Tricks,,How to Start a Blog for Profit,,Follow Our Step-By-Step Guide,,"Follow My Free Step-By-Step Guide. Make a Passive Income, Live Help!",,Guide,,,,Money-Blogging.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r1/,,http://www.buildyourownblog.net/r3/
7+
Best Blogging Website,,"A Simple, Easy-to-Follow Guide",,Get Your Own Blog Template Today!,,Learn How to Create a Blog,,Read Our Guide to Setup a Blog,,"A Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Chat Support!",,Blogspot,,,,greengeeks.com,,http://www.buildyourownblog.net/make-money-blogging/,,http://www.buildyourownblog.net/r4/
8+
The Best Blog Builder,,Easy Way To Create Your Own Blog.,,Follow Me & Start Your Own Blog.,,What is Blogging?,,Read Our Guide to Build a Blog,,"A Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Help!",,How-To-Blog,,,,BuildYourOwnBlog.net/How-To-Blog,,http://www.buildyourownblog.net/make-money-blogging,,http://www.buildyourownblog.net/make-money-blogging/
9+
How to Make a Blog,,Easy Way To Build Your Own Blog.,,Create Any Type of Blog in 15 Mins!,,How To Make Money From a Blog,,,,"A Dummy Proof Step-By-Step Guide. Start Your Own Blog Today, Live Chat Support!",,Blogger,,,,BuildYourOwnBlog.net/Start-a-Blog,,http://www.buildyourownblog.net/blogspot-lp,,http://www.buildyourownblog.net/welcome-mobile/
10+
HowTo Make Money Blogging,,A Step-By-Step Guide For Beginners,,"Get Your Own Blog, Live Chat Help!",,How to Start a Blog,,,,A Dummy Proof Step-By-Step Guide. Create Any Type of Blog in 15 Mins!,,Create-a-Blog,,,,Blogger.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/,,http://www.greengeeks.com/affiliates/track.php
11+
Start to Blogging on WP,,Easy Way To Start Your Own Blog.,,Follow Me & Start Your Own Blog!,,How to Make Money Blogging,,,,"Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Help!",,,,,,www.buildyourownblog.net,,http://www.buildyourownblog.net/step-7-make-money-blogging/,,http://www.buildyourownblog.net/blogspot-lp
12+
How To Setup a Blog,,Build a Stunning Blog in 20 Minutes,,A Dummy Proof Step-By-Step Guide.,,Guide: How to Start a Blog,,,,"Follow Our Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Help!",,,,,,Passive-Income.BuildYourOwnBlog.net,,http://Wordpress.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r1
13+
How To Setting up a Blog,,Discover The 7 Steps To Blogging.,,"Build a Blog in 15 Mins, Live Help!",,Learn How to Start Blogging,,,,Follow My Free Step-By-Step Guide. Get Your Own Blog Template Today!,,,,,,BuildYourOwnBlog.net/Create-a-Blog,,http://www.buildyourownblog.net/step-7-make-money-blogging,,http://www.buildyourownblog.net/r3
14+
15+
Start a WP Blog Today,,Easy Way To Build Your Won Blog.,,"Get Your Own Blog Now, Live Help!",,How to Become a Pro Blogger,,,,"Follow My Free Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,youtube.com/channel/UCz-EwVCvIkfngDttlKzExNA,,,,http://BuildYourOwnBlog.net
16+
Make Your Own Blog,,Start a Blog That Makes You Money.,,"Build a Blog in 20 Mins, Live Help!",,Best Blog Guide Website,,,,Follow My Free Step-By-Step Guide. Create Any Type of Blog in 15 Mins!,,,,,,Blogspot.com.BuildYourOwnBlog.net,,,,http://www.buildyourownblog.net/step-7-make-money-blogging/
17+
How to Start a Blog,,Build Your Own Blog in 20 Mins.,,Create Any Type of Blog in 16 Mins!,,How To Monetize a Blog,,,,"Follow My Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Help!",,,,,,Blogging-101.BuildYourOwnBlog.net,,,,http://www.buildyourownblog.net/blogspot-lp/
18+
HowTo Make Money Off Blog,,Become a Fashion Blogger Today!,,"Make a Blog in 15 Mins, Live Help!",,Create Your Own Blog Today,,,,"A Dummy Proof Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,Bluehost.com,,,,http://www.buildyourownblog.net/make-money-blogging
19+
Create Your Own Blog,,"Build a Blog in 20 Mins, Live Help.",,Create Any Type of Blog in 20 Mins!,,Best Blogging Website,,,,"Dummy Proof Step-By-Step Guide. Start Any Type of Blog, Live Help!",,,,,,Blogging.BuildYourOwnBlog.net,,,,http://www.youtube.com/watch
20+
Best Free Blog Builder,,A Dummy Proof Step-By-Step Guide,,Follow Me & Set Up Your Own Blog.,,How To Start a Blog Today,,,,"Follow My Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Support!",,,,,,BuildYourOwnBlog.net/Wordpress,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/food-blog
21+
Become a Pro Blogger,,,,Build a Blog in 15 Mins. Live Help!,,How to Blog for Profit,,,,"A Dummy Proof Step-By-Step Guide. Learn How to Start a Blog, Live Chat Support!",,,,,,BuildYourOwnBlog.net/Blog-Guide,,,,http://www.buildyourownblog.net/uncategorized/start-travel-blog/
22+
Make Money Blogging,,,,Make a Blog For Free in 15 Minutes!,,Guide: How to Write a Blog,,,,"A Simple, Easy-to-Follow Guide. Create Any Type of Blog in 15 Mins!",,,,,,Blog.BuildYourOwnBlog.net,,,,http://www.bluehost.com/track/eliran/
23+
How to Create a Blog,,,,"Build Any Type of Blog, Live Help!",,The Best Blog Creator,,,,"Follow My Free Step-By-Step Guide. Create Any Type of Blog, Live Help!",,,,,,BuildYourOwnBlog.net/Build-a-Blog,,,,http://www.buildyourownblog.net/step-7-make-money-blogging
24+
Learn How to Run a Blog,,,,"Make a Blog in 16 Mins, Live Help!",,{KeyWord:Blogging For Beginners},,,,A Dummy Proof Step-By-Step Guide. Follow Me & Start Your Own Blog!,,,,,,BuildYourOwnBlog.net/Setup-Guide,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/food-blog/
25+
Learn How do Blogs Work,,,,Build a Blog in 20 Mins. Live Help!,,WP Blog Development,,,,Follow My Free Step-By-Step Guide. Learn Tons of Blogging Tips &Tricks,,,,,,BuildYourOwnBlog.net/Fashion-Blog,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/fashion-blog
26+
How to Build a Blog,,,,"Start a Blog Today, Live Chat Help!",,Learn How To Setup a Blog,,,,"Dummy Proof Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/travel-blog
27+
How to Write a Blog,,,,Build Any Type of Blog in 15 Mins!,,Setup a Free Blog Website,,,,"Follow Our Free Step-By-Step Guide. Learn How to Make a Blog, Live Chat Support",,,,,,,,,,http://www.buildyourownblog.net/uncategorized/start-food-blog/
28+
How To Blog,,,,"Dummy Proof Guide, Free Live Help!",,Learn How to Write a Blog,,,,"A Dummy Proof Step-By-Step Guide. Start Any Type of Blog, Live Help!",,,,,,,,,,http://www.bluehost.com/track/eliran/{adgroupid}_{creative}_{placement}_{targetid}_{loc_physical_ms}/
29+
The Best Blog Creator,,,,Dummy Proof Step-By-Step Guide!,,How to Setting Up a WP Blog,,,,A Dummy Proof Step-By-Step Guide. Get Your Own Blog Template Today!,,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/travel-blog/
30+
Free Online Blog Builder,,,,"Setup Any Type of Blog, Live Help!",,How To Be a Blogger Today,,,,Follow Our Free Step-By-Step Guide. Learn How to Create a Blog Live Chat Help!,,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/fashion-blog/
31+
Make Money From Blog,,,,"Build a Blog in 16 Mins, Live Help!",,Learn How to Build a Blog Site,,,,"Follow Our Step-By-Step Guide. Learn How to Setup a Blog, Live Chat Support!",,,,,,,,,,http://www.bluehost.com/track/eliran/targetid={targetid}&matchtype={matchtype}&device={device}&campaignid={campaignid}&creative={creative}&adgroupid={adgroupid}&feeditemid={feeditemid}&loc_physical_ms={loc_physical_ms}&loc_interest_ms={loc_interest_ms}&network={network}&devicemodel={devicemodel}&placement={placement}&keyword={keyword}&target={target}&aceid={aceid}&adposition={adposition}/
32+
Start Your Own Blog Today,,,,Follow Me & Start Your Food Blog!,,How to Build a Blog Today,,,,"Dummy Proof Step-By-Step Guide. Learn How to Setup Any Type of Blog, Live Help!",,,,,,,,,,http://www.buildyourownblog.net/uncategorized/start-fashion-blog/
33+
How To Get Paid To Blog,,,,"Make a Blog in 20 Mins, Live Help!",,Guide: How to Make a Blog,,,,"Follow Our Free Step-By-Step Guide. Build Your Own Blog Today, Live Chat Support",,,,,,,,,,http://www.bluehost.com/track/eliran/{adgroupid}_{creative}_{placement}_{loc_physical_ms}/
34+
The Best Blog Maker,,,,Build a Blog in 16 Mins. Live Help!,,How to Create a WP Blog,,,,"A Dummy Proof Step-By-Step Guide. Create Your Own Blog Today, Live Chat Support!",,,,,,,,,,
35+
Ad name: 00040049 - 300x250.gif; 300 x 250,,,,"Start a Blog Today, Free Live Help!",,How To Become a Blogger,,,,"A Dummy Proof Step-By-Step Guide. Set Up Your Own Blog Today, Live Chat Support!",,,,,,,,,,

0 commit comments

Comments
 (0)