added options for ignoring blank lines and corrupt data

Author: jegesh

README.rst

@@ -3,7 +3,7 @@ Random Access File Reader
 
 | This is a small library that allows for reading any given line in a file without having to read all the lines before it
   or load the entire file into memory.  Only the line indexes and lengths are held in memory, which enables random
-  access even on very large files at a very minuscule memory cost.
+  access even on very large files for a very minuscule memory cost.
 
 Installation
 ============
@@ -33,6 +33,10 @@ Usage
     for l in lines:
         print l
 
+| Optional arguments in the constructor:
+ - ``endline_character`` - self-explanatory (default is endline character ``\\n``)
+ - ``ignore_blank_lines`` - if set to ``True``, blank lines in the file will not be read or indexed (default is ``False``)
+
 |
 | Csv example:
 
@@ -51,3 +55,10 @@ Usage
     for l in lines:
         for x in l:
             print x + " = " l[x]
+
+| Optional arguments in the constructor:
+ - ``endline_character`` - self-explanatory (default is endline character ``\\n``)
+ - ``ignore_blank_lines`` - if set to ``True``, blank lines in the file will not be read or indexed (default is ``True``)
+ - ``values_delimiter`` - character used by the csv to separate values within a line (default is ``,``)
+ - ``quotechar`` - character used by the csv to surround values that contain the value delimiting character (default is ``"``)
+ - ``ignore_corrupt`` - if set to ``True``, lines with an invalid length will return blank instead of raising an exception (default is ``False``)

randomAccessReader/__init__.py

@@ -19,13 +19,14 @@
 
 class RandomAccessReader(object):
 
-    def __init__(self, filepath, endline_character='\n'):
+    def __init__(self, filepath, endline_character='\n', ignore_blank_lines=False):
         """
         :param filepath:  Absolute path to file
         :param endline_character: Delimiter for lines. Defaults to newline character (\n)
         """
         self._filepath = filepath
         self._endline = endline_character
+        self._ignore_blanks = ignore_blank_lines
         self._lines = self._get_line_data()
 
     @property
@@ -46,7 +47,8 @@ def _get_line_data(self):
 
             if current == self._endline or current == '':
                 # we've reached the end of the current line
-                lines.append({"position": start_position, "length": current_line})
+                if not self._ignore_blanks or current_line > 0:
+                    lines.append({"position": start_position, "length": current_line})
                 start_position += current_line + 1
                 current_line = 0
                 if current == '':
@@ -75,11 +77,18 @@ def get_lines(self, line_number, amount=1):
 
 class CsvRandomAccessReader(RandomAccessReader):
 
-    def __init__(self, filepath, has_header=True, endline_character='\n', values_delimiter=',', quotechar='"'):
-        super(CsvRandomAccessReader, self).__init__(filepath, endline_character)
+    def __init__(self, filepath, has_header=True, **kwargs):
+        """
+
+        :param filepath:
+        :param has_header:
+        :param kwargs: endline_character='\n', values_delimiter=',', quotechar='"', ignore_corrupt=False, ignore_blank_lines=True
+        """
+        super(CsvRandomAccessReader, self).__init__(filepath, kwargs.get('endline_character','\n'), kwargs.get('ignore_blank_lines', True))
         self._headers = None
-        self._delimiter = values_delimiter
-        self._quotechar = quotechar
+        self._delimiter = kwargs.get('values_delimiter', ',')
+        self._quotechar = kwargs.get('quotechar', '"')
+        self._ignore_bad_lines = kwargs.get('ignore_corrupt', False)
         self.has_header = has_header
         if has_header:
             dialect = self.MyDialect(self._endline, self._quotechar, self._delimiter)
@@ -108,7 +117,9 @@ def _get_line_values(self, line):
         r = csv.reader(b, dialect)
         values = tuple(r.next())
         if len(self._headers) != len(values):
-            raise ValueError("Corrupt csv - header and row have different lengths")
+            if not self._ignore_bad_lines:
+                raise ValueError("Corrupt csv - header and row have different lengths")
+            return None
         return values
 
     def get_line_dicts(self, line_number, amount=1):
@@ -125,7 +136,11 @@ def get_line_dicts(self, line_number, amount=1):
         lines = []
         text_lines = self.get_lines(line_number, amount)
         for x in xrange(amount):
-            lines.append(dict(zip(self._headers, self._get_line_values(text_lines[x]))))
+            vals = self._get_line_values(text_lines[x])
+            if vals is None:
+                lines.append(dict(zip(self._headers, range(len(self._headers)))))
+            else:
+                lines.append(dict(zip(self._headers, vals)))
         return lines
 
     class MyDialect(csv.Dialect):

setup.py

@@ -15,7 +15,7 @@
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='0.2.5',
+    version='0.3.0',
 
     description='A python random access file reader',
     long_description=long_description,

tests/blank_line.csv

@@ -0,0 +1,35 @@
+Ad,a,Description line 1,b,Description line 2,c,Headline 1,d,Headline 2,e,Description,f,Path 1,g,Path 2,dd,Display URL,ee,Destination URL,cc,Final URL
+How to Start Blogging,,A Dummy Proof Step-By-Step Guide.,,"Get Your Own Blog Today, Live Help!",,Guide: How to Create a Blog,,Dummy Proof Step-By-Step Guide,,"Follow Our Free Step-By-Step Guide. Learn How to Create a Blog, Live Chat Help!",,Wordpress,,Create-a-Blog,,BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r4/,,http://www.buildyourownblog.net
+How to Start a Blog Today,,Follow My Free Step-By-Step Guide.,,"Create Any Type of Blog, Live Help!",,How to Make an Online Blog,,A Step-By-Step Guide,,A Step-By-Step Guide For Beginners. Follow Me & Start Your Own Blog.,,Money-From-Blog,,78,,Blogspot.BuildYourOwnBlog.net,,http://www.buildyourownblog.net,,http://www.buildyourownblog.net/r5/
+{KeyWord:Blogging For Dummies},,Free Setup Guide and Blog Tutorials,,"Make a Passive Income, Live Help!",,Learn How to Run a Blog,,Learn How to Start a Blog,,A Dummy Proof Step-By-Step Guide. Learn Tons of Blogging Tips &Tricks,,Passive-Income,,,,Wordpress.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/welcome-mobile/,,http://www.buildyourownblog.net/
+Make Money With a Blog,,Dummy Proof Step-By-Step Guide.,,"Start Any Type of Blog, Live Help!",,How To Make Money With a Blog,,Read Our Guide to Start a Blog,,"Follow Our Free Step-By-Step Guide. Make a Passive Income, Live Chat Support!",,Money,,,,Money.BuildYourOwnBlog.net,,http://Buildyourownblog.net,,http://www.buildyourownblog.net/r1/
+How to Build a Blog Today,,A Step-By-Step Guide For Beginners.,,Learn Tons of Blogging Tips &Tricks,,How to Start a Blog for Profit,,Follow Our Step-By-Step Guide,,"Follow My Free Step-By-Step Guide. Make a Passive Income, Live Help!",,Guide,,,,Money-Blogging.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r1/,,http://www.buildyourownblog.net/r3/
+Best Blogging Website,,"A Simple, Easy-to-Follow Guide",,Get Your Own Blog Template Today!,,Learn How to Create a Blog,,Read Our Guide to Setup a Blog,,"A Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Chat Support!",,Blogspot,,,,greengeeks.com,,http://www.buildyourownblog.net/make-money-blogging/,,http://www.buildyourownblog.net/r4/
+The Best Blog Builder,,Easy Way To Create Your Own Blog.,,Follow Me & Start Your Own Blog.,,What is Blogging?,,Read Our Guide to Build a Blog,,"A Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Help!",,How-To-Blog,,,,BuildYourOwnBlog.net/How-To-Blog,,http://www.buildyourownblog.net/make-money-blogging,,http://www.buildyourownblog.net/make-money-blogging/
+How to Make a Blog,,Easy Way To Build Your Own Blog.,,Create Any Type of Blog in 15 Mins!,,How To Make Money From a Blog,,,,"A Dummy Proof Step-By-Step Guide. Start Your Own Blog Today, Live Chat Support!",,Blogger,,,,BuildYourOwnBlog.net/Start-a-Blog,,http://www.buildyourownblog.net/blogspot-lp,,http://www.buildyourownblog.net/welcome-mobile/
+HowTo Make Money Blogging,,A Step-By-Step Guide For Beginners,,"Get Your Own Blog, Live Chat Help!",,How to Start a Blog,,,,A Dummy Proof Step-By-Step Guide. Create Any Type of Blog in 15 Mins!,,Create-a-Blog,,,,Blogger.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/,,http://www.greengeeks.com/affiliates/track.php
+Start to Blogging on WP,,Easy Way To Start Your Own Blog.,,Follow Me & Start Your Own Blog!,,How to Make Money Blogging,,,,"Dummy Proof Step-By-Step Guide. Create Any Type of Blog, Live Help!",,,,,,www.buildyourownblog.net,,http://www.buildyourownblog.net/step-7-make-money-blogging/,,http://www.buildyourownblog.net/blogspot-lp
+How To Setup a Blog,,Build a Stunning Blog in 20 Minutes,,A Dummy Proof Step-By-Step Guide.,,Guide: How to Start a Blog,,,,"Follow Our Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Help!",,,,,,Passive-Income.BuildYourOwnBlog.net,,http://Wordpress.BuildYourOwnBlog.net,,http://www.buildyourownblog.net/r1
+How To Setting up a Blog,,Discover The 7 Steps To Blogging.,,"Build a Blog in 15 Mins, Live Help!",,Learn How to Start Blogging,,,,Follow My Free Step-By-Step Guide. Get Your Own Blog Template Today!,,,,,,BuildYourOwnBlog.net/Create-a-Blog,,http://www.buildyourownblog.net/step-7-make-money-blogging,,http://www.buildyourownblog.net/r3
+
+Start a WP Blog Today,,Easy Way To Build Your Won Blog.,,"Get Your Own Blog Now, Live Help!",,How to Become a Pro Blogger,,,,"Follow My Free Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,youtube.com/channel/UCz-EwVCvIkfngDttlKzExNA,,,,http://BuildYourOwnBlog.net
+Make Your Own Blog,,Start a Blog That Makes You Money.,,"Build a Blog in 20 Mins, Live Help!",,Best Blog Guide Website,,,,Follow My Free Step-By-Step Guide. Create Any Type of Blog in 15 Mins!,,,,,,Blogspot.com.BuildYourOwnBlog.net,,,,http://www.buildyourownblog.net/step-7-make-money-blogging/
+How to Start a Blog,,Build Your Own Blog in 20 Mins.,,Create Any Type of Blog in 16 Mins!,,How To Monetize a Blog,,,,"Follow My Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Help!",,,,,,Blogging-101.BuildYourOwnBlog.net,,,,http://www.buildyourownblog.net/blogspot-lp/
+HowTo Make Money Off Blog,,Become a Fashion Blogger Today!,,"Make a Blog in 15 Mins, Live Help!",,Create Your Own Blog Today,,,,"A Dummy Proof Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,Bluehost.com,,,,http://www.buildyourownblog.net/make-money-blogging
+Create Your Own Blog,,"Build a Blog in 20 Mins, Live Help.",,Create Any Type of Blog in 20 Mins!,,Best Blogging Website,,,,"Dummy Proof Step-By-Step Guide. Start Any Type of Blog, Live Help!",,,,,,Blogging.BuildYourOwnBlog.net,,,,http://www.youtube.com/watch
+Best Free Blog Builder,,A Dummy Proof Step-By-Step Guide,,Follow Me & Set Up Your Own Blog.,,How To Start a Blog Today,,,,"Follow My Free Step-By-Step Guide. Learn How to Start a Blog, Live Chat Support!",,,,,,BuildYourOwnBlog.net/Wordpress,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/food-blog
+Become a Pro Blogger,,,,Build a Blog in 15 Mins. Live Help!,,How to Blog for Profit,,,,"A Dummy Proof Step-By-Step Guide. Learn How to Start a Blog, Live Chat Support!",,,,,,BuildYourOwnBlog.net/Blog-Guide,,,,http://www.buildyourownblog.net/uncategorized/start-travel-blog/
+Make Money Blogging,,,,Make a Blog For Free in 15 Minutes!,,Guide: How to Write a Blog,,,,"A Simple, Easy-to-Follow Guide. Create Any Type of Blog in 15 Mins!",,,,,,Blog.BuildYourOwnBlog.net,,,,http://www.bluehost.com/track/eliran/
+How to Create a Blog,,,,"Build Any Type of Blog, Live Help!",,The Best Blog Creator,,,,"Follow My Free Step-By-Step Guide. Create Any Type of Blog, Live Help!",,,,,,BuildYourOwnBlog.net/Build-a-Blog,,,,http://www.buildyourownblog.net/step-7-make-money-blogging
+Learn How to Run a Blog,,,,"Make a Blog in 16 Mins, Live Help!",,{KeyWord:Blogging For Beginners},,,,A Dummy Proof Step-By-Step Guide. Follow Me & Start Your Own Blog!,,,,,,BuildYourOwnBlog.net/Setup-Guide,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/food-blog/
+Learn How do Blogs Work,,,,Build a Blog in 20 Mins. Live Help!,,WP Blog Development,,,,Follow My Free Step-By-Step Guide. Learn Tons of Blogging Tips &Tricks,,,,,,BuildYourOwnBlog.net/Fashion-Blog,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/fashion-blog
+How to Build a Blog,,,,"Start a Blog Today, Live Chat Help!",,Learn How To Setup a Blog,,,,"Dummy Proof Step-By-Step Guide. Get Your Own Blog Today, Live Help!",,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/travel-blog
+How to Write a Blog,,,,Build Any Type of Blog in 15 Mins!,,Setup a Free Blog Website,,,,"Follow Our Free Step-By-Step Guide. Learn How to Make a Blog, Live Chat Support",,,,,,,,,,http://www.buildyourownblog.net/uncategorized/start-food-blog/
+How To Blog,,,,"Dummy Proof Guide, Free Live Help!",,Learn How to Write a Blog,,,,"A Dummy Proof Step-By-Step Guide. Start Any Type of Blog, Live Help!",,,,,,,,,,http://www.bluehost.com/track/eliran/{adgroupid}_{creative}_{placement}_{targetid}_{loc_physical_ms}/
+The Best Blog Creator,,,,Dummy Proof Step-By-Step Guide!,,How to Setting Up a WP Blog,,,,A Dummy Proof Step-By-Step Guide. Get Your Own Blog Template Today!,,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/travel-blog/
+Free Online Blog Builder,,,,"Setup Any Type of Blog, Live Help!",,How To Be a Blogger Today,,,,Follow Our Free Step-By-Step Guide. Learn How to Create a Blog Live Chat Help!,,,,,,,,,,http://www.buildyourownblog.net/how-to-start-any-type-of-blog/fashion-blog/
+Make Money From Blog,,,,"Build a Blog in 16 Mins, Live Help!",,Learn How to Build a Blog Site,,,,"Follow Our Step-By-Step Guide. Learn How to Setup a Blog, Live Chat Support!",,,,,,,,,,http://www.bluehost.com/track/eliran/targetid={targetid}&matchtype={matchtype}&device={device}&campaignid={campaignid}&creative={creative}&adgroupid={adgroupid}&feeditemid={feeditemid}&loc_physical_ms={loc_physical_ms}&loc_interest_ms={loc_interest_ms}&network={network}&devicemodel={devicemodel}&placement={placement}&keyword={keyword}&target={target}&aceid={aceid}&adposition={adposition}/
+Start Your Own Blog Today,,,,Follow Me & Start Your Food Blog!,,How to Build a Blog Today,,,,"Dummy Proof Step-By-Step Guide. Learn How to Setup Any Type of Blog, Live Help!",,,,,,,,,,http://www.buildyourownblog.net/uncategorized/start-fashion-blog/
+How To Get Paid To Blog,,,,"Make a Blog in 20 Mins, Live Help!",,Guide: How to Make a Blog,,,,"Follow Our Free Step-By-Step Guide. Build Your Own Blog Today, Live Chat Support",,,,,,,,,,http://www.bluehost.com/track/eliran/{adgroupid}_{creative}_{placement}_{loc_physical_ms}/
+The Best Blog Maker,,,,Build a Blog in 16 Mins. Live Help!,,How to Create a WP Blog,,,,"A Dummy Proof Step-By-Step Guide. Create Your Own Blog Today, Live Chat Support!",,,,,,,,,,
+Ad name: 00040049 - 300x250.gif; 300 x 250,,,,"Start a Blog Today, Free Live Help!",,How To Become a Blogger,,,,"A Dummy Proof Step-By-Step Guide. Set Up Your Own Blog Today, Live Chat Support!",,,,,,,,,,