Merge branch 'develop'

CHANGELOG.md

@@ -1,6 +1,12 @@
+## 4.5.0 - 19 February 2025
+
+- Enhancements
+	- \#392 Add processLargeFile() method to process large XLSX files without reading the entire data into memory
+
 ## 4.4.0 - 8 February 2025
 
-- \#297 readLargeFile() now works on ACF
+- Enhancements
+	- \#297 readLargeFile() now works on ACF
 
 ## 4.3.1 - 29 January 2025
 

ModuleConfig.cfc

@@ -4,7 +4,7 @@ component{
     this.author = "Julian Halliwell";
     this.webURL = "https://github.com/cfsimplicity/spreadsheet-cfml";
     this.description = "CFML Spreadsheet Library";
-    this.version = "4.4.0";
+    this.version = "4.5.0";
     this.autoMapModels = false;
 
     function configure(){

README.md

@@ -152,6 +152,7 @@ You may wish to place the spreadsheet library files in a central location with a
 * [newStreamingXlsx](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/newStreamingXlsx)
 * [newXls](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/newXls)
 * [newXlsx](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/newXlsx)
+* [processLargeFile](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/processLargeFile)
 * [queryToCsv](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/queryToCsv)
 * [readCsv](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/readCsv)
 * [readLargeFile](https://github.com/cfsimplicity/spreadsheet-cfml/wiki/readLargeFile)

Spreadsheet.cfc

@@ -1,7 +1,7 @@
 component accessors="true"{
 
 	//"static"
-	property name="version" default="4.4.0" setter="false";
+	property name="version" default="4.5.0" setter="false";
 	property name="osgiLibBundleVersion" default="5.4.0.0" setter="false"; //first 3 octets = POI version; increment 4th with other jar updates
 	property name="osgiLibBundleSymbolicName" default="spreadsheet-cfml" setter="false";
 	property name="exceptionType" default="cfsimplicity.spreadsheet" setter="false";
@@ -1184,6 +1184,10 @@ component accessors="true"{
 		return new( sheetName=arguments.sheetName, xmlFormat=true );
 	}
 
+	public any function processLargeFile( required string filepath ){
+		return New objects.ProcessLargeFile( this, arguments.filepath );
+	}
+
 	/* row order is not guaranteed if using more than one thread */
 	public string function queryToCsv( required query query, boolean includeHeaderRow=false, string delimiter=",", numeric threads=0 ){
 		return writeCsv()

box.json

@@ -1,7 +1,7 @@
 {
     "name" : "Spreadsheet CFML",
     "slug" : "spreadsheet-cfml",
-    "version" : "4.4.0",
+    "version" : "4.5.0",
     "shortDescription" : "CFML spreadsheet library",
     "author" : "Julian Halliwell",
     "location" : "forgeboxStorage",

helpers/base.cfc

@@ -29,8 +29,8 @@ component{
 		return library().getColumnHelper();
 	}
 
-	any function getCommetHelper(){
-		return library().getCommetHelper();
+	any function getCommentHelper(){
+		return library().getCommentHelper();
 	}
 
 	any function getCsvHelper(){

helpers/exception.cfc

@@ -22,8 +22,13 @@ component extends="base"{
 			Throw( type=library().getExceptionType() & ".invalidReadFormat", message="Invalid format", detail="Supported formats are: 'query', 'html' and 'csv'" );
 	}
 
-	void function throwInvalidFileForReadLargeFileException(){
-		Throw( type=library().getExceptionType() & ".invalidSpreadsheetType", message="Invalid spreadsheet file", detail="readLargeFile() can only be used with XLSX files. The file you are trying to read does not appear to be an XLSX file." );
+	void function throwExceptionIfFileIsInvalidForStreamingReader( required exception ){
+		/*
+			for some reason ACF won't match the exception type as a catch() arg here, i.e.
+			catch( com.github.pjfanning.xlsx.exceptions.ReadException exception ){} hence using an if-test
+		*/
+		if( arguments.exception.type == "com.github.pjfanning.xlsx.exceptions.ReadException" )
+			Throw( type=library().getExceptionType() & ".invalidSpreadsheetType", message="Invalid spreadsheet file", detail="readLargeFile() and processLargeFile() can only be used with XLSX files. The file you are trying to read does not appear to be an XLSX file." );
 	}
 
 	void function throwNonExistentRowException( required numeric rowNumber ){

helpers/streamingReader.cfc

@@ -12,12 +12,7 @@ component extends="base"{
 				return getSheetHelper().sheetToQuery( argumentCollection=arguments.sheetToQueryArgs );
 			}
 			catch( any exception ){
-				/*
-					for some reason ACF won't match the exception type as a catch() arg here, i.e.
-					catch( com.github.pjfanning.xlsx.exceptions.ReadException exception ){}
-				*/
-				if( exception.type == "com.github.pjfanning.xlsx.exceptions.ReadException" )
-					getExceptionHelper().throwInvalidFileForReadLargeFileException();
+				getExceptionHelper().throwExceptionIfFileIsInvalidForStreamingReader( exception );
 				rethrow;
 			}
 			finally{

objects/ProcessLargeFile.cfc

@@ -0,0 +1,123 @@
+component accessors="true"{
+
+  property name="filepath";
+	property name="firstRowIsHeader" type="boolean" default="false";
+	property name="numberOfRowsToSkip" default=0;
+  property name="rowProcessor";
+	property name="sheetName";
+	property name="sheetNumber" default=1;
+  property name="streamingOptions";
+	property name="useVisibleValues" type="boolean" default="false";
+  /* Internal */
+	property name="library" setter="false";
+
+  public ProcessLargeFile function init( required spreadsheetLibrary, required string filepath ){
+		variables.library = arguments.spreadsheetLibrary;
+		variables.library.getFileHelper().throwErrorIFfileNotExists( arguments.filepath );
+		variables.filepath = arguments.filepath;
+    variables.streamingOptions = {};
+		return this;
+	}
+
+  /* Public builder API */
+
+	public ProcessLargeFile function withFirstRowIsHeader( boolean state=true ){
+		variables.firstRowIsHeader = arguments.state;
+		return this;
+	}
+
+  public ProcessLargeFile function withRowProcessor( required function rowProcessor ){
+		variables.rowProcessor = arguments.rowProcessor;
+		return this;
+	}
+
+	public ProcessLargeFile function withPassword( required string password ){
+		variables.streamingOptions.password = arguments.password;
+		return this;
+	}
+
+	public ProcessLargeFile function withSheetName( required string sheetName ){
+		variables.sheetName = arguments.sheetName;
+		return this;
+	}
+
+	public ProcessLargeFile function withSheetNumber( required numeric sheetNumber ){
+		variables.sheetNumber = arguments.sheetNumber;
+		return this;
+	}
+
+	public ProcessLargeFile function withSkipFirstRows( required numeric numberOfRowsToSkip ){
+		if( !IsValid( "integer", arguments.numberOfRowsToSkip ) || ( arguments.numberOfRowsToSkip < 0 ) )
+			Throw( type=variables.library.getExceptionType() & ".invalidArgument", message="Invalid argument to method withSkipFirstRows()", detail="'#arguments.numberOfRowsToSkip#' is not a valid argument to withSkipFirstRows(). Please specify zero or a positive integer" );
+		variables.numberOfRowsToSkip = arguments.numberOfRowsToSkip;
+		return this;
+	}
+
+	public ProcessLargeFile function withStreamingOptions( required struct options ){
+		if( arguments.options.KeyExists( "bufferSize" ) )
+			variables.streamingOptions.bufferSize = Val( arguments.options.bufferSize );
+		if( arguments.options.KeyExists( "rowCacheSize" ) )
+			variables.streamingOptions.rowCacheSize = Val( arguments.options.rowCacheSize );
+		return this;
+	}
+
+	public ProcessLargeFile function withUseVisibleValues( boolean state=true ){
+		variables.useVisibleValues = arguments.state;
+		return this;
+	}
+
+  // final execution
+	public ProcessLargeFile function execute(){
+    lock name="#getFilepath()#" timeout=5 {
+			try{
+				var file = CreateObject( "java", "java.io.FileInputStream" ).init( getFilepath() );
+				var workbook = getLibrary().getStreamingReaderHelper().getBuilder( getStreamingOptions() ).open( file );
+				var rowIterator = getSheetToProcess( workbook ).rowIterator();
+				var currentRecordNumber = 0;
+				var headerRowSkipped = false;
+				var skippedRecords = 0;
+				var rowProcessor = getRowProcessor()
+				var rowDataArgs = {
+					workbook: workbook
+				};
+				if( getUseVisibleValues() )
+					rowDataArgs.returnVisibleValues = true;
+				while( rowIterator.hasNext() ){
+					rowDataArgs.row = rowIterator.next();
+					if( skipThisRecord( skippedRecords ) ){
+						skippedRecords++;
+						continue;
+					}
+					if( getFirstRowIsHeader() && !headerRowSkipped ){
+						headerRowSkipped = true;
+						continue;
+					}
+					var data = getLibrary().getRowHelper().getRowData( argumentCollection=rowDataArgs );
+					if( !IsNull( rowProcessor ) )
+						rowProcessor( data, ++currentRecordNumber );
+				}
+			}
+			catch( any exception ){
+				getLibrary().getExceptionHelper().throwExceptionIfFileIsInvalidForStreamingReader( exception );
+				rethrow;
+			}
+			finally{
+				getLibrary().getFileHelper().closeLocalFileOrStream( local, "file" );
+				getLibrary().getFileHelper().closeLocalFileOrStream( local, "workbook" );
+			}
+		}
+		return this;
+	}
+
+	/* Private */
+	private any function getSheetToProcess( required workbook ){
+		if( !IsNull( getSheetName() ) )
+			return getLibrary().getSheetHelper().getSheetByName( arguments.workbook, getSheetName() );
+		return getLibrary().getSheetHelper().getSheetByNumber( arguments.workbook, getSheetNumber() );
+	}
+
+	private boolean function skipThisRecord( required numeric skippedRecords ){
+		return variables.numberOfRowsToSkip && ( arguments.skippedRecords < variables.numberOfRowsToSkip );
+	}
+
+}

test/specs/addAutoFilter.cfm

@@ -1,33 +1,32 @@
 <cfscript>
-describe( "addAutoFilter", function(){
+describe( "addAutoFilter", ()=>{
 
-	beforeEach( function(){
+	beforeEach( ()=>{
 		var data = QueryNew( "Header1,Header2", "VarChar,VarChar", [ [ "a", "b" ], [ "c", "d" ] ] );
 		var xls = s.workbookFromQuery( data );
 		var xlsx = s.workbookFromQuery( data=data, xmlformat=true );
 		variables.workbooks = [ xls, xlsx ];
-	});
+	})
 
-	it( "Doesn't error when passing valid arguments", function() {
-		workbooks.Each( function( wb ){
+	it( "Doesn't error when passing valid arguments", ()=>{
+		workbooks.Each( ( wb )=>{
 			s.addAutoFilter( wb, "A1:B1" )
 			.addAutoFilter( wb )// default to all cols in first row if no row range passed
 			.addAutoFilter( workbook=wb, row=2 );// allow row to be specified instead of range
-		});
-		
-	});
+		})
+	})
 
-	it( "Doesn't error when passing valid arguments with extra trailing/leading space", function() {
-		workbooks.Each( function( wb ){
+	it( "Doesn't error when passing valid arguments with extra trailing/leading space", ()=>{
+		workbooks.Each( ( wb )=>{
 			s.addAutoFilter( wb, " A1:B1 " );
-		});
-	});
+		})
+	})
 
-	it( "Is chainable", function() {
+	it( "Is chainable", ()=>{
 		workbooks.Each( function( wb ){
 			s.newChainable( wb ).addAutoFilter( " A1:B1 " );
-		});
-	});
+		})
+	})
 
-});	
+})
 </cfscript>