handle saving files without extensions (#54)

Co-authored-by: Burak Targaç <[email protected]>

Author: btargac

.babelrc

@@ -11,7 +11,6 @@
   ],
   "plugins": [
     ["@babel/plugin-transform-runtime", { "corejs": 3 }],
-    "@babel/plugin-proposal-nullish-coalescing-operator",
-    "@babel/plugin-proposal-optional-chaining"
+    "@babel/plugin-proposal-nullish-coalescing-operator"
   ]
 }

.browserslistrc

@@ -1 +1 @@
-electron >= 7.1.7
+electron >= 12.0.2

README.md

@@ -12,6 +12,7 @@ recursively till every item of the array is processed. For example downloading a
 [![Backers on Open Collective](https://opencollective.com/excel-parser-processor/backers/badge.svg)](#backers)
 [![Sponsors on Open Collective](https://opencollective.com/excel-parser-processor/sponsors/badge.svg)](#sponsors)
 [![Open Source Helpers](https://www.codetriage.com/btargac/excel-parser-processor/badges/users.svg)](https://www.codetriage.com/btargac/excel-parser-processor)
+[![CodeFactor][CodeFactor-image]][CodeFactor-url]
 
 #### How to use
 
@@ -116,3 +117,6 @@ MIT © [Burak Targaç](https://github.com/btargac)
 
 [codecov-image]: https://codecov.io/gh/btargac/excel-parser-processor/branch/master/graph/badge.svg
 [codecov-url]: https://codecov.io/gh/btargac/excel-parser-processor
+
+[CodeFactor-image]: https://www.codefactor.io/repository/github/btargac/excel-parser-processor/badge
+[CodeFactor-url]: https://www.codefactor.io/repository/github/btargac/excel-parser-processor

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "excel-parser-processor",
   "productName": "Excel Parser Processor",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Does the tedious processing over all items of a given excel file by converting the rows to an array and process all items of that array recursively",
   "main": "./dist/index.bundle.js",
   "scripts": {
@@ -10,7 +10,7 @@
     "build": "npm-run-all build-main build-renderer",
     "generate-icons": "electron-icon-maker --input=./build-assets/icon.png --output=./build/",
     "start-renderer-dev": "cross-env NODE_ENV=development PROCESS_TYPE=renderer webpack --config webpack.dev.js",
-    "start": "electron ./dist/index.bundle.js",
+    "start": "electron --inspect ./dist/index.bundle.js",
     "test": "jest",
     "test-watch": "jest --coverage --watch",
     "pack": "electron-builder build --dir",
@@ -52,42 +52,42 @@
   },
   "homepage": "https://github.com/btargac/excel-parser-processor#readme",
   "devDependencies": {
-    "@babel/core": "^7.7.7",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.7.4",
-    "@babel/plugin-proposal-optional-chaining": "^7.7.5",
-    "@babel/plugin-transform-runtime": "^7.7.6",
-    "@babel/preset-env": "^7.7.7",
-    "babel-jest": "^24.9.0",
-    "babel-loader": "^8.0.6",
+    "@babel/core": "^7.13.13",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8",
+    "@babel/plugin-transform-runtime": "^7.13.10",
+    "@babel/preset-env": "^7.13.12",
+    "babel-jest": "^26.6.3",
+    "babel-loader": "^8.2.2",
     "clean-webpack-plugin": "^3.0.0",
-    "copy-webpack-plugin": "^5.1.1",
-    "cross-env": "^6.0.3",
-    "css-loader": "^3.4.0",
-    "electron": "^7.2.4",
-    "electron-builder": "^21.2.0",
-    "electron-icon-maker": "^0.0.4",
+    "copy-webpack-plugin": "^8.1.0",
+    "cross-env": "^7.0.3",
+    "css-loader": "^5.2.0",
+    "electron": "^12.0.0",
+    "electron-builder": "^22.10.5",
+    "electron-icon-maker": "^0.0.5",
     "html-webpack-exclude-assets-plugin": "^0.0.7",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.9.0",
-    "mini-css-extract-plugin": "^0.9.0",
-    "node-sass": "^4.13.1",
+    "html-webpack-plugin": "^5.3.1",
+    "jest": "^26.6.3",
+    "mini-css-extract-plugin": "^1.4.0",
+    "node-sass": "^5.0.0",
     "npm-run-all": "^4.1.5",
-    "sass-loader": "^8.0.0",
-    "script-ext-html-webpack-plugin": "^2.1.4",
-    "style-loader": "^1.1.2",
+    "sass-loader": "^11.0.1",
+    "script-ext-html-webpack-plugin": "^2.1.5",
+    "style-loader": "^2.0.0",
     "uglifyjs-webpack-plugin": "^2.2.0",
-    "webpack": "^4.41.5",
-    "webpack-cli": "^3.3.10",
-    "webpack-merge": "^4.2.2"
+    "webpack": "^5.28.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-merge": "^5.7.3"
   },
   "dependencies": {
-    "@babel/runtime-corejs3": "^7.7.7",
+    "@babel/runtime-corejs3": "^7.13.10",
     "@fortawesome/fontawesome": "^1.1.8",
     "@fortawesome/fontawesome-free-solid": "^5.0.13",
-    "electron-fetch": "^1.4.0",
+    "electron-fetch": "^1.7.3",
     "is-url": "^1.2.4",
-    "jquery": "^3.5.0",
-    "node-xlsx": "^0.15.0",
+    "jquery": "^3.6.0",
+    "mime-types": "^2.1.29",
+    "node-xlsx": "^0.16.1",
     "normalize.css": "^8.0.1",
     "opencollective": "^1.0.3"
   },

package.json

@@ -0,0 +1,7 @@
+const extensionRegex = /\.([a-zA-Z0-9]+)$/ig
+
+export default (name, extension) => {
+  const hasExtension = name.match(extensionRegex);
+
+  return hasExtension ? name: `${name}.${extension}`;
+}

src/utils/generateFileName.js

@@ -0,0 +1,24 @@
+import generateFileName from './generateFileName';
+
+test('generateFileName should be a function', () => {
+  expect(typeof generateFileName).toBe('function');
+});
+
+test('should generate the correct file name when file name is extension free', () => {
+  const fileName = generateFileName('sample', 'jpg');
+
+  expect(fileName).toBe('sample.jpg');
+});
+
+
+test('should generate the correct file name when file name has an extension', () => {
+  const fileName = generateFileName('sample.avif', 'avif');
+
+  expect(fileName).toBe('sample.avif');
+});
+
+test('should generate the correct file name when file name is erroneous', () => {
+  const fileName = generateFileName('sample.', 'gif');
+
+  expect(fileName).toBe('sample..gif');
+});

src/utils/generateFileName.spec.js

@@ -1,9 +1,15 @@
 import path from 'path';
 import { createWriteStream, mkdir } from 'fs';
+import { pipeline } from 'stream';
+import { promisify } from 'util';
 import fetch from 'electron-fetch';
 import { URL } from 'url';
 import xlsx from 'node-xlsx';
 import isUrl from 'is-url';
+import mime from 'mime-types';
+
+import generateFileName from './generateFileName';
+const streamPipeline = promisify(pipeline);
 
 let initialItemsLength;
 let processedItemsCount;
@@ -23,24 +29,39 @@ const processItem = async (item, outputPath) => {
   const url = new URL(itemUrl);
   const itemName = newName ? `${newName}${path.extname(url.pathname)}` : path.basename(url.pathname);
 
-  const response = await fetch(itemUrl);
+  // usage of 'If-None-Match' header is just to force the server not to return an 304 since electron net does not
+  // correctly return content-type headers when converting an 304 to 200 internally
+  // see issues https://github.com/electron/electron/issues/27895, https://github.com/electron/electron/pull/21552
+  // and https://github.com/electron/electron/issues/20631
+  const response = await fetch(itemUrl, {headers: {'If-None-Match': null}});
 
   if (response.ok) {
     if (subFolderName) {
       await mkdir(`${outputPath}/${subFolderName}`, { recursive: true }, () => {});
     }
 
-    const dest = createWriteStream(path.join(outputPath, subFolderName ? subFolderName : '', itemName));
-
-    response.body.pipe(dest);
+    const contentType = response.headers.get('content-type');
+    const extension = mime.extension(contentType);
+    const fileName = generateFileName(itemName, extension);
+    // file system flag 'wx' stands for:
+    // Open file for writing. The file is created (if it does not exist), but fails if the path exists. (x causes to throw)
+    const dest = createWriteStream(path.join(outputPath, subFolderName || '', fileName), {flags: 'wx'});
+
+    try {
+      await streamPipeline(response.body, dest);
+    } catch (error) {
+      throw {
+        statusText: error.message,
+        itemInfo: error.path
+      }
+    }
   } else {
     throw {
       status: response.status,
       statusText: response.statusText,
       itemInfo: url.href
     }
   }
-
 };
 
 const processItems = async (rowItems, outputPath, win) => {
@@ -68,8 +89,7 @@ const processItems = async (rowItems, outputPath, win) => {
         });
     });
 
-    await Promise.all(requests)
-      .catch(e => console.log(`Error processing for the batch ${i} - ${e}`));
+    await Promise.allSettled(requests);
   }
 
   const logFileStream = createWriteStream(path.join(

src/utils/processItems.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const { CleanWebpackPlugin } = require('clean-webpack-plugin');
-const CopyWebpackPlugin = require('copy-webpack-plugin');
+const CopyPlugin = require("copy-webpack-plugin");
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 const ScriptExtHtmlWebpackPlugin = require('script-ext-html-webpack-plugin');
@@ -31,18 +31,19 @@ const mainConfig = {
   },
   plugins: [
     cleanDist,
-    new CopyWebpackPlugin([
-      {
-        from: './src/images/',
-        to: 'images',
-        toType: 'dir'
-      },
-      {
-        from: './src/preload.js',
-        to: 'preload.js',
-        toType: 'file'
-      }
-    ])
+    new CopyPlugin({
+      patterns: [
+        {
+          from: './src/images/',
+          to: 'images'
+        },
+        {
+          from: './src/preload.js',
+          to: 'preload.js',
+          toType: 'file'
+        }
+      ],
+    }),
   ],
   module: {
     rules: [

webpack.common.js

@@ -1,5 +1,5 @@
 const path = require('path');
-const merge = require('webpack-merge');
+const { merge } = require('webpack-merge');
 const webpack = require('webpack');
 const { mainConfig, rendererConfig } = require('./webpack.common.js');
 const processType = process.env.PROCESS_TYPE;