Feature: Async queue added (limit 4 parallel minification)

Fixed: jpg to png cache bug

Author: Gleb Mikheev

index.js

@@ -16,6 +16,12 @@ const streamifier = require('streamifier');
 
 const Cache = require('async-disk-cache');
 const cache = new Cache('image-optimize-loader', { supportBuffer: true });
+const TaskQueue = require('task-queue-async');
+const asyncQueue = new TaskQueue({
+  sleepBetweenTasks: 1,
+  concurrency: 4,
+});
+// const concurrency = 0;
 
 function getHashOf(value) {
   return crypto.createHash('sha256').update(value).digest('hex');
@@ -90,69 +96,83 @@ module.exports = function (content) {
 
   const promiseOptimizeImage = Q.defer();
   const promisePrepareImage = Q.defer();
-  const promiseCacheExists = Q.defer();
+  const checkCacheExists = Q.defer();
+
+
+  function startOptimization() {
+    asyncQueue.addTask(function () {
+      // concurrency++;
+      // console.log(`\r optimize:start ${concurrency}`);
+
+      const completePromise = Q.defer();
+      promisePrepareImage.resolve(completePromise);
+      completePromise.promise.then(this.wrapCallback(() => {
+        // concurrency--;
+        // console.log(`\r optimize:finished ${concurrency}`);
+      }));
+    });
+  }
 
   // ensure cache exists
-  promiseCacheExists.promise.then(() => {
+  checkCacheExists.promise.then(() => {
     Q.all([
       cache.has(cacheKey),
       cache.has(`${cacheKey}-checksum`),
     ]).then(checks => {
-      promisePrepareImage.resolve();
-
       // if cache is not found
       if (!checks[0] || !checks[1]) {
-        promisePrepareImage.resolve();
+        startOptimization();
         return;
       }
 
-        // check is cache up to date
+      // check is cache up to date
       cache.get(`${cacheKey}-checksum`).then(results => {
           // if file not changed, return cached value
-        if (results.value === fileHash) {
+        if (results.value.fileHash === fileHash) {
           cache.get(cacheKey).then(cacheEntry => {
+            this.resource = results.value.fileHash;
             promisePrepareImage.reject();
             return callback(null, cacheEntry.value);
           }).catch(callback);
 
           // cache is outdated, create new image
         } else {
-          promisePrepareImage.resolve();
+          startOptimization();
         }
       });
     }).catch(callback);
   });
 
-  promisePrepareImage.promise.then(() => {
+  promisePrepareImage.promise.then((completePromise) => {
     if (settings.optimizer.covertPngToJpg && fileExt === 'png' && isWebpack2) {
       streamifier.createReadStream(content)
         .pipe(new PNG())
         .on('metadata', data => {
           if (data.alpha) {
-            return promiseOptimizeImage.resolve();
+            return promiseOptimizeImage.resolve(completePromise);
           }
 
           // if alpha not found - convert buffer to jpg
-          this.resource = this.resource.replace(/.png$/, '.jpg');
           gm(content).toBuffer('JPG', (err, buffer) => {
             if (err) {
               return callback(err);
             }
 
+            this.resource = this.resource.replace(/.png$/, '.jpg');
             content = buffer;
-            promiseOptimizeImage.resolve();
+            promiseOptimizeImage.resolve(completePromise);
 
             return false;
           });
 
           return false;
         });
     } else {
-      promiseOptimizeImage.resolve();
+      promiseOptimizeImage.resolve(completePromise);
     }
   });
 
-  promiseOptimizeImage.promise.then(() => {
+  promiseOptimizeImage.promise.then((completePromise) => {
     imagemin.buffer(content, {
       plugins: [
         imageminMozjpeg(settings.mozjpeg),
@@ -161,12 +181,17 @@ module.exports = function (content) {
       ],
     }).then(file => {
       cache.set(cacheKey, file);
-      cache.set(`${cacheKey}-checksum`, fileHash);
+      cache.set(`${cacheKey}-checksum`, {
+        fileHash,
+        resource: this.resource,
+      });
+
+      completePromise.resolve();
       callback(null, file);
     }).catch(callback);
   }).catch(callback);
 
-  promiseCacheExists.resolve();
+  checkCacheExists.resolve();
 };
 
 module.exports.raw = true;

package.json

@@ -48,5 +48,6 @@
     "istanbul": "0.4.4",
     "mocha": "2.5.3",
     "webpack": "git://github.com/webpack/webpack.git#6748dca837b6402f056b6ac94ffc3c99f8a25c36"
-  }
-}
+  },
+  "versionify": "0.1.1"
+}

test/app.js

@@ -4,3 +4,46 @@ require('./test.jpg');
 require('./test.png');
 require('./test-noalpha.png');
 require('./test.svg');
+
+require('./test.png?0');
+require('./test.png?1');
+require('./test.png?2');
+require('./test.png?3');
+require('./test.png?4');
+require('./test.png?5');
+require('./test.png?6');
+require('./test.png?7');
+require('./test.png?8');
+require('./test.png?9');
+require('./test.png?10');
+require('./test.png?11');
+require('./test.png?12');
+require('./test.png?13');
+require('./test.png?14');
+require('./test.png?15');
+require('./test.png?16');
+require('./test.png?17');
+require('./test.png?18');
+require('./test.png?19');
+
+require('./test.png?20');
+require('./test.png?21');
+require('./test.png?22');
+require('./test.png?23');
+require('./test.png?24');
+require('./test.png?25');
+require('./test.png?26');
+require('./test.png?27');
+require('./test.png?28');
+require('./test.png?29');
+
+require('./test.png?30');
+require('./test.png?31');
+require('./test.png?32');
+require('./test.png?33');
+require('./test.png?34');
+require('./test.png?35');
+require('./test.png?36');
+require('./test.png?37');
+require('./test.png?38');
+require('./test.png?39');