visual_regression.sh

#!/bin/bash
# This script runs a visual regression test on all the images
# generated by the VexFlow tests.
#
# Prerequisites: ImageMagick
#
# On OSX:   $ brew install imagemagick
# On Linux: $ apt-get install imagemagick
#
# Usage:
#
#  First generate the PNG images from the tests into build/images/.
#
#    ./tools/generate_images.js build          ./build/images/current
#    ./tools/generate_images.js reference      ./build/images/reference
#    ./tools/generate_images.js releases/3.0.9 ./build/images/3.0.9
#
#  Run the visual regression tests against the reference images.
#
#    ./tools/visual_regression.sh ( reference | 3.0.9 | X.Y.Z ) [prefix]
#
#  The optional argument allows you to compare a subset of the images
#  (only those with names starting with the specified prefix).
#
#  Check build/images/diff/results.txt for results. The composite diff
#  images for failed tests are stored in build/images/diff.
#

# PNG viewer on OSX. Switch this to whatever your system uses.
# VIEWER=open

# use . as decimal separator
LC_NUMERIC="en_US.UTF-8"

# Check ImageMagick installation
command -v convert >/dev/null 2>&1 || { echo >&2 "Error: ImageMagick not found."; exit 1; }

# Directories. You might want to change BASE, if you're running from a
# different working directory.
BASE=.
ADIR=$BASE/build/images/$1
ANAME=$1
BDIR=$BASE/build/images/current
BNAME=current
DIFF=$BASE/build/images/diff


# All results are stored in the build/images/diff directory
# The results.txt and warnings.txt contain the output.
mkdir -p $DIFF
RESULTS=$DIFF/results.txt
WARNINGS=$DIFF/warnings.txt

if [ -e "$RESULTS" ]
then
  rm $DIFF/*
fi
touch $RESULTS
touch $RESULTS.fail
touch $WARNINGS

# If no prefix is provided, test all images.
if [ "$2" == "" ]
then
  files=*.png
else
  files=$2*.png
fi


## Sanity checks: some simple checks that the script can run correctly (doesn't validate pngs)
if [ "`basename $PWD`" == "tools" ]
then
  echo Please run this script from the VexFlow base directory.
  exit 1
fi

# Check if some png files are in the right folders and warn if not. doesn't make sure there are actual, usable png images though.
totalImagesB=`ls -1 $BDIR/$files | wc -l | xargs` # xargs trims spaces
if [ $? -ne 0 ] || [ "$totalImagesB" -lt 1 ]
then
  echo Missing images in $BDIR.
  echo Please run \"grunt generate:current\"
  exit 1
fi

totalImagesA=`ls -1 $ADIR/$files | wc -l | xargs`
if [ $? -ne 0 ] || [ "$totalImagesA" -lt 1 ]
then
  echo Missing images in $ADIR.
  echo Please run \"grunt generate:reference\"
  exit 1
fi
# check that #ImagesA == #ImagesB (will continue anyways)
if [ ! "$totalImagesA" -eq "$totalImagesB" ]
then
  echo "Warning: Number of (matching) [$BNAME] images ($totalImagesB) is not the same as [$ANAME] images ($totalImagesA). Continuing anyways."
fi
# ----------------- end of sanity checks -----------------


# Number of simultaneous jobs
nproc=$(sysctl -n hw.physicalcpu 2> /dev/null || nproc)
if [ -n "$NPROC" ]; then
  nproc=$NPROC
fi

echo "Running $totalImagesA tests (nproc=$nproc)..."

function ProgressBar {
    let _progress=(${1}*100/${2}*100)/100
    let _done=(${_progress}*4)/10
    let _left=40-$_done
    _fill=$(printf "%${_done}s")
    _empty=$(printf "%${_left}s")

    printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%%"
}

function diff_image() {
  local image=$1
  local name=`basename $image .png`
  local fileA=$ADIR/$name.png
  local fileB=$BDIR/$name.png
  local diff=$fileB-temp

  if [ ! -e "$fileB" ]
  then
    echo "Warning: $name.png missing in $BDIR." >$diff.warn
    return
  fi

  if [ ! -e "$fileA" ]
  then
    echo "Warning: $name.png missing in $ADIR." >$diff.warn
    return
  fi

  # If the two files are byte-for-byte identical, skip the image comparison below.
  cmp -s $fileA $fileB && return

  cp $fileA $diff-a.png
  cp $fileB $diff-b.png

  # Store the composite diff image.
  # Add the result to results.text
  echo $name >$diff.fail
  # Threshold exceeded, save the diff and the original, current
  cp $diff-diff.png $DIFF/$name.png
  cp $diff-a.png $DIFF/$name'_'$ANAME.png
  cp $diff-b.png $DIFF/$name'_'$BNAME.png
  echo
  echo "Test: $name"
  echo "  Image diff stored in $DIFF/$name.png"
  # $VIEWER "$diff-diff.png" "$diff-a.png" "$diff-b.png"
  # echo 'Hit return to process next image...'
  # read
  rm -f $diff-a.png $diff-b.png $diff-diff.png
}

function wait_jobs () {
  local n=$1
  while [[ "$(jobs -r | wc -l)" -ge "$n" ]] ; do
     # echo ===================================== && jobs -lr
     # wait the oldest job.
     local pid_to_wait=`jobs -rp | head -1`
     # echo wait $pid_to_wait
     wait $pid_to_wait  &> /dev/null
  done
}

count=0
for image in $BDIR/$files
do
  count=$((count + 1))
  ProgressBar ${count} ${totalImagesA}
  wait_jobs $nproc
  diff_image $image &
done
wait

cat $BDIR/*.warn 1>$WARNINGS 2>/dev/null
rm -f $BDIR/*.warn

# Check for files newly built that are not in the reference.
for image in $BDIR/$files
do
  name=`basename $image .png`
  fileA=$ADIR/$name.png
  fileB=$BDIR/$name.png

  if [ ! -e "$ADIR" ]
  then
    echo "  Warning: $name.png missing in $ADIR." >>$WARNINGS
  fi
done

num_warnings=`cat $WARNINGS | wc -l`

cat $BDIR/*.fail 1>$RESULTS.fail 2>/dev/null
num_fails=`cat $RESULTS.fail | wc -l`
rm -f  $BDIR/*.fail

# Sort results
sort $RESULTS.fail >$RESULTS

# The previous cleanup approach (rm -f) triggered the error: Argument list too long.
rm -f $RESULTS.fail

echo
echo Results stored in $DIFF/results.txt
echo All images with a difference, are available in $DIFF.
echo

if [ "$num_warnings" -gt 0 ]
then
  echo
  echo "You have $num_warnings warning(s):"
  cat $WARNINGS
fi

if [ "$num_fails" -gt 0 ]
then
  echo "You have $num_fails fail(s):"
  head -n $num_fails $RESULTS
else
  echo "Success - No differences!"
fi