[RELEASE] iText 7 pdfHTML - 2.1.6

https://github.com/itext/i7j-pdfhtml/releases/tag/2.1.6

* release/2.1.6:
  [RELEASE] 2.1.6-SNAPSHOT -> 2.1.6
  Unignore tests, update cmp files and add TODO
  Exclude internal classes from public javadocs.
  Remove subpackages from the html2pdf pom file.
  Update cmp files
  Deprecate VISIBLE and OVERFLOW in CssConstants
  Add test demonstrating the problem of baseline alignment of checkboxes
  Upload spotbugs report to SonarQube
  Allow overriding default font family easier for DefaultFontProvider
  Make html2pdf tests extend ExtendedITextTest
  Add missing copyright headers
  Add new test demonstrating pdfHTML being too lenient when treating font CSS property
  Move most of CssStyleSheetParserTest from pdfHTML to SXP
  Add missing copyright headers
  Add langAttributeHtmlForTaggedPdfTest
  Add missing copyright headers
  Add tests in html2pdf for surrogate pairs. DEVSIX-3301
  Update cmp-s
  Update License.md for all AGPL products
  Update cmp files
  Add missing copyright headers
  Add new tests: visibleAndHiddenPropertyCssTest
  Improve javadocs.
  Unignore test and update cmp-file
  Remove ToDo remark
  Refresh cmp files and ToDos
  Remove 7.0 related branches from Jenkinsfile
  Refactor code, add comments
  Make substitutional addFont method for addCalligraphFonts
  Override addFont method in DefaultfontProvide
  Change the order of font loading in DefaultFontProvider
  Change font loading order
  [RELEASE] Update dependency versions

Author: iText-CI

Jenkinsfile

@@ -61,7 +61,7 @@ pipeline {
             steps {
                 withMaven(jdk: "${JDK_VERSION}", maven: 'M3', mavenLocalRepo: '.repository') {
                     withSonarQubeEnv('Sonar') {
-                        sh 'mvn --activate-profiles test -DgsExec="${gsExec}" -DcompareExec="${compareExec}" -Dmaven.test.skip=false -Dmaven.test.failure.ignore=false -Dmaven.javadoc.skip=true org.jacoco:jacoco-maven-plugin:prepare-agent verify org.jacoco:jacoco-maven-plugin:report sonar:sonar ' + sonarBranchName + ' ' + sonarBranchTarget
+                        sh 'mvn --activate-profiles test -DgsExec="${gsExec}" -DcompareExec="${compareExec}" -Dmaven.test.skip=false -Dmaven.test.failure.ignore=false -Dmaven.javadoc.skip=true org.jacoco:jacoco-maven-plugin:prepare-agent verify org.jacoco:jacoco-maven-plugin:report -Dsonar.java.spotbugs.reportPaths="target/spotbugs.xml" sonar:sonar ' + sonarBranchName + ' ' + sonarBranchTarget
                     }
                 }
             }
@@ -91,8 +91,6 @@ pipeline {
                 anyOf {
                     branch "master"
                     branch "develop"
-                    branch "7.0"
-                    branch "7.0-master"
                 }
             }
             steps {

LICENSE.md

@@ -1,36 +1,16 @@
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License version 3
-    as published by the Free Software Foundation with the addition of the
-    following permission added to Section 15 as permitted in Section 7(a):
-    FOR ANY PART OF THE COVERED WORK IN WHICH THE COPYRIGHT IS OWNED BY
-    ITEXT GROUP. ITEXT GROUP DISCLAIMS THE WARRANTY OF NON INFRINGEMENT
-    OF THIRD PARTY RIGHTS
+    This program is offered under a commercial and under the AGPL license.
+    For commercial licensing, contact us at https://itextpdf.com/sales.  For AGPL licensing, see below.
 
-    This program is distributed in the hope that it will be useful, but
-    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-    or FITNESS FOR A PARTICULAR PURPOSE.
-    See the GNU Affero General Public License for more details.
-    You should have received a copy of the GNU Affero General Public License
-    along with this program; if not, see http://www.gnu.org/licenses or write to
-    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-    Boston, MA, 02110-1301 USA, or download the license from the following URL:
-    http://itextpdf.com/terms-of-use/
-
-    The interactive user interfaces in modified source and object code versions
-    of this program must display Appropriate Legal Notices, as required under
-    Section 5 of the GNU Affero General Public License.
+    AGPL licensing:
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
 
-    In accordance with Section 7(b) of the GNU Affero General Public License,
-    a covered work must retain the producer line in every PDF that is created
-    or manipulated using iText.
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
 
-    You can be released from the requirements of the license by purchasing
-    a commercial license. Buying such a license is mandatory as soon as you
-    develop commercial activities involving the iText software without
-    disclosing the source code of your own applications.
-    These activities include: offering paid services to customers as an ASP,
-    serving PDFs on the fly in a web application, shipping iText with a closed
-    source product.
-
-    For more information, please contact iText Software Corp. at this
-    address: [email protected]
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.

findbugs-filter.xml

@@ -4,11 +4,11 @@
   <parent>
     <groupId>com.itextpdf</groupId>
     <artifactId>root</artifactId>
-    <version>7.1.8</version>
+    <version>7.1.9</version>
     <relativePath/>
   </parent>
   <artifactId>html2pdf</artifactId>
-  <version>2.1.5</version>
+  <version>2.1.6</version>
   <name>pdfHTML</name>
   <description>pdfHTML is an iText 7 add-on that lets you to parse (X)HTML snippets and the associated CSS and converts
     them to PDF.</description>
@@ -89,16 +89,6 @@
       </resource>
     </resources>
     <plugins>
-      <plugin>
-        <groupId>com.github.spotbugs</groupId>
-        <artifactId>spotbugs-maven-plugin</artifactId>
-        <version>${spotbugs.version}</version>
-        <configuration>
-          <threshold>High</threshold>
-          <failOnError>true</failOnError>
-          <excludeFilterFile>findbugs-filter.xml</excludeFilterFile>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>external.atlassian.jgitflow</groupId>
         <artifactId>jgitflow-maven-plugin</artifactId>
@@ -120,7 +110,12 @@
         <artifactId>maven-javadoc-plugin</artifactId>
         <version>${javadoc.version}</version>
         <configuration>
-          <subpackages>com.itextpdf.html2pdf</subpackages>
+          <sourceFileExcludes>
+            <sourceFileExclude>com/itextpdf/html2pdf/attach/impl/layout/BodyHtmlStylesContainer.java</sourceFileExclude>
+
+            <!-- ProductInfo -->
+            <sourceFileExclude>**/Html2PdfProductInfo.java</sourceFileExclude>
+          </sourceFileExcludes>
           <groups>
             <group>
               <title>pdfHTML</title>

pom.xml

@@ -65,12 +65,11 @@ This file is part of the iText (R) project.
  * This is how this worker processes the <p> tag:
  * <ul>
  * <li> if the worker meets a text or an inline element, it processes them with a help of
- * the {@link com.itextpdf.html2pdf.attach.util.WaitingInlineElementsHelper} instance</li>
+ * the {@link com.itextpdf.html2pdf.attach.util.WaitingInlineElementsHelper} instance
  *
  * <li> if the worker meets a block element without inline displaying or
  * an inline element with the {@code display: block} style, it wraps all the content which hasn't been handled yet
  * into a {@code com.itextpdf.layout.element.Paragraph} object and adds this paragraph to the resultant {@code com.itextpdf.layout.element.Div} object
- * </li>
  * </ul>
  */
 public class PTagWorker implements ITagWorker, IDisplayAware {

src/main/java/com/itextpdf/html2pdf/attach/impl/tags/PTagWorker.java

@@ -56,6 +56,7 @@ This file is part of the iText (R) project.
 import com.itextpdf.layout.element.ListItem;
 import com.itextpdf.layout.element.Paragraph;
 import com.itextpdf.layout.element.Text;
+import com.itextpdf.styledxmlparser.css.CommonCssConstants;
 import com.itextpdf.styledxmlparser.util.WhiteSpaceUtil;
 
 import java.util.ArrayList;
@@ -178,7 +179,7 @@ public void flushHangingLeaves(IPropertyContainer container) {
         Paragraph p = createLeavesContainer();
         if (p != null) {
             Map<String, String> map = new HashMap<>();
-            map.put(CssConstants.OVERFLOW, CssConstants.VISIBLE);
+            map.put(CssConstants.OVERFLOW, CommonCssConstants.VISIBLE);
             OverflowApplierUtil.applyOverflow(map, p);
             if (container instanceof Document) {
                 ((Document) container).add(p);

src/main/java/com/itextpdf/html2pdf/attach/util/WaitingInlineElementsHelper.java

@@ -94,7 +94,8 @@ public class CssConstants extends CommonCssConstants {
     /** The Constant OUTLINE_OFFSET. */
     public static final String OUTLINE_OFFSET = "outline-offset";
 
-    /** The Constant OVERFLOW. */
+    /** The Constant OVERFLOW. @Deprecated in favour of CommonCssConstants.OVERFLOW */
+    @Deprecated
     public static final String OVERFLOW = "overflow";
 
     /** The Constant OVERFLOW_X. */
@@ -121,7 +122,10 @@ public class CssConstants extends CommonCssConstants {
     /** The Constant VERTICAL_ALIGN. */
     public static final String VERTICAL_ALIGN = "vertical-align";
 
-    /** The Constant VISIBLE. */
+    /** The Constant VISIBLE.
+     * @deprecated in favour of StyledXmlParser
+     */
+    @Deprecated
     public static final String VISIBLE = "visible";
 
     // property values
@@ -262,7 +266,7 @@ public class CssConstants extends CommonCssConstants {
 
     /** The Constant OVERFLOW_VALUES. */
     public static final Set<String> OVERFLOW_VALUES = new HashSet<>(
-            Arrays.asList(new String[] {VISIBLE, HIDDEN, SCROLL, AUTO}));
+            Arrays.asList(new String[] {CommonCssConstants.VISIBLE, HIDDEN, SCROLL, AUTO}));
 
     // pseudo-elements
 

src/main/java/com/itextpdf/html2pdf/css/CssConstants.java

@@ -53,6 +53,7 @@ This file is part of the iText (R) project.
 import java.io.InputStream;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.List;
 
 /**
  * The default {@link BasicFontProvider} for pdfHTML, that, as opposed to
@@ -66,6 +67,8 @@ public class DefaultFontProvider extends BasicFontProvider {
      */
     private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DefaultFontProvider.class);
 
+    private static final String DEFAULT_FONT_FAMILY = "Times";
+
     /** The path to the shipped fonts. */
     private static final String SHIPPED_FONT_RESOURCE_PATH = "com/itextpdf/html2pdf/font/";
 
@@ -92,6 +95,12 @@ public class DefaultFontProvider extends BasicFontProvider {
     private static final Range FREE_FONT_RANGE = new RangeBuilder()
             .addRange(0, 0x058F).addRange(0x0E80, Integer.MAX_VALUE).create();
 
+
+    //we want to add free fonts to font provider before calligraph fonts. However, the existing public API states
+    // that addCalligraphFonts() should be used first to load calligraph fonts and to define the range for loading free fonts.
+    // In order to maintain backward compatibility, this temporary field is used to stash calligraph fonts before free fonts are loaded.
+    private List<byte[]> calligraphyFontsTempList = new ArrayList<>();
+
     /**
      * Creates a new {@link DefaultFontProvider} instance.
      */
@@ -107,10 +116,30 @@ public DefaultFontProvider() {
      * @param registerSystemFonts use true if you want to register the system fonts (can require quite some resources)
      */
     public DefaultFontProvider(boolean registerStandardPdfFonts, boolean registerShippedFreeFonts, boolean registerSystemFonts) {
-        super(registerStandardPdfFonts, registerSystemFonts);
+        this(registerStandardPdfFonts, registerShippedFreeFonts, registerSystemFonts, DEFAULT_FONT_FAMILY);
+    }
+
+    /**
+     * Creates a new {@link DefaultFontProvider} instance.
+     *
+     * @param registerStandardPdfFonts use true if you want to register the standard Type 1 fonts (can't be embedded)
+     * @param registerShippedFreeFonts use true if you want to register the shipped fonts (can be embedded)
+     * @param registerSystemFonts use true if you want to register the system fonts (can require quite some resources)
+     * @param defaultFontFamily default font family
+     */
+    public DefaultFontProvider(boolean registerStandardPdfFonts, boolean registerShippedFreeFonts, boolean registerSystemFonts, String defaultFontFamily) {
+        super(registerStandardPdfFonts, registerSystemFonts, defaultFontFamily);
         if (registerShippedFreeFonts) {
-            addShippedFreeFonts(addCalligraphFonts());
+            addAllAvailableFonts(addCalligraphFonts());
+        }
+    }
+
+    private void addAllAvailableFonts(Range rangeToLoad) {
+        addShippedFreeFonts(rangeToLoad);
+        for(byte[] fontData : calligraphyFontsTempList) {
+            addFont(fontData, null);
         }
+        calligraphyFontsTempList = null;
     }
 
     /**
@@ -148,8 +177,9 @@ protected Range addCalligraphFonts() {
             try {
                 Method m = klass.getMethod(methodName);
                 ArrayList<byte[]> fontStreams = (ArrayList<byte[]>) m.invoke(null, null);
-                for (byte[] font : fontStreams)
-                    addFont(font);
+                for (byte[] font : fontStreams) {
+                    this.calligraphyFontsTempList.add(font);
+                }
                 // here we return a unicode range that excludes the loaded from the calligraph module fonts
                 // i.e. the unicode range that is to be rendered with standard or shipped free fonts
                 return FREE_FONT_RANGE;

src/main/java/com/itextpdf/html2pdf/resolver/font/DefaultFontProvider.java

@@ -0,0 +1,114 @@
+/*
+    This file is part of the iText (R) project.
+    Copyright (c) 1998-2019 iText Group NV
+    Authors: iText Software.
+
+    This program is offered under a commercial and under the AGPL license.
+    For commercial licensing, contact us at https://itextpdf.com/sales.  For AGPL licensing, see below.
+
+    AGPL licensing:
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+package com.itextpdf.html2pdf;
+
+import com.itextpdf.io.LogMessageConstant;
+import com.itextpdf.kernel.utils.CompareTool;
+import com.itextpdf.test.ExtendedITextTest;
+import com.itextpdf.test.annotations.LogMessage;
+import com.itextpdf.test.annotations.LogMessages;
+import com.itextpdf.test.annotations.type.IntegrationTest;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.File;
+import java.io.IOException;
+
+@Category(IntegrationTest.class)
+public class SurrogatePairsTests extends ExtendedITextTest {
+    public static final String sourceFolder = "./src/test/resources/com/itextpdf/html2pdf/SurrogatePairsTests/";
+    public static final String destinationFolder = "./target/test/com/itextpdf/html2pdf/SurrogatePairsTests/";
+    private static final String TYPOGRAPHY_WARNING = "Cannot find pdfCalligraph module, which was implicitly required by one of the layout properties";
+
+    @BeforeClass
+    public static void beforeClass() {
+        createOrClearDestinationFolder(destinationFolder);
+    }
+
+    @Test
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2)})
+    public void surrogatePairFrom2Chars() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePairFrom2Chars.html"),
+                new File(destinationFolder + "surrogatePairFrom2Chars.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePairFrom2Chars.pdf",
+                sourceFolder + "cmp_surrogatePairFrom2Chars.pdf", destinationFolder));
+    }
+
+    @Test
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2)})
+    public void surrogatePair2Pairs() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePair2Pairs.html"),
+                new File(destinationFolder + "surrogatePair2Pairs.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePair2Pairs.pdf",
+                sourceFolder + "cmp_surrogatePair2Pairs.pdf", destinationFolder));
+    }
+
+    @Test
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2)})
+    public void surrogatePairFullCharacter() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePairFullCharacter.html"),
+                new File(destinationFolder + "surrogatePairFullCharacter.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePairFullCharacter.pdf",
+                sourceFolder + "cmp_surrogatePairFullCharacter.pdf", destinationFolder));
+    }
+
+    @Test
+    //TODO DEVSIX-3307
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2),
+            @LogMessage(messageTemplate = LogMessageConstant.FONT_SUBSET_ISSUE)})
+    public void surrogatePairCombingFullSurrs() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePairCombingFullSurrs.html"),
+                new File(destinationFolder + "surrogatePairCombingFullSurrs.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePairCombingFullSurrs.pdf",
+                sourceFolder + "cmp_surrogatePairCombingFullSurrs.pdf", destinationFolder));
+    }
+
+    @Test
+    //TODO DEVSIX-3307
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2),
+            @LogMessage(messageTemplate = LogMessageConstant.FONT_SUBSET_ISSUE)})
+    public void surrogatePairCombingFullSurrsWithNoSurrs() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePairCombingFullSurrsWithNoSurrs.html"),
+                new File(destinationFolder + "surrogatePairCombingFullSurrsWithNoSurrs.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePairCombingFullSurrsWithNoSurrs.pdf",
+                sourceFolder + "cmp_surrogatePairCombingFullSurrsWithNoSurrs.pdf", destinationFolder));
+    }
+
+    @Test
+    //TODO DEVSIX-3307
+    @LogMessages(messages = {
+            @LogMessage(messageTemplate = TYPOGRAPHY_WARNING, count = 2)})
+    public void surrogatePairCombinationOf3TypesPairs() throws IOException, InterruptedException {
+        HtmlConverter.convertToPdf(new File(sourceFolder + "surrogatePairCombinationOf3TypesPairs.html"),
+                new File(destinationFolder + "surrogatePairCombinationOf3TypesPairs.pdf"));
+        Assert.assertNull(new CompareTool().compareByContent(destinationFolder + "surrogatePairCombinationOf3TypesPairs.pdf",
+                sourceFolder + "cmp_surrogatePairCombinationOf3TypesPairs.pdf", destinationFolder));
+    }
+}