Merge branch 'release/1.0.3'

code4craft · Feb 10, 2025 · cfae008 · cfae008
2 parents 1cd199b + f4a8825
commit cfae008
Show file tree

Hide file tree

Showing 15 changed files with 30 additions and 12 deletions.
diff --git a/LICENSE b/LICENSE
@@ -176,7 +176,7 @@ recommend that a file or class name and description of purpose be included on
 the same "printed page" as the copyright notice for easier identification within
 third-party archives.
 
-   Copyright 2013 code4craft
+   Copyright 2025 code4craft
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/pom.xml b/pom.xml
@@ -12,7 +12,7 @@
         <version>2.2.1</version>
     </parent>
     <groupId>us.codecraft</groupId>
-    <version>1.0.2</version>
+    <version>1.0.3</version>
     <packaging>pom</packaging>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

diff --git a/webmagic-core/pom.xml b/webmagic-core/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java
@@ -36,7 +36,7 @@
  */
 public class HttpClientGenerator {
 
-    private transient Logger logger = LoggerFactory.getLogger(getClass());
+    private Logger logger = LoggerFactory.getLogger(getClass());
 
     private PoolingHttpClientConnectionManager connectionManager;
 

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/selector/HtmlNode.java b/webmagic-core/src/main/java/us/codecraft/webmagic/selector/HtmlNode.java
@@ -31,6 +31,11 @@ public Selectable smartContent() {
         return select(smartContentSelector, getSourceTexts());
     }
 
+    public Selectable smartContent(int threshold) {
+        SmartContentSelector smartContentSelector = Selectors.smartContent(threshold);
+        return select(smartContentSelector, getSourceTexts());
+    }
+
     @Override
     public Selectable links() {
         return selectElements(new LinksSelector());

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectors.java b/webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectors.java
@@ -20,6 +20,10 @@ public static SmartContentSelector smartContent() {
         return new SmartContentSelector();
     }
 
+    public static SmartContentSelector smartContent(int threshold) {
+        return new SmartContentSelector(threshold);
+    }
+
     public static CssSelector $(String expr) {
         return new CssSelector(expr);
     }

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java b/webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java
@@ -16,9 +16,15 @@
 @Experimental
 public class SmartContentSelector implements Selector {
 
+    private int threshold = 86;
+
     public SmartContentSelector() {
     }
 
+    public SmartContentSelector(int threshold) {
+        this.threshold = threshold;
+    }
+
     @Override
     public String select(String html) {
         html = html.replaceAll("(?is)<!DOCTYPE.*?>", "");
@@ -29,7 +35,6 @@ public String select(String html) {
         html = html.replaceAll("(?is)<.*?>", "");
         List<String> lines;
         int blocksWidth =3;
-        int threshold =86;
         int start;
         int end;
         StringBuilder text = new StringBuilder();

diff --git a/webmagic-coverage/pom.xml b/webmagic-coverage/pom.xml
@@ -10,7 +10,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
 
     <artifactId>webmagic-coverage</artifactId>

diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java
@@ -6,6 +6,7 @@
 import us.codecraft.webmagic.Request;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.selector.PlainText;
+import us.codecraft.webmagic.utils.HttpConstant;
 
 import java.io.*;
 
@@ -96,7 +97,7 @@ public Page download(Request request, Task task) {
                 page.setRawText(content);
                 page.setUrl(new PlainText(request.getUrl()));
                 page.setRequest(request);
-                page.setStatusCode(200);
+                page.setStatusCode(HttpConstant.StatusCode.CODE_200);
             }
             onSuccess(page, task);
         } catch (Exception e) {

diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-saxon/pom.xml b/webmagic-saxon/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-scripts/pom.xml b/webmagic-scripts/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-selenium/pom.xml b/webmagic-selenium/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic</artifactId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/...-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java b/...-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
@@ -14,9 +14,11 @@
 import us.codecraft.webmagic.downloader.AbstractDownloader;
 import us.codecraft.webmagic.selector.Html;
 import us.codecraft.webmagic.selector.PlainText;
+import us.codecraft.webmagic.utils.HttpConstant;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.http.HttpRequest;
 import java.util.Map;
 
 /**
@@ -111,6 +113,7 @@ public Page download(Request request, Task task) {
             page.setHtml(new Html(content, request.getUrl()));
             page.setUrl(new PlainText(request.getUrl()));
             page.setRequest(request);
+            page.setStatusCode(HttpConstant.StatusCode.CODE_200);
             onSuccess(page, task);
         } catch (Exception e) {
             logger.warn("download page {} error", request.getUrl(), e);