From 3d5ed098968a91f648addcab9b538d63014b6ca7 Mon Sep 17 00:00:00 2001
From: ricky50575 <molesg@yeah.net>
Date: Mon, 6 Feb 2023 15:35:31 +0800
Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E7=94=A8Scrapy=E6=A1=86=E6=9E=B6?=
 =?UTF-8?q?=E5=AE=9E=E7=8E=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore                     |   3 +
 README.md                      |  11 +++
 lightnovel/__init__.py         |  35 ++++++++
 lightnovel/items.py            |  51 ++++++++++++
 lightnovel/middlewares.py      | 143 +++++++++++++++++++++++++++++++++
 lightnovel/pipelines.py        | 137 +++++++++++++++++++++++++++++++
 lightnovel/settings.py         | 101 +++++++++++++++++++++++
 lightnovel/spiders/__init__.py |   4 +
 lightnovel/spiders/libi.py     | 112 ++++++++++++++++++++++++++
 lightnovel/spiders/wenku8.py   |  96 ++++++++++++++++++++++
 scrapy.cfg                     |  11 +++
 11 files changed, 704 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 README.md
 create mode 100644 lightnovel/__init__.py
 create mode 100644 lightnovel/items.py
 create mode 100644 lightnovel/middlewares.py
 create mode 100644 lightnovel/pipelines.py
 create mode 100644 lightnovel/settings.py
 create mode 100644 lightnovel/spiders/__init__.py
 create mode 100644 lightnovel/spiders/libi.py
 create mode 100644 lightnovel/spiders/wenku8.py
 create mode 100644 scrapy.cfg

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a1df691
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+img
+proxy
+*.db
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..71866ee
--- /dev/null
+++ b/README.md
@@ -0,0 +1,11 @@
+## 注意事项
+
+1. 在重新开始一次采集前，需要手动删除`job`目录下的对应工作进度文件，再启动采集。
+2. 尽管自定义下载器中间件`LightnovelDownloaderMiddleware`能够处理网络中断引发的异常，但依然建议在发生网络中断时立即暂停采集任务。
+3. 请勿在采集器工作时强行中断下载进程，否则可能导致请求丢失或响应无法被处理。
+
+## 已知问题
+
+### libi.py
+
+由于[PSYCOME炼爱学狱 第五卷 与杀人机共度体育灾 第五项 炼狱的爱吼、狂乱的死鸣 Behemoth feat.Leviathan（4）_哔哩轻小说 (linovelib.com)](https://www.linovelib.com/novel/94/13214_4.html)源代码的105行存在一个异常的`<Style>`字符串，网页不能被正确解析，在浏览器中表现为网页内容在此处被截断，在采集器中表现为`libi.py`模块的第86行抛出迭代器索引溢出异常并中止了该部作品的数据采集，即《PSYCOME炼爱学狱》从第五卷《与杀人机共度体育灾》的第五项《炼狱的爱吼、狂乱的死鸣 Behemoth feat.Leviathan》开始的内容不能被采集。同样的内容在[第五卷 与杀人机共度体育灾 第五项 炼狱的爱吼、狂乱的死鸣 Behemoth feat.Leviathan-PSYCOME炼爱学狱(恋狱剧场)-Fami通文库-轻小说文库 (wenku8.net)](https://www.wenku8.net/novel/1/1488/69360.htm)源代码的第966行中为`&lt;Style&gt;`，网页显示正常。怀疑哩哔轻小说亦是使用了爬虫程序从轻小说文库上获取了小说正文，而且没有对字符做转义。由于是网页代码错误引发的异常，根本原因不在采集器，因此该问题不会被修复。
\ No newline at end of file
diff --git a/lightnovel/__init__.py b/lightnovel/__init__.py
new file mode 100644
index 0000000..5962c6a
--- /dev/null
+++ b/lightnovel/__init__.py
@@ -0,0 +1,35 @@
+import pymysql
+from dbutils.persistent_db import PersistentDB
+import requests
+
+class MysqlPool:
+    _pool = None
+
+    @classmethod
+    def connect(cls):
+        if cls._pool == None:
+            cls._pool = PersistentDB(
+                creator=pymysql,
+                host='127.0.0.1',
+                port=3306,
+                user='root',
+                password='',
+                database='lightnovel_spider'
+            )
+        return cls._pool.connection()
+
+def isOnline(proxy:str=None) -> bool:
+    if proxy == None:
+        PROXIES = None
+    else:
+        PROXIES = {
+            'http': proxy,
+            'https': proxy
+        }
+    def test(url:str) -> bool:
+        try:
+            requests.get(url, timeout=1, proxies=PROXIES)
+            return True
+        except requests.RequestException as e:
+            return False
+    return test('http://www.baidu.com') or test('http://www.bing.com')
\ No newline at end of file
diff --git a/lightnovel/items.py b/lightnovel/items.py
new file mode 100644
index 0000000..75b62ad
--- /dev/null
+++ b/lightnovel/items.py
@@ -0,0 +1,51 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class LightnovelItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass
+
+class Wenku8BookItem(scrapy.Item):
+    aid = scrapy.Field()
+    title = scrapy.Field()
+    author = scrapy.Field()
+    category = scrapy.Field()
+    tags = scrapy.Field()
+    description = scrapy.Field()
+
+class Wenku8ChapterItem(scrapy.Item):
+    aid = scrapy.Field()
+    cid = scrapy.Field()
+    volume = scrapy.Field()
+    chapter = scrapy.Field()
+    content = scrapy.Field()
+
+class Wenku8ImageItem(scrapy.Item):
+    cid = scrapy.Field()
+    filename = scrapy.Field()
+    content = scrapy.Field()
+
+class LibiBookItem(scrapy.Item):
+    bid = scrapy.Field()
+    title = scrapy.Field()
+    author = scrapy.Field()
+    category = scrapy.Field()
+    description = scrapy.Field()
+
+class LibiChapterItem(scrapy.Item):
+    bid = scrapy.Field()
+    cid = scrapy.Field()
+    volume = scrapy.Field()
+    chapter = scrapy.Field()
+    content = scrapy.Field()
+
+class LibiImageItem(scrapy.Item):
+    cid = scrapy.Field()
+    filename = scrapy.Field()
+    content = scrapy.Field()
\ No newline at end of file
diff --git a/lightnovel/middlewares.py b/lightnovel/middlewares.py
new file mode 100644
index 0000000..e75d29d
--- /dev/null
+++ b/lightnovel/middlewares.py
@@ -0,0 +1,143 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+from scrapy import Request, Spider
+from twisted.internet import defer
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
+from twisted.web.client import ResponseFailed
+from scrapy.core.downloader.handlers.http11 import TunnelError
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+from lightnovel import isOnline
+
+class LightnovelSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class LightnovelDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls(crawler.settings)
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    # IOError is raised by the HttpCompression middleware when trying to
+    # decompress an empty response
+    NETWORK_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
+                           ConnectionRefusedError, ConnectionDone, ConnectError,
+                           ConnectionLost, TCPTimedOutError, ResponseFailed,
+                           IOError, TunnelError)
+
+    def __init__(self, settings) -> None:
+        self.proxy_addr = settings.get('PROXY_ADDR', None)
+        self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+
+    def process_request(self, request:Request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+
+        # 设置代理
+        if self.proxy_addr != None:
+            request.meta['proxy'] = self.proxy_addr
+        
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request:Request, exception, spider:Spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        if isinstance(exception, self.NETWORK_EXCEPTIONS):
+            if isOnline(self.proxy_addr):
+                if request.meta.get('network_retry_times', 0) < 1:
+                    request.meta['network_retry_times'] = request.meta.get('network_retry_times', 0) + 1
+                else:
+                    return None
+            else:
+                request.meta['network_retry_times'] = 0
+                spider.logger.warn('网络连接异常，正在等待连接……')
+            if request.meta.get('retry_times', 0) > 0:
+                request.meta['retry_times'] = 0
+                request.priority = request.priority - self.priority_adjust
+            return request
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/lightnovel/pipelines.py b/lightnovel/pipelines.py
new file mode 100644
index 0000000..57048a6
--- /dev/null
+++ b/lightnovel/pipelines.py
@@ -0,0 +1,137 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+import os
+import pymysql
+from lightnovel import MysqlPool
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+from scrapy import Spider, Item
+from lightnovel.items import *
+
+class LightnovelPipeline:
+    def process_item(self, item, spider):
+        return item
+
+class Wenku8Pipeline:
+    def process_item(self, item:Item, spider):
+        if isinstance(item, Wenku8BookItem):
+            adapter = ItemAdapter(item)
+            aid = adapter['aid']
+            title = adapter['title']
+            author = adapter['author']
+            category = adapter['category']
+            tags = adapter['tags']
+            description = adapter['description']
+            with MysqlPool.connect() as conn:
+                with conn.cursor() as cur:
+                    try:
+                        cur.execute('INSERT IGNORE INTO wenku8_books(aid,title,author,category,tags,description) VALUES (%s,%s,%s,%s,%s,%s)', (aid,title,author,category,tags,description))
+                        conn.commit()
+                    except pymysql.err.Error as e:
+                        conn.rollback()
+                        raise e
+        elif isinstance(item, Wenku8ChapterItem):
+            adapter = ItemAdapter(item)
+            aid = adapter['aid']
+            cid = adapter['cid']
+            volume = adapter['volume']
+            chapter = adapter['chapter']
+            content = adapter['content']
+            with MysqlPool.connect() as conn:
+                with conn.cursor() as cur:
+                    try:
+                        cur.execute('INSERT IGNORE INTO wenku8_chapters(aid,cid,volume,chapter,content) VALUES (%s,%s,%s,%s,%s)', (aid,cid,volume,chapter,content))
+                        conn.commit()
+                    except pymysql.err.Error as e:
+                        conn.rollback()
+                        raise e
+        elif isinstance(item, Wenku8ImageItem):
+            adapter = ItemAdapter(item)
+            cid = adapter['cid']
+            filename = adapter['filename']
+            content = adapter['content']
+            try:
+                os.mkdir('./img')
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/wenku8')
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/wenku8/' + str(int(cid/10000)))
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/wenku8/' + str(int(cid/10000)) + '/' + str(cid))
+            except FileExistsError:
+                pass
+            with open('./img/wenku8/' + str(int(cid/10000)) + '/' + str(cid) + '/' + filename, 'wb') as f:
+                f.write(content)
+        return item
+
+class LibiPipeline:
+    def open_spider(self, spider:Spider):
+        pass
+
+    def process_item(self, item:Item, spider):
+        if isinstance(item, LibiBookItem):
+            adapter = ItemAdapter(item)
+            bid = adapter['bid']
+            title = adapter['title']
+            author = adapter['author']
+            category = adapter['category']
+            description = adapter['description']
+            with MysqlPool.connect() as conn:
+                with conn.cursor() as cur:
+                    try:
+                        cur.execute('INSERT IGNORE INTO libi_books(bid,title,author,category,description) VALUES (%s,%s,%s,%s,%s)', (bid,title,author,category,description))
+                        conn.commit()
+                    except pymysql.err.Error as e:
+                        conn.rollback()
+                        raise e
+        elif isinstance(item, LibiChapterItem):
+            adapter = ItemAdapter(item)
+            bid = adapter['bid']
+            cid = adapter['cid']
+            volume = adapter['volume']
+            chapter = adapter['chapter']
+            content = adapter['content']
+            with MysqlPool.connect() as conn:
+                with conn.cursor() as cur:
+                    try:
+                        cur.execute('INSERT IGNORE INTO libi_chapters(bid,cid,volume,chapter,content) VALUES (%s,%s,%s,%s,%s)', (bid,cid,volume,chapter,content))
+                        conn.commit()
+                    except pymysql.err.Error as e:
+                        conn.rollback()
+                        raise e
+        elif isinstance(item, LibiImageItem):
+            adapter = ItemAdapter(item)
+            cid = adapter['cid']
+            filename = adapter['filename']
+            content = adapter['content']
+            try:
+                os.mkdir('./img')
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/libi')
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/libi/' + str(int(cid/10000)))
+            except FileExistsError:
+                pass
+            try:
+                os.mkdir('./img/libi/' + str(int(cid/10000)) + '/' + str(cid))
+            except FileExistsError:
+                pass
+            with open('./img/libi/' + str(int(cid/10000)) + '/' + str(cid) + '/' + filename, 'wb') as f:
+                f.write(content)
+        return item
+
+    def close_spider(self, spider):
+        pass
\ No newline at end of file
diff --git a/lightnovel/settings.py b/lightnovel/settings.py
new file mode 100644
index 0000000..5c59abb
--- /dev/null
+++ b/lightnovel/settings.py
@@ -0,0 +1,101 @@
+# Scrapy settings for lightnovel project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'lightnovel'
+
+SPIDER_MODULES = ['lightnovel.spiders']
+NEWSPIDER_MODULE = 'lightnovel.spiders'
+
+LOG_LEVEL = 'INFO'
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52 LightnovelSpider/3.0'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+DOWNLOAD_DELAY = 0
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+DEFAULT_REQUEST_HEADERS = {
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+  'Accept-Language': 'zh,ja,en',
+}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'lightnovel.middlewares.LightnovelSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+DOWNLOADER_MIDDLEWARES = {
+   'lightnovel.middlewares.LightnovelDownloaderMiddleware': 543,
+}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'lightnovel.pipelines.LightnovelPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+AUTOTHROTTLE_START_DELAY = 1
+# The maximum download delay to be set in case of high latencies
+AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+AUTOTHROTTLE_TARGET_CONCURRENCY = 32.0
+# Enable showing throttling stats for every response received:
+AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+
+# 设置重试中间件
+RETRY_ENABLED = True
+RETRY_TIMES = 4
+RETRY_PRIORITY_ADJUST = +1
+
+# 设置代理
+PROXY_ADDR = 'http://127.0.0.1:15800'
\ No newline at end of file
diff --git a/lightnovel/spiders/__init__.py b/lightnovel/spiders/__init__.py
new file mode 100644
index 0000000..ebd689a
--- /dev/null
+++ b/lightnovel/spiders/__init__.py
@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
diff --git a/lightnovel/spiders/libi.py b/lightnovel/spiders/libi.py
new file mode 100644
index 0000000..e367c28
--- /dev/null
+++ b/lightnovel/spiders/libi.py
@@ -0,0 +1,112 @@
+import scrapy
+from scrapy.http import Response
+from lightnovel.items import LibiBookItem, LibiChapterItem, LibiImageItem
+import pymysql
+from lightnovel import MysqlPool
+import html2text
+
+class LibiSpider(scrapy.Spider):
+    name = 'libi'
+    custom_settings = {
+        'ITEM_PIPELINES': {'lightnovel.pipelines.LibiPipeline': 300},
+        'JOBDIR': 'job/libi'
+        }
+
+    COOKIES = ''''''
+    cookiesDict = {i.split("=")[0]:i.split("=")[-1] for i in COOKIES.split("; ")}
+
+    def start_requests(self):
+        yield scrapy.Request('https://www.linovelib.com/top/postdate/1.html', cookies=self.cookiesDict)
+
+    def parse(self, response:Response):
+        totalPageNum = int(response.css('a.last::text').get())
+        self.logger.info('总页数：%d', totalPageNum)
+        for i in range(totalPageNum):
+            curPageNum = i+1
+            yield scrapy.Request('https://www.linovelib.com/top/postdate/'+str(curPageNum)+'.html', cookies=self.cookiesDict, callback=self.parse_list, dont_filter=True)
+
+    def parse_list(self, response:Response):
+        for selector in response.css('.rank_d_list'):
+            bid = title = author = category = description = None
+            bid = int(selector.css('.rank_d_b_name a::attr(href)').get().split('/')[-1].split('.')[0])
+            title = selector.css('.rank_d_b_name a::text').get()
+            author = selector.css('.rank_d_b_cate a::text')[0].get()
+            category = selector.css('.rank_d_b_cate a::text')[1].get()
+            description = selector.css('.rank_d_b_info::text').get()
+            libiBookItem = LibiBookItem()
+            libiBookItem['bid'] = bid
+            libiBookItem['title'] = title
+            libiBookItem['author'] = author
+            libiBookItem['category'] = category
+            libiBookItem['description'] = description
+            self.logger.info('%d %s %s %s', bid, title, author, category)
+            yield libiBookItem
+            yield scrapy.Request('https://www.linovelib.com/novel/'+str(bid)+'/catalog', cookies=self.cookiesDict, callback=self.parse_catalog)
+
+    def parse_catalog(self, response:Response):
+        catalogCount = len(response.css('.chapter-list li.col-4 a::attr(href)').getall())
+        bid = int(response.request.url.split('/')[-2])
+        with MysqlPool.connect() as conn:
+            with conn.cursor() as cur:
+                try:
+                    cur.execute('SELECT COUNT(cid) FROM libi_chapters WHERE bid=%s', (bid,))
+                    savedCount = cur.fetchone()[0]
+                except pymysql.err.Error as e:
+                    raise e
+        if catalogCount > savedCount:
+            url = 'https://www.linovelib.com' + response.css('.chapter-list li.col-4 a::attr(href)').get()
+            yield scrapy.Request(url=url, cookies=self.cookiesDict, callback=self.parse_chapter)
+
+    def parse_chapter(self, response:Response, lastLibiChapterItem:LibiChapterItem=None):
+        h = html2text.HTML2Text()
+
+        def convertCharaters(s:str):
+            '''将pctheme.js文件的218行通过beautifier.io反混淆后得到，共100对转换字符'''
+            return s.replace('', '的').replace('', '一').replace('', '是').replace('', '了').replace('', '我').replace('', '不').replace('', '人').replace('', '在').replace('', '他').replace('', '有').replace('', '这').replace('', '个').replace('', '上').replace('', '们').replace('', '来').replace('', '到').replace('', '时').replace('', '大').replace('', '地').replace('', '为').replace('', '子').replace('', '中').replace('', '你').replace('', '说').replace('', '生').replace('', '国').replace('', '年').replace('', '着').replace('', '就').replace('', '那').replace('', '和').replace('', '要').replace('', '她').replace('', '出').replace('', '也').replace('', '得').replace('', '里').replace('', '后').replace('', '自').replace('', '以').replace('', '会').replace('', '家').replace('', '可').replace('', '下').replace('', '而').replace('', '过').replace('', '天').replace('', '去').replace('', '能').replace('', '对').replace('', '小').replace('', '多').replace('', '然').replace('', '于').replace('', '心').replace('', '学').replace('', '么').replace('', '之').replace('', '都').replace('', '好').replace('', '看').replace('', '起').replace('', '发').replace('', '当').replace('', '没').replace('', '成').replace('', '只').replace('', '如').replace('', '事').replace('', '把').replace('', '还').replace('', '用').replace('', '第').replace('', '样').replace('', '道').replace('', '想').replace('', '作').replace('', '种').replace('', '开').replace('', '美').replace('', '乳').replace('', '阴').replace('', '液').replace('', '茎').replace('', '欲').replace('', '呻').replace('', '肉').replace('', '交').replace('', '性').replace('', '胸').replace('', '私').replace('', '穴').replace('', '淫').replace('', '臀').replace('', '舔').replace('', '射').replace('', '脱').replace('', '裸').replace('', '骚').replace('', '唇')
+
+        bid = cid = volume = chapter = content = None
+        if lastLibiChapterItem != None:
+            bid = lastLibiChapterItem['bid']
+            cid = lastLibiChapterItem['cid']
+            volume = lastLibiChapterItem['volume']
+            chapter = lastLibiChapterItem['chapter']
+            content = lastLibiChapterItem['content'] + h.handle(convertCharaters(response.css('#TextContent').get()))
+        else:
+            bid = int(response.request.url.split('/')[-2])
+            cid = int(response.request.url.split('/')[-1].split('.')[0])
+            volume = response.css('.chepnav::text')[2].get().replace('>', '').strip()
+            chapter = response.css('h1::text').get()
+            content = h.handle(convertCharaters(response.css('#TextContent').get()))
+        libiChapterItem = LibiChapterItem()
+        libiChapterItem['bid'] = bid
+        libiChapterItem['cid'] = cid
+        libiChapterItem['volume'] = volume
+        libiChapterItem['chapter'] = chapter
+        libiChapterItem['content'] = content
+        nextPageName = response.css('.mlfy_page a::text')[4].get()
+        nextPageUrl = 'https://www.linovelib.com' + response.css('.mlfy_page a::attr(href)')[2].get()
+        if nextPageName == '下一页':
+            yield scrapy.Request(nextPageUrl, cookies=self.cookiesDict, callback=self.parse_chapter, cb_kwargs=dict(lastLibiChapterItem=libiChapterItem))
+        else:
+            self.logger.info('%d %d %s %s', bid, cid, volume, chapter)
+            yield libiChapterItem
+            if nextPageName == '下一章':
+                yield scrapy.Request(nextPageUrl, cookies=self.cookiesDict, callback=self.parse_chapter)
+        imgUrls = response.css('#TextContent img::attr(src)').getall()
+        for imgUrl in imgUrls:
+            if imgUrl[0:7]=='http://' or imgUrl[0:8]=='https://':
+                yield scrapy.Request(imgUrl, cookies=self.cookiesDict, callback=self.parse_image, cb_kwargs=dict(cid=cid))
+            elif imgUrl[0:2]=='//':
+                yield scrapy.Request('https:' + imgUrl, cookies=self.cookiesDict, callback=self.parse_image, cb_kwargs=dict(cid=cid))
+            elif imgUrl[0]=='/':
+                yield scrapy.Request('https://www.linovelib.com' + imgUrl, cookies=self.cookiesDict, callback=self.parse_image, cb_kwargs=dict(cid=cid))
+
+    def parse_image(self, response:Response, cid:int):
+        filename = response.request.url.split('/')[-1]
+        content = response.body
+        libiImageItem = LibiImageItem()
+        libiImageItem['cid'] = cid
+        libiImageItem['filename'] = filename
+        libiImageItem['content'] = content
+        self.logger.info('%d %s', cid, filename)
+        yield libiImageItem
\ No newline at end of file
diff --git a/lightnovel/spiders/wenku8.py b/lightnovel/spiders/wenku8.py
new file mode 100644
index 0000000..11575ab
--- /dev/null
+++ b/lightnovel/spiders/wenku8.py
@@ -0,0 +1,96 @@
+import scrapy
+from scrapy.http import Response
+from lightnovel.items import Wenku8BookItem, Wenku8ChapterItem, Wenku8ImageItem
+import pymysql
+from lightnovel import MysqlPool
+
+class Wenku8Spider(scrapy.Spider):
+    name = 'wenku8'
+    custom_settings = {
+        'ITEM_PIPELINES': {'lightnovel.pipelines.Wenku8Pipeline': 300},
+        'JOBDIR': 'job/wenku8'
+        }
+
+    COOKIES = ''''''
+    cookiesDict = {i.split("=")[0]:i.split("=")[-1] for i in COOKIES.split("; ")}
+
+    def start_requests(self):
+        yield scrapy.Request('https://www.wenku8.net/modules/article/articlelist.php', cookies=self.cookiesDict)
+
+    def parse(self, response:Response):
+        totalPageNum = int(response.css('a.last::text').get())
+        self.logger.info('总页数：%d', totalPageNum)
+        for i in range(totalPageNum):
+            curPageNum = i+1
+            yield scrapy.Request('https://www.wenku8.net/modules/article/articlelist.php?page=%d' % (curPageNum,), cookies=self.cookiesDict, callback=self.parse_list, dont_filter=True)
+
+    def parse_list(self, response:Response):
+        for selector in response.css('td div div b'):
+            aid = int(selector.css('a::attr(href)').get().split('/')[-1].split('.')[0])
+            yield scrapy.Request('https://www.wenku8.net/modules/article/articleinfo.php?id=%d' % (aid,), cookies=self.cookiesDict, callback=self.parse_info)
+
+    def parse_info(self, response:Response):
+        aid = title = author = category = tags = description = None
+        aid = int(response.request.url.split('id=')[-1])
+        title = response.css('span b::text').get()
+        author = response.css('td[width="20%"]::text')[1].get().split('：')[-1]
+        category = response.css('td[width="20%"]::text')[0].get().split('：')[-1]
+        tags = response.css('span.hottext b::text')[0].get().split('：')[-1]
+        description = '\n\n'.join(textline.strip() for textline in response.css('div#content table')[2].css('span')[-1].css('*::text').getall())
+        wenku8BookItem = Wenku8BookItem()
+        wenku8BookItem['aid'] = aid
+        wenku8BookItem['title'] = title
+        wenku8BookItem['author'] = author
+        wenku8BookItem['category'] = category
+        wenku8BookItem['tags'] = tags
+        wenku8BookItem['description'] = description
+        self.logger.info('%d %s %s %s', aid, title, author, category)
+        yield wenku8BookItem
+        yield scrapy.Request('https://www.wenku8.net/modules/article/reader.php?aid=%d' % aid, cookies=self.cookiesDict, callback=self.parse_catalog)
+
+    def parse_catalog(self, response:Response):
+        volume = chapter = url = None
+        for td in response.css('td'):
+            if td.css('td::attr(class)').get() == 'vcss':
+                volume = td.css('td::text').get()
+            elif td.css('td::attr(class)').get() == 'ccss':
+                chapter = td.css('a::text').get(None)
+                url = td.css('a::attr(href)').get(None)
+                if chapter!=None and url!=None:
+                    cid = int(url.split('cid=')[-1])
+                    with MysqlPool.connect() as conn:
+                        with conn.cursor() as cur:
+                            try:
+                                cur.execute('SELECT COUNT(cid) FROM wenku8_chapters WHERE cid=%s', (cid,))
+                                savedCount = cur.fetchone()[0]
+                            except pymysql.err.Error as e:
+                                raise e
+                    if savedCount == 0:
+                        yield scrapy.Request(url, cookies=self.cookiesDict, callback=self.parse_chapter, cb_kwargs=dict(volume=volume, chapter=chapter))
+
+    def parse_chapter(self, response:Response, volume:str, chapter:str):
+        aid = cid = content = None
+        aid = int(response.request.url.split('aid=')[-1].split('&')[0])
+        cid = int(response.request.url.split('cid=')[-1])
+        content = '\n'.join(textline.strip() for textline in response.css('div#content::text').getall()).strip()
+        wenku8ChapterItem = Wenku8ChapterItem()
+        wenku8ChapterItem['aid'] = aid
+        wenku8ChapterItem['cid'] = cid
+        wenku8ChapterItem['volume'] = volume
+        wenku8ChapterItem['chapter'] = chapter
+        wenku8ChapterItem['content'] = content
+        self.logger.info('%d %d %s %s', aid, cid, volume, chapter)
+        yield wenku8ChapterItem
+        imgUrls = response.css('div#content img::attr(src)').getall()
+        for imgUrl in imgUrls:
+            yield scrapy.Request(imgUrl, cookies=self.cookiesDict, callback=self.parse_image, cb_kwargs=dict(cid=cid))
+
+    def parse_image(self, response:Response, cid:int):
+        filename = response.request.url.split('/')[-1]
+        content = response.body
+        wenku8ImageItem = Wenku8ImageItem()
+        wenku8ImageItem['cid'] = cid
+        wenku8ImageItem['filename'] = filename
+        wenku8ImageItem['content'] = content
+        self.logger.info('%d %s', cid, filename)
+        yield wenku8ImageItem
\ No newline at end of file
diff --git a/scrapy.cfg b/scrapy.cfg
new file mode 100644
index 0000000..c21982b
--- /dev/null
+++ b/scrapy.cfg
@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = lightnovel.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = lightnovel