调整azure pipeline

zlzforever · zlzforever · commit 23326a66ddb3 · 2021-02-17T20:08:59.000+08:00
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -39,7 +39,7 @@ steps:
 
   - task: DotNetCoreInstaller@0
     inputs:
-      version: '3.1.102'
+      version: '5.0.102'
 
   - script: sh runtests.sh
     displayName: 'RunTestCases'
diff --git a/src/DotnetSpider.HBase/HBaseStorage.cs b/src/DotnetSpider.HBase/HBaseStorage.cs
@@ -49,6 +49,11 @@ public static HBaseStorage CreateFromOptions(IConfiguration configuration)
 			return storage;
 		}
 
+		public override Task InitializeAsync()
+		{
+			return Task.CompletedTask;
+		}
+
 		public HBaseStorage(string restServer)
 		{
 			var uri = new Uri(restServer);
diff --git a/src/DotnetSpider.Mongo/MongoEntityStorage.cs b/src/DotnetSpider.Mongo/MongoEntityStorage.cs
@@ -50,6 +50,11 @@ internal MongoEntityStorage(IMongoClient mongoClient)
 			_client = mongoClient;
 		}
 
+		public override Task InitializeAsync()
+		{
+			return Task.CompletedTask;
+		}
+
 		protected override async Task HandleAsync(DataFlowContext context,
 			IDictionary<Type, ICollection<dynamic>> entities)
 		{
diff --git a/src/DotnetSpider.MySql/MySqlEntityStorage.cs b/src/DotnetSpider.MySql/MySqlEntityStorage.cs
@@ -1,6 +1,7 @@
 using System.Data;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using DotnetSpider.DataFlow;
 using DotnetSpider.DataFlow.Storage;
 using Microsoft.Extensions.Configuration;
@@ -38,6 +39,11 @@ public MySqlEntityStorage(StorageMode mode,
 		{
 		}
 
+		public override Task InitializeAsync()
+		{
+			return Task.CompletedTask;
+		}
+
 		/// <summary>
 		/// 创建数据库连接接口
 		/// </summary>
diff --git a/src/DotnetSpider.MySql/MySqlFileEntityStorage.cs b/src/DotnetSpider.MySql/MySqlFileEntityStorage.cs
@@ -47,6 +47,11 @@ public class MySqlFileEntityStorage : EntityFileStorageBase
 
 		public MySqlFileType MySqlFileType { get; set; }
 
+		public override Task InitializeAsync()
+		{
+			return Task.CompletedTask;
+		}
+
 		/// <summary>
 		/// 根据配置返回存储器
 		/// </summary>
diff --git a/src/DotnetSpider.PostgreSql/PostgreSqlEntityStorage.cs b/src/DotnetSpider.PostgreSql/PostgreSqlEntityStorage.cs
@@ -2,6 +2,7 @@
 using System.Data;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using Dapper;
 using DotnetSpider.DataFlow;
 using DotnetSpider.DataFlow.Storage;
@@ -32,6 +33,11 @@ public static IDataFlow CreateFromOptions(IConfiguration configuration)
 			};
 		}
 
+		public override Task InitializeAsync()
+		{
+			return Task.CompletedTask;
+		}
+
 		/// <summary>
 		/// 创建数据库和表
 		/// </summary>
diff --git a/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj b/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj
@@ -25,8 +25,4 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
-
-  <ItemGroup>
-    <Folder Include="logs" />
-  </ItemGroup>
 </Project>
diff --git a/src/DotnetSpider.Sample/samples/BaseUsage.cs b/src/DotnetSpider.Sample/samples/BaseUsage.cs
@@ -31,18 +31,20 @@ public static async Task RunAsync()
 
 		class MyDataParser : DataParser
 		{
-			public MyDataParser()
-			{
-				AddRequiredValidator("cnblogs\\.com");
-				AddFollowRequestQuerier(Selectors.XPath("."));
-			}
-
 			protected override Task ParseAsync(DataFlowContext context)
 			{
 				context.AddData("URL", context.Request.RequestUri);
 				context.AddData("Title", context.Selectable.XPath(".//title")?.Value);
 				return Task.CompletedTask;
 			}
+
+			public override Task InitializeAsync()
+			{
+				AddRequiredValidator("cnblogs\\.com");
+				AddFollowRequestQuerier(Selectors.XPath("."));
+
+				return Task.CompletedTask;
+			}
 		}
 
 		public BaseUsageSpider(IOptions<SpiderOptions> options, DependenceServices services,
diff --git a/src/DotnetSpider.Sample/samples/CnblogsSpider.cs b/src/DotnetSpider.Sample/samples/CnblogsSpider.cs
@@ -51,6 +51,11 @@ protected override SpiderId CreateSpiderId()
 
 		protected class MyConsoleStorage : DataFlowBase
 		{
+			public override Task InitializeAsync()
+			{
+				return Task.CompletedTask;
+			}
+
 			public override Task HandleAsync(DataFlowContext context)
 			{
 				if (IsNullOrEmpty(context))
@@ -72,7 +77,7 @@ public override Task HandleAsync(DataFlowContext context)
 
 		protected class ListNewsParser : DataParser
 		{
-			public ListNewsParser()
+			public override Task InitializeAsync()
 			{
 				// AddRequiredValidator("news\\.cnblogs\\.com/n/page");
 				AddRequiredValidator((request =>
@@ -83,6 +88,7 @@ public ListNewsParser()
 				}));
 				// if you want to collect every pages
 				// AddFollowRequestQuerier(Selectors.XPath(".//div[@class='pager']"));
+				return Task.CompletedTask;
 			}
 
 			protected override Task ParseAsync(DataFlowContext context)
@@ -113,9 +119,10 @@ protected override Task ParseAsync(DataFlowContext context)
 
 		protected class NewsParser : DataParser
 		{
-			public NewsParser()
+			public override Task InitializeAsync()
 			{
 				AddRequiredValidator("news\\.cnblogs\\.com/n/\\d+");
+				return Task.CompletedTask;
 			}
 
 			protected override Task ParseAsync(DataFlowContext context)
diff --git a/src/DotnetSpider.Sample/samples/DatabaseSpider.cs b/src/DotnetSpider.Sample/samples/DatabaseSpider.cs
@@ -31,7 +31,7 @@ public DatabaseSpider(IOptions<SpiderOptions> options, DependenceServices servic
 		{
 		}
 
-		protected override async Task InitializeAsync(CancellationToken stoppingToken)
+		protected override async Task InitializeAsync(CancellationToken stoppingToken = default)
 		{
 			AddDataFlow(new ListNewsParser());
 			AddDataFlow(new NewsParser());
@@ -41,7 +41,7 @@ protected override async Task InitializeAsync(CancellationToken stoppingToken)
 
 		class MyStorage : DataFlowBase
 		{
-			public override async Task InitAsync()
+			public override async Task InitializeAsync()
 			{
 				await using var conn =
 					new MySqlConnection(
diff --git a/src/DotnetSpider.Sample/samples/DistributedSpider.cs b/src/DotnetSpider.Sample/samples/DistributedSpider.cs
@@ -20,7 +20,5 @@ public static async Task RunAsync()
 			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
 			await builder.Build().RunAsync();
 		}
-
-
 	}
 }
diff --git a/src/DotnetSpider.Sample/samples/ProxySpider.cs b/src/DotnetSpider.Sample/samples/ProxySpider.cs
@@ -30,9 +30,10 @@ public static async Task RunAsync()
 			});
 			builder.UseDownloader<HttpClientDownloader>();
 			builder.UseSerilog();
+			// Open fiddler and make sure the listen port is 8866
 			builder.UseProxy<FiddlerProxySupplier, DefaultProxyValidator>(x =>
 			{
-				x.ProxyTestUrl = "http://192.168.31.200:8866";
+				x.ProxyTestUrl = "http://localhost:8866";
 			});
 			builder.IgnoreServerCertificateError();
 			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
diff --git a/src/DotnetSpider.Sample/samples/SpeedSpider.cs b/src/DotnetSpider.Sample/samples/SpeedSpider.cs
@@ -56,6 +56,11 @@ protected class MyDataFlow : DataFlowBase
 		{
 			private int _downloadCount;
 
+			public override Task InitializeAsync()
+			{
+				return Task.CompletedTask;
+			}
+
 			public override Task HandleAsync(DataFlowContext context)
 			{
 				Interlocked.Increment(ref _downloadCount);
diff --git a/src/DotnetSpider.Sample/samples/SpeedSpider2.cs b/src/DotnetSpider.Sample/samples/SpeedSpider2.cs
@@ -60,10 +60,10 @@ protected class MyDataFlow : DataFlowBase
 
 			private DateTime _start;
 
-			public override Task InitAsync()
+			public override Task InitializeAsync()
 			{
 				_start = DateTime.Now;
-				return base.InitAsync();
+				return Task.CompletedTask;
 			}
 
 			public override Task HandleAsync(DataFlowContext context)
diff --git a/src/DotnetSpider.Sample/samples/WholeSiteSpider.cs b/src/DotnetSpider.Sample/samples/WholeSiteSpider.cs
@@ -49,10 +49,11 @@ protected override SpiderId CreateSpiderId()
 
 		class MyDataParser : DataParser
 		{
-			public MyDataParser()
+			public override Task InitializeAsync()
 			{
 				AddRequiredValidator("cnblogs\\.com");
 				AddFollowRequestQuerier(Selectors.XPath("."));
+				return Task.CompletedTask;
 			}
 
 			protected override Task ParseAsync(DataFlowContext context)
diff --git a/src/DotnetSpider.Tests/DataParser2.cs b/src/DotnetSpider.Tests/DataParser2.cs
@@ -0,0 +1,9 @@
+using DotnetSpider.DataFlow.Parser;
+using DotnetSpider.DataFlow.Storage;
+
+namespace DotnetSpider.Tests
+{
+	public class DataParser2<T>: DataParser<T> where T : EntityBase<T>, new()
+	{
+	}
+}
diff --git a/src/DotnetSpider.Tests/DataParserTests.cs b/src/DotnetSpider.Tests/DataParserTests.cs
@@ -75,6 +75,11 @@ protected override Task ParseAsync(DataFlowContext context)
 			{
 				return Task.CompletedTask;
 			}
+
+			public override Task InitializeAsync()
+			{
+				return Task.CompletedTask;
+			}
 		}
 	}
 }
diff --git a/src/DotnetSpider.Tests/DataParserTests`.cs b/src/DotnetSpider.Tests/DataParserTests`.cs
@@ -41,6 +41,7 @@ public async Task ParseEntity()
 				new Response {Content = new ByteArrayContent(File.ReadAllBytes("Jd.html"))});
 
 			var parser = new DataParser<Product>();
+			await parser.InitializeAsync();
 			parser.UseHtmlSelectableBuilder();
 			await parser.HandleAsync(dataContext);
 
@@ -85,7 +86,7 @@ public async Task SingleEntitySelector()
 
 			var parser = new DataParser<N>();
 			parser.UseHtmlSelectableBuilder();
-
+			await parser.InitializeAsync();
 			await parser.HandleAsync(dataContext);
 
 			var results = (List<N>)dataContext.GetData(typeof(N));
@@ -105,7 +106,7 @@ public async Task MultiEntitySelector()
 					new Response {Content = new ByteArrayContent(Encoding.UTF8.GetBytes(Html))});
 
 			var parser = new DataParser<E>();
-
+			await parser.InitializeAsync();
 			await parser.HandleAsync(dataContext);
 
 			var results = (List<E>)dataContext.GetData(typeof(E));
diff --git a/src/DotnetSpider.Tests/MySqlEntityStorageTests.cs b/src/DotnetSpider.Tests/MySqlEntityStorageTests.cs
@@ -135,7 +135,7 @@ public async Task CreateTable()
 			await conn.ExecuteAsync($"drop table if exists {Escape}test{Escape}.{Escape}createtable{Escape};");
 			{
 				var storage = CreateStorage(StorageMode.Insert);
-				await storage.InitAsync();
+				await storage.InitializeAsync();
 
 				var context = new DataFlowContext(null, new SpiderOptions(),
 					new Request(), new Response());
diff --git a/src/DotnetSpider/DataFlow/DataFlowBase.cs b/src/DotnetSpider/DataFlow/DataFlowBase.cs
@@ -15,10 +15,7 @@ public abstract class DataFlowBase : IDataFlow
 		/// 初始化
 		/// </summary>
 		/// <returns></returns>
-		public virtual Task InitAsync()
-		{
-			return Task.CompletedTask;
-		}
+		public abstract Task InitializeAsync();
 
 		public void SetLogger(ILogger logger)
 		{
diff --git a/src/DotnetSpider/DataFlow/IDataFlow.cs b/src/DotnetSpider/DataFlow/IDataFlow.cs
@@ -13,7 +13,7 @@ public interface IDataFlow : IDisposable
         /// 初始化
         /// </summary>
         /// <returns></returns>
-        Task InitAsync();
+        Task InitializeAsync();
 
         /// <summary>
         /// 设置日志
diff --git a/src/DotnetSpider/DataFlow/Parser/DataParser.cs b/src/DotnetSpider/DataFlow/Parser/DataParser.cs
@@ -69,20 +69,22 @@ public virtual void AddRequiredValidator(string pattern)
 		protected virtual void AddParsedResult<T>(DataFlowContext context, IEnumerable<T> results)
 			where T : EntityBase<T>, new()
 		{
-			if (results != null)
+			if (results == null)
 			{
-				var type = typeof(T);
-				var items = context.GetData(type);
-				if (items == null)
-				{
-					var list = new List<T>();
-					list.AddRange(results);
-					context.AddData(type, list);
-				}
-				else
-				{
-					items.AddRange(results);
-				}
+				return;
+			}
+
+			var type = typeof(T);
+			var items = context.GetData(type);
+			if (items == null)
+			{
+				var list = new List<T>();
+				list.AddRange(results);
+				context.AddData(type, list);
+			}
+			else
+			{
+				items.AddRange(results);
 			}
 		}
 
@@ -121,7 +123,6 @@ public override async Task HandleAsync(DataFlowContext context)
 				return;
 			}
 
-			var request = context.Request;
 			if (context.Selectable == null)
 			{
 				if (SelectableBuilder != null)
@@ -166,14 +167,14 @@ public override async Task HandleAsync(DataFlowContext context)
 				}
 			}
 
-			foreach (var followRequest in requests)
+			foreach (var request in requests)
 			{
-				if (IsValidRequest(followRequest))
+				if (IsValidRequest(request))
 				{
 					// 在此强制设制 Owner, 防止用户忘记导致出错
-					followRequest.Owner = request.Owner;
-					followRequest.Agent = context.Response.Agent;
-					context.AddFollowRequests(followRequest);
+					request.Owner = context.Request.Owner;
+					request.Agent = context.Response.Agent;
+					context.AddFollowRequests(request);
 				}
 			}
 		}
diff --git a/src/DotnetSpider/DataFlow/Parser/DataParser`.cs b/src/DotnetSpider/DataFlow/Parser/DataParser`.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/ConsoleEntityStorage.cs b/src/DotnetSpider/DataFlow/Storage/ConsoleEntityStorage.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/ConsoleStorage.cs b/src/DotnetSpider/DataFlow/Storage/ConsoleStorage.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/FileStorageBase.cs b/src/DotnetSpider/DataFlow/Storage/FileStorageBase.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/ImageStorage.cs b/src/DotnetSpider/DataFlow/Storage/ImageStorage.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/JsonEntityFileStorage.cs b/src/DotnetSpider/DataFlow/Storage/JsonEntityFileStorage.cs
diff --git a/src/DotnetSpider/DataFlow/Storage/SqlServerEntityStorage.cs b/src/DotnetSpider/DataFlow/Storage/SqlServerEntityStorage.cs
diff --git a/src/DotnetSpider/Spider.cs b/src/DotnetSpider/Spider.cs

Original file line number	Diff line number	Diff line change
`@@ -49,6 +49,11 @@ public static HBaseStorage CreateFromOptions(IConfiguration configuration)`
`49`	`49`	`return storage;`
`50`	`50`	`}`
`51`	`51`
	`52`	`+ public override Task InitializeAsync()`
	`53`	`+ {`
	`54`	`+ return Task.CompletedTask;`
	`55`	`+ }`
	`56`	`+`
`52`	`57`	`public HBaseStorage(string restServer)`
`53`	`58`	`{`
`54`	`59`	`var uri = new Uri(restServer);`
Original file line number	Diff line number	Diff line change
`@@ -50,6 +50,11 @@ internal MongoEntityStorage(IMongoClient mongoClient)`
`50`	`50`	`_client = mongoClient;`
`51`	`51`	`}`
`52`	`52`
	`53`	`+ public override Task InitializeAsync()`
	`54`	`+ {`
	`55`	`+ return Task.CompletedTask;`
	`56`	`+ }`
	`57`	`+`
`53`	`58`	`protected override async Task HandleAsync(DataFlowContext context,`
`54`	`59`	`IDictionary<Type, ICollection<dynamic>> entities)`
`55`	`60`	`{`
Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,11 @@ protected override SpiderId CreateSpiderId()`
`51`	`51`
`52`	`52`	`protected class MyConsoleStorage : DataFlowBase`
`53`	`53`	`{`
	`54`	`+ public override Task InitializeAsync()`
	`55`	`+ {`
	`56`	`+ return Task.CompletedTask;`
	`57`	`+ }`
	`58`	`+`
`54`	`59`	`public override Task HandleAsync(DataFlowContext context)`
`55`	`60`	`{`
`56`	`61`	`if (IsNullOrEmpty(context))`
`@@ -72,7 +77,7 @@ public override Task HandleAsync(DataFlowContext context)`
`72`	`77`
`73`	`78`	`protected class ListNewsParser : DataParser`
`74`	`79`	`{`
`75`		`- public ListNewsParser()`
	`80`	`+ public override Task InitializeAsync()`
`76`	`81`	`{`
`77`	`82`	`// AddRequiredValidator("news\\.cnblogs\\.com/n/page");`
`78`	`83`	`AddRequiredValidator((request =>`
`@@ -83,6 +88,7 @@ public ListNewsParser()`
`83`	`88`	`}));`
`84`	`89`	`// if you want to collect every pages`
`85`	`90`	`// AddFollowRequestQuerier(Selectors.XPath(".//div[@class='pager']"));`
	`91`	`+ return Task.CompletedTask;`
`86`	`92`	`}`
`87`	`93`
`88`	`94`	`protected override Task ParseAsync(DataFlowContext context)`
`@@ -113,9 +119,10 @@ protected override Task ParseAsync(DataFlowContext context)`
`113`	`119`
`114`	`120`	`protected class NewsParser : DataParser`
`115`	`121`	`{`
`116`		`- public NewsParser()`
	`122`	`+ public override Task InitializeAsync()`
`117`	`123`	`{`
`118`	`124`	`AddRequiredValidator("news\\.cnblogs\\.com/n/\\d+");`
	`125`	`+ return Task.CompletedTask;`
`119`	`126`	`}`
`120`	`127`
`121`	`128`	`protected override Task ParseAsync(DataFlowContext context)`
Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ public DatabaseSpider(IOptions<SpiderOptions> options, DependenceServices servic`
`31`	`31`	`{`
`32`	`32`	`}`
`33`	`33`
`34`		`- protected override async Task InitializeAsync(CancellationToken stoppingToken)`
	`34`	`+ protected override async Task InitializeAsync(CancellationToken stoppingToken = default)`
`35`	`35`	`{`
`36`	`36`	`AddDataFlow(new ListNewsParser());`
`37`	`37`	`AddDataFlow(new NewsParser());`
`@@ -41,7 +41,7 @@ protected override async Task InitializeAsync(CancellationToken stoppingToken)`
`41`	`41`
`42`	`42`	`class MyStorage : DataFlowBase`
`43`	`43`	`{`
`44`		`- public override async Task InitAsync()`
	`44`	`+ public override async Task InitializeAsync()`
`45`	`45`	`{`
`46`	`46`	`await using var conn =`
`47`	`47`	`new MySqlConnection(`