Skip to content

Commit f578c85

Browse files
znamenapznamenap
committed
Fix #395 : SftpClient Enumerates Rather Than Accumulates Directory Items
1 parent cefdc20 commit f578c85

File tree

3 files changed

+325
-18
lines changed

3 files changed

+325
-18
lines changed

‎src/Renci.SshNet.Tests/Classes/SftpClientTest.EnumerateDirectory.cs

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
using Microsoft.VisualStudio.TestTools.UnitTesting;
2+
using Renci.SshNet.Common;
3+
using Renci.SshNet.Tests.Common;
4+
using Renci.SshNet.Tests.Properties;
5+
using System;
6+
using System.Diagnostics;
7+
using System.Linq;
8+
9+
namespace Renci.SshNet.Tests.Classes
10+
{
11+
/// <summary>
12+
/// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
13+
/// </summary>
14+
public partial class SftpClientTest : TestBase
15+
{
16+
[TestMethod]
17+
[TestCategory("Sftp")]
18+
[ExpectedException(typeof(SshConnectionException))]
19+
public void Test_Sftp_EnumerateDirectory_Without_Connecting()
20+
{
21+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
22+
{
23+
var files = sftp.EnumerateDirectory(".");
24+
foreach (var file in files)
25+
{
26+
Debug.WriteLine(file.FullName);
27+
}
28+
}
29+
}
30+
31+
[TestMethod]
32+
[TestCategory("Sftp")]
33+
[TestCategory("integration")]
34+
[ExpectedException(typeof(SftpPermissionDeniedException))]
35+
public void Test_Sftp_EnumerateDirectory_Permission_Denied()
36+
{
37+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
38+
{
39+
sftp.Connect();
40+
41+
var files = sftp.EnumerateDirectory("/root");
42+
foreach (var file in files)
43+
{
44+
Debug.WriteLine(file.FullName);
45+
}
46+
47+
sftp.Disconnect();
48+
}
49+
}
50+
51+
[TestMethod]
52+
[TestCategory("Sftp")]
53+
[TestCategory("integration")]
54+
[ExpectedException(typeof(SftpPathNotFoundException))]
55+
public void Test_Sftp_EnumerateDirectory_Not_Exists()
56+
{
57+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
58+
{
59+
sftp.Connect();
60+
61+
var files = sftp.EnumerateDirectory("/asdfgh");
62+
foreach (var file in files)
63+
{
64+
Debug.WriteLine(file.FullName);
65+
}
66+
67+
sftp.Disconnect();
68+
}
69+
}
70+
71+
[TestMethod]
72+
[TestCategory("Sftp")]
73+
[TestCategory("integration")]
74+
public void Test_Sftp_EnumerateDirectory_Current()
75+
{
76+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
77+
{
78+
sftp.Connect();
79+
80+
var files = sftp.EnumerateDirectory(".");
81+
82+
Assert.IsTrue(files.Count() > 0);
83+
84+
foreach (var file in files)
85+
{
86+
Debug.WriteLine(file.FullName);
87+
}
88+
89+
sftp.Disconnect();
90+
}
91+
}
92+
93+
[TestMethod]
94+
[TestCategory("Sftp")]
95+
[TestCategory("integration")]
96+
public void Test_Sftp_EnumerateDirectory_Empty()
97+
{
98+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
99+
{
100+
sftp.Connect();
101+
102+
var files = sftp.EnumerateDirectory(string.Empty);
103+
104+
Assert.IsTrue(files.Count() > 0);
105+
106+
foreach (var file in files)
107+
{
108+
Debug.WriteLine(file.FullName);
109+
}
110+
111+
sftp.Disconnect();
112+
}
113+
}
114+
115+
[TestMethod]
116+
[TestCategory("Sftp")]
117+
[TestCategory("integration")]
118+
[Description("Test passing null to EnumerateDirectory.")]
119+
[ExpectedException(typeof(ArgumentNullException))]
120+
public void Test_Sftp_EnumerateDirectory_Null()
121+
{
122+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
123+
{
124+
sftp.Connect();
125+
126+
var files = sftp.EnumerateDirectory(null);
127+
128+
Assert.IsTrue(files.Count() > 0);
129+
130+
foreach (var file in files)
131+
{
132+
Debug.WriteLine(file.FullName);
133+
}
134+
135+
sftp.Disconnect();
136+
}
137+
}
138+
139+
[TestMethod]
140+
[TestCategory("Sftp")]
141+
[TestCategory("integration")]
142+
public void Test_Sftp_EnumerateDirectory_HugeDirectory()
143+
{
144+
var stopwatch = Stopwatch.StartNew();
145+
try
146+
{
147+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
148+
{
149+
sftp.Connect();
150+
sftp.ChangeDirectory("/home/" + Resources.USERNAME);
151+
152+
var count = 10000;
153+
// Create 10000 directory items
154+
for (int i = 0; i < count; i++)
155+
{
156+
sftp.CreateDirectory(string.Format("test_{0}", i));
157+
}
158+
Debug.WriteLine("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds);
159+
160+
stopwatch.Restart();
161+
var files = sftp.EnumerateDirectory(".");
162+
Debug.WriteLine("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds);
163+
164+
// Ensure that directory has at least 10000 items
165+
stopwatch.Restart();
166+
var actualCount = files.Count();
167+
Assert.IsTrue(actualCount >= 10000);
168+
Debug.WriteLine("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds);
169+
170+
sftp.Disconnect();
171+
}
172+
}
173+
finally
174+
{
175+
stopwatch.Restart();
176+
RemoveAllFiles();
177+
stopwatch.Stop();
178+
Debug.WriteLine("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds);
179+
}
180+
}
181+
182+
[TestMethod]
183+
[TestCategory("Sftp")]
184+
[TestCategory("integration")]
185+
[ExpectedException(typeof(SshConnectionException))]
186+
public void Test_Sftp_EnumerateDirectory_After_Disconnected()
187+
{
188+
try {
189+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
190+
{
191+
sftp.Connect();
192+
193+
sftp.CreateDirectory("test_at_dsiposed");
194+
195+
var files = sftp.EnumerateDirectory(".").Take(1);
196+
197+
sftp.Disconnect();
198+
199+
// Must fail on disconnected session.
200+
var count = files.Count();
201+
}
202+
}
203+
finally
204+
{
205+
RemoveAllFiles();
206+
}
207+
}
208+
}
209+
}

‎src/Renci.SshNet.Tests/Classes/SftpClientTest.ListDirectory.cs

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -140,26 +140,42 @@ public void Test_Sftp_ListDirectory_Null()
140140
[TestCategory("integration")]
141141
public void Test_Sftp_ListDirectory_HugeDirectory()
142142
{
143-
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
143+
var stopwatch = Stopwatch.StartNew();
144+
try
144145
{
145-
sftp.Connect();
146-
147-
// Create 10000 directory items
148-
for (int i = 0; i < 10000; i++)
146+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
149147
{
150-
sftp.CreateDirectory(string.Format("test_{0}", i));
151-
Debug.WriteLine("Created " + i);
148+
sftp.Connect();
149+
sftp.ChangeDirectory("/home/" + Resources.USERNAME);
150+
151+
var count = 10000;
152+
// Create 10000 directory items
153+
for (int i = 0; i < count; i++)
154+
{
155+
sftp.CreateDirectory(string.Format("test_{0}", i));
156+
}
157+
Debug.WriteLine("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds);
158+
159+
stopwatch.Restart();
160+
var files = sftp.ListDirectory(".");
161+
Debug.WriteLine("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds);
162+
163+
// Ensure that directory has at least 10000 items
164+
stopwatch.Restart();
165+
var actualCount = files.Count();
166+
Assert.IsTrue(actualCount >= 10000);
167+
Debug.WriteLine("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds);
168+
169+
sftp.Disconnect();
152170
}
153-
154-
var files = sftp.ListDirectory(".");
155-
156-
// Ensure that directory has at least 10000 items
157-
Assert.IsTrue(files.Count() > 10000);
158-
159-
sftp.Disconnect();
160171
}
161-
162-
RemoveAllFiles();
172+
finally
173+
{
174+
stopwatch.Restart();
175+
RemoveAllFiles();
176+
stopwatch.Stop();
177+
Debug.WriteLine("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds);
178+
}
163179
}
164180

165181
[TestMethod]

‎src/Renci.SshNet/SftpClient.cs

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,33 @@ public IEnumerable<SftpFile> EndListDirectory(IAsyncResult asyncResult)
537537
return ar.EndInvoke();
538538
}
539539

540+
/// <summary>
541+
/// Enumerates files and directories in remote directory.
542+
/// </summary>
543+
/// <remarks>
544+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
545+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
546+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
547+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
548+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
549+
/// </remarks>
550+
/// <param name="path">The path.</param>
551+
/// <param name="listCallback">The list callback.</param>
552+
/// <returns>
553+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
554+
/// </returns>
555+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
556+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
557+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
558+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
559+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
560+
public IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null)
561+
{
562+
CheckDisposed();
563+
564+
return InternalEnumerateDirectory(path, listCallback);
565+
}
566+
540567
/// <summary>
541568
/// Gets reference to remote file or directory.
542569
/// </summary>
@@ -1974,15 +2001,15 @@ private IEnumerable<FileInfo> InternalSynchronizeDirectories(string sourcePath,
19742001
return uploadedFiles;
19752002
}
19762003

1977-
#endregion
2004+
#endregion
19782005

19792006
/// <summary>
19802007
/// Internals the list directory.
19812008
/// </summary>
19822009
/// <param name="path">The path.</param>
19832010
/// <param name="listCallback">The list callback.</param>
19842011
/// <returns>
1985-
/// A list of files in the specfied directory.
2012+
/// A list of files in the specified directory.
19862013
/// </returns>
19872014
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
19882015
/// <exception cref="SshConnectionException">Client not connected.</exception>
@@ -2027,6 +2054,61 @@ private IEnumerable<SftpFile> InternalListDirectory(string path, Action<int> lis
20272054
return result;
20282055
}
20292056

2057+
/// <summary>
2058+
/// Internals the list directory.
2059+
/// </summary>
2060+
/// <param name="path">The path.</param>
2061+
/// <param name="listCallback">The list callback.</param>
2062+
/// <returns>
2063+
/// A list of files in the specified directory.
2064+
/// </returns>
2065+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
2066+
/// <exception cref="SshConnectionException">Client not connected.</exception>
2067+
private IEnumerable<SftpFile> InternalEnumerateDirectory(string path, Action<int> listCallback)
2068+
{
2069+
if (path == null)
2070+
throw new ArgumentNullException("path");
2071+
2072+
if (_sftpSession == null)
2073+
throw new SshConnectionException("Client not connected.");
2074+
2075+
var fullPath = _sftpSession.GetCanonicalPath(path);
2076+
2077+
var handle = _sftpSession.RequestOpenDir(fullPath);
2078+
2079+
var basePath = fullPath;
2080+
2081+
if (!basePath.EndsWith("/"))
2082+
basePath = string.Format("{0}/", fullPath);
2083+
2084+
try
2085+
{
2086+
int count = 0;
2087+
var files = _sftpSession.RequestReadDir(handle);
2088+
2089+
while (files != null)
2090+
{
2091+
count += files.Length;
2092+
// Call callback to report number of files read
2093+
if (listCallback != null)
2094+
{
2095+
// Execute callback on different thread
2096+
ThreadAbstraction.ExecuteThread(() => listCallback(count));
2097+
}
2098+
foreach (var file in files)
2099+
{
2100+
var fullName = string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, file.Key);
2101+
yield return new SftpFile(_sftpSession, fullName, file.Value);
2102+
}
2103+
files = _sftpSession.RequestReadDir(handle);
2104+
}
2105+
}
2106+
finally
2107+
{
2108+
_sftpSession.RequestClose(handle);
2109+
}
2110+
}
2111+
20302112
/// <summary>
20312113
/// Internals the download file.
20322114
/// </summary>

0 commit comments

Comments
 (0)