python - PySFTP 连接有效，但 get() 失败

Question

我可以建立连接，pysftp但我无法开始s.get()工作。

连接工作正常：

import pysftp
s = pysftp.Connection(host="xxx", username="xxx", password="xxx")

我还可以使用s.chdir("/path/to/target")并显示我想通过s.listdir()和接收True的文件s.isfile("/path/to/target/file.xxx")。

然而，执行s.get("/path/to/target/file.xxx")yield IOError: Folder not found: C:\some\other\folder\file.xxx。请注意，原始/path/to/target/文件（也显示在 WinSCP 等 SFTP 客户端中）现在似乎指向具有 Windows 语法的不同文件夹。

追溯：

IOError                                   Traceback (most recent call last)
<ipython-input-31-9d6fc4a6dd9d> in <module>()
----> 1 s.get(r'/path/to/target/file.xxx')

C:\Miniconda3\envs\py\lib\site-packages\pysftp.pyc in get(self, remotepath, localpath, callback, preserve_mtime)
    231             sftpattrs = self._sftp.stat(remotepath)
    232 
--> 233         self._sftp.get(remotepath, localpath, callback=callback)
    234         if preserve_mtime:
    235             os.utime(localpath, (sftpattrs.st_atime, sftpattrs.st_mtime))

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_client.pyc in get(self, remotepath, localpath, callback)
    718         file_size = self.stat(remotepath).st_size
    719         with open(localpath, 'wb') as fl:
--> 720             size = self.getfo(remotepath, fl, callback)
    721         s = os.stat(localpath)
    722         if s.st_size != size:

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_client.pyc in getfo(self, remotepath, fl, callback)
    688         with self.open(remotepath, 'rb') as fr:
    689             file_size = self.stat(remotepath).st_size
--> 690             fr.prefetch()
    691             size = 0
    692             while True:

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_file.pyc in prefetch(self)
    394         .. versionadded:: 1.5.1
    395         """
--> 396         size = self.stat().st_size
    397         # queue up async reads for the rest of the file
    398         chunks = []

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_file.pyc in stat(self)
    237         :return: an `.SFTPAttributes` object containing attributes about this file.
    238         """
--> 239         t, msg = self.sftp._request(CMD_FSTAT, self.handle)
    240         if t != CMD_ATTRS:
    241             raise SFTPError('Expected attributes')

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_client.pyc in _request(self, t, *arg)
    727     def _request(self, t, *arg):
    728         num = self._async_request(type(None), t, *arg)
--> 729         return self._read_response(num)
    730 
    731     def _async_request(self, fileobj, t, *arg):

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_client.pyc in _read_response(self, waitfor)
    774                 # synchronous
    775                 if t == CMD_STATUS:
--> 776                     self._convert_status(msg)
    777                 return t, msg
    778             if fileobj is not type(None):

C:\Miniconda3\envs\py\lib\site-packages\paramiko\sftp_client.pyc in _convert_status(self, msg)
    804             raise IOError(errno.EACCES, text)
    805         else:
--> 806             raise IOError(text)
    807 
    808     def _adjust_cwd(self, path):
IOError: Folder not found: C:\some\other\folder\file.xxx

使用 WinSCP 成功下载的日志：

[...]
. 2014-10-09 10:39:22.146 Listing directory "/path/to/target".
> 2014-10-09 10:39:22.146 Type: SSH_FXP_OPENDIR, Size: 77, Number: 3595
< 2014-10-09 10:39:22.146 Type: SSH_FXP_STATUS, Size: 17, Number: 3332
. 2014-10-09 10:39:22.146 Discarding reserved response
< 2014-10-09 10:39:22.332 Type: SSH_FXP_HANDLE, Size: 12, Number: 3595
> 2014-10-09 10:39:22.333 Type: SSH_FXP_READDIR, Size: 12, Number: 3852
< 2014-10-09 10:39:22.518 Type: SSH_FXP_NAME, Size: 1043, Number: 3852
> 2014-10-09 10:39:22.650 Type: SSH_FXP_READDIR, Size: 12, Number: 4108
< 2014-10-09 10:39:22.785 Type: SSH_FXP_STATUS, Size: 28, Number: 4108
< 2014-10-09 10:39:22.786 Status code: 1
> 2014-10-09 10:39:22.786 Type: SSH_FXP_CLOSE, Size: 12, Number: 4356
. 2014-10-09 10:39:22.786 ..;D;0;2013-11-05T06:20:22.000Z;"" [0];"" [0];r-x------;0
[...]
. 2014-10-09 10:39:22.786 file.xxx;-;3087870;2014-10-09T00:00:13.000Z;"" [0];"" [0];r-x------;0
[...]
. 2014-10-09 10:39:27.310 File: '/path/to/target/file.xxx' [2014-10-09T00:00:13.000Z] [3087870]
. 2014-10-09 10:39:27.322 Copying "/path/to/target/file.xxx" to local directory started.
. 2014-10-09 10:39:27.322 Binary transfer mode selected.
. 2014-10-09 10:39:27.344 Checking existence of partially transfered file.
. 2014-10-09 10:39:27.344 Opening remote file.
> 2014-10-09 10:39:27.344 Type: SSH_FXP_OPEN, Size: 129, Number: 4611
< 2014-10-09 10:39:27.344 Type: SSH_FXP_STATUS, Size: 17, Number: 4356
. 2014-10-09 10:39:27.344 Discarding reserved response
< 2014-10-09 10:39:27.518 Type: SSH_FXP_HANDLE, Size: 12, Number: 4611
> 2014-10-09 10:39:27.518 Type: SSH_FXP_FSTAT, Size: 16, Number: 4872
< 2014-10-09 10:39:27.658 Type: SSH_FXP_STATUS, Size: 112, Number: 4872
. 2014-10-09 10:39:27.770 Confirming overwriting of file.


. 2014-10-09 10:39:27.781 Asking user:

[...]

> 2014-10-09 10:39:30.736 Type: SSH_FXP_READ, Size: 24, Number: 5125
< 2014-10-09 10:39:40.496 Status code: 1
. 2014-10-09 10:39:40.539 222 skipped SSH_FXP_WRITE, SSH_FXP_READ, SSH_FXP_DATA and SSH_FXP_STATUS packets.
> 2014-10-09 10:39:40.539 Type: SSH_FXP_CLOSE, Size: 12, Number: 34052
< 2014-10-09 10:39:40.539 Type: SSH_FXP_STATUS, Size: 28, Number: 33285
< 2014-10-09 10:39:40.632 Type: SSH_FXP_STATUS, Size: 28, Number: 33541
< 2014-10-09 10:39:40.753 Type: SSH_FXP_STATUS, Size: 28, Number: 33797
. 2014-10-09 10:39:40.910 Preserving timestamp [2014-10-09T00:00:13.000Z]
. 2014-10-09 10:41:10.393 Closing connection.
. 2014-10-09 10:41:10.393 Sending special code: 12
. 2014-10-09 10:41:10.393 Sent EOF message

从 SFTP-3 WinSCP 下载日志：

. 2014-10-10 10:39:36.882 --------------------------------------------------------------------------
. 2014-10-10 10:39:36.883 WinSCP Version 5.5.1 (Build 3970) [...]
[...]
. 2014-10-10 10:39:36.884 SFTP Bugs: A,A
. 2014-10-10 10:39:36.884 SFTP Server: default
. 2014-10-10 10:39:36.884 Local directory: C:\Users, Remote directory: /path/to/target/somefolder, Update: Yes, Cache: Yes
. 2014-10-10 10:39:36.884 Cache directory changes: Yes, Permanent: Yes
. 2014-10-10 10:39:36.884 DST mode: 1; Timezone offset: 0h 0m
. 2014-10-10 10:39:36.884 --------------------------------------------------------------------------
[...]
. 2014-10-10 10:39:51.049 --------------------------------------------------------------------------
. 2014-10-10 10:39:51.050 Using SFTP protocol.
. 2014-10-10 10:39:51.050 Doing startup conversation with host.
> 2014-10-10 10:39:51.104 Type: SSH_FXP_INIT, Size: 5, Number: -1
< 2014-10-10 10:39:51.256 Type: SSH_FXP_VERSION, Size: 5, Number: -1
. 2014-10-10 10:39:51.256 SFTP version 3 negotiated.
. 2014-10-10 10:39:51.256 We believe the server has signed timestamps bug
. 2014-10-10 10:39:51.256 We will use UTF-8 strings when appropriate
. 2014-10-10 10:39:51.268 Changing directory to "/path/to/target/somefolder".
. 2014-10-10 10:39:51.268 Getting real path for '/path/to/target/somefolder'
> 2014-10-10 10:39:51.268 Type: SSH_FXP_REALPATH, Size: 41, Number: 16
< 2014-10-10 10:39:53.424 Type: SSH_FXP_NAME, Size: 141, Number: 16
. 2014-10-10 10:39:53.424 Real path is '/path/to/target/somefolder'
. 2014-10-10 10:39:53.424 Trying to open directory "/path/to/target/somefolder".
> 2014-10-10 10:39:53.424 Type: SSH_FXP_LSTAT, Size: 41, Number: 263
< 2014-10-10 10:39:53.594 Type: SSH_FXP_ATTRS, Size: 21, Number: 263
. 2014-10-10 10:39:53.594 Getting current directory name.
. 2014-10-10 10:39:53.743 Listing directory "/path/to/target/somefolder".
> 2014-10-10 10:39:53.743 Type: SSH_FXP_OPENDIR, Size: 41, Number: 523
< 2014-10-10 10:39:53.894 Type: SSH_FXP_HANDLE, Size: 12, Number: 523
> 2014-10-10 10:39:53.894 Type: SSH_FXP_READDIR, Size: 12, Number: 780
< 2014-10-10 10:39:54.403 Type: SSH_FXP_NAME, Size: 16407, Number: 780
> 2014-10-10 10:39:54.403 Type: SSH_FXP_READDIR, Size: 12, Number: 1036
< 2014-10-10 10:39:54.915 Type: SSH_FXP_NAME, Size: 4111, Number: 1036
> 2014-10-10 10:39:54.916 Type: SSH_FXP_READDIR, Size: 12, Number: 1292
< 2014-10-10 10:39:55.067 Type: SSH_FXP_STATUS, Size: 28, Number: 1292
< 2014-10-10 10:39:55.067 Status code: 1
> 2014-10-10 10:39:55.067 Type: SSH_FXP_CLOSE, Size: 12, Number: 1540
[...]
. 2014-10-10 10:39:55.164 Startup conversation with host finished.
. 2014-10-10 10:40:23.673 Cached directory change via ".." to "/path/to/target".
. 2014-10-10 10:40:23.730 Getting current directory name.
. 2014-10-10 10:40:23.730 Listing directory "/path/to/target".
> 2014-10-10 10:40:23.730 Type: SSH_FXP_OPENDIR, Size: 32, Number: 1803
< 2014-10-10 10:40:23.730 Type: SSH_FXP_STATUS, Size: 17, Number: 1540
. 2014-10-10 10:40:23.731 Discarding reserved response
< 2014-10-10 10:40:23.927 Type: SSH_FXP_HANDLE, Size: 12, Number: 1803
> 2014-10-10 10:40:24.022 Type: SSH_FXP_READDIR, Size: 12, Number: 2060
< 2014-10-10 10:40:26.323 Type: SSH_FXP_NAME, Size: 1247, Number: 2060
> 2014-10-10 10:40:26.381 Type: SSH_FXP_READDIR, Size: 12, Number: 2316
< 2014-10-10 10:40:26.531 Type: SSH_FXP_STATUS, Size: 28, Number: 2316
< 2014-10-10 10:40:26.531 Status code: 1
> 2014-10-10 10:40:26.531 Type: SSH_FXP_CLOSE, Size: 12, Number: 2564
[...]
. 2014-10-10 10:40:47.716 File: '/path/to/target/otherfolder/file.xxx' [2014-10-10T00:00:10.000Z] [3087870]
. 2014-10-10 10:40:47.742 Copying "/path/to/target/otherfolder/file.xxx" to local directory started.
. 2014-10-10 10:40:47.742 Binary transfer mode selected.
. 2014-10-10 10:40:47.774 Checking existence of partially transfered file.
. 2014-10-10 10:40:47.774 Opening remote file.
> 2014-10-10 10:40:47.774 Type: SSH_FXP_OPEN, Size: 128, Number: 3843
< 2014-10-10 10:40:47.774 Type: SSH_FXP_STATUS, Size: 17, Number: 3588
. 2014-10-10 10:40:47.774 Discarding reserved response
< 2014-10-10 10:40:47.993 Type: SSH_FXP_HANDLE, Size: 12, Number: 3843
> 2014-10-10 10:40:48.179 Type: SSH_FXP_FSTAT, Size: 12, Number: 4104
< 2014-10-10 10:40:48.335 Type: SSH_FXP_STATUS, Size: 112, Number: 4104
. 2014-10-10 10:40:48.336 Confirming overwriting of file.
. 2014-10-10 10:40:48.446 Asking user:
[...]
< 2014-10-10 10:41:10.204 Status code: 1
. 2014-10-10 10:41:10.241 222 skipped SSH_FXP_WRITE, SSH_FXP_READ, SSH_FXP_DATA and SSH_FXP_STATUS packets.
> 2014-10-10 10:41:10.241 Type: SSH_FXP_CLOSE, Size: 12, Number: 33284
< 2014-10-10 10:41:10.347 Type: SSH_FXP_STATUS, Size: 28, Number: 32517
< 2014-10-10 10:41:10.347 Type: SSH_FXP_STATUS, Size: 28, Number: 32773
< 2014-10-10 10:41:10.669 Type: SSH_FXP_STATUS, Size: 28, Number: 33029
. 2014-10-10 10:41:10.812 Preserving timestamp [2014-10-10T00:00:10.000Z]
. 2014-10-10 10:42:26.145 Closing connection.
. 2014-10-10 10:42:26.195 Sending special code: 12
. 2014-10-10 10:42:26.196 Sent EOF message

有任何想法吗？

score 2 · Accepted Answer

编辑：我发现了一个更高性能的解决方法，它依赖于猴子修补 paramiko 代码。同样，这不是一个很好的解决方案，但它得到了我需要的东西。我正在尝试以更通用的方式解决此问题，然后将其提交回 paramiko 源。

在我向您展示代码之前，请注意，这依赖于对 paramiko 库进行猴子修补，因此对 SFTPFile.stat 的每次调用将始终返回使用 fake_stat 修补的最后一个文件名的统计结果。

仅当您知道自己在做什么时才使用它

虽然它似乎适用于一次下载单个文件的简单脚本，但我不保证它适用于任何更复杂的用例。

import paramiko.sftp_file

with pysftp.Connection(host, username=self.user, password=self.password) as sftp:
        try:
            with sftp.cd(location):
                def fake_stat(sftpFile):                                                                                                                                                       
                    return sftpFile.sftp.stat(latest_file.filename)                                                                                                                            

                paramiko.sftp_file.SFTPFile.stat = fake_stat                                                                                                                                   

                sftp.get(filename, out_filename)                                                                                                                 
            return self.filename    
        except Exception as e:
            logging.fatal(e.message)
            raise

我可以确认 Martin 的分析：只有通过 paramiko 进行的底层 stat 调用才会导致错误。我发现的一种解决方法是自己手动执行复制。当然，这有一个非常慢的缺点，因为预取调用确实有助于减少每个块的往返延迟。

MB = 1024
BLOCK_SIZE = 4 * MB


with pysftp.Connection(host, username=self.user, password=self.password) as sftp:
        try:
            with sftp.cd(location):
                with sftp.open(remote_filename) as remote, open(local_filename), 'w') as local:
                    result = chardet.detect(remote.read(BLOCK_SIZE))
                    encoding = result['encoding']
                    remote.seek(0)

                    if encoding == 'ascii':
                        # Ascii is the most gimped encoding there is.                                                                                                                      
                        # Latin1 is strictly better, because at least its byte-for-byte.                                                                                                   
                        # Apologies to purists.                                                                                                                                            
                        print 'using encoding latin1'
                        encoding = 'latin1'

                    while True:
                        buf = remote.read(BLOCK_SIZE)
                        if not buf:
                            break

                        line = codecs.decode(buf, encoding, 'replace')
                        local.write(codecs.encode(line, 'utf-8'))

                        if len(buf) < BLOCK_SIZE:
                            break
        except Exception as e:
            logging.fatal(e.message)
            raise

python - PySFTP 连接有效，但 get() 失败

1 回答 1

Related

Reference