Mailing List Archive

From: Mike Frysinger <vapier@chromium.org>

Add support to the sftp-server for the copy-data extension defined in
[1]. This provides for efficient copying on the server side otherwise
clients have to emulate it by downloading the file before uploading it
to the new path. For large files or slow networks, this can be
prohibitively slow. Now doing a file copy is limited only by the server
I/O.

For clients that provide virtual file systems on top of SFTP, this can
provide significant performance gains.

[1] https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7
---
Rebased onto the latest tree.

PROTOCOL | 39 +++++++++++++++++++++-
sftp-server.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 130 insertions(+), 1 deletion(-)

diff --git a/PROTOCOL b/PROTOCOL
index e6a7d60eef0b..36192341959b 100644
--- a/PROTOCOL
+++ b/PROTOCOL
@@ -492,7 +492,7 @@ This request asks the server to call fsync(2) on an open file handle.
string "fsync@openssh.com"
string handle

-One receiving this request, a server will call fsync(handle_fd) and will
+On receiving this request, a server will call fsync(handle_fd) and will
respond with a SSH_FXP_STATUS message.

This extension is advertised in the SSH_FXP_VERSION hello with version
@@ -576,6 +576,43 @@ Its reply is the same format as that of SSH2_FXP_REALPATH.
This extension is advertised in the SSH_FXP_VERSION hello with version
"1".

+4.10. sftp: Extension request "copy-data"
+
+This request asks the server to copy data from one open file handle and
+write it to a different open file handle. This avoids needing to transfer
+the data across the network twice (a download followed by an upload).
+
+ byte SSH_FXP_EXTENDED
+ uint32 id
+ string "copy-data"
+ string read-from-handle
+ uint64 read-from-offset
+ uint64 read-data-length
+ string write-to-handle
+ uint64 write-to-offset
+
+The server will copy read-data-length bytes starting from
+read-from-offset from the read-from-handle and write them to
+write-to-handle starting from write-to-offset, and then respond with a
+SSH_FXP_STATUS message.
+
+It's equivalent to issuing a series of SSH_FXP_READ requests on
+read-from-handle and a series of requests of SSH_FXP_WRITE on
+write-to-handle.
+
+If read-from-handle and write-to-handle are the same, the server will
+fail the request and respond with a SSH_FX_INVALID_PARAMETER message.
+
+If read-data-length is 0, then the server will read data from the
+read-from-handle until EOF is reached.
+
+This extension is advertised in the SSH_FXP_VERSION hello with version
+"1".
+
+This request is identical to the "copy-data" request documented in:
+
+https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7
+
5. Miscellaneous changes

5.1 Public key format
diff --git a/sftp-server.c b/sftp-server.c
index ec91e9c2d33f..6d228ec2a9d5 100644
--- a/sftp-server.c
+++ b/sftp-server.c
@@ -44,6 +44,7 @@
#include <unistd.h>
#include <stdarg.h>

+#include "atomicio.h"
#include "xmalloc.h"
#include "sshbuf.h"
#include "ssherr.h"
@@ -119,6 +120,7 @@ static void process_extended_fsync(u_int32_t id);
static void process_extended_lsetstat(u_int32_t id);
static void process_extended_limits(u_int32_t id);
static void process_extended_expand(u_int32_t id);
+static void process_extended_copy_data(u_int32_t id);
static void process_extended(u_int32_t id);

struct sftp_handler {
@@ -164,6 +166,7 @@ static const struct sftp_handler extended_handlers[] = {
{ "limits", "limits@openssh.com", 0, process_extended_limits, 0 },
{ "expand-path", "expand-path@openssh.com", 0,
process_extended_expand, 0 },
+ { "copy-data", "copy-data", 0, process_extended_copy_data, 1 },
{ NULL, NULL, 0, NULL, 0 }
};

@@ -713,6 +716,7 @@ process_init(void)
compose_extension(msg, "lsetstat@openssh.com", "1");
compose_extension(msg, "limits@openssh.com", "1");
compose_extension(msg, "expand-path@openssh.com", "1");
+ compose_extension(msg, "copy-data", "1");

send_msg(msg);
sshbuf_free(msg);
@@ -1584,6 +1588,94 @@ process_extended_expand(u_int32_t id)
free(path);
}

+static void
+process_extended_copy_data(u_int32_t id)
+{
+ u_char buf[64*1024];
+ int read_handle, read_fd, write_handle, write_fd;
+ u_int64_t len, read_off, read_len, write_off;
+ int r, copy_until_eof, status = SSH2_FX_OP_UNSUPPORTED;
+ size_t ret;
+
+ if ((r = get_handle(iqueue, &read_handle)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &read_off)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &read_len)) != 0 ||
+ (r = get_handle(iqueue, &write_handle)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &write_off)) != 0)
+ fatal("%s: buffer error: %s", __func__, ssh_err(r));
+
+ debug("request %u: copy-data from \"%s\" (handle %d) off %llu len %llu "
+ "to \"%s\" (handle %d) off %llu",
+ id, handle_to_name(read_handle), read_handle,
+ (unsigned long long)read_off, (unsigned long long)read_len,
+ handle_to_name(write_handle), write_handle,
+ (unsigned long long)write_off);
+
+ /* For read length of 0, we read until EOF. */
+ if (read_len == 0) {
+ read_len = (u_int64_t)-1 - read_off;
+ copy_until_eof = 1;
+ } else
+ copy_until_eof = 0;
+
+ read_fd = handle_to_fd(read_handle);
+ write_fd = handle_to_fd(write_handle);
+
+ /* Disallow reading & writing to the same handle or same path or dirs */
+ if (read_handle == write_handle || read_fd < 0 || write_fd < 0 ||
+ !strcmp(handle_to_name(read_handle), handle_to_name(write_handle))) {
+ status = SSH2_FX_FAILURE;
+ goto out;
+ }
+
+ if (lseek(read_fd, read_off, SEEK_SET) < 0) {
+ status = errno_to_portable(errno);
+ error("%s: read_seek failed", __func__);
+ goto out;
+ }
+
+ if (!(handle_to_flags(write_handle) & O_APPEND) &&
+ lseek(write_fd, write_off, SEEK_SET) < 0) {
+ status = errno_to_portable(errno);
+ error("%s: write_seek failed", __func__);
+ goto out;
+ }
+
+ /* Process the request in chunks. */
+ while (read_len > 0 || copy_until_eof) {
+ len = MINIMUM(sizeof(buf), read_len);
+ read_len -= len;
+
+ ret = atomicio(read, read_fd, buf, len);
+ if (ret == 0 && errno == EPIPE) {
+ status = copy_until_eof ? SSH2_FX_OK : SSH2_FX_EOF;
+ break;
+ } else if (ret == 0) {
+ status = errno_to_portable(errno);
+ error("%s: read failed: %s", __func__, strerror(errno));
+ break;
+ }
+ len = ret;
+ handle_update_read(read_handle, len);
+
+ ret = atomicio(vwrite, write_fd, buf, len);
+ if (ret != len) {
+ status = errno_to_portable(errno);
+ error("%s: write failed: %llu != %llu: %s", __func__,
+ (unsigned long long)ret, (unsigned long long)len,
+ strerror(errno));
+ break;
+ }
+ handle_update_write(write_handle, len);
+ }
+
+ if (read_len == 0)
+ status = SSH2_FX_OK;
+
+ out:
+ send_status(id, status);
+}
+
static void
process_extended(u_int32_t id)
{
--
2.33.0

_______________________________________________
openssh-unix-dev mailing list
openssh-unix-dev@mindrot.org
https://lists.mindrot.org/mailman/listinfo/openssh-unix-dev

> Add support to the sftp-server for the copy-data extension defined in
> [1]. This provides for efficient copying on the server side otherwise
> clients have to emulate it by downloading the file before uploading it
> to the new path. For large files or slow networks, this can be
> prohibitively slow. Now doing a file copy is limited only by the server
> I/O.
>
> For clients that provide virtual file systems on top of SFTP, this can
> provide significant performance gains.
>
> [1] https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7

any feedback on this one ? it's been kickin around for a few years now :x.
the set i posted in Dec 2021 should still apply cleanly to latest master.
-mike