From 4a61faf75d684eb31c23521bc0e3c3cac5fd1553 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:33 +0800 Subject: [PATCH 1/6] transport-helper: no connection restriction in connect_helper When commit b236752a (Support remote archive from all smart transports, 2009-12-09) added "remote archive" support for "smart transports", it was for transport that supports the ".connect" method. The "connect_helper()" function protected itself from getting called for a transport without the method before calling process_connect_service(), which only worked with the ".connect" method. Later, commit edc9caf7 (transport-helper: introduce stateless-connect, 2018-03-15) added a way for a transport without the ".connect" method to establish a "stateless" connection in protocol v2, where process_connect_service() was taught to handle the ".stateless_connect" method, making the old protection too strict. But commit edc9caf7 forgot to adjust this protection accordingly. Even at the time of commit b236752a, this protection seemed redundant, since process_connect_service() would return 0 if the connection could not be established, and connect_helper() would still die() early. Remove the restriction in connect_helper() and give the function process_connect_service() the opportunity to establish a connection using ".connect" or ".stateless_connect" for protocol v2. So we can connect with a stateless-rpc and do something useful. E.g., in a later commit, implements remote archive for a repository over HTTP protocol. Helped-by: Junio C Hamano Helped-by: Linus Arver Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- transport-helper.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/transport-helper.c b/transport-helper.c index 49811ef176..2e127d24a5 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -662,8 +662,6 @@ static int connect_helper(struct transport *transport, const char *name, /* Get_helper so connect is inited. */ get_helper(transport); - if (!data->connect) - die(_("operation not supported by protocol")); if (!process_connect_service(transport, name, exec)) die(_("can't connect to subservice %s"), name); From 23b7d59a82ce39c0e08a4052de3be9ffbccd99b9 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:34 +0800 Subject: [PATCH 2/6] remote-curl: supports git-upload-archive service Add new service (git-upload-archive) support in remote-curl, so we can support remote archive over HTTP/HTTPS protocols. Differences between git-upload-archive and other services: 1. The git-archive program does not expect to see protocol version and capabilities when connecting to remote-helper, so do not send them in remote-curl for the git-upload-archive service. 2. We need to detect protocol version by calling discover_refs(). Fallback to use the git-upload-pack service (which, like git-upload-archive, is a read-only operation) to discover protocol version. Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- remote-curl.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/remote-curl.c b/remote-curl.c index ef05752ca5..ce6cb8ac05 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1447,8 +1447,14 @@ static int stateless_connect(const char *service_name) * establish a stateless connection, otherwise we need to tell the * client to fallback to using other transport helper functions to * complete their request. + * + * The "git-upload-archive" service is a read-only operation. Fallback + * to use "git-upload-pack" service to discover protocol version. */ - discover = discover_refs(service_name, 0); + if (!strcmp(service_name, "git-upload-archive")) + discover = discover_refs("git-upload-pack", 0); + else + discover = discover_refs(service_name, 0); if (discover->version != protocol_v2) { printf("fallback\n"); fflush(stdout); @@ -1486,9 +1492,11 @@ static int stateless_connect(const char *service_name) /* * Dump the capability listing that we got from the server earlier - * during the info/refs request. + * during the info/refs request. This does not work with the + * "git-upload-archive" service. */ - write_or_die(rpc.in, discover->buf, discover->len); + if (strcmp(service_name, "git-upload-archive")) + write_or_die(rpc.in, discover->buf, discover->len); /* Until we see EOF keep sending POSTs */ while (1) { From 5c858368966f71ffeba78e2fa9aaecec02050f3c Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:35 +0800 Subject: [PATCH 3/6] transport-helper: protocol v2 supports upload-archive We used to support only git-upload-pack service for protocol v2. In order to support remote archive over HTTP/HTTPS protocols, add new service support for git-upload-archive in protocol v2. Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- transport-helper.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transport-helper.c b/transport-helper.c index 2e127d24a5..6fe9f4f208 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -628,7 +628,8 @@ static int process_connect_service(struct transport *transport, ret = run_connect(transport, &cmdbuf); } else if (data->stateless_connect && (get_protocol_version_config() == protocol_v2) && - !strcmp("git-upload-pack", name)) { + (!strcmp("git-upload-pack", name) || + !strcmp("git-upload-archive", name))) { strbuf_addf(&cmdbuf, "stateless-connect %s\n", name); ret = run_connect(transport, &cmdbuf); if (ret) From 24f275ab33cd06f399a87a8384443e74da5f3545 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:36 +0800 Subject: [PATCH 4/6] http-backend: new rpc-service for git-upload-archive Add new rpc-service "upload-archive" in http-backend to add server side support for remote archive over HTTP/HTTPS protocols. Also add new test cases in t5003. In the test case "archive remote http repository", git-archive exits with a non-0 exit code even though we create the archive correctly. It will be fixed in a later commit. Helped-by: Eric Sunshine Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- http-backend.c | 13 ++++++++++--- t/t5003-archive-zip.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/http-backend.c b/http-backend.c index ff07b87e64..1ed1e29d07 100644 --- a/http-backend.c +++ b/http-backend.c @@ -38,6 +38,7 @@ struct rpc_service { static struct rpc_service rpc_service[] = { { "upload-pack", "uploadpack", 1, 1 }, { "receive-pack", "receivepack", 0, -1 }, + { "upload-archive", "uploadarchive", 0, -1 }, }; static struct string_list *get_parameters(void) @@ -639,10 +640,15 @@ static void check_content_type(struct strbuf *hdr, const char *accepted_type) static void service_rpc(struct strbuf *hdr, char *service_name) { - const char *argv[] = {NULL, "--stateless-rpc", ".", NULL}; + struct strvec argv = STRVEC_INIT; struct rpc_service *svc = select_service(hdr, service_name); struct strbuf buf = STRBUF_INIT; + strvec_push(&argv, svc->name); + if (strcmp(service_name, "git-upload-archive")) + strvec_push(&argv, "--stateless-rpc"); + strvec_push(&argv, "."); + strbuf_reset(&buf); strbuf_addf(&buf, "application/x-git-%s-request", svc->name); check_content_type(hdr, buf.buf); @@ -655,9 +661,9 @@ static void service_rpc(struct strbuf *hdr, char *service_name) end_headers(hdr); - argv[0] = svc->name; - run_service(argv, svc->buffer_input); + run_service(argv.v, svc->buffer_input); strbuf_release(&buf); + strvec_clear(&argv); } static int dead; @@ -723,6 +729,7 @@ static struct service_cmd { {"GET", "/objects/pack/pack-[0-9a-f]{64}\\.idx$", get_idx_file}, {"POST", "/git-upload-pack$", service_rpc}, + {"POST", "/git-upload-archive$", service_rpc}, {"POST", "/git-receive-pack$", service_rpc} }; diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index fc499cdff0..6f85bd3463 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -239,4 +239,38 @@ check_zip with_untracked2 check_added with_untracked2 untracked one/untracked check_added with_untracked2 untracked two/untracked +# Test remote archive over HTTP protocol. +# +# Note: this should be the last part of this test suite, because +# by including lib-httpd.sh, the test may end early if httpd tests +# should not be run. +# +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success "setup for HTTP protocol" ' + cp -R bare.git "$HTTPD_DOCUMENT_ROOT_PATH/bare.git" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/bare.git" \ + config http.uploadpack true && + set_askpass user@host pass@host +' + +setup_askpass_helper + +test_expect_success 'remote archive does not work with protocol v1' ' + test_must_fail git -c protocol.version=1 archive \ + --remote="$HTTPD_URL/auth/smart/bare.git" \ + --output=remote-http.zip HEAD >actual 2>&1 && + cat >expect <<-EOF && + fatal: can${SQ}t connect to subservice git-upload-archive + EOF + test_cmp expect actual +' + +test_expect_success 'archive remote http repository' ' + test_must_fail git archive --remote="$HTTPD_URL/auth/smart/bare.git" \ + --output=remote-http.zip HEAD && + test_cmp_bin d.zip remote-http.zip +' + test_done From 35d26e79f8d8c030269a198d07de939bd3f5ea08 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:37 +0800 Subject: [PATCH 5/6] transport-helper: call do_take_over() in connect_helper After successfully connecting to the smart transport by calling process_connect_service() in connect_helper(), run do_take_over() to replace the old vtable with a new one which has methods ready for the smart transport connection. This fixes the exit code of git-archive in test case "archive remote http repository" of t5003. The connect_helper() function is used as the connect method of the vtable in "transport-helper.c", and it is called by transport_connect() in "transport.c" to setup a connection. The only place that we call transport_connect() so far is in "builtin/archive.c". Without running do_take_over(), it may fail to call transport_disconnect() in run_remote_archiver() of "builtin/archive.c". This is because for a stateless connection and a service like "git-upload-archive", the remote helper may receive a SIGPIPE signal and exit early. Call do_take_over() to have a graceful disconnect method, so that we still call transport_disconnect() even if the remote helper exits early. Helped-by: Linus Arver Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- t/t5003-archive-zip.sh | 2 +- transport-helper.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index 6f85bd3463..961c6aac25 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -268,7 +268,7 @@ test_expect_success 'remote archive does not work with protocol v1' ' ' test_expect_success 'archive remote http repository' ' - test_must_fail git archive --remote="$HTTPD_URL/auth/smart/bare.git" \ + git archive --remote="$HTTPD_URL/auth/smart/bare.git" \ --output=remote-http.zip HEAD && test_cmp_bin d.zip remote-http.zip ' diff --git a/transport-helper.c b/transport-helper.c index 6fe9f4f208..91381be622 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -669,6 +669,8 @@ static int connect_helper(struct transport *transport, const char *name, fd[0] = data->helper->out; fd[1] = data->helper->in; + + do_take_over(transport); return 0; } From 176cd68634c4641cc45030dc7f287be47d1809d1 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 21 Jan 2024 21:15:38 +0800 Subject: [PATCH 6/6] transport-helper: call do_take_over() in process_connect The existing pattern among all callers of process_connect() seems to be if (process_connect(...)) { do_take_over(); ... dispatch to the underlying method ... } ... otherwise implement the fallback ... where the return value from process_connect() is the return value of the call it makes to process_connect_service(). Move the call of do_take_over() inside process_connect(), so that calling the process_connect() function is more concise and will not miss do_take_over(). Suggested-by: Junio C Hamano Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- transport-helper.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/transport-helper.c b/transport-helper.c index 91381be622..566f7473df 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -646,6 +646,7 @@ static int process_connect(struct transport *transport, struct helper_data *data = transport->data; const char *name; const char *exec; + int ret; name = for_push ? "git-receive-pack" : "git-upload-pack"; if (for_push) @@ -653,7 +654,10 @@ static int process_connect(struct transport *transport, else exec = data->transport_options.uploadpack; - return process_connect_service(transport, name, exec); + ret = process_connect_service(transport, name, exec); + if (ret) + do_take_over(transport); + return ret; } static int connect_helper(struct transport *transport, const char *name, @@ -685,10 +689,8 @@ static int fetch_refs(struct transport *transport, get_helper(transport); - if (process_connect(transport, 0)) { - do_take_over(transport); + if (process_connect(transport, 0)) return transport->vtable->fetch_refs(transport, nr_heads, to_fetch); - } /* * If we reach here, then the server, the client, and/or the transport @@ -1145,10 +1147,8 @@ static int push_refs(struct transport *transport, { struct helper_data *data = transport->data; - if (process_connect(transport, 1)) { - do_take_over(transport); + if (process_connect(transport, 1)) return transport->vtable->push_refs(transport, remote_refs, flags); - } if (!remote_refs) { fprintf(stderr, @@ -1189,11 +1189,9 @@ static struct ref *get_refs_list(struct transport *transport, int for_push, { get_helper(transport); - if (process_connect(transport, for_push)) { - do_take_over(transport); + if (process_connect(transport, for_push)) return transport->vtable->get_refs_list(transport, for_push, transport_options); - } return get_refs_list_using_list(transport, for_push); } @@ -1277,10 +1275,8 @@ static int get_bundle_uri(struct transport *transport) { get_helper(transport); - if (process_connect(transport, 0)) { - do_take_over(transport); + if (process_connect(transport, 0)) return transport->vtable->get_bundle_uri(transport); - } return -1; }