On Thursday 10 September 2015 16:05:14 Ander Juaristi Alamos wrote:
> > Hi Ander,
> >
> > during the last test I realized that --recursive won't work with FTPS.
>
> Hi Tim,
>
> If been looking through it and I've seen that the '--recursive' option from
> FTP and the one from FTPS follow different code paths.
>
> The code that triggers that difference is at main.c, line 1832:
>
> if ((opt.recursive || opt.page_requisites)
> && (url_scheme (*t) != SCHEME_FTP || url_uses_proxy
> (url_parsed))) {
> ...
> /* Turn opt.follow_ftp on in case of recursive FTP retrieval
> */ if (url_scheme (*t) == SCHEME_FTP)
> opt.follow_ftp = 1;
> ...
> }
>
> If I replace it to include FTPS also:
>
> if ((opt.recursive || opt.page_requisites)
> && ((url_scheme (*t) != SCHEME_FTP && url_scheme (*t) !=
> SCHEME_FTPS)
> || url_uses_proxy (url_parsed)))
>
> {
> ...
> /* Turn opt.follow_ftp on in case of recursive FTP retrieval
> */ if (url_scheme (*t) == SCHEME_FTP || url_scheme (*t) == SCHEME_FTPS)
> opt.follow_ftp = 1;
> ...
> }
>
> Then they both follow the same path. The rationale behind this is that the
> functionality for FTP should behave exactly the same for FTPS.
>
> However, it hangs when downloading the files... Will look further.
>
> Any thoughts?
Hi Ander,
two things that I found.
1. [PATCH 0002] Some FTPS pieces seem to be missing.
I went through the code by searching for SCHEME_FTP and "ftp" and added FTPS
stuff where it was missing. I did not think throughly about what I did - so
please just don't apply those changes blindly.
With those changes, Wget tried to work recursively but hangs on the first
PASSIVE data transfer.
2. [PATCH 0003 ]'using_data_security' in getftp() is a local variable, reset
for each file, and only set if (prot != PROT_CLEAR). I turned the logic and
voila, recursion works. Again, think if this change might break something else
that I didn't test.
Regards, Tim
>From 6e4497efd0d5689a8cc0467c30e185ce208c4437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <[email protected]>
Date: Fri, 11 Sep 2015 10:15:45 +0200
Subject: [PATCH 2/3] Some missing FTPS pieces
---
src/http.c | 3 +++
src/main.c | 4 ++--
src/metalink.h | 2 +-
src/recur.c | 15 ++++++++-------
src/retr.c | 9 ++++++---
src/url.c | 4 ++--
6 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/src/http.c b/src/http.c
index 67085a0..4c1052d 100644
--- a/src/http.c
+++ b/src/http.c
@@ -2742,6 +2742,9 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
case SCHEME_HTTPS:
mres.type = xstrdup ("https");
break;
+ case SCHEME_FTPS:
+ mres.type = xstrdup ("ftps");
+ break;
#endif
case SCHEME_FTP:
mres.type = xstrdup ("ftp");
diff --git a/src/main.c b/src/main.c
index 728084b..4ab9ad9 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1830,12 +1830,12 @@ outputting to a regular file.\n"));
else
{
if ((opt.recursive || opt.page_requisites)
- && (url_scheme (*t) != SCHEME_FTP || url_uses_proxy (url_parsed)))
+ && ((url_scheme (*t) != SCHEME_FTP && url_scheme (*t) != SCHEME_FTPS) || url_uses_proxy (url_parsed)))
{
int old_follow_ftp = opt.follow_ftp;
/* Turn opt.follow_ftp on in case of recursive FTP retrieval */
- if (url_scheme (*t) == SCHEME_FTP)
+ if (url_scheme (*t) == SCHEME_FTP || url_scheme (*t) == SCHEME_FTPS)
opt.follow_ftp = 1;
retrieve_tree (url_parsed, NULL);
diff --git a/src/metalink.h b/src/metalink.h
index ec91b07..e98c210 100644
--- a/src/metalink.h
+++ b/src/metalink.h
@@ -34,7 +34,7 @@ as that of the covered work. */
#ifdef HAVE_SSL
# define RES_TYPE_SUPPORTED(x)\
- ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http") || !strcmp (x, "https"))
+ ((!x) || !strcmp (x, "http") || !strcmp (x, "https") || !strcmp (x, "ftp") || !strcmp (x, "ftps"))
#else
# define RES_TYPE_SUPPORTED(x)\
((!x) || !strcmp (x, "ftp") || !strcmp (x, "http"))
diff --git a/src/recur.c b/src/recur.c
index ce55362..25cdbb7 100644
--- a/src/recur.c
+++ b/src/recur.c
@@ -610,7 +610,7 @@ download_child (const struct urlpos *upos, struct url *parent, int depth,
u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
/* 1. Schemes other than HTTP are normally not recursed into. */
- if (!u_scheme_like_http && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
+ if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP || u->scheme == SCHEME_FTPS) && opt.follow_ftp))
{
DEBUGP (("Not following non-HTTP schemes.\n"));
reason = WG_RR_NONHTTP;
@@ -832,12 +832,13 @@ write_reject_log_url (FILE *fp, const struct url *url)
switch (url->scheme)
{
- case SCHEME_HTTP: scheme_str = "SCHEME_HTTP"; break;
- #ifdef HAVE_SSL
- case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS"; break;
- #endif
- case SCHEME_FTP: scheme_str = "SCHEME_FTP"; break;
- default: scheme_str = "SCHEME_INVALID"; break;
+ case SCHEME_HTTP: scheme_str = "SCHEME_HTTP"; break;
+#ifdef HAVE_SSL
+ case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS"; break;
+ case SCHEME_FTPS: scheme_str = "SCHEME_FTPS"; break;
+#endif
+ case SCHEME_FTP: scheme_str = "SCHEME_FTP"; break;
+ default: scheme_str = "SCHEME_INVALID"; break;
}
fprintf (fp, "%s\t%s\t%s\t%i\t%s\t%s\t%s\t%s",
diff --git a/src/retr.c b/src/retr.c
index 28d1b35..318b09c 100644
--- a/src/retr.c
+++ b/src/retr.c
@@ -837,7 +837,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
FTP. In these cases we must decide whether the text is HTML
according to the suffix. The HTML suffixes are `.html',
`.htm' and a few others, case-insensitive. */
- if (redirection_count && local_file && u->scheme == SCHEME_FTP)
+ if (redirection_count && local_file && (u->scheme == SCHEME_FTP || u->scheme == SCHEME_FTPS))
{
if (has_html_suffix_p (local_file))
*dt |= TEXTHTML;
@@ -1099,12 +1099,12 @@ retrieve_from_file (const char *file, bool html, int *count)
proxy = getproxy (cur_url->url);
if ((opt.recursive || opt.page_requisites)
- && (cur_url->url->scheme != SCHEME_FTP || proxy))
+ && ((cur_url->url->scheme != SCHEME_FTP && cur_url->url->scheme != SCHEME_FTPS) || proxy))
{
int old_follow_ftp = opt.follow_ftp;
/* Turn opt.follow_ftp on in case of recursive FTP retrieval */
- if (cur_url->url->scheme == SCHEME_FTP)
+ if (cur_url->url->scheme == SCHEME_FTP || cur_url->url->scheme == SCHEME_FTPS)
opt.follow_ftp = 1;
status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
@@ -1285,6 +1285,9 @@ getproxy (struct url *u)
case SCHEME_HTTPS:
proxy = opt.https_proxy ? opt.https_proxy : getenv ("https_proxy");
break;
+ case SCHEME_FTPS:
+ proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftps_proxy");
+ break;
#endif
case SCHEME_FTP:
proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftp_proxy");
diff --git a/src/url.c b/src/url.c
index b81ebc5..56079cd 100644
--- a/src/url.c
+++ b/src/url.c
@@ -1779,7 +1779,7 @@ path_simplify (enum url_scheme scheme, char *path)
else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
{
/* Handle "../" by retreating the tortoise by one path
- element -- but not past beggining. */
+ element -- but not past beginning. */
if (t > beg)
{
/* Move backwards until T hits the beginning of the
@@ -1787,7 +1787,7 @@ path_simplify (enum url_scheme scheme, char *path)
for (--t; t > beg && t[-1] != '/'; t--)
;
}
- else if (scheme == SCHEME_FTP)
+ else if (scheme == SCHEME_FTP || scheme == SCHEME_FTPS)
{
/* If we're at the beginning, copy the "../" literally
and move the beginning so a later ".." doesn't remove
--
2.5.1
>From 720a61378ed24861284bc9626a2adba531fbecb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <[email protected]>
Date: Fri, 11 Sep 2015 10:45:01 +0200
Subject: [PATCH 3/3] Let FTPS work recursive
---
src/ftp.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/ftp.c b/src/ftp.c
index cb93838..f9a3316 100644
--- a/src/ftp.c
+++ b/src/ftp.c
@@ -340,7 +340,7 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
* and whether we're actually using any of them
* (encryption at the control connection only,
* or both at control and data connections) */
- bool using_control_security = false, using_data_security = false;
+ bool using_control_security = false, using_data_security = true;
#endif
assert (con != NULL);
@@ -538,8 +538,8 @@ Error in server response, closing control connection.\n"));
if (!opt.server_response)
logputs (LOG_VERBOSE, "done.\n");
- if (prot != PROT_CLEAR)
- using_data_security = true;
+ if (prot == PROT_CLEAR)
+ using_data_security = false;
}
}
#endif
--
2.5.1
