Add tls to ftp (#1581)

Author: bartvaneswhiffle

fsspec/implementations/ftp.py

@@ -2,7 +2,7 @@
 import sys
 import uuid
 import warnings
-from ftplib import FTP, Error, error_perm
+from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
 
 from ..spec import AbstractBufferedFile, AbstractFileSystem
@@ -27,6 +27,7 @@ def __init__(
         tempdir=None,
         timeout=30,
         encoding="utf-8",
+        tls=False,
         **kwargs,
     ):
         """
@@ -56,28 +57,38 @@ def __init__(
             Timeout of the ftp connection in seconds
         encoding: str
             Encoding to use for directories and filenames in FTP connection
+        tls: bool
+            Use FTP-TLS, by default False
         """
         super().__init__(**kwargs)
         self.host = host
         self.port = port
         self.tempdir = tempdir or "/tmp"
-        self.cred = username, password, acct
+        self.cred = username or "", password or "", acct or ""
+        print(self.cred)
         self.timeout = timeout
         self.encoding = encoding
         if block_size is not None:
             self.blocksize = block_size
         else:
             self.blocksize = 2**16
+        self.tls = tls
         self._connect()
+        if self.tls:
+            self.ftp.prot_p()
 
     def _connect(self):
+        if self.tls:
+            ftp_cls = FTP_TLS
+        else:
+            ftp_cls = FTP
         if sys.version_info >= (3, 9):
-            self.ftp = FTP(timeout=self.timeout, encoding=self.encoding)
+            self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
         elif self.encoding:
             warnings.warn("`encoding` not supported for python<3.9, ignoring")
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         else:
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)
 

fsspec/implementations/reference.py

@@ -997,9 +997,11 @@ def _process_gen(self, gens):
         out = {}
         for gen in gens:
             dimension = {
-                k: v
-                if isinstance(v, list)
-                else range(v.get("start", 0), v["stop"], v.get("step", 1))
+                k: (
+                    v
+                    if isinstance(v, list)
+                    else range(v.get("start", 0), v["stop"], v.get("step", 1))
+                )
                 for k, v in gen["dimensions"].items()
             }
             products = (

fsspec/implementations/tests/ftp_tls.py

@@ -0,0 +1,38 @@
+import os
+
+from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import TLS_FTPHandler
+from pyftpdlib.servers import FTPServer
+
+
+def ftp():
+    """Script to run FTP server that accepts TLS"""
+    # Set up FTP server parameters
+    FTP_HOST = "localhost"
+    FTP_PORT = 2121
+    FTP_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+
+    # Instantiate a dummy authorizer
+    authorizer = DummyAuthorizer()
+    authorizer.add_user(
+        "user",
+        "pass",
+        FTP_DIRECTORY,
+        "elradfmwMT",
+    )
+    authorizer.add_anonymous(FTP_DIRECTORY)
+
+    # Instantiate TLS_FTPHandler with required parameters
+    handler = TLS_FTPHandler
+    handler.certfile = os.path.join(os.path.dirname(__file__), "keycert.pem")
+    handler.authorizer = authorizer
+
+    # Instantiate FTP server with TLS handler and authorizer
+    server = FTPServer((FTP_HOST, FTP_PORT), handler)
+    server.authorizer = authorizer
+
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    ftp()

fsspec/implementations/tests/keycert.pem

@@ -0,0 +1,24 @@
+-----BEGIN EC PARAMETERS-----
+BggqhkjOPQMBBw==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIBTg1e61mzYYPJ+MDkOWCSevnT1HUaaK9iopgTGyDoIuoAoGCCqGSM49
+AwEHoUQDQgAEDy3E+4WgohcRUlaSZBndEZQBTyoRztCSoaDbhZkqsPFBbeaGJ5zA
+E7qX+9LICDezAUsCiq2RYltOqDCsELteiQ==
+-----END EC PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICdzCCAh2gAwIBAgIUNN4kmTSxbLOoQXLFiYOs2XeK1jIwCgYIKoZIzj0EAwIw
+gY8xCzAJBgNVBAYTAk5MMRUwEwYDVQQIDAxadWlkLUhvbGxhbmQxDjAMBgNVBAcM
+BURlbGZ0MRAwDgYDVQQKDAdXaGlmZmxlMQ0wCwYDVQQLDARERVZBMRIwEAYDVQQD
+DAlCYXJ0dmFuRXMxJDAiBgkqhkiG9w0BCQEWFWJhcnQudmFuZXNAd2hpZmZsZS5u
+bDAgFw0yNDA0MTgxMDI0NDFaGA8yMjk4MDIwMTEwMjQ0MVowgY8xCzAJBgNVBAYT
+Ak5MMRUwEwYDVQQIDAxadWlkLUhvbGxhbmQxDjAMBgNVBAcMBURlbGZ0MRAwDgYD
+VQQKDAdXaGlmZmxlMQ0wCwYDVQQLDARERVZBMRIwEAYDVQQDDAlCYXJ0dmFuRXMx
+JDAiBgkqhkiG9w0BCQEWFWJhcnQudmFuZXNAd2hpZmZsZS5ubDBZMBMGByqGSM49
+AgEGCCqGSM49AwEHA0IABA8txPuFoKIXEVJWkmQZ3RGUAU8qEc7QkqGg24WZKrDx
+QW3mhiecwBO6l/vSyAg3swFLAoqtkWJbTqgwrBC7XomjUzBRMB0GA1UdDgQWBBRb
+1nPqritk/P2cbDzTw9SQ9vO7JDAfBgNVHSMEGDAWgBRb1nPqritk/P2cbDzTw9SQ
+9vO7JDAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIBcvCFS4AD3p
+Ix1v8pp3hcMvGFIQLeczh4kXkPfZWvBkAiEAiTCqsdKhZi8k814H6FFkaoQVIjTe
+iUtUlW6RfyDNZ9E=
+-----END CERTIFICATE-----

fsspec/implementations/tests/test_ftp.py

@@ -2,6 +2,7 @@
 import subprocess
 import sys
 import time
+from ftplib import FTP, FTP_TLS
 
 import pytest
 
@@ -17,7 +18,7 @@
 def ftp():
     pytest.importorskip("pyftpdlib")
     P = subprocess.Popen(
-        [sys.executable, "-m", "pyftpdlib", "-d", here],
+        [sys.executable, os.path.join(here, "ftp_tls.py")],
         stderr=subprocess.STDOUT,
         stdout=subprocess.PIPE,
     )
@@ -29,9 +30,31 @@ def ftp():
         P.wait()
 
 
-def test_basic(ftp):
+@pytest.mark.parametrize(
+    "tls,exp_cls",
+    (
+        (False, FTP),
+        (True, FTP_TLS),
+    ),
+)
+def test_tls(ftp, tls, exp_cls):
     host, port = ftp
-    fs = FTPFileSystem(host, port)
+    fs = FTPFileSystem(host, port, tls=tls)
+    assert isinstance(fs.ftp, exp_cls)
+
+
+@pytest.mark.parametrize(
+    "tls,username,password",
+    (
+        (False, "", ""),
+        (True, "", ""),
+        (False, "user", "pass"),
+        (True, "user", "pass"),
+    ),
+)
+def test_basic(ftp, tls, username, password):
+    host, port = ftp
+    fs = FTPFileSystem(host, port, username, password, tls=tls)
     assert fs.ls("/", detail=False) == sorted(os.listdir(here))
     out = fs.cat(f"/{os.path.basename(__file__)}")
     assert out == open(__file__, "rb").read()