repomirror/repomirror/fetcher/ftp.py

import datetime
import ftplib
import logging
import io
import os
import pathlib
##
from . import _base


_logger = logging.getLogger()


class FTP(_base.BaseFetcher):
    type = 'ftp'

    def __init__(self, domain, port, path, dest, owner = None, mtime = False, offset = None, *args, **kwargs):
        super().__init__(domain, port, path, dest, owner = owner, mtime = mtime, offset = offset, *args, **kwargs)
        _logger.debug('Instantiated FTP fetcher')
        self.handler = ftplib.FTP(self.domain)
        _logger.debug('Configured handler for {0}'.format(self.domain))
        self.handler.port = self.port
        _logger.debug('Set port for {0} to {1}'.format(self.domain, self.port))
        self.connected = None

    def _connect(self):
        if not self.connected:
            self.handler.login()
            _logger.debug('Connected to {0}:{1} as Anonymous'.format(self.domain, self.port))
        self.connected = True
        return(None)

    def _disconnect(self):
        if self.connected:
            self.handler.close()
            _logger.debug('Disconnected from {0}:{1} as Anonymous'.format(self.domain, self.port))
        self.connected = False
        return(None)

    def _pathtuple(self, path):
        relpath = path.lstrip('/')
        relpath_stripped = str(pathlib.Path(relpath).relative_to(self.path))
        destdir = os.path.join(self.dest, os.path.dirname(relpath_stripped))
        destpath = os.path.join(self.dest, relpath_stripped)
        return((relpath, destdir, destpath))

    def _prepdir(self, destdir):
        os.makedirs(destdir, mode = 0o0755, exist_ok = True)
        _logger.debug('Created directory {0} (if it did not exist)'.format(destdir))
        if self.owner:
            os.chown(destdir, **self.owner)
            _logger.debug('Chowned {0} to {uid}:{gid}'.format(destdir, **self.owner))
        return()

    def fetch(self):
        def getter(path, relroot):
            _logger.debug('getter invoked with path={0}, relroot={1}'.format(path, relroot))
            if relroot == path:
                parentdir = path
                _logger.debug('relroot and path are the same')
            else:
                parentdir = relroot
                _logger.debug('relroot and path are not the same')
            _logger.debug('parentdir set to {0}'.format(parentdir))
            _logger.debug('Executing LS on {0}'.format(parentdir))
            for itemspec in self.handler.mlsd(parentdir):
                relpath, spec = itemspec
                if relpath in ('.', '..'):
                    continue
                _logger.debug(('Parsing path ('
                               'relroot: {0}, '
                               'path: {1}, '
                               'relpath: {2}) with spec {3}').format(relroot, path, relpath, itemspec))
                ptype = spec['type']
                newpath = os.path.join(parentdir, relpath)
                itemspec = (newpath, itemspec[1])
                if ptype.startswith('OS.unix=slink'):
                    _logger.debug('Fetching symlink {0}'.format(parentdir))
                    self.fetch_symlink(itemspec)
                elif ptype == 'dir':
                    _logger.debug('Fetching dir {0}'.format(parentdir))
                    self.fetch_dir(itemspec)
                    _logger.debug('Recursing getter with relpath={0}, parentdir={1}'.format(relpath, parentdir))
                    getter(relpath, newpath)
                elif ptype == 'file':
                    _logger.debug('Fetching file {0}'.format(parentdir))
                    self.fetch_file(itemspec)
            return(None)
        self._connect()
        getter(self.path, self.path)
        self._disconnect()
        return(None)

    def fetch_content(self, remote_filepath, mtime_only = False):
        self._connect()
        if mtime_only:
            directory, file = os.path.split(remote_filepath)
            parent = '/{0}'.format(directory.lstrip('/'))
            meta = self.handler.mlsd(parent)
            file_info = dict(meta)[file]
            tstmp = file_info['modify']
            content = datetime.datetime.strptime(tstmp, '%Y%m%d%H%M%S')
        else:
            buf = io.BytesIO()
            self.handler.retrbinary('RETR {0}'.format(remote_filepath), buf.write)
            self._disconnect()
            buf.seek(0, 0)
            content = buf.read()
        self._disconnect()
        return(content)

    def fetch_dir(self, pathspec):
        self._connect()
        # Relative to FTP root.
        relpath, destdir, destpath = self._pathtuple(pathspec[0])
        mode = int(pathspec[1]['unix.mode'], 8)
        os.makedirs(destpath, mode = mode, exist_ok = True)
        _logger.debug('Created directory {0} with mode {1} (if it did not exist)'.format(destpath, oct(mode)))
        if self.owner:
            os.chown(destpath, **self.owner)
            _logger.debug('Chowned {0} to {uid}:{gid}'.format(destpath, **self.owner))
        return(None)

    def fetch_file(self, pathspec):
        self._connect()
        relpath, destdir, destpath = self._pathtuple(pathspec[0])
        self._prepdir(destdir)
        with open(destpath, 'wb') as fh:
            self.handler.retrbinary('RETR {0}'.format(relpath), fh.write)
        _logger.debug('Created file {0}'.format(destpath))
        mode = int(pathspec[1]['unix.mode'], 8)
        os.chmod(destpath, mode)
        _logger.debug('Chmodded {0} to {1}'.format(destpath, oct(mode)))
        if self.owner:
            os.chown(destpath, **self.owner)
            _logger.debug('Chowned {0} to {uid}:{gid}'.format(destpath, **self.owner))
        return(None)

    def fetch_symlink(self, pathspec):
        relpath, destdir, destpath = self._pathtuple(pathspec[0])
        self._prepdir(destdir)
        # For symlinks, this is something like: OS.unix=slink:path/to/target
        target = pathspec[1]['type'].split(':', 1)[1]
        # We don't care if the target exists.
        os.symlink(target, destpath)
        _logger.debug('Created symlink {0} -> {1}'.format(destpath, target))
        return(None)