146 lines
5.5 KiB
Python
146 lines
5.5 KiB
Python
import datetime
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import warnings
|
|
##
|
|
_cur_dir = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
|
|
sys.path.append(os.path.abspath(os.path.join(_cur_dir, '..')))
|
|
import constants
|
|
# import logger
|
|
from . import _base
|
|
from . import rsync_returns
|
|
|
|
|
|
_logger = logging.getLogger()
|
|
|
|
|
|
class RSync(_base.BaseFetcher):
|
|
type = 'rsync'
|
|
|
|
def __init__(self,
|
|
domain,
|
|
port,
|
|
path,
|
|
dest,
|
|
rsync_args = None,
|
|
rsync_ignores = None,
|
|
owner = None,
|
|
log = True,
|
|
filechecks = None,
|
|
offset = None,
|
|
mtime = False,
|
|
*args,
|
|
**kwargs):
|
|
super().__init__(domain,
|
|
port,
|
|
path,
|
|
dest,
|
|
owner = owner,
|
|
filechecks = filechecks,
|
|
offset = offset,
|
|
mtime = mtime
|
|
*args,
|
|
**kwargs)
|
|
_logger.debug('Instantiated RSync fetcher')
|
|
if rsync_args:
|
|
self.rsync_args = rsync_args.args[:]
|
|
else:
|
|
self.rsync_args = constants.RSYNC_DEF_ARGS[:]
|
|
_logger.debug('RSync args given: {0}'.format(self.rsync_args))
|
|
self.rsync_ignores = rsync_ignores[:]
|
|
if log:
|
|
# Do I want to do this in subprocess + logging module? Or keep this?
|
|
# It looks a little ugly in the log but it makes more sense than doing it via subprocess just to write it
|
|
# back out.
|
|
_log_path = None
|
|
for h in _logger.handlers:
|
|
if isinstance(h, logging.handlers.RotatingFileHandler):
|
|
_log_path = h.baseFilename
|
|
break
|
|
self.rsync_args.extend(['--verbose',
|
|
'--log-file-format="[RSYNC {0}:{1}]:%l:%f%L"'.format(self.domain, self.port),
|
|
'--log-file={0}'.format(_log_path)])
|
|
|
|
def fetch(self):
|
|
path = self.url.rstrip('/')
|
|
if not path.endswith('/.'):
|
|
path += '/.'
|
|
dest = self.dest
|
|
if not dest.endswith('/.'):
|
|
dest += '/.'
|
|
# Yes, I know it's named "cmd_*str*". Yes, I know it's a *list*.
|
|
cmd_str = ['rsync',
|
|
*self.rsync_args,
|
|
path,
|
|
dest]
|
|
_logger.debug('Running command: {0}'.format(' '.join(cmd_str)))
|
|
cmd = subprocess.run(cmd_str,
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.PIPE)
|
|
stdout = cmd.stdout.decode('utf-8').strip()
|
|
stderr = cmd.stderr.decode('utf-8').strip()
|
|
rtrn = cmd.returncode
|
|
if stdout != '':
|
|
_logger.debug('STDOUT: {0}'.format(stdout))
|
|
if rtrn != 0 and rtrn not in self.rsync_ignores:
|
|
err = rsync_returns.returns[rtrn]
|
|
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
|
|
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
|
|
if stderr != '':
|
|
errmsg += ' an error message: {0}'.format(stderr)
|
|
debugmsg += ' an error message: {0}'.format(stderr)
|
|
if rtrn != 0:
|
|
errmsg += ' with exit status {0} ({1})'.format(rtrn, err)
|
|
debugmsg += ' with exit status {0} ({1})'.format(rtrn, err)
|
|
errmsg += '.'
|
|
_logger.error(errmsg)
|
|
_logger.debug(debugmsg)
|
|
warnings.warn(errmsg)
|
|
return(None)
|
|
|
|
def fetch_content(self, remote_filepath, mtime_only = False):
|
|
tf = tempfile.mkstemp()[1]
|
|
url = os.path.join(self.url.rstrip('/'), remote_filepath.lstrip('/'))
|
|
rsync_args = self.rsync_args[:]
|
|
if mtime_only and not any((('--times' in rsync_args), ('-t' in rsync_args))):
|
|
rsync_args.insert(0, '--times')
|
|
cmd_str = ['rsync',
|
|
*rsync_args,
|
|
url,
|
|
tf]
|
|
_logger.debug('Running command: {0}'.format(' '.join(cmd_str)))
|
|
cmd = subprocess.run(cmd_str,
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.PIPE)
|
|
stdout = cmd.stdout.decode('utf-8').strip()
|
|
stderr = cmd.stderr.decode('utf-8').strip()
|
|
rtrn = cmd.returncode
|
|
if stdout != '':
|
|
_logger.debug('STDOUT: {0}'.format(stdout))
|
|
if rtrn != 0 and rtrn not in self.rsync_ignores:
|
|
err = rsync_returns.returns.get(rtrn, '(UNKNOWN ERROR)')
|
|
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
|
|
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
|
|
if stderr != '':
|
|
errmsg += ' an error message: {0}'.format(stderr)
|
|
debugmsg += ' an error message: {0}'.format(stderr)
|
|
if rtrn != 0:
|
|
errmsg += ' with exit status {0} ({1})'.format(rtrn, err)
|
|
debugmsg += ' with exit status {0} ({1})'.format(rtrn, err)
|
|
errmsg += '.'
|
|
_logger.error(errmsg)
|
|
_logger.debug(debugmsg)
|
|
warnings.warn(errmsg)
|
|
return(b'')
|
|
if mtime_only:
|
|
raw_content = datetime.datetime.fromtimestamp(os.stat(tf).st_mtime)
|
|
else:
|
|
with open(tf, 'rb') as fh:
|
|
raw_content = fh.read()
|
|
os.remove(tf)
|
|
return(raw_content)
|
|
|