From 439e86d8c3f8feea8311edab8a7f8ceccdc86540 Mon Sep 17 00:00:00 2001 From: brent s Date: Sun, 5 Apr 2020 03:26:52 -0400 Subject: [PATCH] args' corresponding functions spec'd out and found (and have workaround for) racetime condition in Vault. --- testing/serverconf.py | 1 + testing/spawn.py | 29 +++++++++----- vaultpass/__init__.py | 83 +++++++++++++++++++++++++++++++++++++++- vaultpass/args.py | 60 +++++++++++++++++++++-------- vaultpass/constants.py | 10 ++++- vaultpass/gpg_handler.py | 11 +++++- vaultpass/mounts.py | 69 +++++++++++++++++++++++++++++++-- 7 files changed, 228 insertions(+), 35 deletions(-) diff --git a/testing/serverconf.py b/testing/serverconf.py index f190f20..bc8e70c 100755 --- a/testing/serverconf.py +++ b/testing/serverconf.py @@ -15,6 +15,7 @@ default_conf = {'listener': [ 'storage': {'file': {'path': './data'}}, 'log_level': 'Debug', # highest is 'Trace' 'pid_file': './test.pid', + 'disable_mlock': True, 'raw_storage_endpoint': True, 'log_format': 'json', # or String 'ui': True} diff --git a/testing/spawn.py b/testing/spawn.py index 369cc8d..86a20cc 100755 --- a/testing/spawn.py +++ b/testing/spawn.py @@ -217,6 +217,9 @@ class VaultSpawner(object): return(None) def cleanup(self): + self._connCheck(bind = False) + if self.is_running: + return(None) storage = self.conf.get('storage', {}).get('file', {}).get('path') if not storage: return(None) @@ -249,7 +252,7 @@ class VaultSpawner(object): mounts['secret'] = 'kv2' mounts['secret_legacy'] = 'kv1' for idx, (mname, mtype) in enumerate(mounts.items()): - opts = None + opts = {} orig_mtype = mtype if mtype.startswith('kv'): opts = {'version': re.sub(r'^kv([0-9]+)$', r'\g<1>', mtype)} @@ -259,9 +262,12 @@ class VaultSpawner(object): path = mname, description = 'Testing mount ({0})'.format(mtype), options = opts) - except hvac.exceptions.InvalidRequest: + # We might have some issues writing secrets on fast machines. + time.sleep(2) + except hvac.exceptions.InvalidRequest as e: # It probably already exists. - pass + print('Exception creating {0}: {1} ({2})'.format(mname, e, e.__class__)) + print(opts) if orig_mtype not in ('kv1', 'kv2', 'cubbyhole'): continue args = {'path': 'test_secret{0}/foo{1}'.format(idx, mname), @@ -279,11 +285,12 @@ class VaultSpawner(object): elif orig_mtype == 'kv2': handler = self.client.secrets.kv.v2.create_or_update_secret try: - handler(**args) + resp = handler(**args) except hvac.exceptions.InvalidPath: print('{0} path invalid'.format(args['path'])) except Exception as e: - print('Exception: {0} ({1})'.format(e, e.__class__)) + print('Exception creating {0} on {1}: {2} ({3})'.format(args['path'], args['mount_point'], e, e.__class__)) + print(args) return(None) def start(self): @@ -324,8 +331,9 @@ class VaultSpawner(object): if self.cmd: self.cmd.kill() else: - import signal - os.kill(self.pid, signal.SIGKILL) + if self.pid: + import signal + os.kill(self.pid, signal.SIGKILL) return(None) @@ -337,7 +345,7 @@ def parseArgs(): help = ('If specified, do not populate with test data (if it doesn\'t exist)')) args.add_argument('-d', '--delete', dest = 'delete_storage', - action = 'store_false', + action = 'store_true', help = ('If specified, delete the storage backend first so a fresh instance is created')) args.add_argument('-c', '--cleanup', dest = 'cleanup', @@ -368,8 +376,9 @@ def main(): s.populate() elif args.oper == 'stop': s.stop() - if args.cleanup: - s.cleanup() + if args.cleanup: + time.sleep(2) + s.cleanup() return(None) diff --git a/vaultpass/__init__.py b/vaultpass/__init__.py index 1038f73..fa0486f 100644 --- a/vaultpass/__init__.py +++ b/vaultpass/__init__.py @@ -8,11 +8,12 @@ from . import auth from . import clipboard from . import config from . import constants +from . import gpg_handler from . import mounts from . import pass_import -class PassMan(object): +class VaultPass(object): client = None auth = None uri = None @@ -64,6 +65,61 @@ class PassMan(object): _logger.debug('Set URI to {0}'.format(self.uri)) return(None) + def convert(self, + mount, + force = False, + gpghome = constants.GPG_HOMEDIR, + pass_dir = constants.PASS_DIR, + *args, **kwargs): + pass # TODO + + def copySecret(self, oldpath, newpath, mount, newmount, force = False, remove_old = False, *args, **kwargs): + pass # TODO + + if remove_old: + self.deleteSecret(oldpath, mount, force = force) + return(None) + + def createSecret(self, secret_dict, path, mount_name, *args, **kwargs): + mtype = self.mount.mounts.get(mount_name) + handler = None + if not mtype: + _logger.error('Could not determine mount type') + _logger.debug('Could not determine mount type for mount {0}'.format(mount_name)) + raise RuntimeError('Could not determine mount type') + args = {'path': path, + 'mount_point': mount_name, + 'secret': secret_dict} + if mtype == 'cubbyhole': + handler = self.mount.cubbyhandler.write_secret + elif mtype == 'kv1': + handler = self.client.secrets.kv.v1.create_or_update_secret + elif mtype == 'kv2': + handler = self.client.secrets.kv.v2.create_or_update_secret + resp = handler(**args) + return(resp) + + def deleteSecret(self, path, mount_name, force = False, recursive = False, *args, **kwargs): + pass # TODO + + def editSecret(self, path, mount, editor = constants.EDITOR, *args, **kwargs): + pass # TODO + + def generateSecret(self, + path, + mount, + symbols = True, + clip = False, + seconds = constants.CLIP_TIMEOUT, + chars = constants.SELECTED_PASS_CHARS, + chars_plain = constants.SELECTED_PASS_NOSYMBOL_CHARS, + in_place = False, + qr = False, + force = False, + length = constants.GENERATED_LENGTH, + *args, **kwargs): + pass # TODO + def getClient(self): auth_xml = self.cfg.xml.find('.//auth') if auth_xml is None: @@ -95,3 +151,28 @@ class PassMan(object): _logger.error('Not initialized') raise RuntimeError('Not initialized') return(None) + + def getSecret(self, path, mount, clip = None, qr = None, seconds = constants.CLIP_TIMEOUT, *args, **kwargs): + pass # TODO + + def initVault(self, *args, **kwargs): + pass # TODO + + def insertSecret(self, + path, + mount, + allow_shouldersurf = False, + multiline = False, + force = False, + confirm = True, + *args, **kwargs): + pass # TODO + + def listSecretNames(self, path, mount, output = None, indent = 4, *args, **kwargs): + pass # TODO + + def searchSecrets(self, pattern, mount, *args, **kwargs): + pass # TODO + + def searchSecretNames(self, pattern, mount, *args, **kwargs): + pass # TODO diff --git a/vaultpass/args.py b/vaultpass/args.py index d31a389..f0dcb71 100644 --- a/vaultpass/args.py +++ b/vaultpass/args.py @@ -18,9 +18,8 @@ def parseArgs(): help = ('The path to your configuration file. Default: ~/.config/vaultpass.xml')) args.add_argument('-m', '--mount', dest = 'mount', - required = False, - help = ('The mount to use in OPERATION. If not specified, assume all mounts we have access ' - 'to/all mounts specified in -c/--config')) + default = 'secret', + help = ('The mount to use in OPERATION. If not specified, assume a mount named "secret"')) # I wish argparse supported default subcommands. It doesn't as of python 3.8. subparser = args.add_subparsers(help = ('Operation to perform'), metavar = 'OPERATION', @@ -78,10 +77,16 @@ def parseArgs(): description = ('Import your existing Pass into Vault'), help = ('Import your existing Pass into Vault')) # CP/COPY + # vp.copySecret() cp.add_argument('-f', '--force', dest = 'force', action = 'store_true', help = ('If specified, replace NEWPATH if it exists')) + cp.add_argument('-m', '--mount', + dest = 'newmount', + nargs = 1, + required = False, + help = ('The mount for the destination. Default is to use the main command\'s -m/--mount')) cp.add_argument('oldpath', metavar = 'OLDPATH', help = ('The original ("source") path for the secret')) @@ -89,6 +94,7 @@ def parseArgs(): metavar = 'NEWPATH', help = ('The new ("destination") path for the secret')) # EDIT + # vp.editSecret() edit.add_argument('-e', '--editor', metavar = '/PATH/TO/EDITOR', dest = 'editor', @@ -100,10 +106,12 @@ def parseArgs(): help = ('Insert a new secret at PATH_TO_SECRET if it does not exist, otherwise edit it using ' 'your default editor (see -e/--editor)')) # FIND/SEARCH + # vp.searchSecretNames() find.add_argument('pattern', metavar = 'NAME_PATTERN', help = ('List secrets\' paths whose names match the regex NAME_PATTERN')) # GENERATE + # vp.generateSecret() # TODO: feature parity with passgen (spaces? etc.) gen.add_argument('-n', '--no-symbols', dest = 'symbols', @@ -160,6 +168,7 @@ def parseArgs(): metavar = 'dummy', help = ('(Unused; kept for compatibility reasons)')) # GREP + # vp.searchSecrets() # I wish argparse supported arbitrary arguments. # It *KIND* of does: https://stackoverflow.com/a/37367814/733214 but then I wouldn't be able to properly grab the # regex pattern without more hackery. So here's to wasting my life. @@ -323,6 +332,7 @@ def parseArgs(): help = ('Regex pattern to search passwords')) # HELP has no arguments. # INIT + # vp.initVault() initvault.add_argument('-p', '--path', dest = 'path', help = ('(Dummy option; kept for compatibility reasons)')) @@ -330,6 +340,7 @@ def parseArgs(): dest = 'gpg_id', help = ('(Dummy option; kept for compatibility reasons)')) # INSERT + # vp.insertSecret() # TODO: if -e/--echo is specified and sys.stdin, use sys.stdin rather than prompt insertval.add_argument('-e', '--echo', dest = 'allow_shouldersurf', @@ -348,11 +359,30 @@ def parseArgs(): action = 'store_false', help = ('If specified, disable password prompt confirmation. ' 'Has no effect if -e/--echo is specified')) + insertval.add_argument('path', + metavar = 'PATH/TO/SECRET', + help = ('The path to the secret')) # LS + # vp.listSecretNames()/vp.mount.print() ? + ls.add_argument('-o', '--output', + dest = 'output', + choices = constants.SUPPORTED_OUTPUT_FORMATS, + metavar = 'OUTPUT_FORMAT', + help = ('The format to output the hierarchy in. ' + 'If specified, must be one of: {0} ' + '(the default is a condensed python ' + 'dict repr)').format(', '.join(constants.SUPPORTED_OUTPUT_FORMATS))) + ls.add_argument('-i', '--indent', + type = int, + default = 4, + dest = 'indent', + help = ('If -o/--output is "pretty", "yaml", or "json", specify the indent level. ' + 'Default is 4')) ls.add_argument('path', metavar = 'PATH/TO/TREE/BASE', help = ('List names of secrets recursively, starting at PATH/TO/TREE/BASE')) # MV + # vp.copySecret(remove_old = True) mv.add_argument('-f', '--force', dest = 'force', action = 'store_true', @@ -364,6 +394,7 @@ def parseArgs(): metavar = 'NEWPATH', help = ('The new ("destination") path for the secret')) # RM + # vp.deleteSecret() # Is this argument even sensible since it isn't a filesystem? rm.add_argument('-r', '--recursive', dest = 'recurse', @@ -377,6 +408,8 @@ def parseArgs(): metavar = 'PATH/TO/SECRET', help = ('The path to the secret or subdirectory')) # SHOW + # vp.getSecret() ? plus QR etc. printing + # TODO: does the default overwrite the None if not specified? show.add_argument('-c', '--clip', nargs = '?', type = int, @@ -387,6 +420,7 @@ def parseArgs(): 'clipboard instead of printing it. ' 'Use 0 for LINE_NUMBER for the entire secret').format(constants.SHOW_CLIP_LINENUM)) show.add_argument('-q', '--qrcode', + dest = 'qr', nargs = '?', type = int, metavar = 'LINE_NUMBER', @@ -406,26 +440,18 @@ def parseArgs(): help = ('The path to the secret')) # VERSION has no args. # IMPORT - def_pass_dir = os.path.abspath(os.path.expanduser(os.environ.get('PASSWORD_STORE_DIR', '~/.password-store'))) - def_gpg_dir = os.path.abspath(os.path.expanduser(constants.SELECTED_GPG_HOMEDIR)) + # vp.convert() importvault.add_argument('-d', '--directory', - default = def_pass_dir, + default = constants.PASS_DIR, metavar = '/PATH/TO/PASSWORD_STORE/DIR', dest = 'pass_dir', - help = ('The path to your Pass data directory. Default: {0}').format(def_pass_dir)) - importvault.add_argument('-k', '--gpg-key-id', - metavar = 'KEY_ID', - dest = 'key_id', - default = constants.PASS_KEY, - help = ('The GPG key ID to use. Default: {0}. ' - '(If None, the default is to first check /PATH/TO/PASSWORD_STORE/DIR/.gpg-id if ' - 'it exists, otherwise use the ' - 'first private key we find)').format(constants.PASS_KEY)) + help = ('The path to your Pass data directory. Default: {0}').format(constants.PASS_DIR)) importvault.add_argument('-H', '--gpg-homedir', - default = def_gpg_dir, + default = constants.GPG_HOMEDIR, dest = 'gpghome', metavar = '/PATH/TO/GNUPG/HOMEDIR', - help = ('The GnuPG "homedir". Default: {0}').format(def_gpg_dir)) + help = ('The GnuPG "homedir". It MUST contain the private key that Pass uses. ' + 'Default: {0}').format(constants.GPG_HOMEDIR)) importvault.add_argument('-f', '--force', dest = 'force', action = 'store_true', diff --git a/vaultpass/constants.py b/vaultpass/constants.py index 77c7e69..8628e5e 100644 --- a/vaultpass/constants.py +++ b/vaultpass/constants.py @@ -4,6 +4,8 @@ import string # These are static. NAME = 'VaultPass' VERSION = '0.0.1' +SUPPORTED_ENGINES = ('kv1', 'kv2', 'cubbyhole') +SUPPORTED_OUTPUT_FORMATS = ('pretty', 'yaml', 'json') ALPHA_LOWER_PASS_CHARS = string.ascii_lowercase ALPHA_UPPER_PASS_CHARS = string.ascii_uppercase ALPHA_PASS_CHARS = ALPHA_LOWER_PASS_CHARS + ALPHA_UPPER_PASS_CHARS @@ -19,9 +21,9 @@ SELECTED_PASS_NOSYMBOL_CHARS = ALPHANUM_PASS_CHARS CLIPBOARD = 'clipboard' GENERATED_LENGTH = 25 # I personally would prefer 32, but Pass compatibility... EDITOR = 'vi' # vi is on ...every? single distro and UNIX/UNIX-like, to my knowledge. -PASS_KEY = None GPG_HOMEDIR = '~/.gnupg' SELECTED_GPG_HOMEDIR = GPG_HOMEDIR +PASS_DIR = '~/.password-store' if not os.environ.get('NO_VAULTPASS_ENVS'): # These are dynamically generated from the environment. @@ -32,5 +34,9 @@ if not os.environ.get('NO_VAULTPASS_ENVS'): CLIPBOARD = os.environ.get('PASSWORD_STORE_X_SELECTION', CLIPBOARD) GENERATED_LENGTH = int(os.environ.get('PASSWORD_STORE_GENERATED_LENGTH', GENERATED_LENGTH)) EDITOR = os.environ.get('EDITOR', EDITOR) - PASS_KEY = os.environ.get('PASSWORD_STORE_KEY', PASS_KEY) SELECTED_GPG_HOMEDIR = os.environ.get('GNUPGHOME', GPG_HOMEDIR) + PASS_DIR = os.environ.get('PASSWORD_STORE_DIR', PASS_DIR) + +# These are made more sane. +PASS_DIR = os.path.abspath(os.path.expanduser(PASS_DIR)) +SELECTED_GPG_HOMEDIR = os.path.abspath(os.path.expanduser(SELECTED_GPG_HOMEDIR)) diff --git a/vaultpass/gpg_handler.py b/vaultpass/gpg_handler.py index a82686e..7955e71 100644 --- a/vaultpass/gpg_handler.py +++ b/vaultpass/gpg_handler.py @@ -18,8 +18,17 @@ class GPG(object): def decrypt(self, fpath): fpath = os.path.abspath(os.path.expanduser(fpath)) + _logger.debug('Opening {0} for decryption'.format(fpath)) with open(fpath, 'rb') as fh: - iobuf = io.BytesIO(fh.read()) + data = fh.read() + decrypted = self.decryptData(data) + return(decrypted) + + def decryptData(self, data): + if isinstance(data, str): + data = data.encode('utf-8') + _logger.debug('Decrypting {0} bytes'.format(len(data))) + iobuf = io.BytesIO(data) iobuf.seek(0, 0) rslt = self.gpg.decrypt(iobuf) decrypted = rslt[0] diff --git a/vaultpass/mounts.py b/vaultpass/mounts.py index 8b1015f..dc0939a 100644 --- a/vaultpass/mounts.py +++ b/vaultpass/mounts.py @@ -1,15 +1,23 @@ +import copy import logging import re import shutil +import time import warnings ## import dpath.util # https://pypi.org/project/dpath/ import hvac.exceptions +## +from . import constants _logger = logging.getLogger() _mount_re = re.compile(r'^(?P.*)/$') _subpath_re = re.compile(r'^/?(?P.*)/$') +_kv_re = re.compile(r'^kv(?:-v)?(?P[0-9]+)$') + + +# TODO: for all write operations, modify handler call to first check if path exists and patch if it does? class CubbyHandler(object): @@ -30,6 +38,18 @@ class CubbyHandler(object): resp = self.client._adapter.get(url = uri) return(resp.json()) + def write_secret(self, path, secret, mount_point = 'cubbyhole'): + path = path.lstrip('/') + args = {'path': '/'.join((mount_point, path))} + for k, v in secret.items(): + if k in args.keys(): + _logger.error('Cannot use reserved secret name') + _logger.debug('Cannot use secret name {0} as it is reserved'.format(k)) + raise ValueError('Cannot use reserved secret name') + args[k] = v + resp = self.client.write(**args) + return(resp) + class MountHandler(object): internal_mounts = ('identity', 'sys') @@ -42,6 +62,38 @@ class MountHandler(object): self.paths = {} self.getSysMounts() + def createMount(self, mount_name, mount_type = 'kv2'): + orig_mtype = mount_type + if mount_type not in constants.SUPPORTED_ENGINES: + _logger.error('Invalid mount type') + _logger.debug(('The mount type {0} is invalid. ' + 'It must be one of: {1}').format(mount_type, ', '.join(constants.SUPPORTED_ENGINES))) + raise ValueError('Invalid mount type') + options = {} + r = _kv_re.search(mount_type) + if r: + mount_type = 'kv' + options['version'] = r.groupdict()['version'] + created = False + try: + self.client.sys.enable_secrets_engine(mount_type, + path = mount_name, + description = 'Created automatically by VaultPass', + options = options) + created = True + except hvac.exceptions.InvalidPath as e: + _logger.error('Invalid path') + _logger.debug('The mount path {0} (type {1}) is invalid: {2}'.format(mount_name, orig_mtype, e)) + raise ValueError('Invalid path') + except hvac.exceptions.InvalidRequest as e: + _logger.error('Invalid request; does mount already exist?') + _logger.debug(('The creation of mount path {0} (type {1}) generated an invalid request: ' + '{2}. Does it already exist?').format(mount_name, orig_mtype, e)) + # Due to how KV2 is created, we can hit a timing/race condition. + if orig_mtype == 'kv2' and created: + time.sleep(2) + return(created) + def getMountType(self, mount): if not self.mounts: self.getSysMounts() @@ -53,7 +105,19 @@ class MountHandler(object): return(mtype) def getSecret(self, path, mount, version = None): - pass + if not self.mounts: + self.getSysMounts() + mtype = self.getMountType(mount) + args = {} + handler = None + if mtype == 'cubbyhole': + handler = self.cubbyhandler.read_secret + elif mtype == 'kv1': + handler = self.client.secrets.kv.v1.read_secret + if mtype == 'kv2': + args['version'] = version + data = self.client.secrets + # TODO def getSecretNames(self, path, mount, version = None): reader = None @@ -214,6 +278,3 @@ class MountHandler(object): elif not output: return(str(self.paths)) return(None) - - def search(self): - pass