From 9784b99585f72e7f466e7922e295474e2f33a9af Mon Sep 17 00:00:00 2001 From: brent s Date: Thu, 9 Apr 2020 15:47:16 -0400 Subject: [PATCH] checking in --- docs/README.adoc | 47 +++++++++---- vaultpass/__init__.py | 153 ++++++++++++++++++++++++++++-------------- vaultpass/args.py | 17 ++++- vaultpass/auth.py | 18 +++-- vaultpass/config.py | 36 ++++++++++ vaultpass/mounts.py | 6 ++ 6 files changed, 209 insertions(+), 68 deletions(-) diff --git a/docs/README.adoc b/docs/README.adoc index 776a0a3..9ceb311 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -27,7 +27,8 @@ bash and backed by GPG. It's fairly barebones in terms of technology but does a VaultPass attempts to bridge the gap between the two. It aims to be a drop-in replacement for the pass CLI utility via subcommands and other operations, but obviously with Vault as a backend instead of GPG-encrypted flatfile hierarchy. -Obviously since the backends are vastly different, total parity is going to be impossible. But I try to get it pretty close. +Obviously since the backends are vastly different, total parity is going to be impossible but I try to get it pretty +close. Important deviations are <>. == Configuration @@ -43,13 +44,13 @@ easier. It's *highly* recommended to use them.] . The root element (`vaultpass`). This element contains attributes describing parsing/validation specifics as well, such as the https://www.w3.org/TR/xml-names/[namespace definitions^] and https://www.w3.org/TR/xmlschema11-1/#xsi_schemaLocation[schema location^].footnote:confheader[] -.. The `server` element.footnote:optelem[This element/attribute/text content is *optional*. See the item's description -for how default values/behaviour are determined.] This element is a container for connection and management of the -Vault server. This consists of: -... A single `uri` element.footnote:optelem[] It should be the same as the **base** URL for your Vault server. -The default (if not specified) is to first check for a **`VAULT_SERVER`** environment variable and, if not found, to use +.. The `server` element. This element is a container for connection and management of the +Vault server and is required (even though it may not have any children). This consists of: +... A single `uri` element.footnote:optelem[This element/attribute/text content is *optional*. See the item's description +for how default values/behaviour are determined.] It should be the same as the **base** URL for your Vault server. +If not specified, the default is to first check for a **`VAULT_ADDR`** environment variable and, if not found, to use `http://localhost:8200/`. -... An unseal directive, which can be used to (attempt to) automatically unseal the server if it is sealed. +... An unseal elementfootnote:optelem[], which can be used to (attempt to) automatically unseal the server if it is sealed. This isn't required, but can assist in automatic operation. One of either:footnote:optelem[] .... `unseal`, the unseal key shard (a Base64 string), or @@ -60,6 +61,9 @@ one of either: .... `authGpg`, an <> config snippet encrypted with GPG. See the section on <>. ... An optional `mounts` container.footnote:optelem[] See the section on <>. +If you would like to initialize Vault with VaultPass, use a self-enclosed <> auth stanza. It will automatically +be replaced once a root token is generated. + Let's look at an example configuration. === Example Configuration @@ -88,10 +92,11 @@ Let's look at an example configuration. ---- -In the above, we can see that it would use the vault server at `http://localhost:8200/` using whatever token is either +In the above, we can see that it would use the Vault server at `http://localhost:8200/` using whatever token is either in the **`VAULT_TOKEN`** environment variable or, if empty, the `~/.vault-token` file. Because an unseal shard was provided, it will be able to attempt to automatically unseal the Vault (assuming its shard will complete the threshold -needed). Because we specify mounts, we do not need permissions in Vault to list `/sys/mounts`. +needed). Because we specify mounts, we do not need permissions in Vault to list `/sys/mounts` (but if our token has +access to do so per its policy, then any automatically discovered will be added). === Auth Vault itself supports a https://www.vaultproject.io/docs/auth/[large number of authentication methods^]. However, in @@ -175,6 +180,10 @@ To determine the behaviour of how this behaves, please refer to the below table. | 4 |token contained in tags, `source` given |Same as **3**; `source` is ignored. |=== +If the Vault instance is not initialized and a `vaultpass init` is called, the configuration file will be updated to +use token auth, populated with the new root token, and populated with the new unseal shard. (The previous configuration +file will be backed up first!). + ===== Example Snippet [source,xml] ---- @@ -399,7 +408,7 @@ alias vaultpass='vaultpass -c ~/.config/alternate.vaultpass.xml' To use the non-aliased command in Bash, you can either invoke the full path: -[source,bash] +[source] ---- /usr/local/bin/vaultpass edit path/to/secret ---- @@ -446,19 +455,33 @@ flags/switches to subcommands. **Some** configuration directives/behaviour may b where supported by Vault/Pass upstream configuration. === Vault Paths Don't Match VaultPass' Paths -=== Issue Description +==== Issue Description Pass and Vault have fundamentally different storage ideas. Pass secrets/passwords are, once decrypted, just plaintext blobs. Vault, on the other hand, uses a key/value type of storage. As a result, this means two things: * The last item in a path in VaultPass is the key name (e.g. the path `foo/bar/baz` in VaultPass would be a Vault path of `foo/bar`, which would then have a **key** named `baz`), and * The **`line-number`** sub-argument is completely irrelevant for things like copying to the clipboard and generating a -QR code (e.g. as in `pass show --clip`**`=line-number`**). +QR code (e.g. as in `pass show --clip=line-number`). ==== Workaround(s) None, aside from not using the `line-number` sub-argument since it's no longer relevant. (You'll get an error if you do.) +=== Unable to specify `line-number` +See <> (_Vault Paths Don't Match VaultPass' Paths_). + +=== Deleting Secrets in KV2 +==== Issue Description +In Pass, because it doesn't have versioning (unless you're using git with your Pass instance). Vault's `kv2` engine, +however, does have versioning. As a result, once a secret is "deleted", it can still be recovered via +https://www.vaultproject.io/docs/secrets/kv/kv-v2/#deleting-and-destroying-data[an `undelete` method^]. If you are +deleting a secret for security reasons, you may want to destroy it instead. VaultPass' delete method uses a delete +rather than a destroy. + +==== Workaround(s) +VaultPass has a new subcommand, `destroy`, which will remove versioned secrets **permanently**. Use with caution, +obviously. If called on a non-KV2 mount's path, it will be the same as the `delete` subcommand. == Submitting a Bug Report/Feature Request Please use https://bugs.square-r00t.net/index.php?do=newtask&project=13[my bugtracker^]. diff --git a/vaultpass/__init__.py b/vaultpass/__init__.py index 4c8911e..99cf57d 100644 --- a/vaultpass/__init__.py +++ b/vaultpass/__init__.py @@ -2,6 +2,7 @@ import logging import tempfile import os import subprocess +import time ## from . import logger _logger = logging.getLogger('VaultPass') @@ -25,13 +26,14 @@ class VaultPass(object): uri = None mount = None - def __init__(self, mount, cfg = '~/.config/vaultpass.xml'): - self.mname = mount + def __init__(self, initialize = False, cfg = '~/.config/vaultpass.xml'): + self.initialize = initialize self.cfg = config.getConfig(cfg) self._getURI() self.getClient() - self._checkSeal() - self._getMount() + if not self.initialize: + self._checkSeal() + self._getMount() def _checkSeal(self): _logger.debug('Checking and attempting unseal if necessary and possible.') @@ -51,48 +53,47 @@ class VaultPass(object): raise RuntimeError('Unable to unseal') return(None) - def _getHandler(self, mount = None, func = 'read', *args, **kwargs): - if func not in ('read', 'write', 'list'): + def _getConfirm(self, msg = None): + if not msg: + msg = 'Are you sure (y/N)? ' + confirm = input(msg) + confirm = confirm.lower().strip() + if confirm.startswith('y'): + return(True) + return(False) + + def _getHandler(self, mount, func = 'read', *args, **kwargs): + if func not in ('read', 'write', 'list', 'delete', 'destroy'): _logger.error('Invalid func') - _logger.debug('Invalid func; must be one of: read, write, list, update') + _logger.debug('Invalid func; must be one of: read, write, list, delete, destroy') raise ValueError('Invalid func') - if not mount: - mount = self.mname mtype = self.mount.getMountType(mount) handler = None - if mtype == 'cubbyhole': - if func == 'read': - handler = self.mount.cubbyhandler.read_secret - elif func == 'write': - handler = self.mount.cubbyhandler.write_secret - elif func == 'list': - handler = self.mount.cubbyhandler.list_secrets - elif mtype == 'kv1': - if func == 'read': - handler = self.client.secrets.kv.v1.read_secret - elif func == 'write': - handler = self.client.secrets.kv.v1.create_or_update_secret - elif func == 'list': - handler = self.client.secrets.kv.v1.list_secrets - elif mtype == 'kv2': - if func == 'read': - handler = self.client.secrets.kv.v2.read_secret_version - elif func == 'write': - handler = self.client.secrets.kv.v2.create_or_update_secret - elif func == 'list': - handler = self.client.secrets.kv.v2.list_secrets + handler_map = {'cubbyhole': {'read': self.mount.cubbyhandler.read_secret, + 'write': self.mount.cubbyhandler.write_secret, + 'list': self.mount.cubbyhandler.list_secrets, + 'delete': self.mount.cubbyhandler.remove_secret, + 'destroy': self.mount.cubbyhandler.remove_secret}, + 'kv1': {'read': self.client.secrets.kv.v1.read_secret, + 'write': self.client.secrets.kv.v1.create_or_update_secret, + 'list': self.client.secrets.kv.v1.list_secrets, + 'delete': self.client.secrets.kv.v1.delete_secret, + 'destroy': self.client.secrets.kv.v1.delete_secret}, + 'kv2': {'read': self.client.secrets.kv.v2.read_secret_version, + 'write': self.client.secrets.kv.v2.create_or_update_secret, + 'list': self.client.secrets.kv.v2.list_secrets, + 'delete': self.client.secrets.kv.v2.delete_secret_versions, + 'destroy': self.client.secrets.kv.v2.destroy_secret_versions}} + handler = handler_map.get(mtype, {}).get(func, None) if not handler: _logger.error('Could not get handler') - _logger.debug('Could not get handler for mount {0}'.format(mount)) + _logger.debug('Could not get handler for function {0} on mount {1} (type {2})'.format(func, mount, mtype)) raise RuntimeError('Could not get handler') return(handler) def _getMount(self): mounts_xml = self.cfg.xml.find('.//mounts') self.mount = mounts.MountHandler(self.client, mounts_xml = mounts_xml) - if self.mname: - # Check that the mount exists - self.mount.getMountType(self.mname) return(None) def _getURI(self): @@ -111,13 +112,20 @@ class VaultPass(object): _logger.debug('Set URI to {0}'.format(self.uri)) return(None) - def _pathExists(self, path, mount = None, *args, **kwargs): - if not mount: - mount = self.mname - exists = False - if self.mount.getPath(path, mount): - exists = True - return(exists) + def _pathExists(self, path, mount, is_secret = False, *args, **kwargs): + kname = None + if is_secret: + lpath = path.split('/') + path = '/'.join(lpath[0:-1]) + kname = lpath[-1] + path_obj = self.mount.getPath(path, mount) + if path_obj: + if not is_secret: + return(True) + else: + if kname in path_obj.keys(): + return(True) + return(False) def convert(self, mount, @@ -127,16 +135,29 @@ class VaultPass(object): *args, **kwargs): pass # TODO - def copySecret(self, oldpath, newpath, mount, newmount, force = False, remove_old = False, *args, **kwargs): + def copySecret(self, oldpath, newpath, mount, newmount = None, force = False, remove_old = False, *args, **kwargs): mtype = self.mount.getMountType(mount) + if not newmount: + newmount = mount + newmtype = mtype + else: + newmtype = self.mount.getMountType(newmount) oldexists = self._pathExists(oldpath, mount = mount) if not oldexists: _logger.error('oldpath does not exist') _logger.debug('The oldpath {0} does not exist'.format(oldpath)) raise ValueError('oldpath does not exist') data = self.getSecret(oldpath, mount) + if not data: + _logger.error('No secret found') + _logger.debug('The secret at path {0} on mount {1} does not exist.'.format(oldpath, mount)) # TODO: left off here - newexists = self._pathExists(newpath, mount = mount) + newexists = self._pathExists(newpath, mount = newmount) + if newexists and not force: + _logger.debug('The newpath {0} exists; prompting for confirmation.'.format(newpath)) + confirm = self._getConfirm('The destination {0} exists. Overwrite (y/N)?'.format(newpath)) + if not confirm: + return(None) if remove_old: self.deleteSecret(oldpath, mount, force = force) @@ -161,7 +182,16 @@ class VaultPass(object): resp = handler(**args) return(resp) - def deleteSecret(self, path, mount_name, force = False, recursive = False, *args, **kwargs): + def deleteSecret(self, path, mount, force = False, recursive = False, *args, **kwargs): + mtype = self.mount.getMountType(mount) + args = {'path': path, + 'mount_point': mount} + handler = self._getHandler(mount, func = 'delete') + is_path = self._pathExists(path, mount) + is_secret = self._pathExists(path, mount, is_secret = True) + + + def destroySecret(self, path, mount, force = False, recursive = False, *args, **kwargs): pass # TODO def editSecret(self, path, mount, editor = constants.EDITOR, *args, **kwargs): @@ -208,7 +238,7 @@ class VaultPass(object): _logger.error('Invalid auth configuration') raise RuntimeError('Invalid auth configuration') self.client = self.auth.client - if not self.client.sys.is_initialized(): + if not self.client.sys.is_initialized() and not self.initialize: _logger.debug('Vault instance is not initialized. Please initialize (and configure, if necessary) first.') _logger.error('Not initialized') raise RuntimeError('Not initialized') @@ -245,6 +275,7 @@ class VaultPass(object): 'qr': qr, 'seconds': seconds, 'printme': printme} + # Add return here? data = self.getSecret(**args) if qr not in (False, None): qrdata, has_x = QR.genQr(data, image = True) @@ -255,9 +286,14 @@ class VaultPass(object): fh.write(qrdata.read()) if printme: _logger.debug('Opening {0} in the default image viwer application'.format(fpath)) - # We intentionally want this to block, as most image viewers will - # unload the image once the file is deleted and we can probably - # elete it before the user can save it elsewhere or scan it with their phone. + # We intentionally want this to block, as most image viewers will unload the image once the file + # is deleted and we can probably delete it faster than the user can save it elsewhere or + # scan it with their phone. + # TODO: we could use Popen() and do a countdown for "seconds" seconds, and then kill the viewer. + # But that breaks compat with Pass' behaviour. + if printme: + print('Now displaying generated QR code. Please close the viewer when done saving/scanning to ' + 'securely clean up the generated file...') cmd = subprocess.run(['xdg-open', fpath], stdout = subprocess.PIPE, stderr = subprocess.PIPE) if cmd.returncode != 0: _logger.error('xdg-open returned non-zero status code') @@ -268,16 +304,35 @@ class VaultPass(object): o = o.decode('utf-8').strip() if o != '': _logger.debug('{0}: {1}'.format(x.upper(), o)) + if printme: + print('Done. Deleting generated file.') os.remove(fpath) elif printme: print(qrdata.read()) qrdata.seek(0, 0) + del(qrdata) if clip not in (False, None): clipboard.pasteClipboard(data, seconds = seconds, clipboard = clipboard, printme = printme) return(data) def initVault(self, *args, **kwargs): - pass # TODO + if not self.client.sys.is_initialized(): + init_rslt = self.client.sys.initialize(secret_shares = 1, secret_threshold = 1) + unseal = init_rslt['keys_base64'][0] + token = init_rslt['root_token'] + self.cfg.updateAuth(unseal, token) + self.client.sys.submit_unseal_key(unseal) + self.client.token = token + # JUST in case. + time.sleep(1) + for mname, mtype in self.mount.mounts.items(): + if mtype == 'cubbyhole': + # There isn't a way to "create" a cubbyhole. + continue + self.mount.createMount(mname, mtype) + self._checkSeal() + self._getMount() + return(None) def insertSecret(self, path, diff --git a/vaultpass/args.py b/vaultpass/args.py index abd422e..fbbeb4c 100644 --- a/vaultpass/args.py +++ b/vaultpass/args.py @@ -67,6 +67,9 @@ def parseArgs(): description = ('Delete a secret'), help = ('Delete a secret'), aliases = ['remove', 'delete']) + destroy = subparser.add_parser('destroy', + description = ('Destroy a secret permanently'), + help = ('Destroy a secret permanently')) show = subparser.add_parser('show', description = ('Print/fetch a secret'), help = ('Print/fetch a secret')) @@ -399,7 +402,7 @@ def parseArgs(): rm.add_argument('-r', '--recursive', dest = 'recurse', action = 'store_true', - help = ('If PATH/TO/SECRET is a directory, delete all subentries')) + help = ('If PATH/TO/SECRET is a directory, delete all subentries along with the path itself')) rm.add_argument('-f', '--force', dest = 'force', action = 'store_true', @@ -407,6 +410,18 @@ def parseArgs(): rm.add_argument('path', metavar = 'PATH/TO/SECRET', help = ('The path to the secret or subdirectory')) + # DESTROY + destroy.add_argument('-r', '--recursive', + dest = 'recurse', + action = 'store_true', + help = ('If PATH/TO/SECRET is a directory, delete all subentries along with the path itself')) + destroy.add_argument('-f', '--force', + dest = 'force', + action = 'store_true', + help = ('If specified, delete all matching path(s) without prompting for confirmation')) + destroy.add_argument('path', + metavar = 'PATH/TO/SECRET', + help = ('The path to the secret or subdirectory')) # SHOW # vp.getSecret(printme = True) # TODO: does the default overwrite the None if not specified? diff --git a/vaultpass/auth.py b/vaultpass/auth.py index 076e89c..1d35e9f 100644 --- a/vaultpass/auth.py +++ b/vaultpass/auth.py @@ -1,5 +1,6 @@ import logging import os +import warnings ## import hvac @@ -111,7 +112,7 @@ class Token(_AuthBase): _logger.debug(('Environment variable {0} was specified as containing the token ' 'but it is empty').format(env_var)) _logger.error('Env var not populated') - raise RuntimeError('Env var not populated') + raise OSError('Env var not populated') return(var) def _getFile(self, fpath): @@ -122,6 +123,7 @@ class Token(_AuthBase): def getClient(self): _token = self.xml.text + chk = True if _token is not None: self.token = _token else: @@ -134,17 +136,20 @@ class Token(_AuthBase): try: self._getEnv('VAULT_TOKEN') break - except Exception as e: + except OSError as e: pass try: self._getFile('~/.vault-token') + _exhausted = True except Exception as e: _exhausted = True if not self.token: _logger.debug(('Unable to automatically determine token from ' - 'environment variable or filesystem defaults')) - _logger.error('Cannot determine token') - raise RuntimeError('Cannot determine token') + 'environment variable or filesystem defaults. Ignore this if you are initializing ' + 'Vault.')) + _logger.warning('Cannot determine token') + warnings.warn('Cannot determine token') + chk = False else: if a.startswith('env:'): e = a.split(':', 1) @@ -156,7 +161,8 @@ class Token(_AuthBase): _logger.info('Initialized client.') self.client.token = self.token _logger.debug('Applied token.') - self.authCheck() + if chk: + self.authCheck() return(None) diff --git a/vaultpass/config.py b/vaultpass/config.py index 3c9493f..7405761 100644 --- a/vaultpass/config.py +++ b/vaultpass/config.py @@ -1,7 +1,9 @@ import copy +import datetime import os import logging import re +import shutil ## from . import gpg_handler import requests @@ -217,6 +219,40 @@ class Config(object): _logger.debug('Rendered string output successfully.') return(strxml) + def updateAuth(self, unseal_shard, token): + nsmap = self.namespaced_xml.nsmap + unseal_ns_xml = self.namespaced_xml.find('.//{{{0}}}unseal'.format(nsmap[None])) + unseal_xml = self.xml.find('.//unseal') + auth_ns_xml = self.namespaced_xml.find('.//{{{0}}}auth'.format(nsmap[None])) + auth_xml = self.xml.find('.//auth') + token_ns_xml = auth_ns_xml.find('.//{{{0}}}token'.format(nsmap[None])) + token_xml = auth_xml.find('.//token') + if token_xml is None: + # Config is using a non-token auth, so we replace it. + newauth_xml = etree.Element('auth') + newauth_ns_xml = etree.Element('auth', nsmap = nsmap) + token_xml = etree.SubElement(newauth_xml, 'token') + token_ns_xml = etree.SubElement(newauth_ns_xml, 'token', nsmap = nsmap) + auth_xml.getparent().replace(auth_xml, newauth_xml) + auth_ns_xml.getparent().replace(auth_ns_xml, newauth_ns_xml) + if unseal_xml is None: + # And we need to add the unseal as well. + server_xml = self.xml.find('.//server') + server_ns_xml = self.namespaced_xml.find('.//{{{0}}}server'.format(nsmap[None])) + unseal_xml = etree.SubElement(server_xml, 'unseal') + unseal_ns_xml = etree.SubElement(server_ns_xml, 'unseal') + unseal_xml.text = unseal_shard + unseal_ns_xml.text = unseal_shard + token_xml.text = token + token_ns_xml.text = token + self.parse() + if isinstance(self, LocalFile): + bakpath = '{0}.bak_{1}'.format(self.source, datetime.datetime.utcnow().timestamp()) + shutil.copy(self.source, bakpath) + with open(self.source, 'wb') as fh: + fh.write(self.toString()) + return(None) + def validate(self): if not self.xsd: self.getXSD() diff --git a/vaultpass/mounts.py b/vaultpass/mounts.py index 37488c5..49f7051 100644 --- a/vaultpass/mounts.py +++ b/vaultpass/mounts.py @@ -43,6 +43,12 @@ class CubbyHandler(object): resp = self.client._adapter.get(url = uri) return(resp.json()) + def remove_secret(self, path, mount_point = 'cubbyhole', *args, **kwargs): + path = path.lstrip('/') + uri = '{0}/{1}'.format(mount_point, path) + resp = self.client._adapter.delete(url = uri) + return(resp.json()) + def write_secret(self, path, secret, mount_point = 'cubbyhole', *args, **kwargs): path = path.lstrip('/') args = {'path': '/'.join((mount_point, path))}