HEX
Server: Apache
System: Linux sg241.singhost.net 2.6.32-896.16.1.lve1.4.51.el6.x86_64 #1 SMP Wed Jan 17 13:19:23 EST 2018 x86_64
User: honghock (909)
PHP: 8.0.30
Disabled: passthru,system,shell_exec,show_source,exec,popen,proc_open
Upload Files
File: //proc/self/root/usr/lib/python2.7/site-packages/salt/returners/couchbase_return.py
# -*- coding: utf-8 -*-
'''
Simple returner for Couchbase. Optional configuration
settings are listed below, along with sane defaults.

.. code-block:: yaml

    couchbase.host:   'salt'
    couchbase.port:   8091
    couchbase.bucket: 'salt'
    couchbase.ttl: 24
    couchbase.password: 'password'
    couchbase.skip_verify_views: False

To use the couchbase returner, append '--return couchbase' to the salt command. ex:

.. code-block:: bash

    salt '*' test.ping --return couchbase

To use the alternative configuration, append '--return_config alternative' to the salt command.

.. versionadded:: 2015.5.0

.. code-block:: bash

    salt '*' test.ping --return couchbase --return_config alternative

To override individual configuration items, append --return_kwargs '{"key:": "value"}' to the salt command.

.. versionadded:: 2016.3.0

.. code-block:: bash

    salt '*' test.ping --return couchbase --return_kwargs '{"bucket": "another-salt"}'


All of the return data will be stored in documents as follows:

JID
===
load: load obj
tgt_minions: list of minions targeted
nocache: should we not cache the return data

JID/MINION_ID
=============
return: return_data
full_ret: full load of job return
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging

try:
    import couchbase
    HAS_DEPS = True
except ImportError:
    HAS_DEPS = False

# Import Salt libs
import salt.utils.jid
import salt.utils.json
import salt.utils.minions

# Import 3rd-party libs
from salt.ext import six

log = logging.getLogger(__name__)


# Define the module's virtual name
__virtualname__ = 'couchbase'

# some globals
COUCHBASE_CONN = None
DESIGN_NAME = 'couchbase_returner'
VERIFIED_VIEWS = False

_json = salt.utils.json.import_json()


def _json_dumps(obj, **kwargs):
    return salt.utils.json.dumps(obj, _json_module=_json)


def __virtual__():
    if not HAS_DEPS:
        return False, 'Could not import couchbase returner; couchbase is not installed.'

    couchbase.set_json_converters(_json_dumps, salt.utils.json.loads)

    return __virtualname__


def _get_options():
    '''
    Get the couchbase options from salt. Apply defaults
    if required.
    '''
    return {'host': __opts__.get('couchbase.host', 'salt'),
            'port': __opts__.get('couchbase.port', 8091),
            'bucket': __opts__.get('couchbase.bucket', 'salt'),
            'password': __opts__.get('couchbase.password', '')}


def _get_connection():
    '''
    Global function to access the couchbase connection (and make it if its closed)
    '''
    global COUCHBASE_CONN
    if COUCHBASE_CONN is None:
        opts = _get_options()
        if opts['password']:
            COUCHBASE_CONN = couchbase.Couchbase.connect(host=opts['host'],
                                                         port=opts['port'],
                                                         bucket=opts['bucket'],
                                                         password=opts['password'])
        else:
            COUCHBASE_CONN = couchbase.Couchbase.connect(host=opts['host'],
                                                         port=opts['port'],
                                                         bucket=opts['bucket'])

    return COUCHBASE_CONN


def _verify_views():
    '''
    Verify that you have the views you need. This can be disabled by
    adding couchbase.skip_verify_views: True in config
    '''
    global VERIFIED_VIEWS

    if VERIFIED_VIEWS or __opts__.get('couchbase.skip_verify_views', False):
        return
    cb_ = _get_connection()
    ddoc = {'views': {'jids': {'map': "function (doc, meta) { if (meta.id.indexOf('/') === -1 && doc.load){ emit(meta.id, null) } }"},
                      'jid_returns': {'map': "function (doc, meta) { if (meta.id.indexOf('/') > -1){ key_parts = meta.id.split('/'); emit(key_parts[0], key_parts[1]); } }"}
                      }
            }

    try:
        curr_ddoc = cb_.design_get(DESIGN_NAME, use_devmode=False).value
        if curr_ddoc['views'] == ddoc['views']:
            VERIFIED_VIEWS = True
            return
    except couchbase.exceptions.HTTPError:
        pass

    cb_.design_create(DESIGN_NAME, ddoc, use_devmode=False)
    VERIFIED_VIEWS = True


def _get_ttl():
    '''
    Return the TTL that we should store our objects with
    '''
    return __opts__.get('couchbase.ttl', 24) * 60 * 60  # keep_jobs is in hours


#TODO: add to returner docs-- this is a new one
def prep_jid(nocache=False, passed_jid=None):
    '''
    Return a job id and prepare the job id directory
    This is the function responsible for making sure jids don't collide (unless
    its passed a jid)
    So do what you have to do to make sure that stays the case
    '''
    if passed_jid is None:
        jid = salt.utils.jid.gen_jid(__opts__)
    else:
        jid = passed_jid

    cb_ = _get_connection()

    try:
        cb_.add(six.text_type(jid),
               {'nocache': nocache},
               ttl=_get_ttl(),
               )
    except couchbase.exceptions.KeyExistsError:
        # TODO: some sort of sleep or something? Spinning is generally bad practice
        if passed_jid is None:
            return prep_jid(nocache=nocache)

    return jid


def returner(load):
    '''
    Return data to couchbase bucket
    '''
    cb_ = _get_connection()

    hn_key = '{0}/{1}'.format(load['jid'], load['id'])
    try:
        ret_doc = {'return': load['return'],
                   'full_ret': salt.utils.json.dumps(load)}

        cb_.add(hn_key,
               ret_doc,
               ttl=_get_ttl(),
               )
    except couchbase.exceptions.KeyExistsError:
        log.error(
            'An extra return was detected from minion %s, please verify '
            'the minion, this could be a replay attack', load['id']
        )
        return False


def save_load(jid, clear_load, minion=None):
    '''
    Save the load to the specified jid
    '''
    cb_ = _get_connection()

    try:
        jid_doc = cb_.get(six.text_type(jid))
    except couchbase.exceptions.NotFoundError:
        cb_.add(six.text_type(jid), {}, ttl=_get_ttl())
        jid_doc = cb_.get(six.text_type(jid))

    jid_doc.value['load'] = clear_load
    cb_.replace(six.text_type(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl())

    # if you have a tgt, save that for the UI etc
    if 'tgt' in clear_load and clear_load['tgt'] != '':
        ckminions = salt.utils.minions.CkMinions(__opts__)
        # Retrieve the minions list
        _res = ckminions.check_minions(
            clear_load['tgt'],
            clear_load.get('tgt_type', 'glob')
            )
        minions = _res['minions']
        save_minions(jid, minions)


def save_minions(jid, minions, syndic_id=None):  # pylint: disable=unused-argument
    '''
    Save/update the minion list for a given jid. The syndic_id argument is
    included for API compatibility only.
    '''
    cb_ = _get_connection()

    try:
        jid_doc = cb_.get(six.text_type(jid))
    except couchbase.exceptions.NotFoundError:
        log.warning('Could not write job cache file for jid: %s', jid)
        return False

    # save the minions to a cache so we can see in the UI
    if 'minions' in jid_doc.value:
        jid_doc.value['minions'] = sorted(
            set(jid_doc.value['minions'] + minions)
        )
    else:
        jid_doc.value['minions'] = minions
    cb_.replace(six.text_type(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl())


def get_load(jid):
    '''
    Return the load data that marks a specified jid
    '''
    cb_ = _get_connection()

    try:
        jid_doc = cb_.get(six.text_type(jid))
    except couchbase.exceptions.NotFoundError:
        return {}

    ret = {}
    try:
        ret = jid_doc.value['load']
        ret['Minions'] = jid_doc.value['minions']
    except KeyError as e:
        log.error(e)

    return ret


def get_jid(jid):
    '''
    Return the information returned when the specified job id was executed
    '''
    cb_ = _get_connection()
    _verify_views()

    ret = {}

    for result in cb_.query(DESIGN_NAME, 'jid_returns', key=six.text_type(jid), include_docs=True):
        ret[result.value] = result.doc.value

    return ret


def get_jids():
    '''
    Return a list of all job ids
    '''
    cb_ = _get_connection()
    _verify_views()

    ret = {}

    for result in cb_.query(DESIGN_NAME, 'jids', include_docs=True):
        ret[result.key] = _format_jid_instance(result.key, result.doc.value['load'])

    return ret


def _format_job_instance(job):
    '''
    Return a properly formatted job dict
    '''
    ret = {'Function': job.get('fun', 'unknown-function'),
           'Arguments': list(job.get('arg', [])),
           # unlikely but safeguard from invalid returns
           'Target': job.get('tgt', 'unknown-target'),
           'Target-type': job.get('tgt_type', 'list'),
           'User': job.get('user', 'root')}

    if 'metadata' in job:
        ret['Metadata'] = job.get('metadata', {})
    else:
        if 'kwargs' in job:
            if 'metadata' in job['kwargs']:
                ret['Metadata'] = job['kwargs'].get('metadata', {})
    return ret


def _format_jid_instance(jid, job):
    '''
    Return a properly formatted jid dict
    '''
    ret = _format_job_instance(job)
    ret.update({'StartTime': salt.utils.jid.jid_to_time(jid)})
    return ret