Over a million developers have joined DZone.
Platinum Partner

Happybase Connection Pooling

· DevOps Zone

The DevOps Zone is brought to you by DZone in partnership with Hewlett Packard Enterprise DevOps. Learn where you are in your DevOps journey by taking the HPE/Forrester benchmarking tool.

Wrote a simple connection pool for Happybase using socketpool:

import time
import random
import contextlib
import happybase
from socketpool import ConnectionPool
from socketpool.conn import TcpConnector


class HappybaseConnectionPool(object):
    ''' singleton to share a connection pool per process '''

    _instance = None

    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(HappybaseConnectionPool, cls).__new__(cls, *args, **kwargs)
        return cls._instance

    def __init__(self, host, **options):
        options['host'] = host
        if not hasattr(self, 'pool'):
            self.pool = ConnectionPool(
                factory=HappybaseConnector,
                max_size=options.get('max_size', 10),
                options=options,
            )

    def connection(self, **options):
        return self.pool.connection(**options)

    @contextlib.contextmanager
    def table(self, table_name):
        with self.pool.connection() as connector:
            yield connector.table(table_name)


class HappybaseConnector(TcpConnector):

    def __init__(self, host, port, pool=None, **kwargs):
        self.host = host
        self.port = port
        self.connection = happybase.Connection(self.host, self.port)
        self._connected = True
        # use a 'jiggle' value to make sure there is some
        # randomization to expiry, to avoid many conns expiring very
        # closely together.
        self._life = time.time() - random.randint(0, 10)
        self._pool = pool
        self.logging = kwargs.get('logging')

    def is_connected(self):
        if self._connected and self.connection.transport.isOpen():
            try:
                # isOpen is unreliable, actually try to do something
                self.connection.tables()
                return True
            except:
                pass
        return False

    def handle_exception(self, exception):
        if self.logging:
            self.logging.error(exception)
        else:
            print exception

    def invalidate(self):
        self.connection.close()
        self._connected = False
        self._life = -1

    def open(self):
        pass

    def close(self):
        self.release()

    def __getattr__(self, name):
        if name in ['table', 'tables', 'create_table', 'delete_table',
                'enable_table', 'disable_table', 'is_table_enabled', 'compact_table']:
            return getattr(self.connection, name)
        else:
            raise AttributeError(name)

You can get a single pool object from anywhere in your stack with the following:

pool = HappybaseConnectionPool('localhost', '9090')
with pool.connection() as connection:
     connection.create_table('foobar')

Note: happybase may be adding their own connection pool shortly.

The DevOps Zone is brought to you in partnership with Hewlett Packard Enterprise.  Download Through the DevOps Looking Glass: Learnings from HP's Own Transformation Initiative to get valuable best practices and “lessons learned” straight from the HP IT team to help you start or progress along your own DevOps Journey.

Topics:

Published at DZone with permission of Chase Seibert , DZone MVB .

Opinions expressed by DZone contributors are their own.

{{ parent.title || parent.header.title}}

{{ parent.tldr }}

{{ parent.urlSource.name }}