# -*- coding: utf-8 -*-
'''
TCP transport classes

Wire protocol: "len(payload) msgpack({'head': SOMEHEADER, 'body': SOMEBODY})"

'''

# Import Python Libs
from __future__ import absolute_import
import logging
import msgpack
import socket
import sys
import os
import weakref
import urlparse  # TODO: remove
import time
import traceback
import errno

# Import Salt Libs
import salt.crypt
import salt.utils
import salt.utils.verify
import salt.utils.event
import salt.utils.async
import salt.payload
import salt.exceptions
import salt.transport.frame
import salt.transport.ipc
import salt.transport.client
import salt.transport.server
import salt.transport.mixins.auth
from salt.exceptions import SaltReqTimeoutError, SaltClientError
from salt.transport import iter_transport_opts

# Import Tornado Libs
import tornado
import tornado.tcpserver
import tornado.gen
import tornado.concurrent
import tornado.tcpclient
import tornado.netutil

# Import third party libs
from Crypto.Cipher import PKCS1_OAEP

log = logging.getLogger(__name__)


def _set_tcp_keepalive(sock, opts):
    '''
    Ensure that TCP keepalives are set for the socket.
    '''
    if hasattr(socket, 'SO_KEEPALIVE'):
        if opts.get('tcp_keepalive', False):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
            if hasattr(socket, 'SOL_TCP'):
                if hasattr(socket, 'TCP_KEEPIDLE'):
                    tcp_keepalive_idle = opts.get('tcp_keepalive_idle', -1)
                    if tcp_keepalive_idle > 0:
                        sock.setsockopt(
                            socket.SOL_TCP, socket.TCP_KEEPIDLE,
                            int(tcp_keepalive_idle))
                if hasattr(socket, 'TCP_KEEPCNT'):
                    tcp_keepalive_cnt = opts.get('tcp_keepalive_cnt', -1)
                    if tcp_keepalive_cnt > 0:
                        sock.setsockopt(
                            socket.SOL_TCP, socket.TCP_KEEPCNT,
                            int(tcp_keepalive_cnt))
                if hasattr(socket, 'TCP_KEEPINTVL'):
                    tcp_keepalive_intvl = opts.get('tcp_keepalive_intvl', -1)
                    if tcp_keepalive_intvl > 0:
                        sock.setsockopt(
                            socket.SOL_TCP, socket.TCP_KEEPINTVL,
                            int(tcp_keepalive_intvl))
            if hasattr(socket, 'SIO_KEEPALIVE_VALS'):
                # Windows doesn't support TCP_KEEPIDLE, TCP_KEEPCNT, nor
                # TCP_KEEPINTVL. Instead, it has its own proprietary
                # SIO_KEEPALIVE_VALS.
                tcp_keepalive_idle = opts.get('tcp_keepalive_idle', -1)
                tcp_keepalive_intvl = opts.get('tcp_keepalive_intvl', -1)
                # Windows doesn't support changing something equivalent to
                # TCP_KEEPCNT.
                if tcp_keepalive_idle > 0 or tcp_keepalive_intvl > 0:
                    # Windows defaults may be found by using the link below.
                    # Search for 'KeepAliveTime' and 'KeepAliveInterval'.
                    # https://technet.microsoft.com/en-us/library/bb726981.aspx#EDAA
                    # If one value is set and the other isn't, we still need
                    # to send both values to SIO_KEEPALIVE_VALS and they both
                    # need to be valid. So in that case, use the Windows
                    # default.
                    if tcp_keepalive_idle <= 0:
                        tcp_keepalive_idle = 7200
                    if tcp_keepalive_intvl <= 0:
                        tcp_keepalive_intvl = 1
                    # The values expected are in milliseconds, so multiply by
                    # 1000.
                    sock.ioctl(socket.SIO_KEEPALIVE_VALS, (
                        1, int(tcp_keepalive_idle * 1000),
                        int(tcp_keepalive_intvl * 1000)))
        else:
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)


# TODO: move serial down into message library
class AsyncTCPReqChannel(salt.transport.client.ReqChannel):
    '''
    Encapsulate sending routines to tcp.

    Note: this class returns a singleton
    '''
    # This class is only a singleton per minion/master pair
    # mapping of io_loop -> {key -> channel}
    instance_map = weakref.WeakKeyDictionary()

    def __new__(cls, opts, **kwargs):
        '''
        Only create one instance of channel per __key()
        '''
        # do we have any mapping for this io_loop
        io_loop = kwargs.get('io_loop') or tornado.ioloop.IOLoop.current()
        if io_loop not in cls.instance_map:
            cls.instance_map[io_loop] = weakref.WeakValueDictionary()
        loop_instance_map = cls.instance_map[io_loop]

        key = cls.__key(opts, **kwargs)
        if key not in loop_instance_map:
            log.debug('Initializing new AsyncTCPReqChannel for {0}'.format(key))
            # we need to make a local variable for this, as we are going to store
            # it in a WeakValueDictionary-- which will remove the item if no one
            # references it-- this forces a reference while we return to the caller
            new_obj = object.__new__(cls)
            new_obj.__singleton_init__(opts, **kwargs)
            loop_instance_map[key] = new_obj
        else:
            log.debug('Re-using AsyncTCPReqChannel for {0}'.format(key))
        return loop_instance_map[key]

    @classmethod
    def __key(cls, opts, **kwargs):
        if 'master_uri' in kwargs:
            opts['master_uri'] = kwargs['master_uri']
        return (opts['pki_dir'],     # where the keys are stored
                opts['id'],          # minion ID
                opts['master_uri'],
                kwargs.get('crypt', 'aes'),  # TODO: use the same channel for crypt
                )

    # has to remain empty for singletons, since __init__ will *always* be called
    def __init__(self, opts, **kwargs):
        pass

    # an init for the singleton instance to call
    def __singleton_init__(self, opts, **kwargs):
        self.opts = dict(opts)

        self.serial = salt.payload.Serial(self.opts)

        # crypt defaults to 'aes'
        self.crypt = kwargs.get('crypt', 'aes')

        self.io_loop = kwargs.get('io_loop') or tornado.ioloop.IOLoop.current()

        if self.crypt != 'clear':
            self.auth = salt.crypt.AsyncAuth(self.opts, io_loop=self.io_loop)

        resolver = kwargs.get('resolver')

        parse = urlparse.urlparse(self.opts['master_uri'])
        host, port = parse.netloc.rsplit(':', 1)
        self.master_addr = (host, int(port))
        self._closing = False
        self.message_client = SaltMessageClient(
            self.opts, host, int(port), io_loop=self.io_loop,
            resolver=resolver)

    def close(self):
        if self._closing:
            return
        self._closing = True
        self.message_client.close()

    def __del__(self):
        self.close()

    def _package_load(self, load):
        return {
            'enc': self.crypt,
            'load': load,
        }

    @tornado.gen.coroutine
    def crypted_transfer_decode_dictentry(self, load, dictkey=None, tries=3, timeout=60):
        if not self.auth.authenticated:
            yield self.auth.authenticate()
        ret = yield self.message_client.send(self._package_load(self.auth.crypticle.dumps(load)), timeout=timeout)
        key = self.auth.get_keys()
        cipher = PKCS1_OAEP.new(key)
        aes = cipher.decrypt(ret['key'])
        pcrypt = salt.crypt.Crypticle(self.opts, aes)
        raise tornado.gen.Return(pcrypt.loads(ret[dictkey]))

    @tornado.gen.coroutine
    def _crypted_transfer(self, load, tries=3, timeout=60):
        '''
        In case of authentication errors, try to renegotiate authentication
        and retry the method.
        Indeed, we can fail too early in case of a master restart during a
        minion state execution call
        '''
        @tornado.gen.coroutine
        def _do_transfer():
            data = yield self.message_client.send(self._package_load(self.auth.crypticle.dumps(load)),
                                                  timeout=timeout,
                                                  )
            # we may not have always data
            # as for example for saltcall ret submission, this is a blind
            # communication, we do not subscribe to return events, we just
            # upload the results to the master
            if data:
                data = self.auth.crypticle.loads(data)
            raise tornado.gen.Return(data)

        if not self.auth.authenticated:
            yield self.auth.authenticate()
        try:
            ret = yield _do_transfer()
            raise tornado.gen.Return(ret)
        except salt.crypt.AuthenticationError:
            yield self.auth.authenticate()
            ret = yield _do_transfer()
            raise tornado.gen.Return(ret)

    @tornado.gen.coroutine
    def _uncrypted_transfer(self, load, tries=3, timeout=60):
        ret = yield self.message_client.send(self._package_load(load), timeout=timeout)
        raise tornado.gen.Return(ret)

    @tornado.gen.coroutine
    def send(self, load, tries=3, timeout=60):
        '''
        Send a request, return a future which will complete when we send the message
        '''
        try:
            if self.crypt == 'clear':
                ret = yield self._uncrypted_transfer(load, tries=tries, timeout=timeout)
            else:
                ret = yield self._crypted_transfer(load, tries=tries, timeout=timeout)
        except tornado.iostream.StreamClosedError:
            # Convert to 'SaltClientError' so that clients can handle this
            # exception more appropriately.
            raise SaltClientError('Connection to master lost')
        raise tornado.gen.Return(ret)


class AsyncTCPPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.transport.client.AsyncPubChannel):
    def __init__(self,
                 opts,
                 **kwargs):
        self.opts = opts

        self.serial = salt.payload.Serial(self.opts)

        self.crypt = kwargs.get('crypt', 'aes')
        self.io_loop = kwargs.get('io_loop') or tornado.ioloop.IOLoop.current()
        self.connected = False
        self._closing = False
        self._reconnected = False
        self.event = salt.utils.event.get_event(
            'minion',
            opts=self.opts,
            listen=False
        )

    def close(self):
        if self._closing:
            return
        self._closing = True
        if hasattr(self, 'message_client'):
            self.message_client.close()

    def __del__(self):
        self.close()

    def _package_load(self, load):
        return {
            'enc': self.crypt,
            'load': load,
        }

    @tornado.gen.coroutine
    def send_id(self, tok, force_auth):
        '''
        Send the minion id to the master so that the master may better
        track the connection state of the minion.
        In case of authentication errors, try to renegotiate authentication
        and retry the method.
        '''
        load = {'id': self.opts['id'], 'tok': tok}

        @tornado.gen.coroutine
        def _do_transfer():
            msg = self._package_load(self.auth.crypticle.dumps(load))
            package = salt.transport.frame.frame_msg(msg, header=None)
            yield self.message_client._stream.write(package)
            raise tornado.gen.Return(True)

        if force_auth or not self.auth.authenticated:
            yield self.auth.authenticate()
        try:
            ret = yield _do_transfer()
            raise tornado.gen.Return(ret)
        except salt.crypt.AuthenticationError:
            yield self.auth.authenticate()
            ret = yield _do_transfer()
            raise tornado.gen.Return(ret)

    @tornado.gen.coroutine
    def connect_callback(self, result):
        if self._closing:
            return
        # Force re-auth on reconnect since the master
        # may have been restarted
        yield self.send_id(self.tok, self._reconnected)
        self.connected = True
        self.event.fire_event(
            {'master': self.opts['master']},
            '__master_connected'
        )
        if self._reconnected:
            # On reconnects, fire a master event to notify that the minion is
            # available.
            if self.opts.get('__role') == 'syndic':
                data = 'Syndic {0} started at {1}'.format(
                    self.opts['id'],
                    time.asctime()
                )
                tag = salt.utils.event.tagify(
                    [self.opts['id'], 'start'],
                    'syndic'
                )
            else:
                data = 'Minion {0} started at {1}'.format(
                    self.opts['id'],
                    time.asctime()
                )
                tag = salt.utils.event.tagify(
                    [self.opts['id'], 'start'],
                    'minion'
                )
            load = {'id': self.opts['id'],
                    'cmd': '_minion_event',
                    'pretag': None,
                    'tok': self.tok,
                    'data': data,
                    'tag': tag}
            req_channel = salt.utils.async.SyncWrapper(
                AsyncTCPReqChannel, (self.opts,)
            )
            try:
                req_channel.send(load, timeout=60)
            except salt.exceptions.SaltReqTimeoutError:
                log.info('fire_master failed: master could not be contacted. Request timed out.')
            except Exception:
                log.info('fire_master failed: {0}'.format(
                    traceback.format_exc())
                )
        else:
            self._reconnected = True

    def disconnect_callback(self):
        if self._closing:
            return
        self.connected = False
        self.event.fire_event(
            {'master': self.opts['master']},
            '__master_disconnected'
        )

    @tornado.gen.coroutine
    def connect(self):
        try:
            self.auth = salt.crypt.AsyncAuth(self.opts, io_loop=self.io_loop)
            self.tok = self.auth.gen_token('salt')
            if not self.auth.authenticated:
                yield self.auth.authenticate()
            self.message_client = SaltMessageClient(
                self.opts,
                self.opts['master_ip'],
                int(self.auth.creds['publish_port']),
                io_loop=self.io_loop,
                connect_callback=self.connect_callback,
                disconnect_callback=self.disconnect_callback)
            yield self.message_client.connect()  # wait for the client to be connected
            self.connected = True
        # TODO: better exception handling...
        except KeyboardInterrupt:
            raise
        except:
            raise SaltClientError('Unable to sign_in to master')  # TODO: better error message

    def on_recv(self, callback):
        '''
        Register an on_recv callback
        '''
        if callback is None:
            return self.message_client.on_recv(callback)

        @tornado.gen.coroutine
        def wrap_callback(body):
            if not isinstance(body, dict):
                # TODO: For some reason we need to decode here for things
                #       to work. Fix this.
                body = msgpack.loads(body)
            ret = yield self._decode_payload(body)
            callback(ret)
        return self.message_client.on_recv(wrap_callback)


class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.transport.server.ReqServerChannel):
    # TODO: opts!
    backlog = 5

    def __init__(self, opts):
        salt.transport.server.ReqServerChannel.__init__(self, opts)
        self._socket = None

    @property
    def socket(self):
        return self._socket

    def close(self):
        if self._socket is not None:
            try:
                self._socket.shutdown(socket.SHUT_RDWR)
            except socket.error as exc:
                if exc.errno == errno.ENOTCONN:
                    # We may try to shutdown a socket which is already disconnected.
                    # Ignore this condition and continue.
                    pass
                else:
                    raise exc
            self._socket.close()
            self._socket = None

    def __del__(self):
        self.close()

    def pre_fork(self, process_manager):
        '''
        Pre-fork we need to create the zmq router device
        '''
        salt.transport.mixins.auth.AESReqServerMixin.pre_fork(self, process_manager)
        if not salt.utils.is_windows():
            self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            _set_tcp_keepalive(self._socket, self.opts)
            self._socket.setblocking(0)
            self._socket.bind((self.opts['interface'], int(self.opts['ret_port'])))

    def post_fork(self, payload_handler, io_loop):
        '''
        After forking we need to create all of the local sockets to listen to the
        router

        payload_handler: function to call with your payloads
        '''
        if salt.utils.is_windows():
            self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            _set_tcp_keepalive(self._socket, self.opts)
            self._socket.setblocking(0)
            self._socket.bind((self.opts['interface'], int(self.opts['ret_port'])))
        self.payload_handler = payload_handler
        self.io_loop = io_loop
        self.req_server = SaltMessageServer(self.handle_message, io_loop=self.io_loop)
        self.req_server.add_socket(self._socket)
        self._socket.listen(self.backlog)

        self.serial = salt.payload.Serial(self.opts)
        salt.transport.mixins.auth.AESReqServerMixin.post_fork(self, payload_handler, io_loop)

    @tornado.gen.coroutine
    def handle_message(self, stream, header, payload):
        '''
        Handle incoming messages from underylying tcp streams
        '''
        try:
            payload = self._decode_payload(payload)
        except Exception:
            stream.write(salt.transport.frame.frame_msg('bad load', header=header))
            raise tornado.gen.Return()

        # TODO helper functions to normalize payload?
        if not isinstance(payload, dict) or not isinstance(payload.get('load'), dict):
            yield stream.write(salt.transport.frame.frame_msg(
                'payload and load must be a dict', header=header))
            raise tornado.gen.Return()

        # intercept the "_auth" commands, since the main daemon shouldn't know
        # anything about our key auth
        if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth':
            yield stream.write(salt.transport.frame.frame_msg(
                self._auth(payload['load']), header=header))
            raise tornado.gen.Return()

        # TODO: test
        try:
            ret, req_opts = yield self.payload_handler(payload)
        except Exception as e:
            # always attempt to return an error to the minion
            stream.write('Some exception handling minion payload')
            log.error('Some exception handling a payload from minion', exc_info=True)
            stream.close()
            raise tornado.gen.Return()

        req_fun = req_opts.get('fun', 'send')
        if req_fun == 'send_clear':
            stream.write(salt.transport.frame.frame_msg(ret, header=header))
        elif req_fun == 'send':
            stream.write(salt.transport.frame.frame_msg(self.crypticle.dumps(ret), header=header))
        elif req_fun == 'send_private':
            stream.write(salt.transport.frame.frame_msg(self._encrypt_private(ret,
                                                         req_opts['key'],
                                                         req_opts['tgt'],
                                                         ), header=header))
        else:
            log.error('Unknown req_fun {0}'.format(req_fun))
            # always attempt to return an error to the minion
            stream.write('Server-side exception handling payload')
            stream.close()
        raise tornado.gen.Return()


class SaltMessageServer(tornado.tcpserver.TCPServer, object):
    '''
    Raw TCP server which will receive all of the TCP streams and re-assemble
    messages that are sent through to us
    '''
    def __init__(self, message_handler, *args, **kwargs):
        super(SaltMessageServer, self).__init__(*args, **kwargs)

        self.clients = []
        self.message_handler = message_handler

    @tornado.gen.coroutine
    def handle_stream(self, stream, address):
        '''
        Handle incoming streams and add messages to the incoming queue
        '''
        log.trace('Req client {0} connected'.format(address))
        self.clients.append((stream, address))
        unpacker = msgpack.Unpacker()
        try:
            while True:
                wire_bytes = yield stream.read_bytes(4096, partial=True)
                unpacker.feed(wire_bytes)
                for framed_msg in unpacker:
                    header = framed_msg['head']
                    self.io_loop.spawn_callback(self.message_handler, stream, header, framed_msg['body'])

        except tornado.iostream.StreamClosedError:
            log.trace('req client disconnected {0}'.format(address))
            self.clients.remove((stream, address))
        except Exception as e:
            log.trace('other master-side exception: {0}'.format(e))
            self.clients.remove((stream, address))
            stream.close()

    def shutdown(self):
        '''
        Shutdown the whole server
        '''
        for item in self.clients:
            client, address = item
            client.close()
            self.clients.remove(item)


class TCPClientKeepAlive(tornado.tcpclient.TCPClient):
    '''
    Override _create_stream() in TCPClient to enable keep alive support.
    '''
    def __init__(self, opts, resolver=None, io_loop=None):
        self.opts = opts
        super(TCPClientKeepAlive, self).__init__(
            resolver=resolver, io_loop=io_loop)

    def _create_stream(self, max_buffer_size, af, addr):
        # Always connect in plaintext; we'll convert to ssl if necessary
        # after one connection has completed.
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        _set_tcp_keepalive(sock, self.opts)
        stream = tornado.iostream.IOStream(
            sock,
            io_loop=self.io_loop,
            max_buffer_size=max_buffer_size)
        return stream.connect(addr)


# TODO consolidate with IPCClient
# TODO: limit in-flight messages.
# TODO: singleton? Something to not re-create the tcp connection so much
class SaltMessageClient(object):
    '''
    Low-level message sending client
    '''
    def __init__(self, opts, host, port, io_loop=None, resolver=None,
                 connect_callback=None, disconnect_callback=None):
        self.host = host
        self.port = port
        self.connect_callback = connect_callback
        self.disconnect_callback = disconnect_callback

        self.io_loop = io_loop or tornado.ioloop.IOLoop.current()

        self._tcp_client = TCPClientKeepAlive(
            opts, io_loop=self.io_loop, resolver=resolver)

        self._mid = 1
        self._max_messages = sys.maxint - 1  # number of IDs before we wrap

        # TODO: max queue size
        self.send_queue = []  # queue of messages to be sent
        self.send_future_map = {}  # mapping of request_id -> Future
        self.send_timeout_map = {}  # request_id -> timeout_callback

        self._read_until_future = None
        self._on_recv = None
        self._closing = False
        self._connecting_future = self.connect()
        self.io_loop.spawn_callback(self._stream_return)

    # TODO: timeout inflight sessions
    def close(self):
        if self._closing:
            return
        self._closing = True
        if hasattr(self, '_stream') and not self._stream.closed():
            self._stream.close()
            if self._read_until_future is not None:
                # This will prevent this message from showing up:
                # '[ERROR   ] Future exception was never retrieved:
                # StreamClosedError'
                # This happens because the logic is always waiting to read
                # the next message and the associated read future is marked
                # 'StreamClosedError' when the stream is closed.
                self._read_until_future.exc_info()
        self._tcp_client.close()
        # Clear callback references to allow the object that they belong to
        # to be deleted.
        self.connect_callback = None
        self.disconnect_callback = None

    def __del__(self):
        self.close()

    def connect(self):
        '''
        Ask for this client to reconnect to the origin
        '''
        if hasattr(self, '_connecting_future') and not self._connecting_future.done():
            future = self._connecting_future
        else:
            future = tornado.concurrent.Future()
            self._connecting_future = future
            self.io_loop.add_callback(self._connect)

            # Add the callback only when a new future is created
            if self.connect_callback is not None:
                def handle_future(future):
                    response = future.result()
                    self.io_loop.add_callback(self.connect_callback, response)
                future.add_done_callback(handle_future)

        return future

    # TODO: tcp backoff opts
    @tornado.gen.coroutine
    def _connect(self):
        '''
        Try to connect for the rest of time!
        '''
        while True:
            if self._closing:
                break
            try:
                self._stream = yield self._tcp_client.connect(self.host, self.port)
                self._connecting_future.set_result(True)
                break
            except Exception as e:
                yield tornado.gen.sleep(1)  # TODO: backoff
                #self._connecting_future.set_exception(e)

    @tornado.gen.coroutine
    def _stream_return(self):
        while not self._closing and (
                not self._connecting_future.done() or
                self._connecting_future.result() is not True):
            yield self._connecting_future
        unpacker = msgpack.Unpacker()
        while not self._closing:
            try:
                self._read_until_future = self._stream.read_bytes(4096, partial=True)
                wire_bytes = yield self._read_until_future
                unpacker.feed(wire_bytes)
                for framed_msg in unpacker:
                    header = framed_msg['head']
                    message_id = header.get('mid')

                    if message_id in self.send_future_map:
                        self.send_future_map.pop(message_id).set_result(framed_msg['body'])
                        self.remove_message_timeout(message_id)
                    else:
                        if self._on_recv is not None:
                            self.io_loop.spawn_callback(self._on_recv, header, framed_msg['body'])
                        else:
                            log.error('Got response for message_id {0} that we are not tracking'.format(message_id))
            except tornado.iostream.StreamClosedError as e:
                log.debug('tcp stream to {0}:{1} closed, unable to recv'.format(self.host, self.port))
                for future in self.send_future_map.itervalues():
                    future.set_exception(e)
                self.send_future_map = {}
                if self._closing:
                    return
                if self.disconnect_callback:
                    self.disconnect_callback()
                # if the last connect finished, then we need to make a new one
                if self._connecting_future.done():
                    self._connecting_future = self.connect()
                yield self._connecting_future
            except Exception as e:
                log.error('Exception parsing response', exc_info=True)
                for future in self.send_future_map.itervalues():
                    future.set_exception(e)
                self.send_future_map = {}
                if self._closing:
                    return
                if self.disconnect_callback:
                    self.disconnect_callback()
                # if the last connect finished, then we need to make a new one
                if self._connecting_future.done():
                    self._connecting_future = self.connect()
                yield self._connecting_future

    @tornado.gen.coroutine
    def _stream_send(self):
        while not self._connecting_future.done() or self._connecting_future.result() is not True:
            yield self._connecting_future
        while len(self.send_queue) > 0:
            message_id, item = self.send_queue[0]
            try:
                yield self._stream.write(item)
                del self.send_queue[0]
            # if the connection is dead, lets fail this send, and make sure we
            # attempt to reconnect
            except tornado.iostream.StreamClosedError as e:
                if message_id in self.send_future_map:
                    self.send_future_map.pop(message_id).set_exception(e)
                self.remove_message_timeout(message_id)
                del self.send_queue[0]
                if self._closing:
                    return
                if self.disconnect_callback:
                    self.disconnect_callback()
                # if the last connect finished, then we need to make a new one
                if self._connecting_future.done():
                    self._connecting_future = self.connect()
                yield self._connecting_future

    def _message_id(self):
        wrap = False
        while self._mid in self.send_future_map:
            if self._mid >= self._max_messages:
                if wrap:
                    # this shouldn't ever happen, but just in case
                    raise Exception('Unable to find available messageid')
                self._mid = 1
                wrap = True
            else:
                self._mid += 1

        return self._mid

    # TODO: return a message object which takes care of multiplexing?
    def on_recv(self, callback):
        '''
        Register a callback for received messages (that we didn't initiate)
        '''
        if callback is None:
            self._on_recv = callback
        else:
            def wrap_recv(header, body):
                callback(body)
            self._on_recv = wrap_recv

    def remove_message_timeout(self, message_id):
        if message_id not in self.send_timeout_map:
            return
        timeout = self.send_timeout_map.pop(message_id)
        self.io_loop.remove_timeout(timeout)

    def timeout_message(self, message_id):
        if message_id in self.send_timeout_map:
            del self.send_timeout_map[message_id]
        if message_id in self.send_future_map:
            self.send_future_map.pop(message_id).set_exception(
                SaltReqTimeoutError('Message timed out')
            )

    def send(self, msg, timeout=None, callback=None):
        '''
        Send given message, and return a future
        '''
        message_id = self._message_id()
        header = {'mid': message_id}

        future = tornado.concurrent.Future()
        if callback is not None:
            def handle_future(future):
                response = future.result()
                self.io_loop.add_callback(callback, response)
            future.add_done_callback(handle_future)
        # Add this future to the mapping
        self.send_future_map[message_id] = future

        if timeout is not None:
            send_timeout = self.io_loop.call_later(timeout, self.timeout_message, message_id)
            self.send_timeout_map[message_id] = send_timeout

        # if we don't have a send queue, we need to spawn the callback to do the sending
        if len(self.send_queue) == 0:
            self.io_loop.spawn_callback(self._stream_send)
        self.send_queue.append((message_id, salt.transport.frame.frame_msg(msg, header=header)))
        return future


class Subscriber(object):
    '''
    Client object for use with the TCP publisher server
    '''
    def __init__(self, stream, address):
        self.stream = stream
        self.address = address
        self._closing = False
        self._read_until_future = None
        self.id_ = None

    def close(self):
        if self._closing:
            return
        self._closing = True
        if not self.stream.closed():
            self.stream.close()
            if self._read_until_future is not None:
                # This will prevent this message from showing up:
                # '[ERROR   ] Future exception was never retrieved:
                # StreamClosedError'
                # This happens because the logic is always waiting to read
                # the next message and the associated read future is marked
                # 'StreamClosedError' when the stream is closed.
                self._read_until_future.exc_info()

    def __del__(self):
        self.close()


class PubServer(tornado.tcpserver.TCPServer, object):
    '''
    TCP publisher
    '''
    def __init__(self, opts, io_loop=None):
        super(PubServer, self).__init__(io_loop=io_loop)
        self.opts = opts
        self._closing = False
        self.clients = set()
        self.aes_funcs = salt.master.AESFuncs(self.opts)
        self.present = {}
        self.presence_events = False
        if self.opts.get('presence_events', False):
            tcp_only = True
            for transport, _ in iter_transport_opts(self.opts):
                if transport != 'tcp':
                    tcp_only = False
            if tcp_only:
                # Only when the transport is TCP only, the presence events will
                # be handled here. Otherwise, it will be handled in the
                # 'Maintenance' process.
                self.presence_events = True

        if self.presence_events:
            self.event = salt.utils.event.get_event(
                'master',
                opts=self.opts,
                listen=False
            )

    def close(self):
        if self._closing:
            return
        self._closing = True

    def __del__(self):
        self.close()

    def _add_client_present(self, client):
        id_ = client.id_
        if id_ in self.present:
            clients = self.present[id_]
            clients.add(client)
        else:
            self.present[id_] = set([client])
            if self.presence_events:
                data = {'new': [id_],
                        'lost': []}
                self.event.fire_event(
                    data,
                    salt.utils.event.tagify('change', 'presence')
                )
                data = {'present': list(self.present.keys())}
                self.event.fire_event(
                    data,
                    salt.utils.event.tagify('present', 'presence')
                )

    def _remove_client_present(self, client):
        id_ = client.id_
        if id_ is None or id_ not in self.present:
            # This is possible if _remove_client_present() is invoked
            # before the minion's id is validated.
            return

        clients = self.present[id_]
        if client not in clients:
            # Since _remove_client_present() is potentially called from
            # _stream_read() and/or publish_payload(), it is possible for
            # it to be called twice, in which case we will get here.
            # This is not an abnormal case, so no logging is required.
            return

        clients.remove(client)
        if len(clients) == 0:
            del self.present[id_]
            if self.presence_events:
                data = {'new': [],
                        'lost': [id_]}
                self.event.fire_event(
                    data,
                    salt.utils.event.tagify('change', 'presence')
                )
                data = {'present': list(self.present.keys())}
                self.event.fire_event(
                    data,
                    salt.utils.event.tagify('present', 'presence')
                )

    @tornado.gen.coroutine
    def _stream_read(self, client):
        unpacker = msgpack.Unpacker()
        while not self._closing:
            try:
                client._read_until_future = client.stream.read_bytes(4096, partial=True)
                wire_bytes = yield client._read_until_future
                unpacker.feed(wire_bytes)
                for framed_msg in unpacker:
                    body = framed_msg['body']
                    if body['enc'] != 'aes':
                        # We only accept 'aes' encoded messages for 'id'
                        continue
                    crypticle = salt.crypt.Crypticle(self.opts, salt.master.SMaster.secrets['aes']['secret'].value)
                    load = crypticle.loads(body['load'])
                    if not self.aes_funcs.verify_minion(load['id'], load['tok']):
                        continue
                    client.id_ = load['id']
                    self._add_client_present(client)
            except tornado.iostream.StreamClosedError as e:
                log.debug('tcp stream to {0} closed, unable to recv'.format(client.address))
                client.close()
                self._remove_client_present(client)
                self.clients.discard(client)
                break
            except Exception as e:
                log.error('Exception parsing response', exc_info=True)
                continue

    def handle_stream(self, stream, address):
        log.trace('Subscriber at {0} connected'.format(address))
        client = Subscriber(stream, address)
        self.clients.add(client)
        self.io_loop.spawn_callback(self._stream_read, client)

    # TODO: ACK the publish through IPC
    @tornado.gen.coroutine
    def publish_payload(self, package, _):
        log.debug('TCP PubServer sending payload: {0}'.format(package))
        payload = salt.transport.frame.frame_msg(package['payload'])

        to_remove = []
        if 'topic_lst' in package:
            topic_lst = package['topic_lst']
            for topic in topic_lst:
                if topic in self.present:
                    # This will rarely be a list of more than 1 item. It will
                    # be more than 1 item if the minion disconnects from the
                    # master in an unclean manner (eg cable yank), then
                    # restarts and the master is yet to detect the disconnect
                    # via TCP keep-alive.
                    for client in self.present[topic]:
                        try:
                            # Write the packed str
                            f = client.stream.write(payload)
                            self.io_loop.add_future(f, lambda f: True)
                        except tornado.iostream.StreamClosedError:
                            to_remove.append(client)
                else:
                    log.debug('Publish target {0} not connected'.format(topic))
        else:
            for client in self.clients:
                try:
                    # Write the packed str
                    f = client.stream.write(payload)
                    self.io_loop.add_future(f, lambda f: True)
                except tornado.iostream.StreamClosedError:
                    to_remove.append(client)
        for client in to_remove:
            log.debug('Subscriber at {0} has disconnected from publisher'.format(client.address))
            client.close()
            self._remove_client_present(client)
            self.clients.discard(client)
        log.trace('TCP PubServer finished publishing payload')


class TCPPubServerChannel(salt.transport.server.PubServerChannel):
    # TODO: opts!
    # Based on default used in tornado.netutil.bind_sockets()
    backlog = 128

    def __init__(self, opts):
        self.opts = opts
        self.serial = salt.payload.Serial(self.opts)  # TODO: in init?
        self.io_loop = None

    def __setstate__(self, state):
        salt.master.SMaster.secrets = state['secrets']
        self.__init__(state['opts'])

    def __getstate__(self):
        return {'opts': self.opts,
                'secrets': salt.master.SMaster.secrets}

    def _publish_daemon(self, log_queue=None):
        '''
        Bind to the interface specified in the configuration file
        '''
        salt.utils.appendproctitle(self.__class__.__name__)

        if log_queue is not None:
            salt.log.setup.set_multiprocessing_logging_queue(log_queue)
        salt.log.setup.setup_multiprocessing_logging(log_queue)

        # Check if io_loop was set outside
        if self.io_loop is None:
            self.io_loop = tornado.ioloop.IOLoop.current()

        # Spin up the publisher
        pub_server = PubServer(self.opts, io_loop=self.io_loop)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        _set_tcp_keepalive(sock, self.opts)
        sock.setblocking(0)
        sock.bind((self.opts['interface'], int(self.opts['publish_port'])))
        sock.listen(self.backlog)
        # pub_server will take ownership of the socket
        pub_server.add_socket(sock)

        # Set up Salt IPC server
        if self.opts.get('ipc_mode', '') == 'tcp':
            pull_uri = int(self.opts.get('tcp_master_publish_pull', 4514))
        else:
            pull_uri = os.path.join(self.opts['sock_dir'], 'publish_pull.ipc')

        pull_sock = salt.transport.ipc.IPCMessageServer(
            pull_uri,
            io_loop=self.io_loop,
            payload_handler=pub_server.publish_payload,
        )

        # Securely create socket
        log.info('Starting the Salt Puller on {0}'.format(pull_uri))
        old_umask = os.umask(0o177)
        try:
            pull_sock.start()
        finally:
            os.umask(old_umask)

        # run forever
        try:
            self.io_loop.start()
        except (KeyboardInterrupt, SystemExit):
            salt.log.setup.shutdown_multiprocessing_logging()

    def pre_fork(self, process_manager):
        '''
        Do anything necessary pre-fork. Since this is on the master side this will
        primarily be used to create IPC channels and create our daemon process to
        do the actual publishing
        '''
        kwargs = {}
        if salt.utils.is_windows():
            kwargs['log_queue'] = (
                salt.log.setup.get_multiprocessing_logging_queue()
            )

        process_manager.add_process(self._publish_daemon, kwargs=kwargs)

    def publish(self, load):
        '''
        Publish "load" to minions
        '''
        payload = {'enc': 'aes'}

        crypticle = salt.crypt.Crypticle(self.opts, salt.master.SMaster.secrets['aes']['secret'].value)
        payload['load'] = crypticle.dumps(load)
        if self.opts['sign_pub_messages']:
            master_pem_path = os.path.join(self.opts['pki_dir'], 'master.pem')
            log.debug("Signing data packet")
            payload['sig'] = salt.crypt.sign_message(master_pem_path, payload['load'])
        # Use the Salt IPC server
        if self.opts.get('ipc_mode', '') == 'tcp':
            pull_uri = int(self.opts.get('tcp_master_publish_pull', 4514))
        else:
            pull_uri = os.path.join(self.opts['sock_dir'], 'publish_pull.ipc')
        # TODO: switch to the actual async interface
        #pub_sock = salt.transport.ipc.IPCMessageClient(self.opts, io_loop=self.io_loop)
        pub_sock = salt.utils.async.SyncWrapper(
            salt.transport.ipc.IPCMessageClient,
            (pull_uri,)
        )
        pub_sock.connect()

        int_payload = {'payload': self.serial.dumps(payload)}

        # add some targeting stuff for lists only (for now)
        if load['tgt_type'] == 'list':
            int_payload['topic_lst'] = load['tgt']
        # Send it over IPC!
        pub_sock.send(int_payload)
