We are using ZeroMQ for a project, with the following architecture:
This is basically our module we wrote in C for using ZeroMQ:
#define RET_ERROR(x) syslog(LOG_ERR, x "%s", zmq_strerror(errno)); cleanupZMQ(); return false;static zmq_ctx_t ctx = NULL;static zmq_sock_t pub_sock = NULL;static zmq_sock_t sub_sock = NULL;static zmq_sock_t sub_mon_sock = NULL;static zmq_sock_t pub_mon_sock = NULL;static char* subscriptions[] = { REQ_TOPIC };static size_t subscriptions_count = (sizeof(subscriptions) / sizeof(subscriptions[0]));static inline void cleanupZMQ() { if (pub_sock) { zmq_close(pub_sock); } if (sub_sock) { zmq_close(sub_sock); } if (sub_mon_sock) { zmq_close(sub_mon_sock); } if (pub_mon_sock) { zmq_close(pub_mon_sock); } if (ctx) { zmq_ctx_destroy(ctx); }}static bool waitForConnect(zmq_sock_t monitor) { zmq_msg_t msg; bool ret = true; bool connected = false; bool handshaked = false; do { zmq_msg_init(&msg); int rc = zmq_msg_recv(&msg, monitor, 0); if (rc < 0) { RET_ERROR("Error! Can not receive first frame from monitor: ") } uint8_t* data = (uint8_t*)zmq_msg_data(&msg); uint16_t event = *((uint16_t*)data); uint32_t value = *((uint32_t*)data+2); zmq_msg_init(&msg); rc = zmq_msg_recv(&msg, monitor, 0); if (rc < 0) { RET_ERROR("Error! Can not receive second frame from monitor: ") } char* addr = (char*)zmq_msg_data(&msg); syslog(LOG_DEBUG, "Event: %u, Value: %u, Addr: %s", event, value, addr); if (event == ZMQ_EVENT_CONNECTED) { syslog(LOG_INFO, "Connected to '%s'.", addr); connected = true; } else if (event == ZMQ_EVENT_CONNECT_DELAYED) { syslog(LOG_NOTICE, "Connecting delayed!"); } else if (event == ZMQ_EVENT_CONNECT_RETRIED) { syslog(LOG_NOTICE, "Connecting retried!"); } else if ((event == ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL) || (event == ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL) || (event == ZMQ_EVENT_HANDSHAKE_FAILED_AUTH)) { syslog(LOG_ERR, "Error! Handshake with '%s' failed: %s", addr, zmq_strerror(value)); handshaked = true; ret = false; } else if (event == ZMQ_EVENT_HANDSHAKE_SUCCEEDED) { syslog(LOG_INFO, "Handshake with '%s' succeded.", addr); handshaked = true; ret = true; } else { syslog(LOG_NOTICE, "Unexpected event: %u", event); } } while(!(connected && handshaked)); zmq_msg_close(&msg); return ret;}bool startZMQ() { if (ctx == NULL) { ctx = zmq_ctx_new(); if (ctx == NULL) { RET_ERROR("Error! Can not open ZMQ context: "); } } else { syslog(LOG_INFO, "ZMQ is already started."); } if (sub_sock == NULL) { sub_sock = zmq_socket(ctx, ZMQ_SUB); if (sub_sock == NULL) { RET_ERROR("Error! Can not open ZMQ sub socket: "); } if (zmq_socket_monitor(sub_sock, SUB_MON_ADDR, ZMQ_EVENT_ALL)) { RET_ERROR("Error! Can not monitor ZMQ sub socket: "); } sub_mon_sock = zmq_socket(ctx, ZMQ_PAIR); if (sub_mon_sock == NULL) { RET_ERROR("Error! Can not open ZMQ sub-monitor socket: "); } if (zmq_connect(sub_mon_sock, SUB_MON_ADDR)) { RET_ERROR("Error! Can not connect ZMQ sub-monitor socket: "); } for (size_t i=0; i<subscriptions_count; i++) { if (zmq_setsockopt(sub_sock, ZMQ_SUBSCRIBE, subscriptions[i], strlen(subscriptions[i]))) { syslog(LOG_ERR, "Error! Can not subscribe to topic '%s': %s", subscriptions[i], zmq_strerror(errno)); } else { syslog(LOG_INFO, "Subscribed to '%s'.", subscriptions[i]); } } if (zmq_connect(sub_sock, SUB_ADDR)) { RET_ERROR("Error! Can not connect ZMQ sub socket: "); } waitForConnect(sub_mon_sock); } else { syslog(LOG_INFO, "Subscriber socket is already open."); } if (pub_sock == NULL) { pub_sock = zmq_socket(ctx, ZMQ_PUB); if (pub_sock == NULL) { RET_ERROR("Error! Can not open ZMQ pub socket: "); } if (zmq_socket_monitor(pub_sock, PUB_MON_ADDR, ZMQ_EVENT_ALL)) { RET_ERROR("Error! Can not monitor ZMQ pub socket: "); } pub_mon_sock = zmq_socket(ctx, ZMQ_PAIR); if (pub_mon_sock == NULL) { RET_ERROR("Error! Can not open ZMQ pub-monitor socket: "); } if (zmq_connect(pub_mon_sock, PUB_MON_ADDR)) { RET_ERROR("Error! Can not connect ZMQ pub-monitor socket: "); } if (zmq_connect(pub_sock, PUB_ADDR)) { RET_ERROR("Error! Can not connect ZMQ pub socket: "); } waitForConnect(pub_mon_sock); } else { syslog(LOG_INFO, "Publisher socket is already open."); } sleep(3); return true;}size_t sendZMQMsg(const char* topic, size_t topic_len, const msg_buffer_t msg, size_t msg_len) { size_t sended = 0; int rc = zmq_send(pub_sock, topic, topic_len, ZMQ_SNDMORE); if (rc < 0) { syslog(LOG_ERR, "Error! Could not send ZMQ topic: %s", zmq_strerror(errno)); return 0; } sended += rc; rc = zmq_send(pub_sock, msg, msg_len, 0); if (rc < 0) { syslog(LOG_ERR, "Error! Could not send ZMQ message: %s", zmq_strerror(errno)); return 0; } sended += rc; return sended;}void endZMQ() { cleanupZMQ();}As you can see, we had to add an sleep(3) at the end of our startZMQ() function. Without this the first few messages sent would get lost.
Of course we know about this 'slow joiner syndrome'. We ensure that the broker is ready before connecting anything to it and the subscribers are connected (also with the three second delay) before the publishers. But still, we have to wait these three seconds before the publishers can use their sockets.The publishers and subscribers do not know each other due to the central broker and we do not want that they have to connect directly to each other, since we have a lot of both parts and if anyone had to directly connect to anyone else, the system would basically unmaintainable.
We found this Question and this one and of course we read the Guide, especially this Part, where in the guide themself a sleep(1) was used and than a polling with a second socket-pair is recommended.Is there really no other way in this library to check if your socket is ready, than polling it?
As you can see we already catching the zmq-events in our waitForConnect function using zmq-monitor-sockets. Should this not be enough? Are we missing something here?