corosync  3.0.2
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
147  void *knet_context;
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
179  char *nozzle_ipaddr;
180  char *nozzle_prefix;
181  char *nozzle_macaddr;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 static void totemknet_start_merge_detect_timeout(
199  void *knet_context);
200 
201 static void totemknet_stop_merge_detect_timeout(
202  void *knet_context);
203 
204 static void log_flush_messages (
205  void *knet_context);
206 
207 static void totemknet_instance_initialize (struct totemknet_instance *instance)
208 {
209  int res;
210 
211  memset (instance, 0, sizeof (struct totemknet_instance));
212  res = pthread_mutex_init(&instance->log_mutex, NULL);
213  /*
214  * There is not too much else what can be done.
215  */
216  assert(res == 0);
217 }
218 
219 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
220 do { \
221  (void)pthread_mutex_lock(&instance->log_mutex); \
222  instance->totemknet_log_printf ( \
223  level, subsys, function, file, line, \
224  (const char *)format, ##args); \
225  (void)pthread_mutex_unlock(&instance->log_mutex); \
226 } while (0);
227 
228 #define knet_log_printf(level, format, args...) \
229 do { \
230  knet_log_printf_lock ( \
231  level, instance->totemknet_subsys_id, \
232  __FUNCTION__, __FILE__, __LINE__, \
233  (const char *)format, ##args); \
234 } while (0);
235 
236 #define libknet_log_printf(level, format, args...) \
237 do { \
238  knet_log_printf_lock ( \
239  level, instance->knet_subsys_id, \
240  __FUNCTION__, "libknet.h", __LINE__, \
241  (const char *)format, ##args); \
242 } while (0);
243 
244 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
245 do { \
246  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
247  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
248  instance->totemknet_log_printf ( \
249  level, instance->totemknet_subsys_id, \
250  __FUNCTION__, __FILE__, __LINE__, \
251  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
252  } while(0)
253 
254 
255 #ifdef HAVE_LIBNOZZLE
256 static inline int is_ether_addr_multicast(const uint8_t *addr)
257 {
258  return (addr[0] & 0x01);
259 }
260 static inline int is_ether_addr_zero(const uint8_t *addr)
261 {
262  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
263 }
264 
265 static int ether_host_filter_fn(void *private_data,
266  const unsigned char *outdata,
267  ssize_t outdata_len,
268  uint8_t tx_rx,
269  knet_node_id_t this_host_id,
270  knet_node_id_t src_host_id,
271  int8_t *channel,
272  knet_node_id_t *dst_host_ids,
273  size_t *dst_host_ids_entries)
274 {
275  struct ether_header *eth_h = (struct ether_header *)outdata;
276  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
277  uint16_t dst_host_id;
278 
279  if (is_ether_addr_zero(dst_mac))
280  return -1;
281 
282  if (is_ether_addr_multicast(dst_mac)) {
283  return 1;
284  }
285 
286  memmove(&dst_host_id, &dst_mac[4], 2);
287 
288  dst_host_ids[0] = ntohs(dst_host_id);
289  *dst_host_ids_entries = 1;
290 
291  return 0;
292 }
293 #endif
294 
295 static int dst_host_filter_callback_fn(void *private_data,
296  const unsigned char *outdata,
297  ssize_t outdata_len,
298  uint8_t tx_rx,
299  knet_node_id_t this_host_id,
300  knet_node_id_t src_host_id,
301  int8_t *channel,
302  knet_node_id_t *dst_host_ids,
303  size_t *dst_host_ids_entries)
304 {
305  struct totem_message_header *header = (struct totem_message_header *)outdata;
306  int res;
307 
308 #ifdef HAVE_LIBNOZZLE
309  if (*channel != 0) {
310  return ether_host_filter_fn(private_data,
311  outdata, outdata_len,
312  tx_rx,
313  this_host_id, src_host_id,
314  channel,
315  dst_host_ids,
316  dst_host_ids_entries);
317  }
318 #endif
319  if (header->target_nodeid) {
320  dst_host_ids[0] = header->target_nodeid;
321  *dst_host_ids_entries = 1;
322  res = 0; /* unicast message */
323  }
324  else {
325  *dst_host_ids_entries = 0;
326  res = 1; /* multicast message */
327  }
328  return res;
329 }
330 
331 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
332 {
333  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
334 
335  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
336  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
337  knet_handle_remove_datafd(instance->knet_handle, datafd);
338  }
339 }
340 
341 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
342 {
343  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
344 
345  // TODO: what? if anything.
346  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
347 }
348 
349 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
350 {
351  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
352  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
353 
354  /* We don't need to tell corosync the actual knet MTU */
355 // instance->totemknet_mtu_changed(instance->context, data_mtu);
356 }
357 
359  void *knet_context,
360  const char *cipher_type,
361  const char *hash_type)
362 {
363  return (0);
364 }
365 
366 
367 static inline void ucast_sendmsg (
368  struct totemknet_instance *instance,
369  struct totem_ip_address *system_to,
370  const void *msg,
371  unsigned int msg_len)
372 {
373  int res = 0;
374  struct totem_message_header *header = (struct totem_message_header *)msg;
375  struct msghdr msg_ucast;
376  struct iovec iovec;
377 
378  header->target_nodeid = system_to->nodeid;
379 
380  iovec.iov_base = (void *)msg;
381  iovec.iov_len = msg_len;
382 
383  /*
384  * Build unicast message
385  */
386  memset(&msg_ucast, 0, sizeof(msg_ucast));
387  msg_ucast.msg_iov = (void *)&iovec;
388  msg_ucast.msg_iovlen = 1;
389 #ifdef HAVE_MSGHDR_CONTROL
390  msg_ucast.msg_control = 0;
391 #endif
392 #ifdef HAVE_MSGHDR_CONTROLLEN
393  msg_ucast.msg_controllen = 0;
394 #endif
395 #ifdef HAVE_MSGHDR_FLAGS
396  msg_ucast.msg_flags = 0;
397 #endif
398 #ifdef HAVE_MSGHDR_ACCRIGHTS
399  msg_ucast.msg_accrights = NULL;
400 #endif
401 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
402  msg_ucast.msg_accrightslen = 0;
403 #endif
404 
405  /*
406  * Transmit unicast message
407  * An error here is recovered by totemsrp
408  */
409 
410  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
411  if (res < 0) {
413  "sendmsg(ucast) failed (non-critical)");
414  }
415 }
416 
417 static inline void mcast_sendmsg (
418  struct totemknet_instance *instance,
419  const void *msg,
420  unsigned int msg_len,
421  int only_active)
422 {
423  int res;
424  struct totem_message_header *header = (struct totem_message_header *)msg;
425  struct msghdr msg_mcast;
426  struct iovec iovec;
427 
428  iovec.iov_base = (void *)msg;
429  iovec.iov_len = msg_len;
430 
431  header->target_nodeid = 0;
432 
433  /*
434  * Build multicast message
435  */
436  memset(&msg_mcast, 0, sizeof(msg_mcast));
437  msg_mcast.msg_iov = (void *)&iovec;
438  msg_mcast.msg_iovlen = 1;
439 #ifdef HAVE_MSGHDR_CONTROL
440  msg_mcast.msg_control = 0;
441 #endif
442 #ifdef HAVE_MSGHDR_CONTROLLEN
443  msg_mcast.msg_controllen = 0;
444 #endif
445 #ifdef HAVE_MSGHDR_FLAGS
446  msg_mcast.msg_flags = 0;
447 #endif
448 #ifdef HAVE_MSGHDR_ACCRIGHTS
449  msg_mcast.msg_accrights = NULL;
450 #endif
451 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
452  msg_mcast.msg_accrightslen = 0;
453 #endif
454 
455 
456 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
457 
458  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
459  if (res < msg_len) {
460  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
461  }
462 
463  if (!only_active || instance->send_merge_detect_message) {
464  /*
465  * Current message was sent to all nodes
466  */
468  instance->send_merge_detect_message = 0;
469  }
470 }
471 
472 static int node_compare(const void *aptr, const void *bptr)
473 {
474  uint16_t a,b;
475 
476  a = *(uint16_t *)aptr;
477  b = *(uint16_t *)bptr;
478 
479  return a > b;
480 }
481 
483  char ***status,
484  unsigned int *iface_count)
485 {
486  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
487  struct knet_link_status link_status;
488  knet_node_id_t host_list[KNET_MAX_HOST];
489  uint8_t link_list[KNET_MAX_LINK];
490  size_t num_hosts;
491  size_t num_links;
492  size_t link_idx;
493  int i,j;
494  char *ptr;
495  int res = 0;
496 
497  /*
498  * Don't do the whole 'link_info' bit if the caller just wants
499  * a count of interfaces.
500  */
501  if (status) {
502 
503  res = knet_host_get_host_list(instance->knet_handle,
504  host_list, &num_hosts);
505  if (res) {
506  return (-1);
507  }
508  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
509 
510  for (i=0; i<INTERFACE_MAX; i++) {
511  memset(instance->link_status[i], 'n', CFG_INTERFACE_STATUS_MAX_LEN-1);
512  instance->link_status[i][num_hosts] = '\0';
513  }
514 
515  /* This is all a bit "inside-out" because "status" is a set of strings per link
516  * and knet orders things by host
517  */
518  for (j=0; j<num_hosts; j++) {
519  res = knet_link_get_link_list(instance->knet_handle,
520  host_list[j], link_list, &num_links);
521  if (res) {
522  return (-1);
523  }
524 
525  link_idx = 0;
526  for (i=0; i < num_links; i++) {
527  /*
528  * Skip over links that are unconfigured to corosync. This is basically
529  * link0 if corosync isn't using it for comms, as we will still
530  * have it set up for loopback.
531  */
532  if (!instance->totem_config->interfaces[link_list[i]].configured) {
533  continue;
534  }
535  ptr = instance->link_status[link_idx++];
536 
537  res = knet_link_get_status(instance->knet_handle,
538  host_list[j],
539  link_list[i],
540  &link_status,
541  sizeof(link_status));
542  if (res == 0) {
543  ptr[j] = '0' + (link_status.enabled |
544  link_status.connected<<1 |
545  link_status.dynconnected<<2);
546  }
547  else {
548  ptr[j] = '?';
549  }
550  }
551  }
552  *status = instance->link_status;
553  }
554 
555  *iface_count = INTERFACE_MAX;
556 
557  return (res);
558 }
559 
561  void *knet_context)
562 {
563  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
564  int res = 0;
565  int i,j;
566  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
567  uint8_t links[KNET_MAX_LINK];
568  size_t num_nodes;
569  size_t num_links;
570 
571  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
572 
573  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
574  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
575 
576  /*
577  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
578  */
579  res = knet_handle_setfwd(instance->knet_handle, 0);
580  if (res) {
581  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
582  }
583 
584  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
585  if (res) {
586  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
587  /* Crash out anyway */
588  goto finalise_error;
589  }
590 
591  /* Tidily shut down all nodes & links. */
592  for (i=0; i<num_nodes; i++) {
593 
594  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
595  if (res) {
596  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
597  goto finalise_error;
598  }
599  for (j=0; j<num_links; j++) {
600  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
601  if (res) {
602  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
603  }
604  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
605  if (res) {
606  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
607  }
608  }
609  res = knet_host_remove(instance->knet_handle, nodes[i]);
610  if (res) {
611  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
612  }
613  }
614 
615 finalise_error:
616  res = knet_handle_free(instance->knet_handle);
617  if (res) {
618  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
619  }
620 
621  totemknet_stop_merge_detect_timeout(instance);
622 
623  log_flush_messages(instance);
624 
625  /*
626  * Error is deliberately ignored
627  */
628  (void)pthread_mutex_destroy(&instance->log_mutex);
629 
630  return (res);
631 }
632 
633 static int log_deliver_fn (
634  int fd,
635  int revents,
636  void *data)
637 {
638  struct totemknet_instance *instance = (struct totemknet_instance *)data;
639  char buffer[sizeof(struct knet_log_msg)*4];
640  char *bufptr = buffer;
641  int done = 0;
642  int len;
643 
644  len = read(fd, buffer, sizeof(buffer));
645  while (done < len) {
646  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
647  switch (msg->msglevel) {
648  case KNET_LOG_ERR:
650  knet_log_get_subsystem_name(msg->subsystem),
651  msg->msg);
652  break;
653  case KNET_LOG_WARN:
655  knet_log_get_subsystem_name(msg->subsystem),
656  msg->msg);
657  break;
658  case KNET_LOG_INFO:
660  knet_log_get_subsystem_name(msg->subsystem),
661  msg->msg);
662  break;
663  case KNET_LOG_DEBUG:
665  knet_log_get_subsystem_name(msg->subsystem),
666  msg->msg);
667  break;
668  }
669  bufptr += sizeof(struct knet_log_msg);
670  done += sizeof(struct knet_log_msg);
671  }
672  return 0;
673 }
674 
675 static int data_deliver_fn (
676  int fd,
677  int revents,
678  void *data)
679 {
680  struct totemknet_instance *instance = (struct totemknet_instance *)data;
681  struct msghdr msg_hdr;
682  struct iovec iov_recv;
683  struct sockaddr_storage system_from;
684  ssize_t msg_len;
685  int truncated_packet;
686 
687  iov_recv.iov_base = instance->iov_buffer;
688  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
689 
690  msg_hdr.msg_name = &system_from;
691  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
692  msg_hdr.msg_iov = &iov_recv;
693  msg_hdr.msg_iovlen = 1;
694 #ifdef HAVE_MSGHDR_CONTROL
695  msg_hdr.msg_control = 0;
696 #endif
697 #ifdef HAVE_MSGHDR_CONTROLLEN
698  msg_hdr.msg_controllen = 0;
699 #endif
700 #ifdef HAVE_MSGHDR_FLAGS
701  msg_hdr.msg_flags = 0;
702 #endif
703 #ifdef HAVE_MSGHDR_ACCRIGHTS
704  msg_hdr.msg_accrights = NULL;
705 #endif
706 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
707  msg_hdr.msg_accrightslen = 0;
708 #endif
709 
710  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
711  if (msg_len <= 0) {
712  return (0);
713  }
714 
715  truncated_packet = 0;
716 
717 #ifdef HAVE_MSGHDR_FLAGS
718  if (msg_hdr.msg_flags & MSG_TRUNC) {
719  truncated_packet = 1;
720  }
721 #else
722  /*
723  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
724  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
725  */
726  if (bytes_received == KNET_MAX_PACKET_SIZE) {
727  truncated_packet = 1;
728  }
729 #endif
730 
731  if (truncated_packet) {
733  "Received too big message. This may be because something bad is happening"
734  "on the network (attack?), or you tried join more nodes than corosync is"
735  "compiled with (%u) or bug in the code (bad estimation of "
736  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
737  return (0);
738  }
739 
740  /*
741  * Handle incoming message
742  */
743  instance->totemknet_deliver_fn (
744  instance->context,
745  instance->iov_buffer,
746  msg_len,
747  &system_from);
748 
749  return (0);
750 }
751 
752 static void timer_function_netif_check_timeout (
753  void *data)
754 {
755  struct totemknet_instance *instance = (struct totemknet_instance *)data;
756  int i;
757 
758  for (i=0; i < INTERFACE_MAX; i++) {
759  if (!instance->totem_config->interfaces[i].configured) {
760  continue;
761  }
762  instance->totemknet_iface_change_fn (instance->context,
763  &instance->my_ids[i],
764  i);
765  }
766 }
767 
768 static void knet_set_access_list_config(struct totemknet_instance *instance)
769 {
770 #ifdef HAVE_KNET_ACCESS_LIST
771  uint32_t value;
772  cs_error_t err;
773 
774  value = instance->totem_config->block_unlisted_ips;
775  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
776 
777  err = knet_handle_enable_access_lists(instance->knet_handle, value);
778  if (err) {
779  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
780  }
781 #endif
782 }
783 
784 
785 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
786 static void totemknet_refresh_config(
787  int32_t event,
788  const char *key_name,
789  struct icmap_notify_value new_val,
790  struct icmap_notify_value old_val,
791  void *user_data)
792 {
793  uint8_t reloading;
794  uint32_t value;
795  uint32_t link_no;
796  size_t num_nodes;
797  knet_node_id_t host_ids[KNET_MAX_HOST];
798  int i;
799  int err;
800  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
801 
802  ENTER();
803 
804  /*
805  * If a full reload is in progress then don't do anything until it's done and
806  * can reconfigure it all atomically
807  */
808  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
809  return;
810  }
811 
812  knet_set_access_list_config(instance);
813 
814  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
815 
817  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
818  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
819  if (err) {
820  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
821  }
822  }
823 
824  /* Configure link parameters for each node */
825  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
826  if (err != 0) {
827  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
828  }
829 
830  for (i=0; i<num_nodes; i++) {
831  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
832  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
833  continue;
834  }
835 
836  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
837  instance->totem_config->interfaces[link_no].knet_ping_interval,
838  instance->totem_config->interfaces[link_no].knet_ping_timeout,
839  instance->totem_config->interfaces[link_no].knet_ping_precision);
840  if (err) {
841  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
842  }
843  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
844  instance->totem_config->interfaces[link_no].knet_pong_count);
845  if (err) {
846  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
847  }
848  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
849  instance->totem_config->interfaces[link_no].knet_link_priority);
850  if (err) {
851  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
852  }
853 
854  }
855  }
856 
857  LEAVE();
858 }
859 
860 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
861 {
862  icmap_track_t icmap_track_totem = NULL;
863  icmap_track_t icmap_track_reload = NULL;
864 
865  ENTER();
866 
867  icmap_track_add("totem.",
869  totemknet_refresh_config,
870  instance,
871  &icmap_track_totem);
872 
873  icmap_track_add("config.totemconfig_reload_in_progress",
875  totemknet_refresh_config,
876  instance,
877  &icmap_track_reload);
878 
879  LEAVE();
880 }
881 
882 /*
883  * Create an instance
884  */
886  qb_loop_t *poll_handle,
887  void **knet_context,
888  struct totem_config *totem_config,
889  totemsrp_stats_t *stats,
890  void *context,
891 
892  void (*deliver_fn) (
893  void *context,
894  const void *msg,
895  unsigned int msg_len,
896  const struct sockaddr_storage *system_from),
897 
898  void (*iface_change_fn) (
899  void *context,
900  const struct totem_ip_address *iface_address,
901  unsigned int link_no),
902 
903  void (*mtu_changed) (
904  void *context,
905  int net_mtu),
906 
907  void (*target_set_completed) (
908  void *context))
909 {
910  struct totemknet_instance *instance;
911  int8_t channel=0;
912  int res;
913  int i;
914 
915  instance = malloc (sizeof (struct totemknet_instance));
916  if (instance == NULL) {
917  return (-1);
918  }
919 
920  totemknet_instance_initialize (instance);
921 
922  instance->totem_config = totem_config;
923 
924  /*
925  * Configure logging
926  */
927  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
934 
935  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
936 
937  /*
938  * Initialize local variables for totemknet
939  */
940 
941  instance->our_nodeid = instance->totem_config->node_id;
942 
943  for (i=0; i< INTERFACE_MAX; i++) {
944  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
945  instance->my_ids[i].nodeid = instance->our_nodeid;
946  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
947 
948  /* Needed for totemsrp */
949  totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
950  }
951 
952  instance->poll_handle = poll_handle;
953 
954  instance->context = context;
955  instance->totemknet_deliver_fn = deliver_fn;
956 
957  instance->totemknet_iface_change_fn = iface_change_fn;
958 
959  instance->totemknet_mtu_changed = mtu_changed;
960 
961  instance->totemknet_target_set_completed = target_set_completed;
962 
963  instance->loopback_link = 0;
964 
965  res = pipe(instance->logpipes);
966  if (res == -1) {
967  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
968  goto exit_error;
969  }
970  fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK);
971  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK);
972 
973 #if !defined(KNET_API_VER) || (KNET_API_VER == 1)
974  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
975 #endif
976 #if KNET_API_VER == 2
977  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
978 #endif
979 
980  if (!instance->knet_handle) {
981  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
982  goto exit_error;
983  }
984 
985  knet_set_access_list_config(instance);
986 
987  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
988  if (res) {
989  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
990  }
991  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
992  if (res) {
993  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
994  }
995  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
996  if (res) {
997  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
998  }
999  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1000  if (res) {
1001  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1002  }
1003  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1004  if (res) {
1005  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1006  }
1007  global_instance = instance;
1008 
1009  /* Get an fd into knet */
1010  instance->knet_fd = 0;
1011  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1012  if (res) {
1013  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1014  goto exit_error;
1015  }
1016 
1017  /* Enable crypto if requested */
1018  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1019  struct knet_handle_crypto_cfg crypto_cfg;
1020 
1021  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
1022  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
1023  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
1024  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1025  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1026 
1027  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1028  if (res == -1) {
1029  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1030  goto exit_error;
1031  }
1032  if (res == -2) {
1033  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1034  goto exit_error;
1035  }
1036  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
1037  }
1038 
1039  /* Set up compression */
1040  totemknet_reconfigure(instance, instance->totem_config);
1041 
1042  knet_handle_setfwd(instance->knet_handle, 1);
1043 
1044  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1045  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1046  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1047  }
1048  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1049  instance->link_mode = KNET_LINK_POLICY_RR;
1050  }
1051 
1052  for (i=0; i<INTERFACE_MAX; i++) {
1053  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1054  if (!instance->link_status[i]) {
1055  goto exit_error;
1056  }
1057  }
1058 
1059  qb_loop_poll_add (instance->poll_handle,
1060  QB_LOOP_MED,
1061  instance->logpipes[0],
1062  POLLIN, instance, log_deliver_fn);
1063 
1064  qb_loop_poll_add (instance->poll_handle,
1065  QB_LOOP_HIGH,
1066  instance->knet_fd,
1067  POLLIN, instance, data_deliver_fn);
1068 
1069  /*
1070  * Upper layer isn't ready to receive message because it hasn't
1071  * initialized yet. Add short timer to check the interfaces.
1072  */
1073  qb_loop_timer_add (instance->poll_handle,
1074  QB_LOOP_MED,
1075  100*QB_TIME_NS_IN_MSEC,
1076  (void *)instance,
1077  timer_function_netif_check_timeout,
1078  &instance->timer_netif_check_timeout);
1079 
1080  totemknet_start_merge_detect_timeout(instance);
1081 
1082  /* Start listening for config changes */
1083  totemknet_add_config_notifications(instance);
1084 
1085  /* Add stats keys to icmap */
1087 
1088  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1089  *knet_context = instance;
1090 
1091  return (0);
1092 
1093 exit_error:
1094  log_flush_messages(instance);
1095  free(instance);
1096  return (-1);
1097 }
1098 
1100 {
1101  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1102  return malloc(KNET_MAX_PACKET_SIZE + 512);
1103 }
1104 
1106 {
1107  return free (ptr);
1108 }
1109 
1111  void *knet_context,
1112  int processor_count)
1113 {
1114  return (0);
1115 }
1116 
1117 int totemknet_recv_flush (void *knet_context)
1118 {
1119  return (0);
1120 }
1121 
1122 int totemknet_send_flush (void *knet_context)
1123 {
1124  return (0);
1125 }
1126 
1128  void *knet_context,
1129  const void *msg,
1130  unsigned int msg_len)
1131 {
1132  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1133  int res = 0;
1134 
1135  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1136 
1137  return (res);
1138 }
1140  void *knet_context,
1141  const void *msg,
1142  unsigned int msg_len)
1143 {
1144  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1145  int res = 0;
1146 
1147  mcast_sendmsg (instance, msg, msg_len, 0);
1148 
1149  return (res);
1150 }
1151 
1153  void *knet_context,
1154  const void *msg,
1155  unsigned int msg_len)
1156 {
1157  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1158  int res = 0;
1159 
1160  mcast_sendmsg (instance, msg, msg_len, 1);
1161 
1162  return (res);
1163 }
1164 
1165 
1166 extern int totemknet_iface_check (void *knet_context)
1167 {
1168  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1169  int res = 0;
1170 
1171  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1172 
1173  return (res);
1174 }
1175 
1176 extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1177 {
1178  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1179 
1180  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1181 }
1182 
1184  void *knet_context,
1185  unsigned int nodeid)
1186 {
1187  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1188  int res = 0;
1189 
1190  instance->token_target.nodeid = nodeid;
1191 
1192  instance->totemknet_target_set_completed (instance->context);
1193 
1194  return (res);
1195 }
1196 
1198  void *knet_context)
1199 {
1200  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1201  unsigned int res;
1202  struct sockaddr_storage system_from;
1203  struct msghdr msg_hdr;
1204  struct iovec iov_recv;
1205  struct pollfd ufd;
1206  int nfds;
1207  int msg_processed = 0;
1208 
1209  iov_recv.iov_base = instance->iov_buffer;
1210  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1211 
1212  msg_hdr.msg_name = &system_from;
1213  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1214  msg_hdr.msg_iov = &iov_recv;
1215  msg_hdr.msg_iovlen = 1;
1216 #ifdef HAVE_MSGHDR_CONTROL
1217  msg_hdr.msg_control = 0;
1218 #endif
1219 #ifdef HAVE_MSGHDR_CONTROLLEN
1220  msg_hdr.msg_controllen = 0;
1221 #endif
1222 #ifdef HAVE_MSGHDR_FLAGS
1223  msg_hdr.msg_flags = 0;
1224 #endif
1225 #ifdef HAVE_MSGHDR_ACCRIGHTS
1226  msg_msg_hdr.msg_accrights = NULL;
1227 #endif
1228 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1229  msg_msg_hdr.msg_accrightslen = 0;
1230 #endif
1231 
1232  do {
1233  ufd.fd = instance->knet_fd;
1234  ufd.events = POLLIN;
1235  nfds = poll (&ufd, 1, 0);
1236  if (nfds == 1 && ufd.revents & POLLIN) {
1237  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1238  if (res != -1) {
1239  msg_processed = 1;
1240  } else {
1241  msg_processed = -1;
1242  }
1243  }
1244  } while (nfds == 1);
1245 
1246  return (msg_processed);
1247 }
1248 
1249 int totemknet_iface_set (void *knet_context,
1250  const struct totem_ip_address *local_addr,
1251  unsigned short ip_port,
1252  unsigned int iface_no)
1253 {
1254  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1255 
1256  totemip_copy(&instance->my_ids[iface_no], local_addr);
1257 
1258  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1259 
1260  instance->ip_port[iface_no] = ip_port;
1261 
1262  return 0;
1263 }
1264 
1265 
1267  void *knet_context,
1268  const struct totem_ip_address *local,
1269  const struct totem_ip_address *member,
1270  int link_no)
1271 {
1272  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1273  int err;
1274  int port = instance->ip_port[link_no];
1275  struct sockaddr_storage remote_ss;
1276  struct sockaddr_storage local_ss;
1277  int addrlen;
1278  int i;
1279  int host_found = 0;
1280  knet_node_id_t host_ids[KNET_MAX_HOST];
1281  size_t num_host_ids;
1282 
1283  /* Only create 1 loopback link and use link 0 */
1284  if (member->nodeid == instance->our_nodeid) {
1285  if (!instance->loopback_link) {
1286  link_no = 0;
1287  instance->loopback_link = 1;
1288  } else {
1289  /* Already done */
1290  return 0;
1291  }
1292  }
1293 
1294  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1295  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1296 
1297 
1298  /* Only add the host if it doesn't already exist in knet */
1299  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1300  if (err) {
1301  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1302  return -1;
1303  }
1304  for (i=0; i<num_host_ids; i++) {
1305  if (host_ids[i] == member->nodeid) {
1306  host_found = 1;
1307  }
1308  }
1309 
1310  if (!host_found) {
1311  err = knet_host_add(instance->knet_handle, member->nodeid);
1312  if (err != 0 && errno != EEXIST) {
1313  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1314  return -1;
1315  }
1316  } else {
1317  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1318  }
1319 
1320 
1321  if (err == 0) {
1322  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1323  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1324  return -1;
1325  }
1326  }
1327 
1328  memset(&local_ss, 0, sizeof(local_ss));
1329  /* Casts to remove const */
1330  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1331  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1332 
1333  if (member->nodeid == instance->our_nodeid) {
1334  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1335 
1336  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1337  KNET_TRANSPORT_LOOPBACK,
1338  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1339  }
1340  else {
1341  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1342  instance->totem_config->interfaces[link_no].knet_transport,
1343  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1344  }
1345  if (err) {
1346  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1347  return -1;
1348  }
1349 
1350  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1351  instance->totem_config->interfaces[link_no].knet_link_priority);
1352 
1353  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1354  instance->totem_config->interfaces[link_no].knet_link_priority);
1355  if (err) {
1356  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1357  }
1358 
1359  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1360  get done in totemknet_refresh_config */
1361  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1362  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1363  instance->totem_config->interfaces[link_no].knet_ping_interval,
1364  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1365  instance->totem_config->interfaces[link_no].knet_ping_precision);
1366  if (err) {
1367  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1368  }
1369  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1370  instance->totem_config->interfaces[link_no].knet_pong_count);
1371  if (err) {
1372  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1373  }
1374  }
1375 
1376  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1377  if (err) {
1378  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1379  return -1;
1380  }
1381 
1382  /* register stats */
1383  stats_knet_add_member(member->nodeid, link_no);
1384  return (0);
1385 }
1386 
1388  void *knet_context,
1389  const struct totem_ip_address *token_target,
1390  int link_no)
1391 {
1392  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1393  int res;
1394  uint8_t link_list[KNET_MAX_LINK];
1395  size_t num_links;
1396 
1397  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1398 
1399  /* Don't remove the link with the loopback on it until we shut down */
1400  if (token_target->nodeid == instance->our_nodeid) {
1401  return 0;
1402  }
1403 
1404  /* Tidy stats */
1405  stats_knet_del_member(token_target->nodeid, link_no);
1406 
1407  /* Remove the link first */
1408  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1409  if (res != 0) {
1410  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1411  return res;
1412  }
1413 
1414  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1415  if (res != 0) {
1416  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1417  return res;
1418  }
1419 
1420  /* If this is the last link, then remove the node */
1421  res = knet_link_get_link_list(instance->knet_handle,
1422  token_target->nodeid, link_list, &num_links);
1423  if (res) {
1424  return (0); /* not really failure */
1425  }
1426 
1427  if (num_links == 0) {
1428  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1429  }
1430  return res;
1431 }
1432 
1434  void *knet_context)
1435 {
1436  return (0);
1437 }
1438 
1440  void *knet_context,
1441  struct totem_config *totem_config)
1442 {
1443  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1444  struct knet_handle_compress_cfg compress_cfg;
1445  int res = 0;
1446 
1447  if (totem_config->knet_compression_model) {
1448  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1449  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1450  compress_cfg.compress_level = totem_config->knet_compression_level;
1451 
1452  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1453  if (res) {
1454  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1455  }
1456  }
1457 
1458 #ifdef HAVE_LIBNOZZLE
1459  /* Set up nozzle device(s). Return code is ignored, because unability
1460  * configure nozzle is not fatal problem, errors are logged and
1461  * there is not much else we can do */
1462  (void)setup_nozzle(instance);
1463 #endif
1464  return (res);
1465 }
1466 
1467 
1469  void *knet_context)
1470 {
1471  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1472 
1473  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1474 }
1475 
1476 /* For the stats module */
1478  knet_node_id_t node, uint8_t link_no,
1479  struct knet_link_status *status)
1480 {
1481  int res;
1482  int ret = CS_OK;
1483 
1484  /* We are probably not using knet */
1485  if (!global_instance) {
1486  return CS_ERR_NOT_EXIST;
1487  }
1488 
1489  if (link_no >= INTERFACE_MAX) {
1490  return CS_ERR_NOT_EXIST; /* Invalid link number */
1491  }
1492 
1493  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1494  if (res) {
1495  switch (errno) {
1496  case EINVAL:
1497  ret = CS_ERR_INVALID_PARAM;
1498  break;
1499  case EBUSY:
1500  ret = CS_ERR_BUSY;
1501  break;
1502  case EDEADLK:
1503  ret = CS_ERR_TRY_AGAIN;
1504  break;
1505  default:
1506  ret = CS_ERR_LIBRARY;
1507  break;
1508  }
1509  }
1510 
1511  return (ret);
1512 }
1513 
1515  struct knet_handle_stats *stats)
1516 {
1517  /* We are probably not using knet */
1518  if (!global_instance) {
1519  return CS_ERR_NOT_EXIST;
1520  }
1521 
1522  return knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1523 }
1524 
1525 static void timer_function_merge_detect_timeout (
1526  void *data)
1527 {
1528  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1529 
1530  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1531  instance->send_merge_detect_message = 1;
1532  }
1533 
1535 
1536  totemknet_start_merge_detect_timeout(instance);
1537 }
1538 
1539 static void totemknet_start_merge_detect_timeout(
1540  void *knet_context)
1541 {
1542  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1543 
1544  qb_loop_timer_add(instance->poll_handle,
1545  QB_LOOP_MED,
1546  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1547  (void *)instance,
1548  timer_function_merge_detect_timeout,
1549  &instance->timer_merge_detect_timeout);
1550 
1551 }
1552 
1553 static void totemknet_stop_merge_detect_timeout(
1554  void *knet_context)
1555 {
1556  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1557 
1558  qb_loop_timer_del(instance->poll_handle,
1559  instance->timer_merge_detect_timeout);
1560 }
1561 
1562 static void log_flush_messages (void *knet_context)
1563 {
1564  struct pollfd pfd;
1565  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1566  int cont;
1567 
1568  cont = 1;
1569 
1570  while (cont) {
1571  pfd.fd = instance->logpipes[0];
1572  pfd.events = POLLIN;
1573  pfd.revents = 0;
1574 
1575  if ((poll(&pfd, 1, 0) > 0) &&
1576  (pfd.revents & POLLIN) &&
1577  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1578  cont = 1;
1579  } else {
1580  cont = 0;
1581  }
1582  }
1583 }
1584 
1585 
1586 #ifdef HAVE_LIBNOZZLE
1587 #define NOZZLE_NAME "nozzle.name"
1588 #define NOZZLE_IPADDR "nozzle.ipaddr"
1589 #define NOZZLE_PREFIX "nozzle.ipprefix"
1590 #define NOZZLE_MACADDR "nozzle.macaddr"
1591 
1592 #define NOZZLE_CHANNEL 1
1593 
1594 
1595 static char *get_nozzle_script_dir(void *knet_context)
1596 {
1597  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1598  char filename[PATH_MAX + FILENAME_MAX + 1];
1599  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1600  int res;
1601  const char *dirname_res;
1602 
1603  /*
1604  * Build script directory based on corosync.conf file location
1605  */
1606  res = snprintf(filename, sizeof(filename), "%s",
1608  if (res >= sizeof(filename)) {
1609  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1610  return NULL;
1611  }
1612 
1613  dirname_res = dirname(filename);
1614 
1615  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1616  dirname_res, "updown.d");
1617  if (res >= sizeof(updown_dirname)) {
1618  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1619  return NULL;
1620  }
1621  return updown_dirname;
1622 }
1623 
1624 /*
1625  * Deliberately doesn't return the status as caller doesn't care.
1626  * The result will be logged though
1627  */
1628 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1629 {
1630  int res;
1631  char *exec_string;
1632 
1633  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1634  if (res == -1 && errno != ENOENT) {
1635  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1636  } else if (res == -2) {
1637  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1638  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1639  }
1640 }
1641 
1642 /*
1643  * Reparse IP address to add in our node ID
1644  * IPv6 addresses must end in '::'
1645  * IPv4 addresses must just be valid
1646  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1647  *
1648  * Returns the modified IP address as a string to pass into libnozzle
1649  */
1650 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1651  const char *input_addr,
1652  const char *prefix, int nodeid,
1653  char *output_addr, size_t output_len)
1654 {
1655  char *coloncolon;
1656  int bits;
1657  int max_prefix = 64;
1658  uint32_t nodeid_mask;
1659  uint32_t addr_mask;
1660  uint32_t masked_nodeid;
1661  struct in_addr *addr;
1662  struct totem_ip_address totemip;
1663 
1664  coloncolon = strstr(input_addr, "::");
1665  if (!coloncolon) {
1666  max_prefix = 30;
1667  }
1668 
1669  bits = atoi(prefix);
1670  if (bits < 8 || bits > max_prefix) {
1671  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1672  return -1;
1673  }
1674 
1675  /* IPv6 is easy */
1676  if (coloncolon) {
1677  memcpy(output_addr, input_addr, coloncolon-input_addr);
1678  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1679  return 0;
1680  }
1681 
1682  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1683  * add in the masked_nodeid and 'print' it out again
1684  */
1685  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1686  addr_mask = UINT32_MAX ^ nodeid_mask;
1687  masked_nodeid = nodeid & nodeid_mask;
1688 
1689  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1690  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1691  return -1;
1692  }
1693  addr = (struct in_addr *)&totemip.addr;
1694  addr->s_addr &= htonl(addr_mask);
1695  addr->s_addr |= htonl(masked_nodeid);
1696 
1697  inet_ntop(AF_INET, addr, output_addr, output_len);
1698  return 0;
1699 }
1700 
1701 static int create_nozzle_device(void *knet_context, const char *name,
1702  const char *ipaddr, const char *prefix,
1703  const char *macaddr)
1704 {
1705  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1706  char device_name[IFNAMSIZ+1];
1707  size_t size = IFNAMSIZ;
1708  int8_t channel = NOZZLE_CHANNEL;
1709  nozzle_t nozzle_dev;
1710  int nozzle_fd;
1711  int res;
1712  char *updown_dir;
1713  char parsed_ipaddr[INET6_ADDRSTRLEN];
1714  char mac[19];
1715 
1716  memset(device_name, 0, size);
1717  memset(&mac, 0, sizeof(mac));
1718  strncpy(device_name, name, size);
1719 
1720  updown_dir = get_nozzle_script_dir(knet_context);
1721  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
1722 
1723  nozzle_dev = nozzle_open(device_name, size, updown_dir);
1724  if (!nozzle_dev) {
1725  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
1726  return -1;
1727  }
1728  instance->nozzle_handle = nozzle_dev;
1729 
1730  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
1731  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
1732  goto out_clean;
1733  }
1734 
1735  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
1736  /* Prints its own errors */
1737  goto out_clean;
1738  }
1739  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
1740  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
1741  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
1742  goto out_clean;
1743  }
1744 
1745  nozzle_fd = nozzle_get_fd(nozzle_dev);
1746  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
1747 
1748  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
1749  if (res != 0) {
1750  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
1751  goto out_clean;
1752  }
1753 
1754  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
1755 
1756  res = nozzle_set_up(nozzle_dev);
1757  if (res != 0) {
1758  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
1759  goto out_clean;
1760  }
1761  run_nozzle_script(instance, NOZZLE_UP, "up");
1762 
1763  return 0;
1764 
1765 out_clean:
1766  nozzle_close(nozzle_dev);
1767  return -1;
1768 }
1769 
1770 static int remove_nozzle_device(void *knet_context)
1771 {
1772  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1773  int res;
1774  int datafd;
1775 
1776  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
1777  if (res != 0) {
1778  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1779  return -1;
1780  }
1781 
1782  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
1783  if (res != 0) {
1784  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1785  return -1;
1786  }
1787 
1788  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
1789  res = nozzle_set_down(instance->nozzle_handle);
1790  if (res != 0) {
1791  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
1792  return -1;
1793  }
1794  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
1795 
1796  res = nozzle_close(instance->nozzle_handle);
1797  if (res != 0) {
1798  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
1799  return -1;
1800  }
1801  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
1802  return 0;
1803 }
1804 
1805 static void free_nozzle(struct totemknet_instance *instance)
1806 {
1807  free(instance->nozzle_name);
1808  free(instance->nozzle_ipaddr);
1809  free(instance->nozzle_prefix);
1810  free(instance->nozzle_macaddr);
1811 
1812  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
1813  instance->nozzle_macaddr = NULL;
1814 }
1815 
1816 static int setup_nozzle(void *knet_context)
1817 {
1818  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1819  char *ipaddr_str = NULL;
1820  char *name_str = NULL;
1821  char *prefix_str = NULL;
1822  char *macaddr_str = NULL;
1823  char mac[32];
1824  int name_res;
1825  int macaddr_res;
1826  int res = -1;
1827 
1828  /*
1829  * Return value ignored on purpose. icmap_get_string changes
1830  * ipaddr_str/prefix_str only on success.
1831  */
1832  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
1833  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
1834  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
1835  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
1836 
1837  /* Is is being removed? */
1838  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
1839  remove_nozzle_device(instance);
1840  free_nozzle(instance);
1841  goto out_free;
1842  }
1843 
1844  if (!name_str) {
1845  /* no nozzle */
1846  goto out_free;
1847  }
1848 
1849  if (!ipaddr_str) {
1850  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
1851  goto out_free;
1852  }
1853 
1854  if (!prefix_str) {
1855  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
1856  goto out_free;
1857  }
1858 
1859  if (macaddr_str && strlen(macaddr_str) != 17) {
1860  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
1861  goto out_free;
1862  }
1863  if (!macaddr_str) {
1864  macaddr_str = (char*)"54:54:01:00:00:00";
1865  }
1866 
1867  if (instance->nozzle_name &&
1868  (strcmp(name_str, instance->nozzle_name) == 0) &&
1869  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
1870  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
1871  (instance->nozzle_macaddr == NULL ||
1872  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
1873  /* Nothing has changed */
1874  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
1875  goto out_free;
1876  }
1877 
1878  /* Add nodeid into MAC address */
1879  memcpy(mac, macaddr_str, 12);
1880  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
1881  instance->our_nodeid >> 8,
1882  instance->our_nodeid & 0xFF);
1883  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
1884 
1885  if (name_res == CS_OK && name_str) {
1886  /* Reconfigure */
1887  if (instance->nozzle_name) {
1888  remove_nozzle_device(instance);
1889  free_nozzle(instance);
1890  }
1891 
1892  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
1893  mac);
1894 
1895  instance->nozzle_name = strdup(name_str);
1896  instance->nozzle_ipaddr = strdup(ipaddr_str);
1897  instance->nozzle_prefix = strdup(prefix_str);
1898  instance->nozzle_macaddr = strdup(macaddr_str);
1899  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
1900  !instance->nozzle_prefix) {
1901  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
1902  /*
1903  * This 'free' will cause a complete reconfigure of the device next time we reload
1904  * but will also let the the current device keep working until then.
1905  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
1906  */
1907  free_nozzle(instance);
1908  }
1909  }
1910 
1911 out_free:
1912  free(name_str);
1913  free(ipaddr_str);
1914  free(prefix_str);
1915  if (macaddr_res == CS_OK) {
1916  free(macaddr_str);
1917  }
1918 
1919  return res;
1920 }
1921 #endif // HAVE_LIBNOZZLE
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
int knet_ping_precision
Definition: totem.h:92
int totemknet_log_level_security
Definition: totemknet.c:124
qb_loop_t * poll_handle
Definition: totemknet.c:96
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1387
#define MSG_NOSIGNAL
Definition: totemknet.c:83
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
int knet_link_priority
Definition: totem.h:89
uint32_t value
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
struct totem_interface * interfaces
Definition: totem.h:158
void stats_knet_add_handle(void)
Definition: stats.c:637
int totemknet_log_level_error
Definition: totemknet.c:126
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:236
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
The totem_ip_address struct.
Definition: coroapi.h:111
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:264
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:114
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:228
int send_merge_detect_message
Definition: totemknet.c:169
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:560
uint32_t knet_compression_threshold
Definition: totem.h:226
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
unsigned int knet_pmtud_interval
Definition: totem.h:162
char * crypto_model
Definition: totem.h:218
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1477
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1105
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1468
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:244
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:624
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
unsigned int private_key_len
Definition: totem.h:169
int totemknet_log_level_notice
Definition: totemknet.c:130
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
#define INTERFACE_MAX
Definition: coroapi.h:88
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:358
unsigned int block_unlisted_ips
Definition: totem.h:236
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1139
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1166
void(*) void knet_context)
Definition: totemknet.c:145
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:826
unsigned int node_id
Definition: totem.h:160
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
uint8_t configured
Definition: totem.h:87
int totemknet_log_level_warning
Definition: totemknet.c:128
char * knet_compression_model
Definition: totem.h:224
const char * corosync_get_config_file(void)
Definition: main.c:204
void * user_data
Definition: sam.c:127
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1183
struct totem_config * totem_config
Definition: totemknet.c:161
unsigned int nodeid
Definition: coroapi.h:112
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1439
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1249
int knet_transport
Definition: totem.h:94
char * crypto_hash_type
Definition: totem.h:222
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1110
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1117
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1122
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
struct totem_ip_address boundto
Definition: totem.h:83
typedef __attribute__
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:850
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1514
struct totemknet_instance * global_instance
Definition: totemknet.c:187
struct totem_message_header header
Definition: totemsrp.c:260
uint16_t ip_port
Definition: totem.h:85
int knet_compression_level
Definition: totem.h:228
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:314
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
#define ENTER
Definition: logsys.h:324
unsigned int net_mtu
Definition: totem.h:202
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
struct totem_ip_address token_target
Definition: totemknet.c:163
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
pthread_mutex_t log_mutex
Definition: totemknet.c:176
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1266
int knet_pong_count
Definition: totem.h:93
struct totemknet_instance * instance
Definition: totemknet.c:192
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:880
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
int knet_ping_interval
Definition: totem.h:90
const void * msg
Definition: totemknet.c:190
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
int knet_ping_timeout
Definition: totem.h:91
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:272
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:614
struct srp_addr system_from
Definition: totemsrp.c:261
char * crypto_cipher_type
Definition: totem.h:220
char type
Definition: totem.h:55
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1176
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1433
unsigned int merge_timeout
Definition: totem.h:190
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1152
int totemknet_log_level_debug
Definition: totemknet.c:132
unsigned int target_nodeid
Definition: totem.h:130
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1127
struct totem_ip_address bindnet
Definition: totem.h:82
unsigned int nodeid
Definition: coroapi.h:75
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1197
unsigned int msg_len
Definition: totemknet.c:191
#define LEAVE
Definition: logsys.h:325
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:885
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:482
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1151
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem.nodeid", "totem.version", ...
Definition: icmap.h:85
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1099
knet_handle_t knet_handle
Definition: totemknet.c:98