# # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright 2021 Tintri by DDN, Inc. All rights reserved. # PROG = in.mpathd ROOTFS_PROG = $(PROG) OBJS = mpd_tables.o mpd_main.o mpd_probe.o DEFAULTFILES = mpathd.dfl SVCMETHOD = net-ipmp MANIFEST = network-ipmp.xml include ../../../Makefile.cmd ROOTCMDDIR = $(ROOT)/lib/inet ROOTMANIFESTDIR = $(ROOTSVCNETWORK) POFILE = $(PROG).po POFILES = $(OBJS:%.o=%.po) CSTD = $(CSTD_GNU99) # # We need access to the ancillary data features which are only available # via the SUS standards. Further, C99 support requires SUSv3 or higher. # CPPFLAGS += -D_XOPEN_SOURCE=600 -D__EXTENSIONS__ LDLIBS += -lsocket -lnsl -lsysevent -lnvpair -lipmp -linetutil -ldlpi CERRWARN += $(CNOWARN_UNINIT) # # Instrument in.mpathd with CTF data to ease debugging. # CTFCONVERT_HOOK = && $(CTFCONVERT_O) CTFMERGE_HOOK = && $(CTFMERGE) -L VERSION -o $@ $(OBJS) $(OBJS) : CFLAGS += $(CTF_FLAGS) .KEEP_STATE: all: $(PROG) $(PROG): $(OBJS) $(LINK.c) -o $@ $(OBJS) $(LDLIBS) $(CTFMERGE_HOOK) $(POST_PROCESS) check: $(CHKMANIFEST) install: all $(ROOTCMD) $(ROOTMANIFEST) $(ROOTSVCMETHOD) \ $(ROOTETCDEFAULTFILES) clean: $(RM) $(OBJS) $(POFILE): $(POFILES) $(RM) $@ $(CAT) $(POFILES) > $@ include ../../../Makefile.targ * Copyright (c) 1987 Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms are permitted * provided that the above copyright notice and this paragraph are * duplicated in all such forms and that any documentation, * advertising materials, and other materials related to such * distribution and use acknowledge that the software was developed * by the University of California, Berkeley. The name of the * University may not be used to endorse or promote products derived * from this software without specific prior written permission. * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. PORTIONS OF MPATHD COMMAND FUNCTIONALITY # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License, Version 1.0 only # (the "License"). You may not use this file except in compliance # with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # Copyright 2000 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # ident "%Z%%M% %I% %E% SMI" # # Time taken by mpathd to detect a NIC failure in ms. The minimum time # that can be specified is 100 ms. # FAILURE_DETECTION_TIME=10000 # # Failback is enabled by default. To disable failback turn off this option # FAILBACK=yes # # By default only interfaces configured as part of multipathing groups # are tracked. Turn off this option to track all network interfaces # on the system # TRACK_INTERFACES_ONLY_WITH_GROUPS=yes /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _MPD_DEFS_H #define _MPD_DEFS_H #ifdef __cplusplus extern "C" { #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Debug flags */ #define D_ALL 0xffff /* enable all debug */ #define D_PROBE 0x0001 /* probe mechanism */ #define D_FAILREP 0x0002 /* failure/repair mechanism */ #define D_PHYINT 0x0004 /* phyint table */ #define D_LOGINT 0x0008 /* logint table */ #define D_TARGET 0x0010 /* target table */ #define D_TIMER 0x0020 /* Timer mechanism */ #define D_PKTBAD 0x0040 /* Malformed packet */ #define D_LINKNOTE 0x0080 /* Link up/down notifications */ /* * Need a common header file that defines the 2 constants below. * Many applications need them. */ #define IF_SEPARATOR ':' #define IPV6_MAX_HOPS 255 /* * General parameters for phyint failure/repair detection */ #define NUM_PROBE_FAILS 5 /* NUM_PROBE_FAILS probe failures */ /* trigger NIC failure detection */ #define NUM_PROBE_REPAIRS 10 /* NUM_PROBE_REPAIRS probe repairs */ /* trigger NIC repair detection */ #define MIN_RANDOM_FACTOR 0.5 /* Randomization factors to */ #define MAX_RANDOM_FACTOR 1.0 /* determine probe send time */ #define MIN_PROBE_TARGETS 3 /* Minimum number of targets */ #define MAX_PROBE_TARGETS 5 /* Maximum number of targets */ /* * A target that is declared slow is usable again after MIN_RECOVERY_TIME ns */ #define MIN_RECOVERY_TIME (60000000000LL) /* (In ns) 60 secs */ /* * If the Failure Detection Time (FDT) is bumped up because the target CRTT * is high, it won't be reduced for the next MIN_SETTLING_TIME ns, to prevent * flapping of FDT */ #define MIN_SETTLING_TIME (60000000000LL) /* (In ns) 60 secs */ /* * An admin or script might place a phyint in a group before assigning a test * address. To give them time to configure a test address, we delay whining * about it being missing for TESTADDR_CONF_TIME seconds. */ #define TESTADDR_CONF_TIME 20 /* * The circular probe stats array should be able to hold enough * samples to detect phyint failure, target failure, phyint repair * and target repair. */ #define PROBE_STATS_COUNT \ ((uint16_t)(NUM_PROBE_REPAIRS * MAX_PROBE_TARGETS + 2)) #define FAILURE_DETECTION_TIME 10000 /* Default is 10 s */ #define MIN_FAILURE_DETECTION_TIME 100 /* Minimum is 100 ms */ #define FAILURE_DETECTION_QP 40 /* quiet period, in seconds */ #define NEXT_FDT_MULTIPLE 2 /* Raise or lower the FDT by this */ /* factor when required */ #define LOWER_FDT_TRIGGER 4 /* Lower the FDT if crtt is less */ /* than FDT / LOWER_FDT_TRIGGER */ #define EXCEPTION_FACTOR 2 /* The exception target has a crtt */ /* greater by this factor */ #define IF_SCAN_INTERVAL 20000 /* Do initifs() every 20 secs */ /* Return a random number from a range inclusive of the endpoints */ #define GET_RANDOM(LOW, HIGH) (random() % ((HIGH) - (LOW) + 1) + (LOW)) #define TIMER_INFINITY 0x7FFFFFFFU /* Never time out */ /* * Comparing unsigned 32 bit time values in a circular 32-bit sequence space */ #define TIME_GE(a, b) ((int32_t)((a) - (b)) >= 0) #define TIME_GT(a, b) ((int32_t)((a) - (b)) > 0) #define TIME_LT(a, b) ((int32_t)((a) - (b)) < 0) #define TIME_LE(a, b) ((int32_t)((a) - (b)) <= 0) /* * Comparing unsigned 16 bit sequence numbers in a circular 16-bit * sequence space */ #define SEQ_GE(a, b) ((int16_t)((a) - (b)) >= (int16_t)0) #define SEQ_GT(a, b) ((int16_t)((a) - (b)) > (int16_t)0) #define SEQ_LT(a, b) ((int16_t)((a) - (b)) < (int16_t)0) #define SEQ_LE(a, b) ((int16_t)((a) - (b)) <= (int16_t)0) #define AF_OTHER(af) ((af) == AF_INET ? AF_INET6 : AF_INET) #define AF_STR(af) ((af) == AF_INET ? "inet" : "inet6") /* * Globals */ extern boolean_t failback_enabled; /* cmd option to disable failbacks */ extern boolean_t track_all_phyints; /* cmd option to track all phyints */ /* all times below in millisec */ extern int user_probe_interval; /* interval between probes, as */ /* derived from user specified fdt */ extern int user_failure_detection_time; /* User specified fdt */ extern int ifsock_v4; /* IPv4 socket for ioctls */ extern int ifsock_v6; /* IPv6 socket for ioctls */ extern int debug; /* debug option */ extern boolean_t cleanup_started; /* true if we're shutting down */ extern boolean_t handle_link_notifications; /* * Function prototypes */ extern void timer_schedule(uint_t delay); extern void logmsg(int pri, const char *fmt, ...); extern void logperror(const char *str); extern int poll_add(int fd); extern int poll_remove(int fd); extern uint64_t getcurrentsec(void); extern uint_t getcurrenttime(void); #define logerr(...) logmsg(LOG_ERR, __VA_ARGS__) #define logtrace(...) logmsg(LOG_INFO, __VA_ARGS__) #define logdebug(...) logmsg(LOG_DEBUG, __VA_ARGS__) #ifdef __cplusplus } #endif #endif /* _MPD_DEFS_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2021 Tintri by DDN, Inc. All rights reserved. */ #include "mpd_defs.h" #include "mpd_tables.h" int debug = 0; /* Debug flag */ static int pollfd_num = 0; /* Num. of poll descriptors */ static struct pollfd *pollfds = NULL; /* Array of poll descriptors */ /* All times below in ms */ int user_failure_detection_time; /* user specified failure detection */ /* time (fdt) */ int user_probe_interval; /* derived from user specified fdt */ /* * Structure to store mib2 information returned by the kernel. * This is used to process routing table information. */ typedef struct mib_item_s { struct mib_item_s *mi_next; struct opthdr mi_opthdr; void *mi_valp; } mib_item_t; static int rtsock_v4; /* AF_INET routing socket */ static int rtsock_v6; /* AF_INET6 routing socket */ int ifsock_v4 = -1; /* IPv4 socket for ioctls */ int ifsock_v6 = -1; /* IPv6 socket for ioctls */ static int lsock_v4; /* Listen socket to detect mpathd */ static int lsock_v6; /* Listen socket to detect mpathd */ static int mibfd = -1; /* fd to get mib info */ static boolean_t force_mcast = _B_FALSE; /* Only for test purposes */ static uint_t last_initifs_time; /* Time when initifs was last run */ static char **argv0; /* Saved for re-exec on SIGHUP */ boolean_t handle_link_notifications = _B_TRUE; static int ipRouteEntrySize; /* Size of IPv4 route entry */ static int ipv6RouteEntrySize; /* Size of IPv6 route entry */ static void initlog(void); static void run_timeouts(void); static void initifs(void); static void check_if_removed(struct phyint_instance *pii); static void select_test_ifs(void); static void update_router_list(mib_item_t *item); static void mib_get_constants(mib_item_t *item); static int mibwalk(void (*proc)(mib_item_t *)); static void ire_process_v4(mib2_ipRouteEntry_t *buf, size_t len); static void ire_process_v6(mib2_ipv6RouteEntry_t *buf, size_t len); static void router_add_common(int af, char *ifname, struct in6_addr nexthop); static void init_router_targets(); static void cleanup(void); static int setup_listener(int af); static void check_config(void); static void check_testconfig(void); static void check_addr_unique(struct phyint_instance *, struct sockaddr_storage *); static void init_host_targets(void); static void dup_host_targets(struct phyint_instance *desired_pii); static void loopback_cmd(int sock, int family); static boolean_t daemonize(void); static int closefunc(void *, int); static unsigned int process_cmd(int newfd, union mi_commands *mpi); static unsigned int process_query(int fd, mi_query_t *miq); static unsigned int send_addrinfo(int fd, ipmp_addrinfo_t *adinfop); static unsigned int send_groupinfo(int fd, ipmp_groupinfo_t *grinfop); static unsigned int send_grouplist(int fd, ipmp_grouplist_t *grlistp); static unsigned int send_ifinfo(int fd, ipmp_ifinfo_t *ifinfop); static unsigned int send_result(int fd, unsigned int error, int syserror); addrlist_t *localaddrs; /* * Return the current time in milliseconds (from an arbitrary reference) * truncated to fit into an int. Truncation is ok since we are interested * only in differences and not the absolute values. */ uint_t getcurrenttime(void) { uint_t cur_time; /* In ms */ /* * Use of a non-user-adjustable source of time is * required. However millisecond precision is sufficient. * divide by 10^6 */ cur_time = (uint_t)(gethrtime() / 1000000LL); return (cur_time); } uint64_t getcurrentsec(void) { return (gethrtime() / NANOSEC); } /* * Add fd to the set being polled. Returns 0 if ok; -1 if failed. */ int poll_add(int fd) { int i; int new_num; struct pollfd *newfds; retry: /* Check if already present */ for (i = 0; i < pollfd_num; i++) { if (pollfds[i].fd == fd) return (0); } /* Check for empty spot already present */ for (i = 0; i < pollfd_num; i++) { if (pollfds[i].fd == -1) { pollfds[i].fd = fd; return (0); } } /* Allocate space for 32 more fds and initialize to -1 */ new_num = pollfd_num + 32; newfds = realloc(pollfds, new_num * sizeof (struct pollfd)); if (newfds == NULL) { logperror("poll_add: realloc"); return (-1); } for (i = pollfd_num; i < new_num; i++) { newfds[i].fd = -1; newfds[i].events = POLLIN; } pollfd_num = new_num; pollfds = newfds; goto retry; } /* * Remove fd from the set being polled. Returns 0 if ok; -1 if failed. */ int poll_remove(int fd) { int i; /* Check if already present */ for (i = 0; i < pollfd_num; i++) { if (pollfds[i].fd == fd) { pollfds[i].fd = -1; return (0); } } return (-1); } /* * Extract information about the phyint instance. If the phyint instance still * exists in the kernel then set pii_in_use, else clear it. check_if_removed() * will use it to detect phyint instances that don't exist any longer and * remove them, from our database of phyint instances. * Return value: * returns true if the phyint instance exists in the kernel, * returns false otherwise */ static boolean_t pii_process(int af, char *name, struct phyint_instance **pii_p) { int err; struct phyint_instance *pii; struct phyint_instance *pii_other; if (debug & D_PHYINT) logdebug("pii_process(%s %s)\n", AF_STR(af), name); pii = phyint_inst_lookup(af, name); if (pii == NULL) { /* * Phyint instance does not exist in our tables, * create new phyint instance */ pii = phyint_inst_init_from_k(af, name); } else { /* Phyint exists in our tables */ err = phyint_inst_update_from_k(pii); switch (err) { case PI_IOCTL_ERROR: /* Some ioctl error. don't change anything */ pii->pii_in_use = 1; break; case PI_GROUP_CHANGED: case PI_IFINDEX_CHANGED: /* * Interface index or group membership has changed. * Delete the old state and recreate based on the new * state (it may no longer be in a group). */ pii_other = phyint_inst_other(pii); if (pii_other != NULL) phyint_inst_delete(pii_other); phyint_inst_delete(pii); pii = phyint_inst_init_from_k(af, name); break; case PI_DELETED: /* Phyint instance has disappeared from kernel */ pii->pii_in_use = 0; break; case PI_OK: /* Phyint instance exists and is fine */ pii->pii_in_use = 1; break; default: /* Unknown status */ logerr("pii_process: Unknown status %d\n", err); break; } } *pii_p = pii; if (pii != NULL) return (pii->pii_in_use ? _B_TRUE : _B_FALSE); else return (_B_FALSE); } /* * Scan all interfaces to detect changes as well as new and deleted interfaces */ static void initifs() { int i, nlifr; int af; char *cp; char *buf; int sockfd; uint64_t flags; struct lifnum lifn; struct lifconf lifc; struct lifreq lifreq; struct lifreq *lifr; struct logint *li; struct phyint_instance *pii; struct phyint_instance *next_pii; struct phyint_group *pg, *next_pg; char pi_name[LIFNAMSIZ + 1]; if (debug & D_PHYINT) logdebug("initifs: Scanning interfaces\n"); last_initifs_time = getcurrenttime(); /* * Free the existing local address list; we'll build a new list below. */ addrlist_free(&localaddrs); /* * Mark the interfaces so that we can find phyints and logints * which have disappeared from the kernel. pii_process() and * logint_init_from_k() will set {pii,li}_in_use when they find * the interface in the kernel. Also, clear dupaddr bit on probe * logint. check_addr_unique() will set the dupaddr bit on the * probe logint, if the testaddress is not unique. */ for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pii->pii_in_use = 0; for (li = pii->pii_logint; li != NULL; li = li->li_next) { li->li_in_use = 0; if (pii->pii_probe_logint == li) li->li_dupaddr = 0; } } /* * As above, mark groups so that we can detect IPMP interfaces which * have been removed from the kernel. Also, delete the group address * list since we'll iteratively recreate it below. */ for (pg = phyint_groups; pg != NULL; pg = pg->pg_next) { pg->pg_in_use = _B_FALSE; addrlist_free(&pg->pg_addrs); } lifn.lifn_family = AF_UNSPEC; lifn.lifn_flags = LIFC_ALLZONES | LIFC_UNDER_IPMP; again: if (ioctl(ifsock_v4, SIOCGLIFNUM, (char *)&lifn) < 0) { logperror("initifs: ioctl (get interface count)"); return; } /* * Pad the interface count to detect when additional interfaces have * been configured between SIOCGLIFNUM and SIOCGLIFCONF. */ lifn.lifn_count += 4; if ((buf = calloc(lifn.lifn_count, sizeof (struct lifreq))) == NULL) { logperror("initifs: calloc"); return; } lifc.lifc_family = AF_UNSPEC; lifc.lifc_flags = LIFC_ALLZONES | LIFC_UNDER_IPMP; lifc.lifc_len = lifn.lifn_count * sizeof (struct lifreq); lifc.lifc_buf = buf; if (ioctl(ifsock_v4, SIOCGLIFCONF, (char *)&lifc) < 0) { logperror("initifs: ioctl (get interface configuration)"); free(buf); return; } /* * If every lifr_req slot is taken, then additional interfaces must * have been plumbed between the SIOCGLIFNUM and the SIOCGLIFCONF. * Recalculate to make sure we didn't miss any interfaces. */ nlifr = lifc.lifc_len / sizeof (struct lifreq); if (nlifr >= lifn.lifn_count) { free(buf); goto again; } /* * Walk through the lifreqs returned by SIOGGLIFCONF, and refresh the * global list of addresses, phyint groups, phyints, and logints. */ for (lifr = lifc.lifc_req, i = 0; i < nlifr; i++, lifr++) { af = lifr->lifr_addr.ss_family; sockfd = (af == AF_INET) ? ifsock_v4 : ifsock_v6; (void) strlcpy(lifreq.lifr_name, lifr->lifr_name, LIFNAMSIZ); if (ioctl(sockfd, SIOCGLIFFLAGS, &lifreq) == -1) { if (errno != ENXIO) logperror("initifs: ioctl (SIOCGLIFFLAGS)"); continue; } flags = lifreq.lifr_flags; /* * If the address is IFF_UP, add it to the local address list. * (We ignore addresses that aren't IFF_UP since another node * might legitimately have that address IFF_UP.) */ if (flags & IFF_UP) { (void) addrlist_add(&localaddrs, lifr->lifr_name, flags, &lifr->lifr_addr); } /* * If this address is on an IPMP meta-interface, update our * phyint_group information (either by recording that group * still exists or creating a new group), and track what * group the address is part of. */ if (flags & IFF_IPMP) { if (ioctl(sockfd, SIOCGLIFGROUPNAME, &lifreq) == -1) { if (errno != ENXIO) logperror("initifs: ioctl " "(SIOCGLIFGROUPNAME)"); continue; } pg = phyint_group_lookup(lifreq.lifr_groupname); if (pg == NULL) { pg = phyint_group_create(lifreq.lifr_groupname); if (pg == NULL) { logerr("initifs: cannot create group " "%s\n", lifreq.lifr_groupname); continue; } phyint_group_insert(pg); } pg->pg_in_use = _B_TRUE; /* * Add this to the group's list of data addresses. */ if (!addrlist_add(&pg->pg_addrs, lifr->lifr_name, flags, &lifr->lifr_addr)) { logerr("initifs: insufficient memory to track " "data address information for %s\n", lifr->lifr_name); } continue; } /* * This isn't an address on an IPMP meta-interface, so it's * either on an underlying interface or not related to any * group. Update our phyint and logint information (via * pii_process() and logint_init_from_k()) -- but first, * convert the logint name to a phyint name so we can call * pii_process(). */ (void) strlcpy(pi_name, lifr->lifr_name, sizeof (pi_name)); if ((cp = strchr(pi_name, IF_SEPARATOR)) != NULL) *cp = '\0'; if (pii_process(af, pi_name, &pii)) { /* The phyint is fine. So process the logint */ logint_init_from_k(pii, lifr->lifr_name); check_addr_unique(pii, &lifr->lifr_addr); } } free(buf); /* * Scan for groups, phyints and logints that have disappeared from the * kernel, and delete them. */ for (pii = phyint_instances; pii != NULL; pii = next_pii) { next_pii = pii->pii_next; check_if_removed(pii); } for (pg = phyint_groups; pg != NULL; pg = next_pg) { next_pg = pg->pg_next; if (!pg->pg_in_use) { phyint_group_delete(pg); continue; } /* * Refresh the group's state. This is necessary since the * group's state is defined by the set of usable interfaces in * the group, and an interface is considered unusable if all * of its addresses are down. When an address goes down/up, * the RTM_DELADDR/RTM_NEWADDR brings us through here. */ phyint_group_refresh_state(pg); } /* * Select a test address for sending probes on each phyint instance */ select_test_ifs(); /* * Handle link up/down notifications. */ process_link_state_changes(); } /* * Check that a given test address is unique across all of the interfaces in a * group. (e.g., IPv6 link-locals may not be inherently unique, and binding * to such an (IFF_NOFAILOVER) address can produce unexpected results.) * Any issues will be reported by check_testconfig(). */ static void check_addr_unique(struct phyint_instance *ourpii, struct sockaddr_storage *ss) { struct phyint *pi; struct phyint_group *pg; struct in6_addr addr; struct phyint_instance *pii; struct sockaddr_in *sin; if (ss->ss_family == AF_INET) { sin = (struct sockaddr_in *)ss; IN6_INADDR_TO_V4MAPPED(&sin->sin_addr, &addr); } else { assert(ss->ss_family == AF_INET6); addr = ((struct sockaddr_in6 *)ss)->sin6_addr; } /* * For anonymous groups, every interface is assumed to be on its own * link, so there is no chance of overlapping addresses. */ pg = ourpii->pii_phyint->pi_group; if (pg == phyint_anongroup) return; /* * Walk the list of phyint instances in the group and check for test * addresses matching ours. Of course, we skip ourself. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { pii = PHYINT_INSTANCE(pi, ss->ss_family); if (pii == NULL || pii == ourpii || pii->pii_probe_logint == NULL) continue; /* * If this test address is not unique, set the dupaddr bit. */ if (IN6_ARE_ADDR_EQUAL(&addr, &pii->pii_probe_logint->li_addr)) pii->pii_probe_logint->li_dupaddr = 1; } } /* * Stop probing an interface. Called when an interface is offlined. * The probe socket is closed on each interface instance, and the * interface state set to PI_OFFLINE. */ void stop_probing(struct phyint *pi) { struct phyint_instance *pii; pii = pi->pi_v4; if (pii != NULL) { if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); pii->pii_probe_logint = NULL; } pii = pi->pi_v6; if (pii != NULL) { if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); pii->pii_probe_logint = NULL; } phyint_chstate(pi, PI_OFFLINE); } enum { BAD_TESTFLAGS, OK_TESTFLAGS, BEST_TESTFLAGS }; /* * Rate the provided test flags. By definition, IFF_NOFAILOVER must be set. * IFF_UP must also be set so that the associated address can be used as a * source address. Further, we must be able to exchange packets with local * destinations, so IFF_NOXMIT and IFF_NOLOCAL must be clear. For historical * reasons, we have a proclivity for IFF_DEPRECATED IPv4 test addresses. */ static int rate_testflags(uint64_t flags) { if ((flags & (IFF_NOFAILOVER | IFF_UP)) != (IFF_NOFAILOVER | IFF_UP)) return (BAD_TESTFLAGS); if ((flags & (IFF_NOXMIT | IFF_NOLOCAL)) != 0) return (BAD_TESTFLAGS); if ((flags & (IFF_IPV6 | IFF_DEPRECATED)) == IFF_DEPRECATED) return (BEST_TESTFLAGS); if ((flags & (IFF_IPV6 | IFF_DEPRECATED)) == IFF_IPV6) return (BEST_TESTFLAGS); return (OK_TESTFLAGS); } /* * Attempt to select a test address for each phyint instance. * Call phyint_inst_sockinit() to complete the initializations. */ static void select_test_ifs(void) { struct phyint *pi; struct phyint_instance *pii; struct phyint_instance *next_pii; struct logint *li; struct logint *probe_logint; boolean_t target_scan_reqd = _B_FALSE; int rating; if (debug & D_PHYINT) logdebug("select_test_ifs\n"); /* * For each phyint instance, do the test address selection */ for (pii = phyint_instances; pii != NULL; pii = next_pii) { next_pii = pii->pii_next; probe_logint = NULL; /* * An interface that is offline should not be probed. * IFF_OFFLINE interfaces should always be PI_OFFLINE * unless some other entity has set the offline flag. */ if (pii->pii_phyint->pi_flags & IFF_OFFLINE) { if (pii->pii_phyint->pi_state != PI_OFFLINE) { logerr("shouldn't be probing offline" " interface %s (state is: %u)." " Stopping probes.\n", pii->pii_phyint->pi_name, pii->pii_phyint->pi_state); stop_probing(pii->pii_phyint); } continue; } else { /* * If something cleared IFF_OFFLINE (e.g., by accident * because the SIOCGLIFFLAGS/SIOCSLIFFLAGS sequence is * inherently racy), the phyint may still be offline. * Just ignore it. */ if (pii->pii_phyint->pi_state == PI_OFFLINE) continue; } li = pii->pii_probe_logint; if (li != NULL) { /* * We've already got a test address; only proceed * if it's suboptimal. */ if (rate_testflags(li->li_flags) == BEST_TESTFLAGS) continue; } /* * Walk the logints of this phyint instance, and select * the best available test address */ for (li = pii->pii_logint; li != NULL; li = li->li_next) { /* * Skip 0.0.0.0 addresses, as those are never * actually usable. */ if (pii->pii_af == AF_INET && IN6_IS_ADDR_V4MAPPED_ANY(&li->li_addr)) continue; /* * Skip any IPv6 logints that are not link-local, * since we should always have a link-local address * anyway and in6_data() expects link-local replies. */ if (pii->pii_af == AF_INET6 && !IN6_IS_ADDR_LINKLOCAL(&li->li_addr)) continue; /* * Rate the testflags. If we've found an optimal * match, then break out; otherwise, record the most * recent OK one. */ rating = rate_testflags(li->li_flags); if (rating == BAD_TESTFLAGS) continue; probe_logint = li; if (rating == BEST_TESTFLAGS) break; } /* * If the probe logint has changed, ditch the old one. */ if (pii->pii_probe_logint != NULL && pii->pii_probe_logint != probe_logint) { if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); pii->pii_probe_logint = NULL; } if (probe_logint == NULL) { /* * We don't have a test address; zero out the probe * stats array since it is no longer relevant. * Optimize by checking if it is already zeroed out. */ int pr_ndx; pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); if (pii->pii_probes[pr_ndx].pr_status != PR_UNUSED) { clear_pii_probe_stats(pii); reset_crtt_all(pii->pii_phyint); } continue; } else if (probe_logint == pii->pii_probe_logint) { /* * If we didn't find any new test addr, go to the * next phyint. */ continue; } /* * The phyint is either being assigned a new testaddr * or is being assigned a testaddr for the 1st time. * Need to initialize the phyint socket */ pii->pii_probe_logint = probe_logint; if (!phyint_inst_sockinit(pii)) { if (debug & D_PHYINT) { logdebug("select_test_ifs: " "phyint_sockinit failed\n"); } phyint_inst_delete(pii); continue; } /* * This phyint instance is now enabled for probes; this * impacts our state machine in two ways: * * 1. If we're probe *capable* as well (i.e., we have * probe targets) and the interface is in PI_NOTARGETS, * then transition to PI_RUNNING. * * 2. If we're not probe capable, and the other phyint * instance is also not probe capable, and we were in * PI_RUNNING, then transition to PI_NOTARGETS. * * Also see the state diagram in mpd_probe.c. */ if (PROBE_CAPABLE(pii)) { if (pii->pii_phyint->pi_state == PI_NOTARGETS) phyint_chstate(pii->pii_phyint, PI_RUNNING); } else if (!PROBE_CAPABLE(phyint_inst_other(pii))) { if (pii->pii_phyint->pi_state == PI_RUNNING) phyint_chstate(pii->pii_phyint, PI_NOTARGETS); } /* * If no targets are currently known for this phyint * we need to call init_router_targets. Since * init_router_targets() initializes the list of targets * for all phyints it is done below the loop. */ if (pii->pii_targets == NULL) target_scan_reqd = _B_TRUE; /* * Start the probe timer for this instance. */ if (!pii->pii_basetime_inited && PROBE_ENABLED(pii)) { start_timer(pii); pii->pii_basetime_inited = 1; } } /* * Scan the interface list for any interfaces that are PI_FAILED or * PI_NOTARGETS but no longer enabled to send probes, and call * phyint_check_for_repair() to see if the link state indicates that * the interface should be repaired. Also see the state diagram in * mpd_probe.c. */ for (pi = phyints; pi != NULL; pi = pi->pi_next) { if ((!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) && (pi->pi_state == PI_FAILED || pi->pi_state == PI_NOTARGETS)) { phyint_check_for_repair(pi); } } check_testconfig(); /* * Try to populate the target list. init_router_targets populates * the target list from the routing table. If our target list is * still empty, init_host_targets adds host targets based on the * host target list of other phyints in the group. */ if (target_scan_reqd) { init_router_targets(); init_host_targets(); } } /* * Check test address configuration, and log notices/errors if appropriate. * Note that this function only logs pre-existing conditions (e.g., that * probe-based failure detection is disabled). */ static void check_testconfig(void) { struct phyint *pi; struct logint *li; char abuf[INET6_ADDRSTRLEN]; int pri; for (pi = phyints; pi != NULL; pi = pi->pi_next) { if (pi->pi_flags & IFF_OFFLINE) continue; if (PROBE_ENABLED(pi->pi_v4) || PROBE_ENABLED(pi->pi_v6)) { if (pi->pi_taddrmsg_printed || pi->pi_duptaddrmsg_printed) { if (pi->pi_duptaddrmsg_printed) pri = LOG_ERR; else pri = LOG_INFO; logmsg(pri, "Test address now configured on " "interface %s; enabling probe-based " "failure detection on it\n", pi->pi_name); pi->pi_taddrmsg_printed = 0; pi->pi_duptaddrmsg_printed = 0; } continue; } li = NULL; if (pi->pi_v4 != NULL && pi->pi_v4->pii_probe_logint != NULL && pi->pi_v4->pii_probe_logint->li_dupaddr) li = pi->pi_v4->pii_probe_logint; if (pi->pi_v6 != NULL && pi->pi_v6->pii_probe_logint != NULL && pi->pi_v6->pii_probe_logint->li_dupaddr) li = pi->pi_v6->pii_probe_logint; if (li != NULL && li->li_dupaddr) { if (pi->pi_duptaddrmsg_printed) continue; logerr("Test address %s is not unique in group; " "disabling probe-based failure detection on %s\n", pr_addr(li->li_phyint_inst->pii_af, li->li_addr, abuf, sizeof (abuf)), pi->pi_name); pi->pi_duptaddrmsg_printed = 1; continue; } if (getcurrentsec() < pi->pi_taddrthresh) continue; if (!pi->pi_taddrmsg_printed) { logtrace("No test address configured on interface %s; " "disabling probe-based failure detection on it\n", pi->pi_name); pi->pi_taddrmsg_printed = 1; } } } /* * Check phyint group configuration, to detect any inconsistencies, * and log an error message. This is called from runtimeouts every * 20 secs. But the error message is displayed once. If the * consistency is resolved by the admin, a recovery message is displayed * once. */ static void check_config(void) { struct phyint_group *pg; struct phyint *pi; boolean_t v4_in_group; boolean_t v6_in_group; /* * All phyints of a group must be homogeneous to ensure that they can * take over for one another. If any phyint in a group has IPv4 * plumbed, check that all phyints have IPv4 plumbed. Do a similar * check for IPv6. */ for (pg = phyint_groups; pg != NULL; pg = pg->pg_next) { if (pg == phyint_anongroup) continue; v4_in_group = _B_FALSE; v6_in_group = _B_FALSE; /* * 1st pass. Determine if at least 1 phyint in the group * has IPv4 plumbed and if so set v4_in_group to true. * Repeat similarly for IPv6. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { if (pi->pi_v4 != NULL) v4_in_group = _B_TRUE; if (pi->pi_v6 != NULL) v6_in_group = _B_TRUE; } /* * 2nd pass. If v4_in_group is true, check that phyint * has IPv4 plumbed. Repeat similarly for IPv6. Print * out a message the 1st time only. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { if (pi->pi_flags & IFF_OFFLINE) continue; if (v4_in_group == _B_TRUE && pi->pi_v4 == NULL) { if (!pi->pi_cfgmsg_printed) { logerr("IP interface %s in group %s is" " not plumbed for IPv4, affecting" " IPv4 connectivity\n", pi->pi_name, pi->pi_group->pg_name); pi->pi_cfgmsg_printed = 1; } } else if (v6_in_group == _B_TRUE && pi->pi_v6 == NULL) { if (!pi->pi_cfgmsg_printed) { logerr("IP interface %s in group %s is" " not plumbed for IPv6, affecting" " IPv6 connectivity\n", pi->pi_name, pi->pi_group->pg_name); pi->pi_cfgmsg_printed = 1; } } else { /* * The phyint matches the group configuration, * if we have reached this point. If it was * improperly configured earlier, log an * error recovery message */ if (pi->pi_cfgmsg_printed) { logerr("IP interface %s is now" " consistent with group %s " " and connectivity is restored\n", pi->pi_name, pi->pi_group->pg_name); pi->pi_cfgmsg_printed = 0; } } } } } /* * Timer mechanism using relative time (in milliseconds) from the * previous timer event. Timers exceeding TIMER_INFINITY milliseconds * will fire after TIMER_INFINITY milliseconds. * Unsigned arithmetic note: We assume a 32-bit circular sequence space for * time values. Hence 2 consecutive timer events cannot be spaced farther * than 0x7fffffff. We call this TIMER_INFINITY, and it is the maximum value * that can be passed for the delay parameter of timer_schedule() */ static uint_t timer_next; /* Currently scheduled timeout */ static boolean_t timer_active = _B_FALSE; /* SIGALRM has not yet occurred */ static void timer_init(void) { timer_next = getcurrenttime() + TIMER_INFINITY; /* * The call to run_timeouts() will get the timer started * Since there are no phyints at this point, the timer will * be set for IF_SCAN_INTERVAL ms. */ run_timeouts(); } /* * Make sure the next SIGALRM occurs delay milliseconds from the current * time if not earlier. We are interested only in time differences. */ void timer_schedule(uint_t delay) { uint_t now; struct itimerval itimerval; if (debug & D_TIMER) logdebug("timer_schedule(%u)\n", delay); assert(delay <= TIMER_INFINITY); now = getcurrenttime(); if (delay == 0) { /* Minimum allowed delay */ delay = 1; } /* Will this timer occur before the currently scheduled SIGALRM? */ if (timer_active && TIME_GE(now + delay, timer_next)) { if (debug & D_TIMER) { logdebug("timer_schedule(%u) - no action: " "now %u next %u\n", delay, now, timer_next); } return; } timer_next = now + delay; itimerval.it_value.tv_sec = delay / 1000; itimerval.it_value.tv_usec = (delay % 1000) * 1000; itimerval.it_interval.tv_sec = 0; itimerval.it_interval.tv_usec = 0; if (debug & D_TIMER) { logdebug("timer_schedule(%u): sec %ld usec %ld\n", delay, itimerval.it_value.tv_sec, itimerval.it_value.tv_usec); } timer_active = _B_TRUE; if (setitimer(ITIMER_REAL, &itimerval, NULL) < 0) { logperror("timer_schedule: setitimer"); exit(2); } } static void timer_cancel(void) { struct itimerval itimerval; if (debug & D_TIMER) logdebug("timer_cancel()\n"); bzero(&itimerval, sizeof (itimerval)); if (setitimer(ITIMER_REAL, &itimerval, NULL) < 0) logperror("timer_cancel: setitimer"); } /* * Timer has fired. Determine when the next timer event will occur by asking * all the timer routines. Should not be called from a timer routine. */ static void run_timeouts(void) { uint_t next; uint_t next_event_time; struct phyint_instance *pii; struct phyint_instance *next_pii; static boolean_t timeout_running; /* assert that recursive timeouts don't happen. */ assert(!timeout_running); timeout_running = _B_TRUE; if (debug & D_TIMER) logdebug("run_timeouts()\n"); if ((getcurrenttime() - last_initifs_time) > IF_SCAN_INTERVAL) { initifs(); check_config(); } next = TIMER_INFINITY; for (pii = phyint_instances; pii != NULL; pii = next_pii) { next_pii = pii->pii_next; next_event_time = phyint_inst_timer(pii); if (next_event_time != TIMER_INFINITY && next_event_time < next) next = next_event_time; if (debug & D_TIMER) { logdebug("run_timeouts(%s %s): next scheduled for" " this phyint inst %u, next scheduled global" " %u ms\n", AF_STR(pii->pii_af), pii->pii_phyint->pi_name, next_event_time, next); } } /* * Make sure initifs() is called at least once every * IF_SCAN_INTERVAL, to make sure that we are in sync * with the kernel, in case we have missed any routing * socket messages. */ if (next > IF_SCAN_INTERVAL) next = IF_SCAN_INTERVAL; if (debug & D_TIMER) logdebug("run_timeouts: %u ms\n", next); timer_schedule(next); timeout_running = _B_FALSE; } static int eventpipe_read = -1; /* Used for synchronous signal delivery */ static int eventpipe_write = -1; boolean_t cleanup_started = _B_FALSE; /* true if we're going away */ /* * Ensure that signals are processed synchronously with the rest of * the code by just writing a one character signal number on the pipe. * The poll loop will pick this up and process the signal event. */ static void sig_handler(int signo) { uchar_t buf = (uchar_t)signo; /* * Don't write to pipe if cleanup has already begun. cleanup() * might have closed the pipe already */ if (cleanup_started) return; if (eventpipe_write == -1) { logerr("sig_handler: no pipe found\n"); return; } if (write(eventpipe_write, &buf, sizeof (buf)) < 0) logperror("sig_handler: write"); } extern struct probes_missed probes_missed; /* * Pick up a signal "byte" from the pipe and process it. */ static void in_signal(int fd) { uchar_t buf; uint64_t sent, acked, lost, unacked, unknown; struct phyint_instance *pii; int pr_ndx; switch (read(fd, &buf, sizeof (buf))) { case -1: logperror("in_signal: read"); exit(1); /* NOTREACHED */ case 1: break; case 0: logerr("in_signal: read end of file\n"); exit(1); /* NOTREACHED */ default: logerr("in_signal: read > 1\n"); exit(1); } if (debug & D_TIMER) logdebug("in_signal() got %d\n", buf); switch (buf) { case SIGALRM: if (debug & D_TIMER) { uint_t now = getcurrenttime(); logdebug("in_signal(SIGALRM) delta %u\n", now - timer_next); } timer_active = _B_FALSE; run_timeouts(); break; case SIGUSR1: logdebug("Printing configuration:\n"); /* Print out the internal tables */ phyint_inst_print_all(); /* * Print out the accumulated statistics about missed * probes (happens due to scheduling delay). */ logerr("Missed sending total of %d probes spread over" " %d occurrences\n", probes_missed.pm_nprobes, probes_missed.pm_ntimes); /* * Print out the accumulated statistics about probes * that were sent. */ for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { unacked = 0; acked = pii->pii_cum_stats.acked; lost = pii->pii_cum_stats.lost; sent = pii->pii_cum_stats.sent; unknown = pii->pii_cum_stats.unknown; for (pr_ndx = 0; pr_ndx < PROBE_STATS_COUNT; pr_ndx++) { switch (pii->pii_probes[pr_ndx].pr_status) { case PR_ACKED: acked++; break; case PR_LOST: lost++; break; case PR_UNACKED: unacked++; break; } } logerr("\nProbe stats on (%s %s)\n" "Number of probes sent %lld\n" "Number of probe acks received %lld\n" "Number of probes/acks lost %lld\n" "Number of valid unacknowledged probes %lld\n" "Number of ambiguous probe acks received %lld\n", AF_STR(pii->pii_af), pii->pii_name, sent, acked, lost, unacked, unknown); } break; case SIGHUP: logerr("SIGHUP: restart and reread config file\n"); /* * Cancel the interval timer. Needed since setitimer() uses * alarm() and the time left is inherited across exec(), and * thus the SIGALRM may be delivered before a handler has been * setup, causing in.mpathd to erroneously exit. */ timer_cancel(); cleanup(); (void) execv(argv0[0], argv0); _exit(0177); /* NOTREACHED */ case SIGINT: case SIGTERM: case SIGQUIT: cleanup(); exit(0); /* NOTREACHED */ default: logerr("in_signal: unknown signal: %d\n", buf); } } static void cleanup(void) { struct phyint_instance *pii; struct phyint_instance *next_pii; /* * Make sure that we don't write to eventpipe in * sig_handler() if any signal notably SIGALRM, * occurs after we close the eventpipe descriptor below */ cleanup_started = _B_TRUE; for (pii = phyint_instances; pii != NULL; pii = next_pii) { next_pii = pii->pii_next; phyint_inst_delete(pii); } (void) close(ifsock_v4); (void) close(ifsock_v6); (void) close(rtsock_v4); (void) close(rtsock_v6); (void) close(lsock_v4); (void) close(lsock_v6); (void) close(0); (void) close(1); (void) close(2); (void) close(mibfd); (void) close(eventpipe_read); (void) close(eventpipe_write); } /* * Create pipe for signal delivery and set up signal handlers. */ static void setup_eventpipe(void) { int fds[2]; struct sigaction act; if ((pipe(fds)) < 0) { logperror("setup_eventpipe: pipe"); exit(1); } eventpipe_read = fds[0]; eventpipe_write = fds[1]; if (poll_add(eventpipe_read) == -1) { exit(1); } act.sa_handler = sig_handler; act.sa_flags = SA_RESTART; (void) sigaction(SIGALRM, &act, NULL); (void) sigset(SIGHUP, sig_handler); (void) sigset(SIGUSR1, sig_handler); (void) sigset(SIGTERM, sig_handler); (void) sigset(SIGINT, sig_handler); (void) sigset(SIGQUIT, sig_handler); } /* * Create a routing socket for receiving RTM_IFINFO messages. */ static int setup_rtsock(int af) { int s; int flags; int aware = RTAW_UNDER_IPMP; s = socket(PF_ROUTE, SOCK_RAW, af); if (s == -1) { logperror("setup_rtsock: socket PF_ROUTE"); exit(1); } if (setsockopt(s, SOL_ROUTE, RT_AWARE, &aware, sizeof (aware)) == -1) { logperror("setup_rtsock: setsockopt RT_AWARE"); (void) close(s); exit(1); } if ((flags = fcntl(s, F_GETFL, 0)) < 0) { logperror("setup_rtsock: fcntl F_GETFL"); (void) close(s); exit(1); } if ((fcntl(s, F_SETFL, flags | O_NONBLOCK)) < 0) { logperror("setup_rtsock: fcntl F_SETFL"); (void) close(s); exit(1); } if (poll_add(s) == -1) { (void) close(s); exit(1); } return (s); } /* * Process an RTM_IFINFO message received on a routing socket. * The return value indicates whether a full interface scan is required. * Link up/down notifications are reflected in the IFF_RUNNING flag. * If just the state of the IFF_RUNNING interface flag has changed, a * a full interface scan isn't required. */ static boolean_t process_rtm_ifinfo(if_msghdr_t *ifm, int type) { struct sockaddr_dl *sdl; struct phyint *pi; uint64_t old_flags; struct phyint_instance *pii; assert(ifm->ifm_type == RTM_IFINFO && ifm->ifm_addrs == RTA_IFP); /* * Although the sockaddr_dl structure is directly after the * if_msghdr_t structure. At the time of writing, the size of the * if_msghdr_t structure is different on 32 and 64 bit kernels, due * to the presence of a timeval structure, which contains longs, * in the if_data structure. Anyway, we know where the message ends, * so we work backwards to get the start of the sockaddr_dl structure. */ /*LINTED*/ sdl = (struct sockaddr_dl *)((char *)ifm + ifm->ifm_msglen - sizeof (struct sockaddr_dl)); assert(sdl->sdl_family == AF_LINK); /* * The interface name is in sdl_data. * RTM_IFINFO messages are only generated for logical interface * zero, so there is no colon and logical interface number to * strip from the name. The name is not null terminated, but * there should be enough space in sdl_data to add the null. */ if (sdl->sdl_nlen >= sizeof (sdl->sdl_data)) { if (debug & D_LINKNOTE) logdebug("process_rtm_ifinfo: phyint name too long\n"); return (_B_TRUE); } sdl->sdl_data[sdl->sdl_nlen] = 0; pi = phyint_lookup(sdl->sdl_data); if (pi == NULL) { if (debug & D_LINKNOTE) logdebug("process_rtm_ifinfo: phyint lookup failed" " for %s\n", sdl->sdl_data); return (_B_TRUE); } /* * We want to try and avoid doing a full interface scan for * link state notifications from the datalink layer, as indicated * by the state of the IFF_RUNNING flag. If just the * IFF_RUNNING flag has changed state, the link state changes * are processed without a full scan. * If there is both an IPv4 and IPv6 instance associated with * the physical interface, we will get an RTM_IFINFO message * for each instance. If we just maintained a single copy of * the physical interface flags, it would appear that no flags * had changed when the second message is processed, leading us * to believe that the message wasn't generated by a flags change, * and that a full interface scan is required. * To get around this problem, two additional copies of the flags * are kept, one copy for each instance. These are only used in * this routine. At any one time, all three copies of the flags * should be identical except for the IFF_RUNNING flag. The * copy of the flags in the "phyint" structure is always up to * date. */ pii = (type == AF_INET) ? pi->pi_v4 : pi->pi_v6; if (pii == NULL) { if (debug & D_LINKNOTE) logdebug("process_rtm_ifinfo: no instance of address " "family %s for %s\n", AF_STR(type), pi->pi_name); return (_B_TRUE); } old_flags = pii->pii_flags; pii->pii_flags = PHYINT_FLAGS(ifm->ifm_flags); pi->pi_flags = pii->pii_flags; if (debug & D_LINKNOTE) { logdebug("process_rtm_ifinfo: %s address family: %s, " "old flags: %llx, new flags: %llx\n", pi->pi_name, AF_STR(type), old_flags, pi->pi_flags); } /* * If IFF_STANDBY has changed, indicate that the interface has changed * types and refresh IFF_INACTIVE if need be. */ if ((old_flags ^ pii->pii_flags) & IFF_STANDBY) { phyint_changed(pi); if (pii->pii_flags & IFF_STANDBY) phyint_standby_refresh_inactive(pi); } /* Has just the IFF_RUNNING flag changed state ? */ if ((old_flags ^ pii->pii_flags) != IFF_RUNNING) { struct phyint_instance *pii_other; /* * It wasn't just a link state change. Update * the other instance's copy of the flags. */ pii_other = phyint_inst_other(pii); if (pii_other != NULL) pii_other->pii_flags = pii->pii_flags; return (_B_TRUE); } return (_B_FALSE); } /* * Retrieve as many routing socket messages as possible, and try to * empty the routing sockets. Initiate full scan of targets or interfaces * as needed. * We listen on separate IPv4 an IPv6 sockets so that we can accurately * detect changes in certain flags (see "process_rtm_ifinfo()" above). */ static void process_rtsock(int rtsock_v4, int rtsock_v6) { int nbytes; int64_t msg[2048 / 8]; struct rt_msghdr *rtm; boolean_t need_if_scan = _B_FALSE; boolean_t need_rt_scan = _B_FALSE; boolean_t rtm_ifinfo_seen = _B_FALSE; int type; /* Read as many messages as possible and try to empty the sockets */ for (type = AF_INET; ; type = AF_INET6) { for (;;) { nbytes = read((type == AF_INET) ? rtsock_v4 : rtsock_v6, msg, sizeof (msg)); if (nbytes <= 0) { /* No more messages */ break; } rtm = (struct rt_msghdr *)msg; if (rtm->rtm_version != RTM_VERSION) { logerr("process_rtsock: version %d " "not understood\n", rtm->rtm_version); break; } if (debug & D_PHYINT) { logdebug("process_rtsock: message %d\n", rtm->rtm_type); } switch (rtm->rtm_type) { case RTM_NEWADDR: case RTM_DELADDR: /* * Some logical interface has changed, * have to scan everything to determine * what actually changed. */ need_if_scan = _B_TRUE; break; case RTM_IFINFO: rtm_ifinfo_seen = _B_TRUE; need_if_scan |= process_rtm_ifinfo( (if_msghdr_t *)rtm, type); break; case RTM_ADD: case RTM_DELETE: case RTM_CHANGE: case RTM_OLDADD: case RTM_OLDDEL: need_rt_scan = _B_TRUE; break; default: /* Not interesting */ break; } } if (type == AF_INET6) break; } if (need_if_scan) { if (debug & D_LINKNOTE && rtm_ifinfo_seen) logdebug("process_rtsock: synchronizing with kernel\n"); initifs(); } else if (rtm_ifinfo_seen) { if (debug & D_LINKNOTE) logdebug("process_rtsock: " "link up/down notification(s) seen\n"); process_link_state_changes(); } if (need_rt_scan) init_router_targets(); } /* * Look if the phyint instance or one of its logints have been removed from * the kernel and take appropriate action. * Uses {pii,li}_in_use. */ static void check_if_removed(struct phyint_instance *pii) { struct logint *li; struct logint *next_li; /* Detect phyints that have been removed from the kernel. */ if (!pii->pii_in_use) { logtrace("%s %s has been removed from kernel\n", AF_STR(pii->pii_af), pii->pii_phyint->pi_name); phyint_inst_delete(pii); } else { /* Detect logints that have been removed. */ for (li = pii->pii_logint; li != NULL; li = next_li) { next_li = li->li_next; if (!li->li_in_use) { logint_delete(li); } } } } /* * Parse the supplied mib2 information to extract the routing information * table. Process the routing table to get the list of known onlink routers * and update our database. These onlink routers will serve as probe * targets. */ static void update_router_list(mib_item_t *item) { for (; item != NULL; item = item->mi_next) { if (item->mi_opthdr.name == 0) continue; if (item->mi_opthdr.level == MIB2_IP && item->mi_opthdr.name == MIB2_IP_ROUTE) { ire_process_v4((mib2_ipRouteEntry_t *)item->mi_valp, item->mi_opthdr.len); } else if (item->mi_opthdr.level == MIB2_IP6 && item->mi_opthdr.name == MIB2_IP6_ROUTE) { ire_process_v6((mib2_ipv6RouteEntry_t *)item->mi_valp, item->mi_opthdr.len); } } } /* * Convert octet `octp' to a phyint name and store in `ifname' */ static void oct2ifname(const Octet_t *octp, char *ifname, size_t ifsize) { char *cp; size_t len = MIN(octp->o_length, ifsize - 1); (void) strncpy(ifname, octp->o_bytes, len); ifname[len] = '\0'; if ((cp = strchr(ifname, IF_SEPARATOR)) != NULL) *cp = '\0'; } /* * Examine the IPv4 routing table `buf' for possible targets. For each * possible target, if it's on the same subnet an interface route, pass * it to router_add_common() for further consideration. */ static void ire_process_v4(mib2_ipRouteEntry_t *buf, size_t len) { char ifname[LIFNAMSIZ]; mib2_ipRouteEntry_t *rp, *rp1, *endp; struct in_addr nexthop_v4; struct in6_addr nexthop; if (debug & D_TARGET) logdebug("ire_process_v4(len %d)\n", len); if (len == 0) return; assert((len % ipRouteEntrySize) == 0); endp = buf + (len / ipRouteEntrySize); /* * Scan the routing table entries for any IRE_OFFSUBNET entries, and * cross-reference them with the interface routes to determine if * they're possible probe targets. */ for (rp = buf; rp < endp; rp++) { if (!(rp->ipRouteInfo.re_ire_type & IRE_OFFSUBNET)) continue; /* Get the nexthop address. */ nexthop_v4.s_addr = rp->ipRouteNextHop; /* * Rescan the routing table looking for interface routes that * are on the same subnet, and try to add them. If they're * not relevant (e.g., the interface route isn't part of an * IPMP group, router_add_common() will discard). */ for (rp1 = buf; rp1 < endp; rp1++) { if (!(rp1->ipRouteInfo.re_ire_type & IRE_INTERFACE) || rp1->ipRouteIfIndex.o_length == 0) continue; if ((rp1->ipRouteDest & rp1->ipRouteMask) != (nexthop_v4.s_addr & rp1->ipRouteMask)) continue; oct2ifname(&rp1->ipRouteIfIndex, ifname, LIFNAMSIZ); IN6_INADDR_TO_V4MAPPED(&nexthop_v4, &nexthop); router_add_common(AF_INET, ifname, nexthop); } } } void router_add_common(int af, char *ifname, struct in6_addr nexthop) { struct phyint_instance *pii; struct phyint *pi; if (debug & D_TARGET) logdebug("router_add_common(%s %s)\n", AF_STR(af), ifname); /* * Retrieve the phyint instance; bail if it's not known to us yet. */ pii = phyint_inst_lookup(af, ifname); if (pii == NULL) return; /* * Don't use our own addresses as targets. */ if (own_address(nexthop)) return; /* * If the phyint is part a named group, then add the address to all * members of the group; note that this is suboptimal in the IPv4 case * as it has already been added to all matching interfaces in * ire_process_v4(). Otherwise, add the address only to the phyint * itself, since other phyints in the anongroup may not be on the same * subnet. */ pi = pii->pii_phyint; if (pi->pi_group == phyint_anongroup) { target_add(pii, nexthop, _B_TRUE); } else { pi = pi->pi_group->pg_phyint; for (; pi != NULL; pi = pi->pi_pgnext) target_add(PHYINT_INSTANCE(pi, af), nexthop, _B_TRUE); } } /* * Examine the IPv6 routing table `buf' for possible link-local targets, and * pass any contenders to router_add_common() for further consideration. */ static void ire_process_v6(mib2_ipv6RouteEntry_t *buf, size_t len) { struct lifreq lifr; char ifname[LIFNAMSIZ]; char grname[LIFGRNAMSIZ]; mib2_ipv6RouteEntry_t *rp, *rp1, *endp; struct in6_addr nexthop_v6; if (debug & D_TARGET) logdebug("ire_process_v6(len %d)\n", len); if (len == 0) return; assert((len % ipv6RouteEntrySize) == 0); endp = buf + (len / ipv6RouteEntrySize); /* * Scan the routing table entries for any IRE_OFFSUBNET entries, and * cross-reference them with the interface routes to determine if * they're possible probe targets. */ for (rp = buf; rp < endp; rp++) { if (!(rp->ipv6RouteInfo.re_ire_type & IRE_OFFSUBNET) || !IN6_IS_ADDR_LINKLOCAL(&rp->ipv6RouteNextHop)) continue; /* Get the nexthop address. */ nexthop_v6 = rp->ipv6RouteNextHop; /* * The interface name should always exist for link-locals; * we use it to map this entry to an IPMP group name. */ if (rp->ipv6RouteIfIndex.o_length == 0) continue; oct2ifname(&rp->ipv6RouteIfIndex, lifr.lifr_name, LIFNAMSIZ); if (ioctl(ifsock_v6, SIOCGLIFGROUPNAME, &lifr) == -1 || strlcpy(grname, lifr.lifr_groupname, LIFGRNAMSIZ) == 0) { continue; } /* * Rescan the list of routes for interface routes, and add the * above target to any interfaces in the same IPMP group. */ for (rp1 = buf; rp1 < endp; rp1++) { if (!(rp1->ipv6RouteInfo.re_ire_type & IRE_INTERFACE) || rp1->ipv6RouteIfIndex.o_length == 0) { continue; } oct2ifname(&rp1->ipv6RouteIfIndex, ifname, LIFNAMSIZ); (void) strlcpy(lifr.lifr_name, ifname, LIFNAMSIZ); if (ioctl(ifsock_v6, SIOCGLIFGROUPNAME, &lifr) != -1 && strcmp(lifr.lifr_groupname, grname) == 0) { router_add_common(AF_INET6, ifname, nexthop_v6); } } } } /* * Build a list of target routers, by scanning the routing tables. * It is assumed that interface routes exist, to reach the routers. */ static void init_router_targets(void) { struct target *tg; struct target *next_tg; struct phyint_instance *pii; struct phyint *pi; if (force_mcast) return; for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pi = pii->pii_phyint; /* * Set tg_in_use to false only for router targets. */ if (!pii->pii_targets_are_routers) continue; for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) tg->tg_in_use = 0; } if (mibwalk(update_router_list) == -1) exit(1); for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pi = pii->pii_phyint; if (!pii->pii_targets_are_routers) continue; for (tg = pii->pii_targets; tg != NULL; tg = next_tg) { next_tg = tg->tg_next; /* * If the group has failed, it's likely the route was * removed by an application affected by that failure. * In that case, we keep the target so that we can * reliably repair, at which point we'll refresh the * target list again. */ if (!tg->tg_in_use && !GROUP_FAILED(pi->pi_group)) target_delete(tg); } } } /* * Attempt to assign host targets to any interfaces that do not currently * have probe targets by sharing targets with other interfaces in the group. */ static void init_host_targets(void) { struct phyint_instance *pii; struct phyint_group *pg; for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pg = pii->pii_phyint->pi_group; if (pg != phyint_anongroup && pii->pii_targets == NULL) dup_host_targets(pii); } } /* * Duplicate host targets from other phyints of the group to * the phyint instance 'desired_pii'. */ static void dup_host_targets(struct phyint_instance *desired_pii) { int af; struct phyint *pi; struct phyint_instance *pii; struct target *tg; assert(desired_pii->pii_phyint->pi_group != phyint_anongroup); af = desired_pii->pii_af; /* * For every phyint in the same group as desired_pii, check if * it has any host targets. If so add them to desired_pii. */ for (pi = desired_pii->pii_phyint; pi != NULL; pi = pi->pi_pgnext) { pii = PHYINT_INSTANCE(pi, af); /* * We know that we don't have targets on this phyint instance * since we have been called. But we still check for * pii_targets_are_routers because another phyint instance * could have router targets, since IFF_NOFAILOVER addresses * on different phyint instances may belong to different * subnets. */ if ((pii == NULL) || (pii == desired_pii) || pii->pii_targets_are_routers) continue; for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { target_create(desired_pii, tg->tg_address, _B_FALSE); } } } static void usage(char *cmd) { (void) fprintf(stderr, "usage: %s\n", cmd); } #define MPATHD_DEFAULT_FILE "/etc/default/mpathd" /* Get an option from the /etc/default/mpathd file */ static char * getdefault(char *name) { char namebuf[BUFSIZ]; char *value = NULL; if (defopen(MPATHD_DEFAULT_FILE) == 0) { char *cp; int flags; /* * ignore case */ flags = defcntl(DC_GETFLAGS, 0); TURNOFF(flags, DC_CASE); (void) defcntl(DC_SETFLAGS, flags); /* Add "=" to the name */ (void) strncpy(namebuf, name, sizeof (namebuf) - 2); (void) strncat(namebuf, "=", 2); if ((cp = defread(namebuf)) != NULL) value = strdup(cp); /* close */ (void) defopen((char *)NULL); } return (value); } /* * Command line options below */ boolean_t failback_enabled = _B_TRUE; /* failback enabled/disabled */ boolean_t track_all_phyints = _B_FALSE; /* track all IP interfaces */ static boolean_t adopt = _B_FALSE; static boolean_t foreground = _B_FALSE; int main(int argc, char *argv[]) { int i; int c; struct phyint *pi; struct phyint_instance *pii; char *value; argv0 = argv; /* Saved for re-exec on SIGHUP */ srandom(gethostid()); /* Initialize the random number generator */ /* * NOTE: The messages output by in.mpathd are not suitable for * translation, so we do not call textdomain(). */ (void) setlocale(LC_ALL, ""); /* * Get the user specified value of 'failure detection time' * from /etc/default/mpathd */ value = getdefault("FAILURE_DETECTION_TIME"); if (value != NULL) { user_failure_detection_time = (int)strtol((char *)value, NULL, 0); if (user_failure_detection_time <= 0) { user_failure_detection_time = FAILURE_DETECTION_TIME; logerr("Invalid failure detection time %s, assuming " "default of %d ms\n", value, user_failure_detection_time); } else if (user_failure_detection_time < MIN_FAILURE_DETECTION_TIME) { user_failure_detection_time = MIN_FAILURE_DETECTION_TIME; logerr("Too small failure detection time of %s, " "assuming minimum of %d ms\n", value, user_failure_detection_time); } free(value); } else { /* User has not specified the parameter, Use default value */ user_failure_detection_time = FAILURE_DETECTION_TIME; } /* * This gives the frequency at which probes will be sent. * When fdt ms elapses, we should be able to determine * whether 5 consecutive probes have failed or not. * 1 probe will be sent in every user_probe_interval ms, * randomly anytime in the (0.5 - 1.0) 2nd half of every * user_probe_interval. Thus when we send out probe 'n' we * can be sure that probe 'n - 2' is lost, if we have not * got the ack. (since the probe interval is > crtt). But * probe 'n - 1' may be a valid unacked probe, since the * time between 2 successive probes could be as small as * 0.5 * user_probe_interval. Hence the NUM_PROBE_FAILS + 2 */ user_probe_interval = user_failure_detection_time / (NUM_PROBE_FAILS + 2); /* * Get the user specified value of failback_enabled from * /etc/default/mpathd */ value = getdefault("FAILBACK"); if (value != NULL) { if (strcasecmp(value, "yes") == 0) failback_enabled = _B_TRUE; else if (strcasecmp(value, "no") == 0) failback_enabled = _B_FALSE; else logerr("Invalid value for FAILBACK %s\n", value); free(value); } else { failback_enabled = _B_TRUE; } /* * Get the user specified value of track_all_phyints from * /etc/default/mpathd. The sense is reversed in * TRACK_INTERFACES_ONLY_WITH_GROUPS. */ value = getdefault("TRACK_INTERFACES_ONLY_WITH_GROUPS"); if (value != NULL) { if (strcasecmp(value, "yes") == 0) track_all_phyints = _B_FALSE; else if (strcasecmp(value, "no") == 0) track_all_phyints = _B_TRUE; else logerr("Invalid value for " "TRACK_INTERFACES_ONLY_WITH_GROUPS %s\n", value); free(value); } else { track_all_phyints = _B_FALSE; } while ((c = getopt(argc, argv, "adD:ml")) != EOF) { switch (c) { case 'a': adopt = _B_TRUE; break; case 'm': force_mcast = _B_TRUE; break; case 'd': debug = D_ALL; foreground = _B_TRUE; break; case 'D': i = (int)strtol(optarg, NULL, 0); if (i == 0) { (void) fprintf(stderr, "Bad debug flags: %s\n", optarg); exit(1); } debug |= i; foreground = _B_TRUE; break; case 'l': /* * Turn off link state notification handling. * Undocumented command line flag, for debugging * purposes. */ handle_link_notifications = _B_FALSE; break; default: usage(argv[0]); exit(1); } } /* * The sockets for the loopback command interface should be listening * before we fork and exit in daemonize(). This way, whoever started us * can use the loopback interface as soon as they get a zero exit * status. */ lsock_v4 = setup_listener(AF_INET); lsock_v6 = setup_listener(AF_INET6); if (lsock_v4 < 0 && lsock_v6 < 0) { logerr("main: setup_listener failed for both IPv4 and IPv6\n"); exit(1); } if (!foreground) { if (!daemonize()) { logerr("cannot daemonize\n"); exit(EXIT_FAILURE); } initlog(); } /* * Initializations: * 1. Create ifsock* sockets. These are used for performing SIOC* * ioctls. We have 2 sockets 1 each for IPv4 and IPv6. * 2. Initialize a pipe for handling/recording signal events. * 3. Create the routing sockets, used for listening * to routing / interface changes. * 4. phyint_init() - Initialize physical interface state * (in mpd_tables.c). Must be done before creating interfaces, * which timer_init() does indirectly. * 5. Query kernel for route entry sizes (v4 and v6). * 6. timer_init() - Initialize timer related stuff * 7. initifs() - Initialize our database of all known interfaces * 8. init_router_targets() - Initialize our database of all known * router targets. */ ifsock_v4 = socket(AF_INET, SOCK_DGRAM, 0); if (ifsock_v4 < 0) { logperror("main: IPv4 socket open"); exit(1); } ifsock_v6 = socket(AF_INET6, SOCK_DGRAM, 0); if (ifsock_v6 < 0) { logperror("main: IPv6 socket open"); exit(1); } setup_eventpipe(); rtsock_v4 = setup_rtsock(AF_INET); rtsock_v6 = setup_rtsock(AF_INET6); if (phyint_init() == -1) { logerr("cannot initialize physical interface structures"); exit(1); } if (mibwalk(mib_get_constants) == -1) exit(1); timer_init(); initifs(); /* * If we're operating in "adopt" mode and no interfaces need to be * tracked, shut down (ifconfig(8) will restart us on demand if * interfaces are subsequently put into multipathing groups). */ if (adopt && phyint_instances == NULL) exit(0); /* * Main body. Keep listening for activity on any of the sockets * that we are monitoring and take appropriate action as necessary. * signals are also handled synchronously. */ for (;;) { if (poll(pollfds, pollfd_num, -1) < 0) { if (errno == EINTR) continue; logperror("main: poll"); exit(1); } for (i = 0; i < pollfd_num; i++) { if ((pollfds[i].fd == -1) || !(pollfds[i].revents & POLLIN)) continue; if (pollfds[i].fd == eventpipe_read) { in_signal(eventpipe_read); break; } if (pollfds[i].fd == rtsock_v4 || pollfds[i].fd == rtsock_v6) { process_rtsock(rtsock_v4, rtsock_v6); break; } for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { if (pollfds[i].fd == pii->pii_probe_sock) { if (pii->pii_af == AF_INET) in_data(pii); else in6_data(pii); break; } } for (pi = phyints; pi != NULL; pi = pi->pi_next) { if (pi->pi_notes != 0 && pollfds[i].fd == dlpi_fd(pi->pi_dh)) { (void) dlpi_recv(pi->pi_dh, NULL, NULL, NULL, NULL, 0, NULL); break; } } if (pollfds[i].fd == lsock_v4) loopback_cmd(lsock_v4, AF_INET); else if (pollfds[i].fd == lsock_v6) loopback_cmd(lsock_v6, AF_INET6); } } /* NOTREACHED */ return (EXIT_SUCCESS); } static int setup_listener(int af) { int sock; int on; int len; int ret; struct sockaddr_storage laddr; struct sockaddr_in *sin; struct sockaddr_in6 *sin6; struct in6_addr loopback_addr = IN6ADDR_LOOPBACK_INIT; assert(af == AF_INET || af == AF_INET6); sock = socket(af, SOCK_STREAM, 0); if (sock < 0) { logperror("setup_listener: socket"); exit(1); } on = 1; if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof (on)) < 0) { logperror("setup_listener: setsockopt (SO_REUSEADDR)"); exit(1); } bzero(&laddr, sizeof (laddr)); laddr.ss_family = af; if (af == AF_INET) { sin = (struct sockaddr_in *)&laddr; sin->sin_port = htons(MPATHD_PORT); sin->sin_addr.s_addr = htonl(INADDR_LOOPBACK); len = sizeof (struct sockaddr_in); } else { sin6 = (struct sockaddr_in6 *)&laddr; sin6->sin6_port = htons(MPATHD_PORT); sin6->sin6_addr = loopback_addr; len = sizeof (struct sockaddr_in6); } ret = bind(sock, (struct sockaddr *)&laddr, len); if (ret < 0) { if (errno == EADDRINUSE) { /* * Another instance of mpathd may be already active. */ logerr("main: is another instance of in.mpathd " "already active?\n"); exit(1); } else { (void) close(sock); return (-1); } } if (listen(sock, 30) < 0) { logperror("main: listen"); exit(1); } if (poll_add(sock) == -1) { (void) close(sock); exit(1); } return (sock); } /* * Table of commands and their expected size; used by loopback_cmd(). */ static struct { const char *name; unsigned int size; } commands[] = { { "MI_PING", sizeof (uint32_t) }, { "MI_OFFLINE", sizeof (mi_offline_t) }, { "MI_UNDO_OFFLINE", sizeof (mi_undo_offline_t) }, { "MI_QUERY", sizeof (mi_query_t) } }; /* * Commands received over the loopback interface come here (via libipmp). */ static void loopback_cmd(int sock, int family) { int newfd; ssize_t len; boolean_t is_priv = _B_FALSE; struct sockaddr_storage peer; struct sockaddr_in *peer_sin; struct sockaddr_in6 *peer_sin6; socklen_t peerlen; union mi_commands mpi; char abuf[INET6_ADDRSTRLEN]; uint_t cmd; int retval; peerlen = sizeof (peer); newfd = accept(sock, (struct sockaddr *)&peer, &peerlen); if (newfd < 0) { logperror("loopback_cmd: accept"); return; } switch (family) { case AF_INET: /* * Validate the address and port to make sure that * non privileged processes don't connect and start * talking to us. */ if (peerlen != sizeof (struct sockaddr_in)) { logerr("loopback_cmd: AF_INET peerlen %d\n", peerlen); (void) close(newfd); return; } peer_sin = (struct sockaddr_in *)&peer; is_priv = ntohs(peer_sin->sin_port) < IPPORT_RESERVED; (void) inet_ntop(AF_INET, &peer_sin->sin_addr.s_addr, abuf, sizeof (abuf)); if (ntohl(peer_sin->sin_addr.s_addr) != INADDR_LOOPBACK) { logerr("Attempt to connect from addr %s port %d\n", abuf, ntohs(peer_sin->sin_port)); (void) close(newfd); return; } break; case AF_INET6: if (peerlen != sizeof (struct sockaddr_in6)) { logerr("loopback_cmd: AF_INET6 peerlen %d\n", peerlen); (void) close(newfd); return; } /* * Validate the address and port to make sure that * non privileged processes don't connect and start * talking to us. */ peer_sin6 = (struct sockaddr_in6 *)&peer; is_priv = ntohs(peer_sin6->sin6_port) < IPPORT_RESERVED; (void) inet_ntop(AF_INET6, &peer_sin6->sin6_addr, abuf, sizeof (abuf)); if (!IN6_IS_ADDR_LOOPBACK(&peer_sin6->sin6_addr)) { logerr("Attempt to connect from addr %s port %d\n", abuf, ntohs(peer_sin6->sin6_port)); (void) close(newfd); return; } break; default: logdebug("loopback_cmd: family %d\n", family); (void) close(newfd); return; } /* * The sizeof the 'mpi' buffer corresponds to the maximum size of * all supported commands */ len = read(newfd, &mpi, sizeof (mpi)); /* * In theory, we can receive any sized message for a stream socket, * but we don't expect that to happen for a small message over a * loopback connection. */ if (len < sizeof (uint32_t)) { logerr("loopback_cmd: bad command format or read returns " "partial data %d\n", len); (void) close(newfd); return; } cmd = mpi.mi_command; if (cmd >= MI_NCMD) { logerr("loopback_cmd: unknown command id `%d'\n", cmd); (void) close(newfd); return; } /* * Only MI_PING and MI_QUERY can come from unprivileged sources. */ if (!is_priv && (cmd != MI_QUERY && cmd != MI_PING)) { logerr("Unprivileged request from %s for privileged " "command %s\n", abuf, commands[cmd].name); (void) close(newfd); return; } if (len < commands[cmd].size) { logerr("loopback_cmd: short %s command (expected %d, got %d)\n", commands[cmd].name, commands[cmd].size, len); (void) close(newfd); return; } retval = process_cmd(newfd, &mpi); if (retval != IPMP_SUCCESS) { logerr("failed processing %s: %s\n", commands[cmd].name, ipmp_errmsg(retval)); } (void) close(newfd); } /* * Process the commands received via libipmp. */ static unsigned int process_cmd(int newfd, union mi_commands *mpi) { struct phyint *pi; struct mi_offline *mio; struct mi_undo_offline *miu; unsigned int retval; switch (mpi->mi_command) { case MI_PING: return (send_result(newfd, IPMP_SUCCESS, 0)); case MI_OFFLINE: mio = &mpi->mi_ocmd; pi = phyint_lookup(mio->mio_ifname); if (pi == NULL) return (send_result(newfd, IPMP_EUNKIF, 0)); retval = phyint_offline(pi, mio->mio_min_redundancy); if (retval == IPMP_FAILURE) return (send_result(newfd, IPMP_FAILURE, errno)); return (send_result(newfd, retval, 0)); case MI_UNDO_OFFLINE: miu = &mpi->mi_ucmd; pi = phyint_lookup(miu->miu_ifname); if (pi == NULL) return (send_result(newfd, IPMP_EUNKIF, 0)); retval = phyint_undo_offline(pi); if (retval == IPMP_FAILURE) return (send_result(newfd, IPMP_FAILURE, errno)); return (send_result(newfd, retval, 0)); case MI_QUERY: return (process_query(newfd, &mpi->mi_qcmd)); default: break; } return (send_result(newfd, IPMP_EPROTO, 0)); } /* * Process the query request pointed to by `miq' and send a reply on file * descriptor `fd'. Returns an IPMP error code. */ static unsigned int process_query(int fd, mi_query_t *miq) { ipmp_addrinfo_t *adinfop; ipmp_addrinfolist_t *adlp; ipmp_groupinfo_t *grinfop; ipmp_groupinfolist_t *grlp; ipmp_grouplist_t *grlistp; ipmp_ifinfo_t *ifinfop; ipmp_ifinfolist_t *iflp; ipmp_snap_t *snap; unsigned int retval; switch (miq->miq_inforeq) { case IPMP_ADDRINFO: retval = getgraddrinfo(miq->miq_grname, &miq->miq_addr, &adinfop); if (retval != IPMP_SUCCESS) return (send_result(fd, retval, errno)); retval = send_result(fd, IPMP_SUCCESS, 0); if (retval == IPMP_SUCCESS) retval = send_addrinfo(fd, adinfop); ipmp_freeaddrinfo(adinfop); return (retval); case IPMP_GROUPLIST: retval = getgrouplist(&grlistp); if (retval != IPMP_SUCCESS) return (send_result(fd, retval, errno)); retval = send_result(fd, IPMP_SUCCESS, 0); if (retval == IPMP_SUCCESS) retval = send_grouplist(fd, grlistp); ipmp_freegrouplist(grlistp); return (retval); case IPMP_GROUPINFO: miq->miq_grname[LIFGRNAMSIZ - 1] = '\0'; retval = getgroupinfo(miq->miq_grname, &grinfop); if (retval != IPMP_SUCCESS) return (send_result(fd, retval, errno)); retval = send_result(fd, IPMP_SUCCESS, 0); if (retval == IPMP_SUCCESS) retval = send_groupinfo(fd, grinfop); ipmp_freegroupinfo(grinfop); return (retval); case IPMP_IFINFO: miq->miq_ifname[LIFNAMSIZ - 1] = '\0'; retval = getifinfo(miq->miq_ifname, &ifinfop); if (retval != IPMP_SUCCESS) return (send_result(fd, retval, errno)); retval = send_result(fd, IPMP_SUCCESS, 0); if (retval == IPMP_SUCCESS) retval = send_ifinfo(fd, ifinfop); ipmp_freeifinfo(ifinfop); return (retval); case IPMP_SNAP: /* * Before taking the snapshot, sync with the kernel. */ initifs(); retval = getsnap(&snap); if (retval != IPMP_SUCCESS) return (send_result(fd, retval, errno)); retval = send_result(fd, IPMP_SUCCESS, 0); if (retval != IPMP_SUCCESS) goto out; retval = send_grouplist(fd, snap->sn_grlistp); if (retval != IPMP_SUCCESS) goto out; retval = ipmp_writetlv(fd, IPMP_IFCNT, sizeof (uint32_t), &snap->sn_nif); if (retval != IPMP_SUCCESS) goto out; iflp = snap->sn_ifinfolistp; for (; iflp != NULL; iflp = iflp->ifl_next) { retval = send_ifinfo(fd, iflp->ifl_ifinfop); if (retval != IPMP_SUCCESS) goto out; } retval = ipmp_writetlv(fd, IPMP_GROUPCNT, sizeof (uint32_t), &snap->sn_ngroup); if (retval != IPMP_SUCCESS) goto out; grlp = snap->sn_grinfolistp; for (; grlp != NULL; grlp = grlp->grl_next) { retval = send_groupinfo(fd, grlp->grl_grinfop); if (retval != IPMP_SUCCESS) goto out; } retval = ipmp_writetlv(fd, IPMP_ADDRCNT, sizeof (uint32_t), &snap->sn_naddr); if (retval != IPMP_SUCCESS) goto out; adlp = snap->sn_adinfolistp; for (; adlp != NULL; adlp = adlp->adl_next) { retval = send_addrinfo(fd, adlp->adl_adinfop); if (retval != IPMP_SUCCESS) goto out; } out: ipmp_snap_free(snap); return (retval); default: break; } return (send_result(fd, IPMP_EPROTO, 0)); } /* * Send the group information pointed to by `grinfop' on file descriptor `fd'. * Returns an IPMP error code. */ static unsigned int send_groupinfo(int fd, ipmp_groupinfo_t *grinfop) { ipmp_iflist_t *iflistp = grinfop->gr_iflistp; ipmp_addrlist_t *adlistp = grinfop->gr_adlistp; ipmp_groupinfo_xfer_t grxfer; unsigned int retval; /* * We can't directly transfer an ipmp_groupinfo_t due to the embedded * pointers to ipmp_iflist_t and ipmp_addr_list_t. Copy the data over * to a temporary transfer structure that doesn't have these embedded * pointers. */ memset(&grxfer, 0, sizeof (grxfer)); grxfer.grx_sig = grinfop->gr_sig; grxfer.grx_state = grinfop->gr_state; grxfer.grx_fdt = grinfop->gr_fdt; memcpy(grxfer.grx_name, grinfop->gr_name, sizeof (grxfer.grx_name)); memcpy(grxfer.grx_ifname, grinfop->gr_ifname, sizeof (grxfer.grx_ifname)); memcpy(grxfer.grx_m4ifname, grinfop->gr_m4ifname, sizeof (grxfer.grx_m4ifname)); memcpy(grxfer.grx_m6ifname, grinfop->gr_m6ifname, sizeof (grxfer.grx_m6ifname)); memcpy(grxfer.grx_bcifname, grinfop->gr_bcifname, sizeof (grxfer.grx_bcifname)); retval = ipmp_writetlv(fd, IPMP_GROUPINFO, sizeof (grxfer), &grxfer); if (retval != IPMP_SUCCESS) return (retval); retval = ipmp_writetlv(fd, IPMP_IFLIST, IPMP_IFLIST_SIZE(iflistp->il_nif), iflistp); if (retval != IPMP_SUCCESS) return (retval); return (ipmp_writetlv(fd, IPMP_ADDRLIST, IPMP_ADDRLIST_SIZE(adlistp->al_naddr), adlistp)); } /* * Send the interface information pointed to by `ifinfop' on file descriptor * `fd'. Returns an IPMP error code. */ static unsigned int send_ifinfo(int fd, ipmp_ifinfo_t *ifinfop) { ipmp_addrlist_t *adlist4p = ifinfop->if_targinfo4.it_targlistp; ipmp_addrlist_t *adlist6p = ifinfop->if_targinfo6.it_targlistp; ipmp_ifinfo_xfer_t ifxfer; unsigned int retval; /* * We can't directly tranfer an ipmp_ifinfo_t due to the embedded * ipmp_addrlist_t pointer in if_targinfo_t. Copy the data over to * a temporary transfer structure that doesn't have that embedded * pointer. */ memset(&ifxfer, 0, sizeof (ifxfer)); ifxfer.ifx_state = ifinfop->if_state; ifxfer.ifx_type = ifinfop->if_type; ifxfer.ifx_linkstate = ifinfop->if_linkstate; ifxfer.ifx_probestate = ifinfop->if_probestate; ifxfer.ifx_flags = ifinfop->if_flags; ifxfer.ifx_targinfo4.itx_testaddr = ifinfop->if_targinfo4.it_testaddr; ifxfer.ifx_targinfo4.itx_targmode = ifinfop->if_targinfo4.it_targmode; ifxfer.ifx_targinfo6.itx_testaddr = ifinfop->if_targinfo6.it_testaddr; ifxfer.ifx_targinfo6.itx_targmode = ifinfop->if_targinfo6.it_targmode; memcpy(ifxfer.ifx_name, ifinfop->if_name, sizeof (ifxfer.ifx_name)); memcpy(ifxfer.ifx_group, ifinfop->if_group, sizeof (ifxfer.ifx_group)); memcpy(ifxfer.ifx_targinfo4.itx_name, ifinfop->if_targinfo4.it_name, sizeof (ifxfer.ifx_targinfo4.itx_name)); memcpy(ifxfer.ifx_targinfo6.itx_name, ifinfop->if_targinfo6.it_name, sizeof (ifxfer.ifx_targinfo6.itx_name)); retval = ipmp_writetlv(fd, IPMP_IFINFO, sizeof (ifxfer), &ifxfer); if (retval != IPMP_SUCCESS) return (retval); retval = ipmp_writetlv(fd, IPMP_ADDRLIST, IPMP_ADDRLIST_SIZE(adlist4p->al_naddr), adlist4p); if (retval != IPMP_SUCCESS) return (retval); return (ipmp_writetlv(fd, IPMP_ADDRLIST, IPMP_ADDRLIST_SIZE(adlist6p->al_naddr), adlist6p)); } /* * Send the address information pointed to by `adinfop' on file descriptor * `fd'. Returns an IPMP error code. */ static unsigned int send_addrinfo(int fd, ipmp_addrinfo_t *adinfop) { return (ipmp_writetlv(fd, IPMP_ADDRINFO, sizeof (*adinfop), adinfop)); } /* * Send the group list pointed to by `grlistp' on file descriptor `fd'. * Returns an IPMP error code. */ static unsigned int send_grouplist(int fd, ipmp_grouplist_t *grlistp) { return (ipmp_writetlv(fd, IPMP_GROUPLIST, IPMP_GROUPLIST_SIZE(grlistp->gl_ngroup), grlistp)); } /* * Initialize an mi_result_t structure using `error' and `syserror' and * send it on file descriptor `fd'. Returns an IPMP error code. */ static unsigned int send_result(int fd, unsigned int error, int syserror) { mi_result_t me; me.me_mpathd_error = error; if (error == IPMP_FAILURE) me.me_sys_error = syserror; else me.me_sys_error = 0; return (ipmp_write(fd, &me, sizeof (me))); } /* * Daemonize the process. */ static boolean_t daemonize(void) { switch (fork()) { case -1: return (_B_FALSE); case 0: /* * Lose our controlling terminal, and become both a session * leader and a process group leader. */ if (setsid() == -1) return (_B_FALSE); /* * Under POSIX, a session leader can accidentally (through * open(2)) acquire a controlling terminal if it does not * have one. Just to be safe, fork() again so we are not a * session leader. */ switch (fork()) { case -1: return (_B_FALSE); case 0: (void) chdir("/"); (void) umask(022); (void) fdwalk(closefunc, NULL); break; default: _exit(EXIT_SUCCESS); } break; default: _exit(EXIT_SUCCESS); } return (_B_TRUE); } /* * The parent has created some fds before forking on purpose, keep them open. */ static int closefunc(void *not_used, int fd) { if (fd != lsock_v4 && fd != lsock_v6) (void) close(fd); return (0); } /* LOGGER */ #include /* * Logging routines. All routines log to syslog, unless the daemon is * running in the foreground, in which case the logging goes to stderr. * * The following routines are available: * * logdebug(): A printf-like function for outputting debug messages * (messages at LOG_DEBUG) that are only of use to developers. * * logtrace(): A printf-like function for outputting tracing messages * (messages at LOG_INFO) from the daemon. This is typically used * to log the receipt of interesting network-related conditions. * * logerr(): A printf-like function for outputting error messages * (messages at LOG_ERR) from the daemon. * * logperror*(): A set of functions used to output error messages * (messages at LOG_ERR); these automatically append strerror(errno) * and a newline to the message passed to them. * * NOTE: since the logging functions write to syslog, the messages passed * to them are not eligible for localization. Thus, gettext() must * *not* be used. */ static int logging = 0; static void initlog(void) { logging++; openlog("in.mpathd", LOG_PID, LOG_DAEMON); } /* PRINTFLIKE2 */ void logmsg(int pri, const char *fmt, ...) { va_list ap; va_start(ap, fmt); if (logging) vsyslog(pri, fmt, ap); else (void) vfprintf(stderr, fmt, ap); va_end(ap); } /* PRINTFLIKE1 */ void logperror(const char *str) { if (logging) syslog(LOG_ERR, "%s: %m\n", str); else (void) fprintf(stderr, "%s: %s\n", str, strerror(errno)); } void logperror_pii(struct phyint_instance *pii, const char *str) { if (logging) { syslog(LOG_ERR, "%s (%s %s): %m\n", str, AF_STR(pii->pii_af), pii->pii_phyint->pi_name); } else { (void) fprintf(stderr, "%s (%s %s): %s\n", str, AF_STR(pii->pii_af), pii->pii_phyint->pi_name, strerror(errno)); } } void logperror_li(struct logint *li, const char *str) { struct phyint_instance *pii = li->li_phyint_inst; if (logging) { syslog(LOG_ERR, "%s (%s %s): %m\n", str, AF_STR(pii->pii_af), li->li_name); } else { (void) fprintf(stderr, "%s (%s %s): %s\n", str, AF_STR(pii->pii_af), li->li_name, strerror(errno)); } } void close_probe_socket(struct phyint_instance *pii, boolean_t polled) { if (polled) (void) poll_remove(pii->pii_probe_sock); (void) close(pii->pii_probe_sock); pii->pii_probe_sock = -1; pii->pii_basetime_inited = 0; } boolean_t addrlist_add(addrlist_t **addrsp, const char *name, uint64_t flags, struct sockaddr_storage *ssp) { addrlist_t *addrp; if ((addrp = malloc(sizeof (addrlist_t))) == NULL) return (_B_FALSE); (void) strlcpy(addrp->al_name, name, LIFNAMSIZ); addrp->al_flags = flags; addrp->al_addr = *ssp; addrp->al_next = *addrsp; *addrsp = addrp; return (_B_TRUE); } void addrlist_free(addrlist_t **addrsp) { addrlist_t *addrp, *next_addrp; for (addrp = *addrsp; addrp != NULL; addrp = next_addrp) { next_addrp = addrp->al_next; free(addrp); } *addrsp = NULL; } /* * Send down a T_OPTMGMT_REQ to ip asking for all data in the various * tables defined by mib2.h. Pass the table information returned to the * supplied function. */ static int mibwalk(void (*proc)(mib_item_t *)) { mib_item_t *head_item = NULL; mib_item_t *last_item = NULL; mib_item_t *tmp; struct strbuf ctlbuf, databuf; int flags; int rval; uintptr_t buf[512 / sizeof (uintptr_t)]; struct T_optmgmt_req *tor = (struct T_optmgmt_req *)buf; struct T_optmgmt_ack *toa = (struct T_optmgmt_ack *)buf; struct T_error_ack *tea = (struct T_error_ack *)buf; struct opthdr *req, *optp; int status = -1; if (mibfd == -1) { if ((mibfd = open("/dev/ip", O_RDWR)) < 0) { logperror("mibwalk(): ip open"); return (status); } } tor->PRIM_type = T_SVR4_OPTMGMT_REQ; tor->OPT_offset = sizeof (struct T_optmgmt_req); tor->OPT_length = sizeof (struct opthdr); tor->MGMT_flags = T_CURRENT; /* * Note: we use the special level value below so that IP will return * us information concerning IRE_MARK_TESTHIDDEN routes. */ req = (struct opthdr *)&tor[1]; req->level = EXPER_IP_AND_ALL_IRES; req->name = 0; req->len = 0; ctlbuf.buf = (char *)&buf; ctlbuf.len = tor->OPT_length + tor->OPT_offset; if (putmsg(mibfd, &ctlbuf, NULL, 0) == -1) { logperror("mibwalk(): putmsg(ctl)"); return (status); } /* * The response consists of multiple T_OPTMGMT_ACK msgs, 1 msg for * each table defined in mib2.h. Each T_OPTMGMT_ACK msg contains * a control and data part. The control part contains a struct * T_optmgmt_ack followed by a struct opthdr. The 'opthdr' identifies * the level, name and length of the data in the data part. The * data part contains the actual table data. The last message * is an end-of-data (EOD), consisting of a T_OPTMGMT_ACK and a * single option with zero optlen. */ for (;;) { errno = flags = 0; ctlbuf.maxlen = sizeof (buf); rval = getmsg(mibfd, &ctlbuf, NULL, &flags); if (rval & MORECTL || rval < 0) { if (errno == EINTR) continue; logerr("mibwalk(): getmsg(ctl) ret: %d err: %d\n", rval, errno); goto error; } if (ctlbuf.len < sizeof (t_scalar_t)) { logerr("mibwalk(): ctlbuf.len %d\n", ctlbuf.len); goto error; } switch (toa->PRIM_type) { case T_ERROR_ACK: if (ctlbuf.len < sizeof (struct T_error_ack)) { logerr("mibwalk(): T_ERROR_ACK ctlbuf " "too short: %d\n", ctlbuf.len); goto error; } logerr("mibwalk(): T_ERROR_ACK: TLI_err = 0x%lx: %s\n" " UNIX_err = 0x%lx\n", tea->TLI_error, t_strerror(tea->TLI_error), tea->UNIX_error); goto error; case T_OPTMGMT_ACK: optp = (struct opthdr *)&toa[1]; if (ctlbuf.len < (sizeof (struct T_optmgmt_ack) + sizeof (struct opthdr))) { logerr("mibwalk(): T_OPTMGMT_ACK ctlbuf too " "short: %d\n", ctlbuf.len); goto error; } if (toa->MGMT_flags != T_SUCCESS) { logerr("mibwalk(): MGMT_flags != T_SUCCESS: " "0x%lx\n", toa->MGMT_flags); goto error; } break; default: goto error; } /* The following assert also implies MGMT_flags == T_SUCCESS */ assert(toa->PRIM_type == T_OPTMGMT_ACK); /* * We have reached the end of this T_OPTMGMT_ACK * message. If this is the last message i.e EOD, * break, else process the next T_OPTMGMT_ACK msg. */ if (rval == 0) { if (optp->len == 0 && optp->name == 0 && optp->level == 0) { /* This is the EOD message. */ break; } /* Not EOD but no data to retrieve */ continue; } /* * We should only be here if MOREDATA was set. * Allocate an empty mib_item_t and link into the list * of MIB items. */ if ((tmp = malloc(sizeof (*tmp))) == NULL) { logperror("mibwalk(): malloc() failed."); goto error; } if (last_item != NULL) last_item->mi_next = tmp; else head_item = tmp; last_item = tmp; last_item->mi_next = NULL; last_item->mi_opthdr = *optp; last_item->mi_valp = malloc(optp->len); if (last_item->mi_valp == NULL) { logperror("mibwalk(): malloc() failed."); goto error; } databuf.maxlen = last_item->mi_opthdr.len; databuf.buf = (char *)last_item->mi_valp; databuf.len = 0; /* Retrieve the actual MIB data */ for (;;) { flags = 0; if ((rval = getmsg(mibfd, NULL, &databuf, &flags)) != 0) { if (rval < 0 && errno == EINTR) continue; /* * We shouldn't get MOREDATA here so treat that * as an error. */ logperror("mibwalk(): getmsg(data)"); goto error; } break; } } status = 0; /* Pass the accumulated MIB data to the supplied function pointer */ (*proc)(head_item); error: while (head_item != NULL) { tmp = head_item; head_item = tmp->mi_next; free(tmp->mi_valp); free(tmp); } return (status); } /* * Parse the supplied mib2 information to get the size of routing table * entries. This is needed when running in a branded zone where the * Solaris application environment and the Solaris kernel may not be the * the same release version. */ static void mib_get_constants(mib_item_t *item) { mib2_ip_t *ipv4; mib2_ipv6IfStatsEntry_t *ipv6; for (; item != NULL; item = item->mi_next) { if (item->mi_opthdr.name != 0) continue; if (item->mi_opthdr.level == MIB2_IP) { ipv4 = (mib2_ip_t *)item->mi_valp; ipRouteEntrySize = ipv4->ipRouteEntrySize; } else if (item->mi_opthdr.level == MIB2_IP6) { ipv6 = (mib2_ipv6IfStatsEntry_t *)item->mi_valp; ipv6RouteEntrySize = ipv6->ipv6RouteEntrySize; } } } /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 1987 Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms are permitted * provided that the above copyright notice and this paragraph are * duplicated in all such forms and that any documentation, * advertising materials, and other materials related to such * distribution and use acknowledge that the software was developed * by the University of California, Berkeley. The name of the * University may not be used to endorse or promote products derived * from this software without specific prior written permission. * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. */ #include "mpd_defs.h" #include "mpd_tables.h" /* * Probe types for probe() */ #define PROBE_UNI 0x1234 /* Unicast probe packet */ #define PROBE_MULTI 0x5678 /* Multicast probe packet */ #define PROBE_RTT 0x9abc /* RTT only probe packet */ #define MSEC_PERMIN (60 * MILLISEC) /* Number of milliseconds in a minute */ /* * Format of probe / probe response packets. This is an ICMP Echo request * or ICMP Echo reply. Packet format is same for both IPv4 and IPv6 */ struct pr_icmp { uint8_t pr_icmp_type; /* type field */ uint8_t pr_icmp_code; /* code field */ uint16_t pr_icmp_cksum; /* checksum field */ uint16_t pr_icmp_id; /* Identification */ uint16_t pr_icmp_seq; /* sequence number */ uint64_t pr_icmp_timestamp; /* Time stamp (in ns) */ uint32_t pr_icmp_mtype; /* Message type */ }; static struct in6_addr all_nodes_mcast_v6 = { { 0xff, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } }; static struct in_addr all_nodes_mcast_v4 = { { { 0xe0, 0x0, 0x0, 0x1 } } }; static hrtime_t last_fdt_bumpup_time; /* When FDT was bumped up last */ static void *find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type); static void pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni); static void incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp); static void incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr); static void incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr); static boolean_t check_pg_crtt_improved(struct phyint_group *pg); static boolean_t check_pii_crtt_improved(struct phyint_instance *pii); static boolean_t check_exception_target(struct phyint_instance *pii, struct target *target); static void probe_fail_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_fail_count *pfinfo); static void probe_success_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_success_count *psinfo); static boolean_t phyint_repaired(struct phyint *pi); static boolean_t highest_ack_tg(uint16_t seq, struct target *tg); static int in_cksum(ushort_t *addr, int len); static void reset_snxt_basetimes(void); static int ns2ms(int64_t ns); static int64_t tv2ns(struct timeval *); /* * CRTT - Conservative Round Trip Time Estimate * Probe success - A matching probe reply received before CRTT ms has elapsed * after sending the probe. * Probe failure - No probe reply received and more than CRTT ms has elapsed * after sending the probe. * * TLS - Time last success. Most recent probe ack received at this time. * TFF - Time first fail. The time of the earliest probe failure in * a consecutive series of probe failures. * NUM_PROBE_REPAIRS - Number of consecutive successful probes required * before declaring phyint repair. * NUM_PROBE_FAILS - Number of consecutive probe failures required to * declare a phyint failure. * * Phyint state diagram * * The state of a phyint that is capable of being probed, is completely * specified by the 3-tuple . * * A phyint starts in either PI_RUNNING or PI_OFFLINE, depending on whether * IFF_OFFLINE is set. If the phyint is also configured with a test address * (the common case) and probe targets, then a phyint must also successfully * be able to send and receive probes in order to remain in the PI_RUNNING * state (otherwise, it transitions to PI_FAILED). * * Further, if a PI_RUNNING phyint is configured with a test address but is * unable to find any probe targets, it will transition to the PI_NOTARGETS * state, which indicates that the link is apparently functional but that * in.mpathd is unable to send probes to verify functionality (in this case, * in.mpathd makes the optimistic assumption that the interface is working * correctly and thus does not mark the interface FAILED, but reports it as * IPMP_IF_UNKNOWN through the async events and query interfaces). * * At any point, a phyint may be administratively marked offline via if_mpadm. * In this case, the interface always transitions to PI_OFFLINE, regardless * of its previous state. When the interface is later brought back online, * in.mpathd acts as if the interface is new (and thus it transitions to * PI_RUNNING or PI_FAILED based on the status of the link and the result of * its probes, if probes are sent). * * pi_state - PI_RUNNING or PI_FAILED * PI_RUNNING: The failure detection logic says the phyint is good. * PI_FAILED: The failure detection logic says the phyint has failed. * * pg_state - PG_OK, PG_DEGRADED, or PG_FAILED. * PG_OK: All interfaces in the group are OK. * PG_DEGRADED: Some interfaces in the group are unusable. * PG_FAILED: All interfaces in the group are unusable. * * In the case of router targets, we assume that the current list of * targets obtained from the routing table, is still valid, so the * phyint stat is PI_FAILED. In the case of host targets, we delete the * list of targets, and multicast to the all hosts, to reconstruct the * target list. So the phyints are in the PI_NOTARGETS state. * * I - value of (pi_flags & IFF_INACTIVE) * IFF_INACTIVE: This phyint will not send or receive packets. * Usually, inactive is tied to standby interfaces that are not yet * needed (e.g., no non-standby interfaces in the group have failed). * When failback has been disabled (FAILBACK=no configured), phyint can * also be a non-STANDBY. In this case IFF_INACTIVE is set when phyint * subsequently recovers after a failure. * * Not all 9 possible combinations of the above 3-tuple are possible. * * I is tracked by IP. pi_state is tracked by mpathd. * * pi_state state machine * --------------------------------------------------------------------------- * Event State New State * Action: * --------------------------------------------------------------------------- * IP interface failure (PI_RUNNING, I == 0) -> (PI_FAILED, I == 0) * detection : set IFF_FAILED on this phyint * * IP interface failure (PI_RUNNING, I == 1) -> (PI_FAILED, I == 0) * detection : set IFF_FAILED on this phyint * * IP interface repair (PI_FAILED, I == 0, FAILBACK=yes) * detection -> (PI_RUNNING, I == 0) * : clear IFF_FAILED on this phyint * * IP interface repair (PI_FAILED, I == 0, FAILBACK=no) * detection -> (PI_RUNNING, I == 1) * : clear IFF_FAILED on this phyint * : if failback is disabled set I == 1 * * Group failure (perform on all phyints in the group) * detection PI_RUNNING PI_FAILED * (Router targets) : set IFF_FAILED * * Group failure (perform on all phyints in the group) * detection PI_RUNNING PI_NOTARGETS * (Host targets) : set IFF_FAILED * : delete the target list on all phyints * --------------------------------------------------------------------------- */ struct probes_missed probes_missed; /* * Compose and transmit an ICMP ECHO REQUEST packet. The IP header * will be added on by the kernel. The id field identifies this phyint. * and the sequence number is an increasing (modulo 2^^16) integer. The data * portion holds the time value when the packet is sent. On echo this is * extracted to compute the round-trip time. Three different types of * probe packets are used. * * PROBE_UNI: This type is used to do failure detection / failure recovery * and RTT calculation. PROBE_UNI probes are spaced apart in time, * not less than the current CRTT. pii_probes[] stores data * about these probes. These packets consume sequence number space. * * PROBE_RTT: This type is used to make only rtt measurements. Normally these * are not used. Under heavy network load, the rtt may go up very high, * due to a spike, or may appear to go high, due to extreme scheduling * delays. Once the network stress is removed, mpathd takes long time to * recover, because the probe_interval is already high, and it takes * a long time to send out sufficient number of probes to bring down the * rtt. To avoid this problem, PROBE_RTT probes are sent out every * user_probe_interval ms. and will cause only rtt updates. These packets * do not consume sequence number space nor is information about these * packets stored in the pii_probes[] * * PROBE_MULTI: This type is only used to construct a list of targets, when * no targets are known. The packet is multicast to the all hosts addr. */ static void probe(struct phyint_instance *pii, uint_t probe_type, hrtime_t start_hrtime) { hrtime_t sent_hrtime; struct timeval sent_tv; struct pr_icmp probe_pkt; /* Probe packet */ struct sockaddr_storage targ; /* target address */ uint_t targaddrlen; /* targed address length */ int pr_ndx; /* probe index in pii->pii_probes[] */ boolean_t sent = _B_FALSE; int rval; if (debug & D_TARGET) { logdebug("probe(%s %s %d %lld)\n", AF_STR(pii->pii_af), pii->pii_name, probe_type, start_hrtime); } assert(pii->pii_probe_sock != -1); assert(probe_type == PROBE_UNI || probe_type == PROBE_MULTI || probe_type == PROBE_RTT); probe_pkt.pr_icmp_type = (pii->pii_af == AF_INET) ? ICMP_ECHO_REQUEST : ICMP6_ECHO_REQUEST; probe_pkt.pr_icmp_code = 0; probe_pkt.pr_icmp_cksum = 0; probe_pkt.pr_icmp_seq = htons(pii->pii_snxt); /* * Since there is no need to do arithmetic on the icmpid, * (only equality check is done) pii_icmpid is stored in * network byte order at initialization itself. */ probe_pkt.pr_icmp_id = pii->pii_icmpid; probe_pkt.pr_icmp_timestamp = htonll(start_hrtime); probe_pkt.pr_icmp_mtype = htonl(probe_type); /* * If probe_type is PROBE_MULTI, this packet will be multicast to * the all hosts address. Otherwise it is unicast to the next target. */ assert(probe_type == PROBE_MULTI || ((pii->pii_target_next != NULL) && pii->pii_rtt_target_next != NULL)); bzero(&targ, sizeof (targ)); targ.ss_family = pii->pii_af; if (pii->pii_af == AF_INET6) { struct in6_addr *addr6; addr6 = &((struct sockaddr_in6 *)&targ)->sin6_addr; targaddrlen = sizeof (struct sockaddr_in6); if (probe_type == PROBE_MULTI) { *addr6 = all_nodes_mcast_v6; } else if (probe_type == PROBE_UNI) { *addr6 = pii->pii_target_next->tg_address; } else { /* type is PROBE_RTT */ *addr6 = pii->pii_rtt_target_next->tg_address; } } else { struct in_addr *addr4; addr4 = &((struct sockaddr_in *)&targ)->sin_addr; targaddrlen = sizeof (struct sockaddr_in); if (probe_type == PROBE_MULTI) { *addr4 = all_nodes_mcast_v4; } else if (probe_type == PROBE_UNI) { IN6_V4MAPPED_TO_INADDR( &pii->pii_target_next->tg_address, addr4); } else { /* type is PROBE_RTT */ IN6_V4MAPPED_TO_INADDR( &pii->pii_rtt_target_next->tg_address, addr4); } /* * Compute the IPv4 icmp checksum. Does not cover the IP header. */ probe_pkt.pr_icmp_cksum = in_cksum((ushort_t *)&probe_pkt, (int)sizeof (probe_pkt)); } /* * Use the current time as the time we sent. Not atomic, but the best * we can do from here. */ sent_hrtime = gethrtime(); (void) gettimeofday(&sent_tv, NULL); rval = sendto(pii->pii_probe_sock, &probe_pkt, sizeof (probe_pkt), 0, (struct sockaddr *)&targ, targaddrlen); /* * If the send would block, this may either be transient or a hang in a * lower layer. We pretend the probe was actually sent, the daemon will * not see a reply to the probe and will fail the interface if normal * failure detection criteria are met. */ if (rval == sizeof (probe_pkt) || (rval == -1 && errno == EWOULDBLOCK)) { sent = _B_TRUE; } else { logperror_pii(pii, "probe: probe sendto"); } /* * If this is a PROBE_UNI probe packet being unicast to a target, then * update our tables. We will need this info in processing the probe * response. PROBE_MULTI and PROBE_RTT packets are not used for * the purpose of failure or recovery detection. PROBE_MULTI packets * are only used to construct a list of targets. PROBE_RTT packets are * used only for updating the rtt and not for failure detection. */ if (probe_type == PROBE_UNI && sent) { pr_ndx = pii->pii_probe_next; assert(pr_ndx >= 0 && pr_ndx < PROBE_STATS_COUNT); /* Collect statistics, before we reuse the last slot. */ if (pii->pii_probes[pr_ndx].pr_status == PR_LOST) pii->pii_cum_stats.lost++; else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) pii->pii_cum_stats.acked++; pii->pii_cum_stats.sent++; pii->pii_probes[pr_ndx].pr_id = pii->pii_snxt; pii->pii_probes[pr_ndx].pr_tv_sent = sent_tv; pii->pii_probes[pr_ndx].pr_hrtime_sent = sent_hrtime; pii->pii_probes[pr_ndx].pr_hrtime_start = start_hrtime; pii->pii_probes[pr_ndx].pr_target = pii->pii_target_next; probe_chstate(&pii->pii_probes[pr_ndx], pii, PR_UNACKED); pii->pii_probe_next = PROBE_INDEX_NEXT(pii->pii_probe_next); pii->pii_target_next = target_next(pii->pii_target_next); assert(pii->pii_target_next != NULL); /* * If we have a single variable to denote the next target to * probe for both rtt probes and failure detection probes, we * could end up with a situation where the failure detection * probe targets become disjoint from the rtt probe targets. * Eg. if 2 targets and the actual fdt is double the user * specified fdt. So we have 2 variables. In this scheme * we also reset pii_rtt_target_next for every fdt probe, * though that may not be necessary. */ pii->pii_rtt_target_next = pii->pii_target_next; pii->pii_snxt++; } else if (probe_type == PROBE_RTT) { pii->pii_rtt_target_next = target_next(pii->pii_rtt_target_next); assert(pii->pii_rtt_target_next != NULL); } } /* * Incoming IPv4 data from wire, is received here. Called from main. */ void in_data(struct phyint_instance *pii) { struct sockaddr_in from; struct in6_addr fromaddr; static uint64_t in_packet[(IP_MAXPACKET + 1)/8]; static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8]; struct ip *ip; int iphlen; int len; char abuf[INET_ADDRSTRLEN]; struct msghdr msg; struct iovec iov; struct pr_icmp *reply; struct timeval *recv_tvp; if (debug & D_PROBE) { logdebug("in_data(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } iov.iov_base = (char *)in_packet; iov.iov_len = sizeof (in_packet); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_name = (struct sockaddr *)&from; msg.msg_namelen = sizeof (from); msg.msg_control = ancillary_data; msg.msg_controllen = sizeof (ancillary_data); /* * Poll has already told us that a message is waiting, * on this socket. Read it now. We should not block. */ if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) { logperror_pii(pii, "in_data: recvmsg"); return; } /* * If the datalink has indicated the link is down, don't go * any further. */ if (LINK_DOWN(pii->pii_phyint)) return; /* Get the printable address for error reporting */ (void) inet_ntop(AF_INET, &from.sin_addr, abuf, sizeof (abuf)); /* Ignore packets > 64k or control buffers that don't fit */ if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } /* Make sure packet contains at least minimum ICMP header */ ip = (struct ip *)in_packet; iphlen = ip->ip_hl << 2; if (len < iphlen + ICMP_MINLEN) { if (debug & D_PKTBAD) { logdebug("in_data: packet too short (%d bytes)" " from %s\n", len, abuf); } return; } /* * Subtract the IP hdr length, 'len' will be length of the probe * reply, starting from the icmp hdr. */ len -= iphlen; /* LINTED */ reply = (struct pr_icmp *)((char *)in_packet + iphlen); /* Probe replies are icmp echo replies. Ignore anything else */ if (reply->pr_icmp_type != ICMP_ECHO_REPLY) return; /* * The icmp id should match what we sent, which is stored * in pi_icmpid. The icmp code for reply must be 0. * The reply content must be a struct pr_icmp */ if (reply->pr_icmp_id != pii->pii_icmpid) { /* Not in response to our probe */ return; } if (reply->pr_icmp_code != 0) { logtrace("probe reply code %d from %s on %s\n", reply->pr_icmp_code, abuf, pii->pii_name); return; } if (len < sizeof (struct pr_icmp)) { logtrace("probe reply too short: %d bytes from %s on %s\n", len, abuf, pii->pii_name); return; } recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP); if (recv_tvp == NULL) { logtrace("message without timestamp from %s on %s\n", abuf, pii->pii_name); return; } IN6_INADDR_TO_V4MAPPED(&from.sin_addr, &fromaddr); if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) /* Unicast probe reply */ incoming_echo_reply(pii, reply, fromaddr, recv_tvp); else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) { /* Multicast reply */ incoming_mcast_reply(pii, reply, fromaddr); } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) { incoming_rtt_reply(pii, reply, fromaddr); } else { /* Probably not in response to our probe */ logtrace("probe reply type: %d from %s on %s\n", reply->pr_icmp_mtype, abuf, pii->pii_name); return; } } /* * Incoming IPv6 data from wire is received here. Called from main. */ void in6_data(struct phyint_instance *pii) { struct sockaddr_in6 from; static uint64_t in_packet[(IP_MAXPACKET + 1)/8]; static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8]; int len; char abuf[INET6_ADDRSTRLEN]; struct msghdr msg; struct iovec iov; void *opt; struct pr_icmp *reply; struct timeval *recv_tvp; if (debug & D_PROBE) { logdebug("in6_data(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } iov.iov_base = (char *)in_packet; iov.iov_len = sizeof (in_packet); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_name = (struct sockaddr *)&from; msg.msg_namelen = sizeof (from); msg.msg_control = ancillary_data; msg.msg_controllen = sizeof (ancillary_data); if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) { logperror_pii(pii, "in6_data: recvmsg"); return; } /* * If the datalink has indicated that the link is down, don't go * any further. */ if (LINK_DOWN(pii->pii_phyint)) return; /* Get the printable address for error reporting */ (void) inet_ntop(AF_INET6, &from.sin6_addr, abuf, sizeof (abuf)); if (len < ICMP_MINLEN) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } /* Ignore packets > 64k or control buffers that don't fit */ if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } reply = (struct pr_icmp *)in_packet; if (reply->pr_icmp_type != ICMP6_ECHO_REPLY) return; if (reply->pr_icmp_id != pii->pii_icmpid) { /* Not in response to our probe */ return; } /* * The kernel has already verified the the ICMP checksum. */ if (!IN6_IS_ADDR_LINKLOCAL(&from.sin6_addr)) { logtrace("ICMPv6 echo reply source address not linklocal from " "%s on %s\n", abuf, pii->pii_name); return; } opt = find_ancillary(&msg, IPPROTO_IPV6, IPV6_RTHDR); if (opt != NULL) { /* Can't allow routing headers in probe replies */ logtrace("message with routing header from %s on %s\n", abuf, pii->pii_name); return; } if (reply->pr_icmp_code != 0) { logtrace("probe reply code: %d from %s on %s\n", reply->pr_icmp_code, abuf, pii->pii_name); return; } if (len < (sizeof (struct pr_icmp))) { logtrace("probe reply too short: %d bytes from %s on %s\n", len, abuf, pii->pii_name); return; } recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP); if (recv_tvp == NULL) { logtrace("message without timestamp from %s on %s\n", abuf, pii->pii_name); return; } if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) { incoming_echo_reply(pii, reply, from.sin6_addr, recv_tvp); } else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) { incoming_mcast_reply(pii, reply, from.sin6_addr); } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) { incoming_rtt_reply(pii, reply, from.sin6_addr); } else { /* Probably not in response to our probe */ logtrace("probe reply type: %d from %s on %s\n", reply->pr_icmp_mtype, abuf, pii->pii_name); } } /* * Process the incoming rtt reply, in response to our rtt probe. * Common for both IPv4 and IPv6. Unlike incoming_echo_reply() we don't * have any stored information about the probe we sent. So we don't log * any errors if we receive bad replies. */ static void incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr) { int64_t m; /* rtt measurement in ns */ char abuf[INET6_ADDRSTRLEN]; struct target *target; struct phyint_group *pg; /* Get the printable address for error reporting */ (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)); if (debug & D_PROBE) { logdebug("incoming_rtt_reply: %s %s %s\n", AF_STR(pii->pii_af), pii->pii_name, abuf); } /* Do we know this target ? */ target = target_lookup(pii, fromaddr); if (target == NULL) return; m = (int64_t)(gethrtime() - ntohll(reply->pr_icmp_timestamp)); /* Invalid rtt. It has wrapped around */ if (m < 0) return; /* * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses * The initial few responses after the interface is repaired may * contain high rtt's because they could have been queued up waiting * for ARP/NDP resolution on a failed interface. */ pg = pii->pii_phyint->pi_group; if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg)) return; /* * Update rtt only if the new rtt is lower than the current rtt. * (specified by the 3rd parameter to pi_set_crtt). * If a spike has caused the current probe_interval to be > * user_probe_interval, then this mechanism is used to bring down * the rtt rapidly once the network stress is removed. * If the new rtt is higher than the current rtt, we don't want to * update the rtt. We are having more than 1 outstanding probe and * the increase in rtt we are seeing is being unnecessarily weighted * many times. The regular rtt update will be handled by * incoming_echo_reply() and will take care of any rtt increase. */ pi_set_crtt(target, m, _B_FALSE); if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) && (user_failure_detection_time < pg->pg_fdt) && (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) { /* * If the crtt has now dropped by a factor of LOWER_FT_TRIGGER, * investigate if we can improve the failure detection time to * meet whatever the user specified. */ if (check_pg_crtt_improved(pg)) { pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE, user_failure_detection_time); pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2); if (pii->pii_phyint->pi_group != phyint_anongroup) { logerr("Improved failure detection time %d ms " "on (%s %s) for group \"%s\"\n", pg->pg_fdt, AF_STR(pii->pii_af), pii->pii_name, pii->pii_phyint->pi_group->pg_name); } if (user_failure_detection_time == pg->pg_fdt) { /* Avoid any truncation or rounding errors */ pg->pg_probeint = user_probe_interval; /* * No more rtt probes will be sent. The actual * fdt has dropped to the user specified value. * pii_fd_snxt_basetime and pii_snxt_basetime * will be in sync henceforth. */ reset_snxt_basetimes(); } } } } /* * Process the incoming echo reply, in response to our unicast probe. * Common for both IPv4 and IPv6 */ static void incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp) { int64_t m; /* rtt measurement in ns */ hrtime_t cur_hrtime; /* in ns from some arbitrary point */ char abuf[INET6_ADDRSTRLEN]; int pr_ndx; struct target *target; boolean_t exception; uint64_t pr_icmp_timestamp; uint16_t pr_icmp_seq; struct probe_stats *pr_statp; struct phyint_group *pg = pii->pii_phyint->pi_group; /* Get the printable address for error reporting */ (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)); if (debug & D_PROBE) { logdebug("incoming_echo_reply: %s %s %s seq %u recv_tvp %lld\n", AF_STR(pii->pii_af), pii->pii_name, abuf, ntohs(reply->pr_icmp_seq), tv2ns(recv_tvp)); } pr_icmp_timestamp = ntohll(reply->pr_icmp_timestamp); pr_icmp_seq = ntohs(reply->pr_icmp_seq); /* Reject out of window probe replies */ if (SEQ_GE(pr_icmp_seq, pii->pii_snxt) || SEQ_LT(pr_icmp_seq, pii->pii_snxt - PROBE_STATS_COUNT)) { logtrace("out of window probe seq %u snxt %u on %s from %s\n", pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } cur_hrtime = gethrtime(); m = (int64_t)(cur_hrtime - pr_icmp_timestamp); if (m < 0) { /* * This is a ridiculously high value of rtt. rtt has wrapped * around. Log a message, and ignore the rtt. */ logerr("incoming_echo_reply: rtt wraparound cur_hrtime %lld " "reply timestamp %lld\n", cur_hrtime, pr_icmp_timestamp); } /* * Get the probe index pr_ndx corresponding to the received icmp seq. * number in our pii->pii_probes[] array. The icmp sequence number * pii_snxt corresponds to the probe index pii->pii_probe_next */ pr_ndx = MOD_SUB(pii->pii_probe_next, (uint16_t)(pii->pii_snxt - pr_icmp_seq), PROBE_STATS_COUNT); assert(PR_STATUS_VALID(pii->pii_probes[pr_ndx].pr_status)); target = pii->pii_probes[pr_ndx].pr_target; /* * Perform sanity checks, whether this probe reply that we * have received is genuine */ if (target != NULL) { /* * Compare the src. addr of the received ICMP or ICMPv6 * probe reply with the target address in our tables. */ if (!IN6_ARE_ADDR_EQUAL(&target->tg_address, &fromaddr)) { /* * We don't have any record of having sent a probe to * this target. This is a fake probe reply. Log an error */ logtrace("probe status %d Fake probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) { /* * The address matches, but our tables indicate that * this probe reply has been acked already. So this * is a duplicate probe reply. Log an error */ logtrace("probe status %d Duplicate probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } } else { /* * Target must not be NULL in the PR_UNACKED state */ assert(pii->pii_probes[pr_ndx].pr_status != PR_UNACKED); if (pii->pii_probes[pr_ndx].pr_status == PR_UNUSED) { /* * The probe stats slot is unused. So we didn't * send out any probe to this target. This is a fake. * Log an error. */ logtrace("probe status %d Fake probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); } pii->pii_cum_stats.unknown++; return; } /* * If the rtt does not appear to be right, don't update the * rtt stats. This can happen if the system dropped into the * debugger, or the system was hung or too busy for a * substantial time that we didn't get a chance to run. */ if ((m < 0) || (ns2ms(m) > PROBE_STATS_COUNT * pg->pg_probeint)) { /* * If the probe corresponding to this received response * was truly sent 'm' ns. ago, then this response must * have been rejected by the sequence number checks. The * fact that it has passed the sequence number checks * means that the measured rtt is wrong. We were probably * scheduled long after the packet was received. */ goto out; } /* * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses * The initial few responses after the interface is repaired may * contain high rtt's because they could have been queued up waiting * for ARP/NDP resolution on a failed interface. */ if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg)) goto out; /* * Don't update the Conservative Round Trip Time estimate for this * (phint, target) pair if this is the not the highest ack seq seen * thus far on this target. */ if (!highest_ack_tg(pr_icmp_seq, target)) goto out; /* * Always update the rtt. This is a failure detection probe * and we want to measure both increase / decrease in rtt. */ pi_set_crtt(target, m, _B_TRUE); /* * If the crtt exceeds the average time between probes, * investigate if this slow target is an exception. If so we * can avoid this target and still meet the failure detection * time. Otherwise we can't meet the failure detection time. */ if (target->tg_crtt > pg->pg_probeint) { exception = check_exception_target(pii, target); if (exception) { /* * This target is exceptionally slow. Don't use it * for future probes. check_exception_target() has * made sure that we have at least MIN_PROBE_TARGETS * other active targets */ if (pii->pii_targets_are_routers) { /* * This is a slow router, mark it as slow * and don't use it for further probes. We * don't delete it, since it will be populated * again when we do a router scan. Hence we * need to maintain extra state (unlike the * host case below). Mark it as TG_SLOW. */ if (target->tg_status == TG_ACTIVE) pii->pii_ntargets--; target->tg_status = TG_SLOW; target->tg_latime = gethrtime(); target->tg_rtt_sa = -1; target->tg_crtt = 0; target->tg_rtt_sd = 0; if (pii->pii_target_next == target) { pii->pii_target_next = target_next(target); } } else { /* * the slow target is not a router, we can * just delete it. Send an icmp multicast and * pick the fastest responder that is not * already an active target. target_delete() * adjusts pii->pii_target_next */ target_delete(target); probe(pii, PROBE_MULTI, cur_hrtime); } } else { /* * We can't meet the failure detection time. * Log a message, and update the detection time to * whatever we can achieve. */ pg->pg_probeint = target->tg_crtt * NEXT_FDT_MULTIPLE; pg->pg_fdt = pg->pg_probeint * (NUM_PROBE_FAILS + 2); last_fdt_bumpup_time = gethrtime(); if (pg != phyint_anongroup) { logtrace("Cannot meet requested failure" " detection time of %d ms on (%s %s) new" " failure detection time for group \"%s\"" " is %d ms\n", user_failure_detection_time, AF_STR(pii->pii_af), pii->pii_name, pg->pg_name, pg->pg_fdt); } } } else if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) && (user_failure_detection_time < pg->pg_fdt) && (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) { /* * If the crtt has now dropped by a factor of LOWER_FDT_TRIGGER * investigate if we can improve the failure detection time to * meet whatever the user specified. */ if (check_pg_crtt_improved(pg)) { pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE, user_failure_detection_time); pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2); if (pg != phyint_anongroup) { logtrace("Improved failure detection time %d ms" " on (%s %s) for group \"%s\"\n", pg->pg_fdt, AF_STR(pii->pii_af), pii->pii_name, pg->pg_name); } if (user_failure_detection_time == pg->pg_fdt) { /* Avoid any truncation or rounding errors */ pg->pg_probeint = user_probe_interval; /* * No more rtt probes will be sent. The actual * fdt has dropped to the user specified value. * pii_fd_snxt_basetime and pii_snxt_basetime * will be in sync henceforth. */ reset_snxt_basetimes(); } } } out: pr_statp = &pii->pii_probes[pr_ndx]; pr_statp->pr_hrtime_ackproc = cur_hrtime; pr_statp->pr_hrtime_ackrecv = pr_statp->pr_hrtime_sent + (tv2ns(recv_tvp) - tv2ns(&pr_statp->pr_tv_sent)); probe_chstate(pr_statp, pii, PR_ACKED); /* * Update pii->pii_rack, i.e. the sequence number of the last received * probe response, based on the echo reply we have received now, if * either of the following conditions are satisfied. * a. pii_rack is outside the current receive window of * [pii->pii_snxt - PROBE_STATS_COUNT, pii->pii_snxt). * This means we have not received probe responses for a * long time, and the sequence number has wrapped around. * b. pii_rack is within the current receive window and this echo * reply corresponds to the highest sequence number we have seen * so far. */ if (SEQ_GE(pii->pii_rack, pii->pii_snxt) || SEQ_LT(pii->pii_rack, pii->pii_snxt - PROBE_STATS_COUNT) || SEQ_GT(pr_icmp_seq, pii->pii_rack)) { pii->pii_rack = pr_icmp_seq; } } /* * Returns true if seq is the highest unacknowledged seq for target tg * else returns false */ static boolean_t highest_ack_tg(uint16_t seq, struct target *tg) { struct phyint_instance *pii; int pr_ndx; uint16_t pr_seq; pii = tg->tg_phyint_inst; /* * Get the seq number of the most recent probe sent so far, * and also get the corresponding probe index in the probe stats * array. */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); pr_seq = pii->pii_snxt; pr_seq--; /* * Start from the most recent probe and walk back, trying to find * an acked probe corresponding to target tg. */ for (; pr_ndx != pii->pii_probe_next; pr_ndx = PROBE_INDEX_PREV(pr_ndx), pr_seq--) { if (pii->pii_probes[pr_ndx].pr_target == tg && pii->pii_probes[pr_ndx].pr_status == PR_ACKED) { if (SEQ_GT(pr_seq, seq)) return (_B_FALSE); } } return (_B_TRUE); } /* * Check whether the crtt for the group has improved by a factor of * LOWER_FDT_TRIGGER. Small crtt improvements are ignored to avoid failure * detection time flapping in the face of small crtt changes. */ static boolean_t check_pg_crtt_improved(struct phyint_group *pg) { struct phyint *pi; if (debug & D_PROBE) logdebug("check_pg_crtt_improved()\n"); /* * The crtt for the group is only improved if each phyint_instance * for both ipv4 and ipv6 is improved. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { if (!check_pii_crtt_improved(pi->pi_v4) || !check_pii_crtt_improved(pi->pi_v6)) return (_B_FALSE); } return (_B_TRUE); } /* * Check whether the crtt has improved substantially on this phyint_instance. * Returns _B_TRUE if there's no crtt information available, because pii * is NULL or the phyint_instance is not capable of probing. */ boolean_t check_pii_crtt_improved(struct phyint_instance *pii) { struct target *tg; if (pii == NULL) return (_B_TRUE); if (!PROBE_CAPABLE(pii) || pii->pii_phyint->pi_state == PI_FAILED) return (_B_TRUE); for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { if (tg->tg_status != TG_ACTIVE) continue; if (tg->tg_crtt > (pii->pii_phyint->pi_group->pg_probeint / LOWER_FDT_TRIGGER)) { return (_B_FALSE); } } return (_B_TRUE); } /* * This target responds very slowly to probes. The target's crtt exceeds * the probe interval of its group. Compare against other targets * and determine if this target is an exception, if so return true, else false */ static boolean_t check_exception_target(struct phyint_instance *pii, struct target *target) { struct target *tg; char abuf[INET6_ADDRSTRLEN]; if (debug & D_PROBE) { logdebug("check_exception_target(%s %s target %s)\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, target->tg_address, abuf, sizeof (abuf))); } /* * We should have at least MIN_PROBE_TARGETS + 1 good targets now, * to make a good judgement. Otherwise don't drop this target. */ if (pii->pii_ntargets < MIN_PROBE_TARGETS + 1) return (_B_FALSE); /* * Determine whether only this particular target is slow. * We know that this target's crtt exceeds the group's probe interval. * If all other active targets have a * crtt < (this group's probe interval) / EXCEPTION_FACTOR, * then this target is considered slow. */ for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { if (tg != target && tg->tg_status == TG_ACTIVE) { if (tg->tg_crtt > pii->pii_phyint->pi_group->pg_probeint / EXCEPTION_FACTOR) { return (_B_FALSE); } } } return (_B_TRUE); } /* * Update the target list. The icmp all hosts multicast has given us * some host to which we can send probes. If we already have sufficient * targets, discard it. */ static void incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr) /* ARGSUSED */ { int af; char abuf[INET6_ADDRSTRLEN]; struct phyint *pi; if (debug & D_PROBE) { logdebug("incoming_mcast_reply(%s %s %s)\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf))); } /* * Using host targets is a fallback mechanism. If we have * found a router, don't add this host target. If we already * know MAX_PROBE_TARGETS, don't add another target. */ assert(pii->pii_ntargets <= MAX_PROBE_TARGETS); if (pii->pii_targets != NULL) { if (pii->pii_targets_are_routers || (pii->pii_ntargets == MAX_PROBE_TARGETS)) { return; } } if (IN6_IS_ADDR_UNSPECIFIED(&fromaddr) || IN6_IS_ADDR_V4MAPPED_ANY(&fromaddr)) { /* * Guard against response from 0.0.0.0 * and ::. Log a trace message */ logtrace("probe response from %s on %s\n", pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)), pii->pii_name); return; } /* * This address is one of our own, so reject this address as a * valid probe target. */ af = pii->pii_af; if (own_address(fromaddr)) return; /* * If the phyint is part a named group, then add the address to all * members of the group. Otherwise, add the address only to the * phyint itself, since other phyints in the anongroup may not be on * the same subnet. */ pi = pii->pii_phyint; if (pi->pi_group == phyint_anongroup) { target_add(pii, fromaddr, _B_FALSE); } else { pi = pi->pi_group->pg_phyint; for (; pi != NULL; pi = pi->pi_pgnext) target_add(PHYINT_INSTANCE(pi, af), fromaddr, _B_FALSE); } } /* * Compute CRTT given an existing scaled average, scaled deviation estimate * and a new rtt time. The formula is from Jacobson and Karels' * "Congestion Avoidance and Control" in SIGCOMM '88. The variable names * are the same as those in Appendix A.2 of that paper. * * m = new measurement * sa = scaled RTT average (8 * average estimates) * sv = scaled mean deviation (mdev) of RTT (4 * deviation estimates). * crtt = Conservative round trip time. Used to determine whether probe * has timed out. * * New scaled average and deviation are passed back via sap and svp */ static int64_t compute_crtt(int64_t *sap, int64_t *svp, int64_t m) { int64_t sa = *sap; int64_t sv = *svp; int64_t crtt; int64_t saved_m = m; assert(*sap >= -1); assert(*svp >= 0); if (sa != -1) { /* * Update average estimator: * new rtt = old rtt + 1/8 Error * where Error = m - old rtt * i.e. 8 * new rtt = 8 * old rtt + Error * i.e. new sa = old sa + Error */ m -= sa >> 3; /* m is now Error in estimate. */ if ((sa += m) < 0) { /* Don't allow the smoothed average to be negative. */ sa = 0; } /* * Update deviation estimator: * new mdev = old mdev + 1/4 (abs(Error) - old mdev) * i.e. 4 * new mdev = 4 * old mdev + * (abs(Error) - old mdev) * i.e. new sv = old sv + (abs(Error) - old mdev) */ if (m < 0) m = -m; m -= sv >> 2; sv += m; } else { /* Initialization. This is the first response received. */ sa = (m << 3); sv = (m << 1); } crtt = (sa >> 3) + sv; if (debug & D_PROBE) { logerr("compute_crtt: m = %lld sa = %lld, sv = %lld -> " "crtt = %lld\n", saved_m, sa, sv, crtt); } *sap = sa; *svp = sv; /* * CRTT = average estimates + 4 * deviation estimates * = sa / 8 + sv */ return (crtt); } static void pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni) { struct phyint_instance *pii = tg->tg_phyint_inst; int probe_interval = pii->pii_phyint->pi_group->pg_probeint; int64_t sa = tg->tg_rtt_sa; int64_t sv = tg->tg_rtt_sd; int new_crtt; int i; if (debug & D_PROBE) logdebug("pi_set_crtt: target - m %lld\n", m); /* store the round trip time, in case we need to defer computation */ tg->tg_deferred[tg->tg_num_deferred] = m; new_crtt = ns2ms(compute_crtt(&sa, &sv, m)); /* * If this probe's round trip time would singlehandedly cause an * increase in the group's probe interval consider it suspect. */ if ((new_crtt > probe_interval) && is_probe_uni) { if (debug & D_PROBE) { logdebug("Received a suspect probe on %s, new_crtt =" " %d, probe_interval = %d, num_deferred = %d\n", pii->pii_probe_logint->li_name, new_crtt, probe_interval, tg->tg_num_deferred); } /* * If we've deferred as many rtts as we plan on deferring, then * assume the link really did slow down and process all queued * rtts */ if (tg->tg_num_deferred == MAXDEFERREDRTT) { if (debug & D_PROBE) { logdebug("Received MAXDEFERREDRTT probes which " "would cause an increased probe_interval. " "Integrating queued rtt data points.\n"); } for (i = 0; i <= tg->tg_num_deferred; i++) { tg->tg_crtt = ns2ms(compute_crtt(&tg->tg_rtt_sa, &tg->tg_rtt_sd, tg->tg_deferred[i])); } tg->tg_num_deferred = 0; } else { tg->tg_num_deferred++; } return; } /* * If this is a normal probe, or an RTT probe that would lead to a * reduced CRTT, then update our CRTT data. Further, if this was * a normal probe, pitch any deferred probes since our probes are * again being answered within our CRTT estimates. */ if (is_probe_uni || new_crtt < tg->tg_crtt) { tg->tg_rtt_sa = sa; tg->tg_rtt_sd = sv; tg->tg_crtt = new_crtt; if (is_probe_uni) tg->tg_num_deferred = 0; } } /* * Return a pointer to the specified option buffer. * If not found return NULL. */ static void * find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type) { struct cmsghdr *cmsg; for (cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL; cmsg = CMSG_NXTHDR(msg, cmsg)) { if (cmsg->cmsg_level == cmsg_level && cmsg->cmsg_type == cmsg_type) { return (CMSG_DATA(cmsg)); } } return (NULL); } /* * Try to activate another INACTIVE interface in the same group as `pi'. * Prefer STANDBY INACTIVE to just INACTIVE. */ void phyint_activate_another(struct phyint *pi) { struct phyint *pi2; struct phyint *inactivepi = NULL; if (pi->pi_group == phyint_anongroup) return; for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (pi == pi2 || !phyint_is_functioning(pi2) || !(pi2->pi_flags & IFF_INACTIVE)) continue; inactivepi = pi2; if (pi2->pi_flags & IFF_STANDBY) break; } if (inactivepi != NULL) (void) change_pif_flags(inactivepi, 0, IFF_INACTIVE); } /* * Transition a phyint to PI_RUNNING. The caller must ensure that the * transition is appropriate. Clears IFF_OFFLINE or IFF_FAILED if * appropriate. Also sets IFF_INACTIVE on this or other interfaces as * appropriate (see comment below). Finally, also updates the phyint's group * state to account for the change. */ void phyint_transition_to_running(struct phyint *pi) { struct phyint *pi2; struct phyint *actstandbypi = NULL; uint_t nactive = 0, nnonstandby = 0; boolean_t onlining = (pi->pi_state == PI_OFFLINE); boolean_t initial = (pi->pi_state == PI_INIT); uint64_t set, clear; /* * The interface is running again, but should it or another interface * in the group end up INACTIVE? There are three cases: * * 1. If it's a STANDBY interface, it should be end up INACTIVE if * the group is operating at capacity (i.e., there are at least as * many active interfaces as non-STANDBY interfaces in the group). * No other interfaces should be changed. * * 2. If it's a non-STANDBY interface and we're onlining it or * FAILBACK is enabled, then it should *not* end up INACTIVE. * Further, if the group is above capacity as a result of this * interface, then an active STANDBY interface in the group should * end up INACTIVE. * * 3. If it's a non-STANDBY interface, we're repairing it, and * FAILBACK is disabled, then it should end up INACTIVE *unless* * the group was failed (in which case we have no choice but to * use it). No other interfaces should be changed. */ if (pi->pi_group != phyint_anongroup) { pi2 = pi->pi_group->pg_phyint; for (; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (!(pi2->pi_flags & IFF_STANDBY)) nnonstandby++; if (phyint_is_functioning(pi2) && !(pi2->pi_flags & IFF_INACTIVE)) { nactive++; if (pi2->pi_flags & IFF_STANDBY) actstandbypi = pi2; } } } set = 0; clear = (onlining ? IFF_OFFLINE : IFF_FAILED); if (pi->pi_flags & IFF_STANDBY) { /* case 1 */ if (nactive >= nnonstandby) set |= IFF_INACTIVE; else clear |= IFF_INACTIVE; } else if (onlining || failback_enabled) { /* case 2 */ if (nactive >= nnonstandby && actstandbypi != NULL) (void) change_pif_flags(actstandbypi, IFF_INACTIVE, 0); } else if (!initial && !GROUP_FAILED(pi->pi_group)) { /* case 3 */ set |= IFF_INACTIVE; } (void) change_pif_flags(pi, set, clear); phyint_chstate(pi, PI_RUNNING); /* * Update the group state to account for the change. */ phyint_group_refresh_state(pi->pi_group); } /* * Adjust IFF_INACTIVE on the provided `pi' to trend the group configuration * to have at least one active interface and as many active interfaces as * non-standby interfaces. */ void phyint_standby_refresh_inactive(struct phyint *pi) { struct phyint *pi2; uint_t nactive = 0, nnonstandby = 0; /* * All phyints in the anonymous group are effectively in their own * group and thus active regardless of whether they're marked standby. */ if (pi->pi_group == phyint_anongroup) { (void) change_pif_flags(pi, 0, IFF_INACTIVE); return; } /* * If the phyint isn't functioning we can't consider it. */ if (!phyint_is_functioning(pi)) return; for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (!(pi2->pi_flags & IFF_STANDBY)) nnonstandby++; if (phyint_is_functioning(pi2) && !(pi2->pi_flags & IFF_INACTIVE)) nactive++; } if (nactive == 0 || nactive < nnonstandby) (void) change_pif_flags(pi, 0, IFF_INACTIVE); else if (nactive > nnonstandby) (void) change_pif_flags(pi, IFF_INACTIVE, 0); } /* * See if a previously failed interface has started working again. */ void phyint_check_for_repair(struct phyint *pi) { if (!phyint_repaired(pi)) return; if (pi->pi_group == phyint_anongroup) { logerr("IP interface repair detected on %s\n", pi->pi_name); } else { logerr("IP interface repair detected on %s of group %s\n", pi->pi_name, pi->pi_group->pg_name); } /* * If the interface is PI_OFFLINE, it can't be made PI_RUNNING yet. * So just clear IFF_OFFLINE and defer phyint_transition_to_running() * until it is brought back online. */ if (pi->pi_state == PI_OFFLINE) { (void) change_pif_flags(pi, 0, IFF_FAILED); return; } phyint_transition_to_running(pi); /* calls phyint_chstate() */ } /* * See if an interface has failed, or if the whole group of interfaces has * failed. */ static void phyint_inst_check_for_failure(struct phyint_instance *pii) { struct phyint *pi = pii->pii_phyint; struct phyint *pi2; boolean_t was_active; switch (failure_state(pii)) { case PHYINT_FAILURE: was_active = ((pi->pi_flags & IFF_INACTIVE) == 0); (void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE); if (pi->pi_group == phyint_anongroup) { logerr("IP interface failure detected on %s\n", pii->pii_name); } else { logerr("IP interface failure detected on %s of group" " %s\n", pii->pii_name, pi->pi_group->pg_name); } /* * If the failed interface was active, activate another * INACTIVE interface in the group if possible. */ if (was_active) phyint_activate_another(pi); /* * If the interface is offline, the state change will be * noted when it comes back online. */ if (pi->pi_state != PI_OFFLINE) { phyint_chstate(pi, PI_FAILED); reset_crtt_all(pi); } break; case GROUP_FAILURE: pi2 = pi->pi_group->pg_phyint; for (; pi2 != NULL; pi2 = pi2->pi_pgnext) { (void) change_pif_flags(pi2, IFF_FAILED, IFF_INACTIVE); if (pi2->pi_state == PI_OFFLINE) /* see comment above */ continue; reset_crtt_all(pi2); /* * In the case of host targets, we would have flushed * the targets, and gone to PI_NOTARGETS state. */ if (pi2->pi_state == PI_RUNNING) phyint_chstate(pi2, PI_FAILED); } break; default: break; } } /* * Determines if any timeout event has occurred and returns the number of * milliseconds until the next timeout event for the phyint. Returns * TIMER_INFINITY for "never". */ uint_t phyint_inst_timer(struct phyint_instance *pii) { int pr_ndx; uint_t timeout; struct target *cur_tg; struct probe_stats *pr_statp; struct phyint_instance *pii_other; struct phyint *pi; int valid_unack_count; int i; int interval; uint_t check_time; uint_t cur_time; hrtime_t cur_hrtime; int probe_interval = pii->pii_phyint->pi_group->pg_probeint; cur_hrtime = gethrtime(); cur_time = ns2ms(cur_hrtime); if (debug & D_TIMER) { logdebug("phyint_inst_timer(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } pii_other = phyint_inst_other(pii); if (!PROBE_ENABLED(pii) && !PROBE_ENABLED(pii_other)) { /* * Check to see if we're here due to link up/down flapping; If * enough time has passed, then try to bring the interface * back up; otherwise, schedule a timer to bring it back up * when enough time *has* elapsed. */ pi = pii->pii_phyint; if (pi->pi_state == PI_FAILED && LINK_UP(pi)) { check_time = pi->pi_whenup[pi->pi_whendx] + MSEC_PERMIN; if (check_time > cur_time) return (check_time - cur_time); phyint_check_for_repair(pi); } } /* * If probing is not enabled on this phyint instance, don't proceed. */ if (!PROBE_ENABLED(pii)) return (TIMER_INFINITY); /* * If the timer has fired too soon, probably triggered * by some other phyint instance, return the remaining * time */ if (TIME_LT(cur_time, pii->pii_snxt_time)) return (pii->pii_snxt_time - cur_time); /* * If the link is down, don't send any probes for now. */ if (LINK_DOWN(pii->pii_phyint)) return (TIMER_INFINITY); /* * Randomize the next probe time, between MIN_RANDOM_FACTOR * and MAX_RANDOM_FACTOR with respect to the base probe time. * Base probe time is strictly periodic. */ interval = GET_RANDOM( (int)(MIN_RANDOM_FACTOR * user_probe_interval), (int)(MAX_RANDOM_FACTOR * user_probe_interval)); pii->pii_snxt_time = pii->pii_snxt_basetime + interval; /* * Check if the current time > next time to probe. If so, we missed * sending 1 or more probes, probably due to heavy system load. At least * 'MIN_RANDOM_FACTOR * user_probe_interval' ms has elapsed since we * were scheduled. Make adjustments to the times, in multiples of * user_probe_interval. */ if (TIME_GT(cur_time, pii->pii_snxt_time)) { int n; n = (cur_time - pii->pii_snxt_time) / user_probe_interval; pii->pii_snxt_time += (n + 1) * user_probe_interval; pii->pii_snxt_basetime += (n + 1) * user_probe_interval; logtrace("missed sending %d probes cur_time %u snxt_time %u" " snxt_basetime %u\n", n + 1, cur_time, pii->pii_snxt_time, pii->pii_snxt_basetime); /* Collect statistics about missed probes */ probes_missed.pm_nprobes += n + 1; probes_missed.pm_ntimes++; } pii->pii_snxt_basetime += user_probe_interval; interval = pii->pii_snxt_time - cur_time; if (debug & D_TARGET) { logdebug("cur_time %u snxt_time %u snxt_basetime %u" " interval %u\n", cur_time, pii->pii_snxt_time, pii->pii_snxt_basetime, interval); } /* * If no targets are known, we need to send an ICMP multicast. The * probe type is PROBE_MULTI. We'll check back in 'interval' msec * to see if we found a target. */ if (pii->pii_target_next == NULL) { assert(pii->pii_ntargets == 0); pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; probe(pii, PROBE_MULTI, cur_time); return (interval); } if ((user_probe_interval != probe_interval) && TIME_LT(pii->pii_snxt_time, pii->pii_fd_snxt_basetime)) { /* * the failure detection (fd) probe timer has not yet fired. * Need to send only an rtt probe. The probe type is PROBE_RTT. */ probe(pii, PROBE_RTT, cur_hrtime); return (interval); } /* * the fd probe timer has fired. Need to do all failure * detection / recovery calculations, and then send an fd probe * of type PROBE_UNI. */ if (user_probe_interval == probe_interval) { /* * We could have missed some probes, and then adjusted * pii_snxt_basetime above. Otherwise we could have * blindly added probe_interval to pii_fd_snxt_basetime. */ pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; } else { pii->pii_fd_snxt_basetime += probe_interval; if (TIME_GT(cur_time, pii->pii_fd_snxt_basetime)) { int n; n = (cur_time - pii->pii_fd_snxt_basetime) / probe_interval; pii->pii_fd_snxt_basetime += (n + 1) * probe_interval; } } /* * We can have at most, the latest 2 probes that we sent, in * the PR_UNACKED state. All previous probes sent, are either * PR_LOST or PR_ACKED. An unacknowledged probe is considered * timed out if the probe's time_start + the CRTT < currenttime. * For each of the last 2 probes, examine whether it has timed * out. If so, mark it PR_LOST. The probe stats is a circular array. */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); valid_unack_count = 0; for (i = 0; i < 2; i++) { pr_statp = &pii->pii_probes[pr_ndx]; cur_tg = pii->pii_probes[pr_ndx].pr_target; switch (pr_statp->pr_status) { case PR_ACKED: /* * We received back an ACK, so the switch clearly * is not dropping our traffic, and thus we can * enable failure detection immediately. */ if (pii->pii_fd_hrtime > gethrtime()) { if (debug & D_PROBE) { logdebug("successful probe on %s; " "ending quiet period\n", pii->pii_phyint->pi_name); } pii->pii_fd_hrtime = gethrtime(); } break; case PR_UNACKED: assert(cur_tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use group's probe interval, * which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (cur_tg->tg_crtt != 0) { timeout += cur_tg->tg_crtt; } else { timeout += probe_interval; } if (TIME_LT(timeout, cur_time)) { pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); } else if (i == 1) { /* * We are forced to consider this probe * lost, as we can have at most 2 unack. * probes any time, and we will be sending a * probe at the end of this function. * Normally, we should not be here, but * this can happen if an incoming response * that was considered lost has increased * the crtt for this target, and also bumped * up the FDT. Note that we never cancel or * increase the current pii_time_left, so * when the timer fires, we find 2 valid * unacked probes, and they are yet to timeout */ pr_statp->pr_time_lost = cur_time; probe_chstate(pr_statp, pii, PR_LOST); } else { /* * Only the most recent probe can enter * this 'else' arm. The second most recent * probe must take either of the above arms, * if it is unacked. */ valid_unack_count++; } break; } pr_ndx = PROBE_INDEX_PREV(pr_ndx); } /* * We send out 1 probe randomly in the interval between one half * and one probe interval for the group. Given that the CRTT is always * less than the group's probe interval, we can have at most 1 * unacknowledged probe now. All previous probes are either lost or * acked. */ assert(valid_unack_count == 0 || valid_unack_count == 1); /* * The timer has fired. Take appropriate action depending * on the current state of the phyint. * * PI_RUNNING state - Failure detection * PI_FAILED state - Repair detection */ switch (pii->pii_phyint->pi_state) { case PI_FAILED: /* * If the most recent probe (excluding unacked probes that * are yet to time out) has been acked, check whether the * phyint is now repaired. */ if (pii->pii_rack + valid_unack_count + 1 == pii->pii_snxt) { phyint_check_for_repair(pii->pii_phyint); } break; case PI_RUNNING: /* * It's possible our probes have been lost because of a * spanning-tree mandated quiet period on the switch. If so, * ignore the lost probes. */ if (pii->pii_fd_hrtime - cur_hrtime > 0) break; if (pii->pii_rack + valid_unack_count + 1 != pii->pii_snxt) { /* * We have 1 or more failed probes (excluding unacked * probes that are yet to time out). Determine if the * phyint has failed. */ phyint_inst_check_for_failure(pii); } break; default: logerr("phyint_inst_timer: invalid state %d\n", pii->pii_phyint->pi_state); abort(); } /* * Start the next probe. probe() will also set pii->pii_probe_time_left * to the group's probe interval. If phyint_failed -> target_flush_hosts * was called, the target list may be empty. */ if (pii->pii_target_next != NULL) { probe(pii, PROBE_UNI, cur_hrtime); /* * If we have just the one probe target, and we're not using * router targets, try to find another as we presently have * no resilience. */ if (!pii->pii_targets_are_routers && pii->pii_ntargets == 1) probe(pii, PROBE_MULTI, cur_hrtime); } else { probe(pii, PROBE_MULTI, cur_hrtime); } return (interval); } /* * Start the probe timer for an interface instance. */ void start_timer(struct phyint_instance *pii) { uint32_t interval; /* * Spread the base probe times (pi_snxt_basetime) across phyints * uniformly over the (curtime..curtime + the group's probe_interval). * pi_snxt_basetime is strictly periodic with a frequency of * the group's probe interval. The actual probe time pi_snxt_time * adds some randomness to pi_snxt_basetime and happens in probe(). * For the 1st probe on each phyint after the timer is started, * pi_snxt_time and pi_snxt_basetime are the same. */ interval = GET_RANDOM(0, (int)pii->pii_phyint->pi_group->pg_probeint); pii->pii_snxt_basetime = getcurrenttime() + interval; pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; pii->pii_snxt_time = pii->pii_snxt_basetime; timer_schedule(interval); } /* * Restart the probe timer on an interface instance. */ static void restart_timer(struct phyint_instance *pii) { /* * We don't need to restart the timer if it was never started in * the first place (pii->pii_basetime_inited not set), as the timer * won't have gone off yet. */ if (pii->pii_basetime_inited != 0) { if (debug & D_LINKNOTE) logdebug("restart timer: restarting timer on %s, " "address family %s\n", pii->pii_phyint->pi_name, AF_STR(pii->pii_af)); start_timer(pii); } } static void process_link_state_down(struct phyint *pi) { logerr("The link has gone down on %s\n", pi->pi_name); /* * Clear the probe statistics arrays, we don't want the repair * detection logic relying on probes that were successful prior * to the link going down. */ if (PROBE_CAPABLE(pi->pi_v4)) clear_pii_probe_stats(pi->pi_v4); if (PROBE_CAPABLE(pi->pi_v6)) clear_pii_probe_stats(pi->pi_v6); /* * Check for interface failure. Although we know the interface * has failed, we don't know if all the other interfaces in the * group have failed as well. */ if ((pi->pi_state == PI_RUNNING) || (pi->pi_state != PI_FAILED && !GROUP_FAILED(pi->pi_group))) { if (debug & D_LINKNOTE) { logdebug("process_link_state_down:" " checking for failure on %s\n", pi->pi_name); } if (pi->pi_v4 != NULL) phyint_inst_check_for_failure(pi->pi_v4); else if (pi->pi_v6 != NULL) phyint_inst_check_for_failure(pi->pi_v6); } } static void process_link_state_up(struct phyint *pi) { logerr("The link has come up on %s\n", pi->pi_name); /* * We stopped any running timers on each instance when the link * went down, so restart them. */ if (pi->pi_v4) restart_timer(pi->pi_v4); if (pi->pi_v6) restart_timer(pi->pi_v6); phyint_check_for_repair(pi); pi->pi_whenup[pi->pi_whendx++] = getcurrenttime(); if (pi->pi_whendx == LINK_UP_PERMIN) pi->pi_whendx = 0; } /* * Process any changes in link state passed up from the interfaces. */ void process_link_state_changes(void) { struct phyint *pi; /* Look for interfaces where the link state has just changed */ for (pi = phyints; pi != NULL; pi = pi->pi_next) { boolean_t old_link_state_up = LINK_UP(pi); /* * Except when the "phyint" structure is created, this is * the only place the link state is updated. This allows * this routine to detect changes in link state, rather * than just the current state. */ UPDATE_LINK_STATE(pi); if (LINK_DOWN(pi)) { /* * Has link just gone down? */ if (old_link_state_up) process_link_state_down(pi); } else { /* * Has link just gone back up? */ if (!old_link_state_up) process_link_state_up(pi); } } } void reset_crtt_all(struct phyint *pi) { struct phyint_instance *pii; struct target *tg; pii = pi->pi_v4; if (pii != NULL) { for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { tg->tg_crtt = 0; tg->tg_rtt_sa = -1; tg->tg_rtt_sd = 0; } } pii = pi->pi_v6; if (pii != NULL) { for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { tg->tg_crtt = 0; tg->tg_rtt_sa = -1; tg->tg_rtt_sd = 0; } } } /* * Check if the phyint has failed the last NUM_PROBE_FAILS consecutive * probes on both instances IPv4 and IPv6. * If the interface has failed, return the time of the first probe failure * in "tff". */ static int phyint_inst_probe_failure_state(struct phyint_instance *pii, uint_t *tff) { uint_t pi_tff; struct target *cur_tg; struct probe_fail_count pfinfo; struct phyint_instance *pii_other; int pr_ndx; /* * Get the number of consecutive failed probes on * this phyint across all targets. Also get the number * of consecutive failed probes on this target only */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_fail_info(pii, cur_tg, &pfinfo); /* Get the time of first failure, for later use */ pi_tff = pfinfo.pf_tff; /* * If the current target has not responded to the * last NUM_PROBE_FAILS probes, and other targets are * responding delete this target. Dead gateway detection * will eventually remove this target (if router) from the * routing tables. If that does not occur, we may end * up adding this to our list again. */ if (pfinfo.pf_nfail < NUM_PROBE_FAILS && pfinfo.pf_nfail_tg >= NUM_PROBE_FAILS) { if (pii->pii_targets_are_routers) { if (cur_tg->tg_status == TG_ACTIVE) pii->pii_ntargets--; cur_tg->tg_status = TG_DEAD; cur_tg->tg_crtt = 0; cur_tg->tg_rtt_sa = -1; cur_tg->tg_rtt_sd = 0; if (pii->pii_target_next == cur_tg) pii->pii_target_next = target_next(cur_tg); } else { target_delete(cur_tg); probe(pii, PROBE_MULTI, gethrtime()); } return (PHYINT_OK); } /* * If the phyint has lost NUM_PROBE_FAILS or more * consecutive probes, on both IPv4 and IPv6 protocol * instances of the phyint, then trigger failure * detection, else return false */ if (pfinfo.pf_nfail < NUM_PROBE_FAILS) return (PHYINT_OK); pii_other = phyint_inst_other(pii); if (PROBE_CAPABLE(pii_other)) { probe_fail_info(pii_other, NULL, &pfinfo); if (pfinfo.pf_nfail >= NUM_PROBE_FAILS) { /* * We have NUM_PROBE_FAILS or more failures * on both IPv4 and IPv6. Get the earliest * time when failure was detected on this * phyint across IPv4 and IPv6. */ if (TIME_LT(pfinfo.pf_tff, pi_tff)) pi_tff = pfinfo.pf_tff; } else { /* * This instance has < NUM_PROBE_FAILS failure. * So return false */ return (PHYINT_OK); } } *tff = pi_tff; return (PHYINT_FAILURE); } /* * Check if the link has gone down on this phyint, or it has failed the * last NUM_PROBE_FAILS consecutive probes on both instances IPv4 and IPv6. * Also look at other phyints of this group, for group failures. */ int failure_state(struct phyint_instance *pii) { struct probe_success_count psinfo; uint_t pi2_tls; /* time last success */ uint_t pi_tff; /* time first fail */ struct phyint *pi2; struct phyint *pi; struct phyint_instance *pii2; struct phyint_group *pg; int retval; if (debug & D_FAILREP) logdebug("phyint_failed(%s)\n", pii->pii_name); pi = pii->pii_phyint; pg = pi->pi_group; if (LINK_UP(pi) && phyint_inst_probe_failure_state(pii, &pi_tff) == PHYINT_OK) return (PHYINT_OK); /* * At this point, the link is down, or the phyint is suspect, as it * has lost NUM_PROBE_FAILS or more probes. If the phyint does not * belong to any group, this is a PHYINT_FAILURE. Otherwise, continue * on to determine whether this should be considered a PHYINT_FAILURE * or GROUP_FAILURE. */ if (pg == phyint_anongroup) return (PHYINT_FAILURE); /* * Need to compare against other phyints of the same group * to exclude group failures. If the failure was detected via * probing, then if the time of last success (tls) of any * phyint is more recent than the time of first fail (tff) of the * phyint in question, and the link is up on the phyint, * then it is a phyint failure. Otherwise it is a group failure. * If failure was detected via a link down notification sent from * the driver to IP, we see if any phyints in the group are still * running and haven't received a link down notification. We * will usually be processing the link down notification shortly * after it was received, so there is no point looking at the tls * of other phyints. */ retval = GROUP_FAILURE; for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { /* Exclude ourself from comparison */ if (pi2 == pi) continue; if (LINK_DOWN(pi)) { /* * We use FLAGS_TO_LINK_STATE() to test the flags * directly, rather then LINK_UP() or LINK_DOWN(), as * we may not have got round to processing the link * state for the other phyints in the group yet. * * The check for PI_RUNNING and group failure handles * the case when the group begins to recover. * PI_RUNNING will be set, and group failure cleared * only after receipt of NUM_PROBE_REPAIRS, by which * time the other phyints should have received at * least 1 packet, and so will not have NUM_PROBE_FAILS. */ if ((pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } continue; } if (LINK_DOWN(pi2)) continue; /* * If there's no probe-based failure detection on this * interface, and its link is still up, then it's still * working and thus the group has not failed. */ if (!PROBE_ENABLED(pi2->pi_v4) && !PROBE_ENABLED(pi2->pi_v6)) { retval = PHYINT_FAILURE; break; } /* * Need to compare against both IPv4 and IPv6 instances. */ pii2 = pi2->pi_v4; if (pii2 != NULL) { probe_success_info(pii2, NULL, &psinfo); if (psinfo.ps_tls_valid) { pi2_tls = psinfo.ps_tls; /* * See comment above regarding check * for PI_RUNNING and group failure. */ if (TIME_GT(pi2_tls, pi_tff) && (pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } } } pii2 = pi2->pi_v6; if (pii2 != NULL) { probe_success_info(pii2, NULL, &psinfo); if (psinfo.ps_tls_valid) { pi2_tls = psinfo.ps_tls; /* * See comment above regarding check * for PI_RUNNING and group failure. */ if (TIME_GT(pi2_tls, pi_tff) && (pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } } } } /* * Update the group state to account for the changes. */ phyint_group_refresh_state(pg); return (retval); } /* * Return the information associated with consecutive probe successes * starting with the most recent probe. At most the last 2 probes can be * in the unacknowledged state. All previous probes have either failed * or succeeded. */ static void probe_success_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_success_count *psinfo) { uint_t i; struct probe_stats *pr_statp; uint_t most_recent; uint_t second_most_recent; boolean_t pi_found_failure = _B_FALSE; boolean_t tg_found_failure = _B_FALSE; uint_t now; uint_t timeout; struct target *tg; if (debug & D_FAILREP) logdebug("probe_success_info(%s)\n", pii->pii_name); bzero(psinfo, sizeof (*psinfo)); now = getcurrenttime(); /* * Start with the most recent probe, and count the number * of consecutive probe successes. Latch the number of successes * on hitting a failure. */ most_recent = PROBE_INDEX_PREV(pii->pii_probe_next); second_most_recent = PROBE_INDEX_PREV(most_recent); for (i = most_recent; i != pii->pii_probe_next; i = PROBE_INDEX_PREV(i)) { pr_statp = &pii->pii_probes[i]; switch (pr_statp->pr_status) { case PR_UNACKED: /* * Only the most recent 2 probes can be unacknowledged */ assert(i == most_recent || i == second_most_recent); tg = pr_statp->pr_target; assert(tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use the value of the group's probe * interval which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (tg->tg_crtt != 0) { timeout += tg->tg_crtt; } else { timeout += pii->pii_phyint->pi_group->pg_probeint; } if (TIME_LT(timeout, now)) { /* * We hit a failure. Latch the total number of * recent consecutive successes. */ pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); pi_found_failure = _B_TRUE; if (cur_tg != NULL && tg == cur_tg) { /* * We hit a failure for the desired * target. Latch the number of recent * consecutive successes for this target */ tg_found_failure = _B_TRUE; } } break; case PR_ACKED: /* * Bump up the count of probe successes, if we * have not seen any failure so far. */ if (!pi_found_failure) psinfo->ps_nsucc++; if (cur_tg != NULL && pr_statp->pr_target == cur_tg && !tg_found_failure) { psinfo->ps_nsucc_tg++; } /* * Record the time of last success, if this is * the most recent probe success. */ if (!psinfo->ps_tls_valid) { psinfo->ps_tls = ns2ms(pr_statp->pr_hrtime_ackproc); psinfo->ps_tls_valid = _B_TRUE; } break; case PR_LOST: /* * We hit a failure. Latch the total number of * recent consecutive successes. */ pi_found_failure = _B_TRUE; if (cur_tg != NULL && pr_statp->pr_target == cur_tg) { /* * We hit a failure for the desired target. * Latch the number of recent consecutive * successes for this target */ tg_found_failure = _B_TRUE; } break; default: return; } } } /* * Return the information associated with consecutive probe failures * starting with the most recent probe. Only the last 2 probes can be in the * unacknowledged state. All previous probes have either failed or succeeded. */ static void probe_fail_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_fail_count *pfinfo) { int i; struct probe_stats *pr_statp; boolean_t tg_found_success = _B_FALSE; boolean_t pi_found_success = _B_FALSE; int most_recent; int second_most_recent; uint_t now; uint_t timeout; struct target *tg; if (debug & D_FAILREP) logdebug("probe_fail_info(%s)\n", pii->pii_name); bzero(pfinfo, sizeof (*pfinfo)); now = getcurrenttime(); /* * Start with the most recent probe, and count the number * of consecutive probe failures. Latch the number of failures * on hitting a probe success. */ most_recent = PROBE_INDEX_PREV(pii->pii_probe_next); second_most_recent = PROBE_INDEX_PREV(most_recent); for (i = most_recent; i != pii->pii_probe_next; i = PROBE_INDEX_PREV(i)) { pr_statp = &pii->pii_probes[i]; assert(PR_STATUS_VALID(pr_statp->pr_status)); switch (pr_statp->pr_status) { case PR_UNACKED: /* * Only the most recent 2 probes can be unacknowledged */ assert(i == most_recent || i == second_most_recent); tg = pr_statp->pr_target; /* * Target is guaranteed to exist in the unack. state */ assert(tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use the group's probe interval, * which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (tg->tg_crtt != 0) { timeout += tg->tg_crtt; } else { timeout += pii->pii_phyint->pi_group->pg_probeint; } if (TIME_GT(timeout, now)) break; pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); /* FALLTHRU */ case PR_LOST: if (!pi_found_success) { pfinfo->pf_nfail++; pfinfo->pf_tff = pr_statp->pr_time_lost; } if (cur_tg != NULL && pr_statp->pr_target == cur_tg && !tg_found_success) { pfinfo->pf_nfail_tg++; } break; default: /* * We hit a success or unused slot. Latch the * total number of recent consecutive failures. */ pi_found_success = _B_TRUE; if (cur_tg != NULL && pr_statp->pr_target == cur_tg) { /* * We hit a success for the desired target. * Latch the number of recent consecutive * failures for this target */ tg_found_success = _B_TRUE; } } } } /* * Change the state of probe `pr' on phyint_instance `pii' to state `state'. */ void probe_chstate(struct probe_stats *pr, struct phyint_instance *pii, int state) { if (pr->pr_status == state) return; pr->pr_status = state; (void) probe_state_event(pr, pii); } /* * Check if the phyint has been repaired. If no test address has been * configured, then consider the interface repaired if the link is up (unless * the link is flapping; see below). Otherwise, look for proof of probes * being sent and received. If last NUM_PROBE_REPAIRS probes are fine on * either IPv4 or IPv6 instance, the phyint can be considered repaired. */ static boolean_t phyint_repaired(struct phyint *pi) { struct probe_success_count psinfo; struct phyint_instance *pii; struct target *cur_tg; int pr_ndx; uint_t cur_time; if (debug & D_FAILREP) logdebug("phyint_repaired(%s)\n", pi->pi_name); if (LINK_DOWN(pi)) return (_B_FALSE); /* * If we don't have any test addresses and the link is up, then * consider the interface repaired, unless we've received more than * LINK_UP_PERMIN link up notifications in the last minute, in * which case we keep the link down until we drop back below * the threshold. */ if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) { cur_time = getcurrenttime(); if ((pi->pi_whenup[pi->pi_whendx] == 0 || (cur_time - pi->pi_whenup[pi->pi_whendx]) > MSEC_PERMIN)) { pi->pi_lfmsg_printed = 0; return (_B_TRUE); } if (!pi->pi_lfmsg_printed) { logerr("The link has come up on %s more than %d times " "in the last minute; disabling repair until it " "stabilizes\n", pi->pi_name, LINK_UP_PERMIN); pi->pi_lfmsg_printed = 1; } return (_B_FALSE); } pii = pi->pi_v4; if (PROBE_CAPABLE(pii)) { pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_success_info(pii, cur_tg, &psinfo); if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS || psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS) return (_B_TRUE); } pii = pi->pi_v6; if (PROBE_CAPABLE(pii)) { pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_success_info(pii, cur_tg, &psinfo); if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS || psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS) return (_B_TRUE); } return (_B_FALSE); } /* * Used to set/clear phyint flags, by making a SIOCSLIFFLAGS call. */ boolean_t change_pif_flags(struct phyint *pi, uint64_t set, uint64_t clear) { int ifsock; struct lifreq lifr; uint64_t old_flags; if (debug & D_FAILREP) { logdebug("change_pif_flags(%s): set %llx clear %llx\n", pi->pi_name, set, clear); } if (pi->pi_v4 != NULL) ifsock = ifsock_v4; else ifsock = ifsock_v6; /* * Get the current flags from the kernel, and set/clear the * desired phyint flags. Since we set only phyint flags, we can * do it on either IPv4 or IPv6 instance. */ (void) strlcpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name)); if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) logperror("change_pif_flags: ioctl (get flags)"); return (_B_FALSE); } old_flags = lifr.lifr_flags; lifr.lifr_flags |= set; lifr.lifr_flags &= ~clear; if (old_flags == lifr.lifr_flags) { /* No change in the flags. No need to send ioctl */ return (_B_TRUE); } if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) logperror("change_pif_flags: ioctl (set flags)"); return (_B_FALSE); } /* * Keep pi_flags in synch. with actual flags. Assumes flags are * phyint flags. */ pi->pi_flags |= set; pi->pi_flags &= ~clear; if (pi->pi_v4 != NULL) pi->pi_v4->pii_flags = pi->pi_flags; if (pi->pi_v6 != NULL) pi->pi_v6->pii_flags = pi->pi_flags; return (_B_TRUE); } /* * icmp cksum computation for IPv4. */ static int in_cksum(ushort_t *addr, int len) { register int nleft = len; register ushort_t *w = addr; register ushort_t answer; ushort_t odd_byte = 0; register int sum = 0; /* * Our algorithm is simple, using a 32 bit accumulator (sum), * we add sequential 16 bit words to it, and at the end, fold * back all the carry bits from the top 16 bits into the lower * 16 bits. */ while (nleft > 1) { sum += *w++; nleft -= 2; } /* mop up an odd byte, if necessary */ if (nleft == 1) { *(uchar_t *)(&odd_byte) = *(uchar_t *)w; sum += odd_byte; } /* * add back carry outs from top 16 bits to low 16 bits */ sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */ sum += (sum >> 16); /* add carry */ answer = ~sum; /* truncate to 16 bits */ return (answer); } static void reset_snxt_basetimes(void) { struct phyint_instance *pii; for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; } } /* * Is the address one of our own addresses? Unfortunately, * we cannot check our phyint tables to determine if the address * is our own. This is because, we don't track interfaces that * are not part of any group. We have to either use a 'bind' or * get the complete list of all interfaces using SIOCGLIFCONF, * to do this check. We could also use SIOCTMYADDR. * Bind fails for the local zone address, so we might include local zone * address as target address. If local zone address is a target address * and it is up, it is not possible to detect the interface failure. * SIOCTMYADDR also doesn't consider local zone address as own address. * So, we choose to use SIOCGLIFCONF to collect the local addresses, and they * are stored in `localaddrs' */ boolean_t own_address(struct in6_addr addr) { addrlist_t *addrp; struct sockaddr_storage ss; int af = IN6_IS_ADDR_V4MAPPED(&addr) ? AF_INET : AF_INET6; addr2storage(af, &addr, &ss); for (addrp = localaddrs; addrp != NULL; addrp = addrp->al_next) { if (sockaddrcmp(&ss, &addrp->al_addr)) return (_B_TRUE); } return (_B_FALSE); } static int ns2ms(int64_t ns) { return (NSEC2MSEC(ns)); } static int64_t tv2ns(struct timeval *tvp) { return (tvp->tv_sec * NANOSEC + tvp->tv_usec * 1000); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include "mpd_defs.h" #include "mpd_tables.h" /* * Global list of phyints, phyint instances, phyint groups and the anonymous * group; the latter is initialized in phyint_init(). */ struct phyint *phyints = NULL; struct phyint_instance *phyint_instances = NULL; struct phyint_group *phyint_groups = NULL; struct phyint_group *phyint_anongroup; /* * Grouplist signature; initialized in phyint_init(). */ static uint64_t phyint_grouplistsig; static void phyint_inst_insert(struct phyint_instance *pii); static void phyint_inst_print(struct phyint_instance *pii); static void phyint_insert(struct phyint *pi, struct phyint_group *pg); static void phyint_delete(struct phyint *pi); static boolean_t phyint_is_usable(struct phyint *pi); static void logint_print(struct logint *li); static void logint_insert(struct phyint_instance *pii, struct logint *li); static struct logint *logint_lookup(struct phyint_instance *pii, char *li_name); static void target_print(struct target *tg); static void target_insert(struct phyint_instance *pii, struct target *tg); static struct target *target_first(struct phyint_instance *pii); static struct target *target_select_best(struct phyint_instance *pii); static void target_flush_hosts(struct phyint_group *pg); static void reset_pii_probes(struct phyint_instance *pii, struct target *tg); static boolean_t phyint_inst_v6_sockinit(struct phyint_instance *pii); static boolean_t phyint_inst_v4_sockinit(struct phyint_instance *pii); static int phyint_state_event(struct phyint_group *pg, struct phyint *pi); static int phyint_group_state_event(struct phyint_group *pg); static int phyint_group_change_event(struct phyint_group *pg, ipmp_group_op_t); static int phyint_group_member_event(struct phyint_group *pg, struct phyint *pi, ipmp_if_op_t op); static int logint_upcount(struct phyint *pi); static uint64_t gensig(void); /* Initialize any per-file global state. Returns 0 on success, -1 on failure */ int phyint_init(void) { phyint_grouplistsig = gensig(); if (track_all_phyints) { phyint_anongroup = phyint_group_create(""); if (phyint_anongroup == NULL) return (-1); phyint_group_insert(phyint_anongroup); } return (0); } /* Return the phyint with the given name */ struct phyint * phyint_lookup(const char *name) { struct phyint *pi; if (debug & D_PHYINT) logdebug("phyint_lookup(%s)\n", name); for (pi = phyints; pi != NULL; pi = pi->pi_next) { if (strncmp(pi->pi_name, name, sizeof (pi->pi_name)) == 0) break; } return (pi); } /* * Lookup a phyint in the group that has the same hardware address as `pi', or * NULL if there's none. If `online_only' is set, then only online phyints * are considered when matching. Otherwise, phyints that had been offlined * due to a duplicate hardware address will also be considered. */ static struct phyint * phyint_lookup_hwaddr(struct phyint *pi, boolean_t online_only) { struct phyint *pi2; if (pi->pi_group == phyint_anongroup) return (NULL); for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (pi2 == pi) continue; /* * NOTE: even when online_only is B_FALSE, we ignore phyints * that are administratively offline (rather than offline * because they're dups); when they're brought back online, * they'll be flagged as dups if need be. */ if (pi2->pi_state == PI_OFFLINE && (online_only || !pi2->pi_hwaddrdup)) continue; if (pi2->pi_hwaddrlen == pi->pi_hwaddrlen && bcmp(pi2->pi_hwaddr, pi->pi_hwaddr, pi->pi_hwaddrlen) == 0) return (pi2); } return (NULL); } /* * Respond to DLPI notifications. Currently, this only processes physical * address changes for the phyint passed via `arg' by onlining or offlining * phyints in the group. */ /* ARGSUSED */ static void phyint_link_notify(dlpi_handle_t dh, dlpi_notifyinfo_t *dnip, void *arg) { struct phyint *pi = arg; struct phyint *oduppi = NULL, *duppi = NULL; assert((dnip->dni_note & pi->pi_notes) != 0); if (dnip->dni_note != DL_NOTE_PHYS_ADDR) return; assert(dnip->dni_physaddrlen <= DLPI_PHYSADDR_MAX); /* * If our hardware address hasn't changed, there's nothing to do. */ if (pi->pi_hwaddrlen == dnip->dni_physaddrlen && bcmp(pi->pi_hwaddr, dnip->dni_physaddr, pi->pi_hwaddrlen) == 0) return; oduppi = phyint_lookup_hwaddr(pi, _B_FALSE); pi->pi_hwaddrlen = dnip->dni_physaddrlen; (void) memcpy(pi->pi_hwaddr, dnip->dni_physaddr, pi->pi_hwaddrlen); duppi = phyint_lookup_hwaddr(pi, _B_FALSE); if (oduppi != NULL || pi->pi_hwaddrdup) { /* * Our old hardware address was a duplicate. If we'd been * offlined because of it, and our new hardware address is not * a duplicate, then bring us online. Otherwise, `oduppi' * must've been the one brought offline; bring it online. */ if (pi->pi_hwaddrdup) { if (duppi == NULL) (void) phyint_undo_offline(pi); } else { assert(oduppi->pi_hwaddrdup); (void) phyint_undo_offline(oduppi); } } if (duppi != NULL && !pi->pi_hwaddrdup) { /* * Our new hardware address was a duplicate and we're not * yet flagged as a duplicate; bring us offline. */ pi->pi_hwaddrdup = _B_TRUE; (void) phyint_offline(pi, 0); } } /* * Initialize information about the underlying link for `pi', and set us * up to be notified about future changes. Returns _B_TRUE on success. */ boolean_t phyint_link_init(struct phyint *pi) { int retval; uint_t notes; const char *errmsg; dlpi_notifyid_t id; pi->pi_notes = 0; retval = dlpi_open(pi->pi_name, &pi->pi_dh, 0); if (retval != DLPI_SUCCESS) { pi->pi_dh = NULL; errmsg = "cannot open"; goto failed; } pi->pi_hwaddrlen = DLPI_PHYSADDR_MAX; retval = dlpi_get_physaddr(pi->pi_dh, DL_CURR_PHYS_ADDR, pi->pi_hwaddr, &pi->pi_hwaddrlen); if (retval != DLPI_SUCCESS) { errmsg = "cannot get hardware address"; goto failed; } /* * Check if the link supports DLPI link state notifications. For * historical reasons, the actual changes are tracked through routing * sockets, so we immediately disable the notification upon success. */ notes = DL_NOTE_LINK_UP | DL_NOTE_LINK_DOWN; retval = dlpi_enabnotify(pi->pi_dh, notes, phyint_link_notify, pi, &id); if (retval == DLPI_SUCCESS) { (void) dlpi_disabnotify(pi->pi_dh, id, NULL); pi->pi_notes |= notes; } /* * Enable notification of hardware address changes to keep pi_hwaddr * up-to-date and track if we need to offline/undo-offline phyints. */ notes = DL_NOTE_PHYS_ADDR; retval = dlpi_enabnotify(pi->pi_dh, notes, phyint_link_notify, pi, &id); if (retval == DLPI_SUCCESS && poll_add(dlpi_fd(pi->pi_dh)) == 0) pi->pi_notes |= notes; return (_B_TRUE); failed: logerr("%s: %s: %s\n", pi->pi_name, errmsg, dlpi_strerror(retval)); if (pi->pi_dh != NULL) { dlpi_close(pi->pi_dh); pi->pi_dh = NULL; } return (_B_FALSE); } /* * Close use of link on `pi'. */ void phyint_link_close(struct phyint *pi) { if (pi->pi_notes & DL_NOTE_PHYS_ADDR) { (void) poll_remove(dlpi_fd(pi->pi_dh)); pi->pi_notes &= ~DL_NOTE_PHYS_ADDR; } /* * NOTE: we don't clear pi_notes here so that iflinkstate() can still * properly report the link state even when offline (which is possible * since we use IFF_RUNNING to track link state). */ dlpi_close(pi->pi_dh); pi->pi_dh = NULL; } /* Return the phyint instance with the given name and the given family */ struct phyint_instance * phyint_inst_lookup(int af, char *name) { struct phyint *pi; if (debug & D_PHYINT) logdebug("phyint_inst_lookup(%s %s)\n", AF_STR(af), name); assert(af == AF_INET || af == AF_INET6); pi = phyint_lookup(name); if (pi == NULL) return (NULL); return (PHYINT_INSTANCE(pi, af)); } struct phyint_group * phyint_group_lookup(const char *pg_name) { struct phyint_group *pg; if (debug & D_PHYINT) logdebug("phyint_group_lookup(%s)\n", pg_name); for (pg = phyint_groups; pg != NULL; pg = pg->pg_next) { if (strncmp(pg->pg_name, pg_name, sizeof (pg->pg_name)) == 0) break; } return (pg); } /* * Insert the phyint in the linked list of all phyints. If the phyint belongs * to some group, insert it in the phyint group list. */ static void phyint_insert(struct phyint *pi, struct phyint_group *pg) { if (debug & D_PHYINT) logdebug("phyint_insert(%s '%s')\n", pi->pi_name, pg->pg_name); /* Insert the phyint at the head of the 'all phyints' list */ pi->pi_next = phyints; pi->pi_prev = NULL; if (phyints != NULL) phyints->pi_prev = pi; phyints = pi; /* * Insert the phyint at the head of the 'phyint_group members' list * of the phyint group to which it belongs. */ pi->pi_pgnext = NULL; pi->pi_pgprev = NULL; pi->pi_group = pg; pi->pi_pgnext = pg->pg_phyint; if (pi->pi_pgnext != NULL) pi->pi_pgnext->pi_pgprev = pi; pg->pg_phyint = pi; /* Refresh the group state now that this phyint has been added */ phyint_group_refresh_state(pg); pg->pg_sig++; (void) phyint_group_member_event(pg, pi, IPMP_IF_ADD); } /* Insert the phyint instance in the linked list of all phyint instances. */ static void phyint_inst_insert(struct phyint_instance *pii) { if (debug & D_PHYINT) { logdebug("phyint_inst_insert(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } /* * Insert the phyint at the head of the 'all phyint instances' list. */ pii->pii_next = phyint_instances; pii->pii_prev = NULL; if (phyint_instances != NULL) phyint_instances->pii_prev = pii; phyint_instances = pii; } /* * Create a new phyint with the given parameters. Also insert it into * the list of all phyints and the list of phyint group members by calling * phyint_insert(). */ static struct phyint * phyint_create(char *pi_name, struct phyint_group *pg, uint_t ifindex, uint64_t flags) { struct phyint *pi; pi = calloc(1, sizeof (struct phyint)); if (pi == NULL) { logperror("phyint_create: calloc"); return (NULL); } /* * Record the phyint values. */ (void) strlcpy(pi->pi_name, pi_name, sizeof (pi->pi_name)); pi->pi_taddrthresh = getcurrentsec() + TESTADDR_CONF_TIME; pi->pi_ifindex = ifindex; pi->pi_icmpid = htons(((getpid() & 0xFF) << 8) | (ifindex & 0xFF)); pi->pi_state = PI_INIT; pi->pi_flags = PHYINT_FLAGS(flags); /* * Initialize the link state. The link state is initialized to * up, so that if the link is down when IPMP starts monitoring * the interface, it will appear as though there has been a * transition from the link up to link down. This avoids * having to treat this situation as a special case. */ INIT_LINK_STATE(pi); if (!phyint_link_init(pi)) { free(pi); return (NULL); } /* * Insert the phyint in the list of all phyints, and the * list of phyint group members */ phyint_insert(pi, pg); return (pi); } /* * Create a new phyint instance belonging to the phyint 'pi' and address * family 'af'. Also insert it into the list of all phyint instances by * calling phyint_inst_insert(). */ static struct phyint_instance * phyint_inst_create(struct phyint *pi, int af) { struct phyint_instance *pii; pii = calloc(1, sizeof (struct phyint_instance)); if (pii == NULL) { logperror("phyint_inst_create: calloc"); return (NULL); } /* * Attach the phyint instance to the phyint. * Set the back pointers as well */ pii->pii_phyint = pi; if (af == AF_INET) pi->pi_v4 = pii; else pi->pi_v6 = pii; pii->pii_in_use = 1; pii->pii_probe_sock = -1; pii->pii_snxt = 1; pii->pii_af = af; pii->pii_fd_hrtime = gethrtime() + (FAILURE_DETECTION_QP * (hrtime_t)NANOSEC); pii->pii_flags = pi->pi_flags; /* Insert the phyint instance in the list of all phyint instances. */ phyint_inst_insert(pii); return (pii); } /* * Change the state of phyint `pi' to state `state'. */ void phyint_chstate(struct phyint *pi, enum pi_state state) { /* * To simplify things, some callers always set a given state * regardless of the previous state of the phyint (e.g., setting * PI_RUNNING when it's already set). We shouldn't bother * generating an event or consuming a signature for these, since * the actual state of the interface is unchanged. */ if (pi->pi_state == state) return; pi->pi_state = state; phyint_changed(pi); } /* * Note that `pi' has changed state. */ void phyint_changed(struct phyint *pi) { pi->pi_group->pg_sig++; (void) phyint_state_event(pi->pi_group, pi); } /* * Insert the phyint group in the linked list of all phyint groups * at the head of the list */ void phyint_group_insert(struct phyint_group *pg) { pg->pg_next = phyint_groups; pg->pg_prev = NULL; if (phyint_groups != NULL) phyint_groups->pg_prev = pg; phyint_groups = pg; phyint_grouplistsig++; (void) phyint_group_change_event(pg, IPMP_GROUP_ADD); } /* * Create a new phyint group called 'name'. */ struct phyint_group * phyint_group_create(const char *name) { struct phyint_group *pg; if (debug & D_PHYINT) logdebug("phyint_group_create(%s)\n", name); pg = calloc(1, sizeof (struct phyint_group)); if (pg == NULL) { logperror("phyint_group_create: calloc"); return (NULL); } (void) strlcpy(pg->pg_name, name, sizeof (pg->pg_name)); pg->pg_sig = gensig(); pg->pg_fdt = user_failure_detection_time; pg->pg_probeint = user_probe_interval; pg->pg_in_use = _B_TRUE; /* * Normal groups always start in the PG_FAILED state since they * have no active interfaces. In contrast, anonymous groups are * heterogeneous and thus always PG_OK. */ pg->pg_state = (name[0] == '\0' ? PG_OK : PG_FAILED); return (pg); } /* * Change the state of the phyint group `pg' to state `state'. */ void phyint_group_chstate(struct phyint_group *pg, enum pg_state state) { assert(pg != phyint_anongroup); /* * To simplify things, some callers always set a given state * regardless of the previous state of the group (e.g., setting * PG_DEGRADED when it's already set). We shouldn't bother * generating an event or consuming a signature for these, since * the actual state of the group is unchanged. */ if (pg->pg_state == state) return; pg->pg_state = state; switch (state) { case PG_FAILED: /* * We can never know with certainty that a group has * failed. It is possible that all known targets have * failed simultaneously, and new targets have come up * instead. If the targets are routers then router * discovery will kick in, and we will see the new routers * thru routing socket messages. But if the targets are * hosts, we have to discover it by multicast. So flush * all the host targets. The next probe will send out a * multicast echo request. If this is a group failure, we * will still not see any response, otherwise the group * will be repaired after we get NUM_PROBE_REPAIRS * consecutive unicast replies on any phyint. */ target_flush_hosts(pg); break; case PG_OK: case PG_DEGRADED: break; default: logerr("phyint_group_chstate: invalid group state %d; " "aborting\n", state); abort(); } pg->pg_sig++; (void) phyint_group_state_event(pg); } /* * Create a new phyint instance and initialize it from the values supplied by * the kernel. Always check for ENXIO before logging any error, because the * interface could have vanished after completion of SIOCGLIFCONF. * Return values: * pointer to the phyint instance on success * NULL on failure Eg. if the phyint instance is not found in the kernel */ struct phyint_instance * phyint_inst_init_from_k(int af, char *pi_name) { char pg_name[LIFNAMSIZ + 1]; int ifsock; uint_t ifindex; uint64_t flags; struct lifreq lifr; struct phyint *pi; struct phyint_instance *pii; boolean_t pi_created; struct phyint_group *pg; retry: pii = NULL; pi = NULL; pg = NULL; pi_created = _B_FALSE; if (debug & D_PHYINT) { logdebug("phyint_inst_init_from_k(%s %s)\n", AF_STR(af), pi_name); } assert(af == AF_INET || af == AF_INET6); /* Get the socket for doing ioctls */ ifsock = (af == AF_INET) ? ifsock_v4 : ifsock_v6; /* * Get the interface flags. Ignore virtual interfaces, IPMP * meta-interfaces, point-to-point interfaces, and interfaces * that can't support multicast. */ (void) strlcpy(lifr.lifr_name, pi_name, sizeof (lifr.lifr_name)); if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) { logperror("phyint_inst_init_from_k:" " ioctl (get flags)"); } return (NULL); } flags = lifr.lifr_flags; if (!(flags & IFF_MULTICAST) || (flags & (IFF_VIRTUAL|IFF_IPMP|IFF_POINTOPOINT))) return (NULL); /* * Get the ifindex for recording later in our tables, in case we need * to create a new phyint. */ if (ioctl(ifsock, SIOCGLIFINDEX, (char *)&lifr) < 0) { if (errno != ENXIO) { logperror("phyint_inst_init_from_k: " " ioctl (get lifindex)"); } return (NULL); } ifindex = lifr.lifr_index; /* * Get the phyint group name of this phyint, from the kernel. */ if (ioctl(ifsock, SIOCGLIFGROUPNAME, (char *)&lifr) < 0) { if (errno != ENXIO) { logperror("phyint_inst_init_from_k: " "ioctl (get group name)"); } return (NULL); } (void) strlcpy(pg_name, lifr.lifr_groupname, sizeof (pg_name)); /* * If the phyint is not part of any group, pg_name is the * null string. If 'track_all_phyints' is false, there is no * need to create a phyint. */ if (pg_name[0] == '\0' && !track_all_phyints) { /* * If the IFF_FAILED, IFF_INACTIVE, or IFF_OFFLINE flags are * set, reset them. These flags shouldn't be set if in.mpathd * isn't tracking the interface. */ if ((flags & (IFF_FAILED | IFF_INACTIVE | IFF_OFFLINE))) { lifr.lifr_flags = flags & ~(IFF_FAILED | IFF_INACTIVE | IFF_OFFLINE); if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) { logperror("phyint_inst_init_from_k:" " ioctl (set flags)"); } } } return (NULL); } /* * We need to create a new phyint instance. We may also need to * create the group if e.g. the SIOCGLIFCONF loop in initifs() found * an underlying interface before it found its IPMP meta-interface. * Note that we keep any created groups even if phyint_inst_from_k() * fails since a group's existence is not dependent on the ability of * in.mpathd to the track the group's interfaces. */ if ((pg = phyint_group_lookup(pg_name)) == NULL) { if ((pg = phyint_group_create(pg_name)) == NULL) { logerr("phyint_inst_init_from_k: cannot create group " "%s\n", pg_name); return (NULL); } phyint_group_insert(pg); } /* * Lookup the phyint. If the phyint does not exist create it. */ pi = phyint_lookup(pi_name); if (pi == NULL) { pi = phyint_create(pi_name, pg, ifindex, flags); if (pi == NULL) { logerr("phyint_inst_init_from_k:" " unable to create phyint %s\n", pi_name); return (NULL); } pi_created = _B_TRUE; } else { /* The phyint exists already. */ assert(pi_created == _B_FALSE); /* * Normally we should see consistent values for the IPv4 and * IPv6 instances, for phyint properties. If we don't, it * means things have changed underneath us, and we should * resync our tables with the kernel. Check whether the * interface index has changed. If so, it is most likely * the interface has been unplumbed and replumbed, * while we are yet to update our tables. Do it now. */ if (pi->pi_ifindex != ifindex) { phyint_inst_delete(PHYINT_INSTANCE(pi, AF_OTHER(af))); goto retry; } assert(PHYINT_INSTANCE(pi, af) == NULL); /* * If the group name seen by the IPv4 and IPv6 instances * are different, it is most likely the groupname has * changed, while we are yet to update our tables. Do it now. */ if (strcmp(pi->pi_group->pg_name, pg_name) != 0) { phyint_inst_delete(PHYINT_INSTANCE(pi, AF_OTHER(af))); goto retry; } } /* * Create a new phyint instance, corresponding to the 'af' * passed in. */ pii = phyint_inst_create(pi, af); if (pii == NULL) { logerr("phyint_inst_init_from_k: unable to create" "phyint inst %s\n", pi->pi_name); if (pi_created) phyint_delete(pi); return (NULL); } /* * NOTE: the change_pif_flags() implementation requires a phyint * instance before it can function, so a number of tasks that would * otherwise be done in phyint_create() are deferred to here. */ if (pi_created) { /* * If the interface is offline, set the state to PI_OFFLINE. * Otherwise, optimistically consider this interface running. * Later (in process_link_state_changes()), we will adjust * this to match the current state of the link. Further, if * test addresses are subsequently assigned, we will * transition to PI_NOTARGETS and then to either PI_RUNNING or * PI_FAILED depending on the probe results. */ if (pi->pi_flags & IFF_OFFLINE) { phyint_chstate(pi, PI_OFFLINE); } else { /* calls phyint_chstate() */ phyint_transition_to_running(pi); } /* * If this a standby phyint, determine whether it should be * IFF_INACTIVE. */ if (pi->pi_flags & IFF_STANDBY) phyint_standby_refresh_inactive(pi); /* * If this phyint does not have a unique hardware address in its * group, offline it. */ if (phyint_lookup_hwaddr(pi, _B_TRUE) != NULL) { pi->pi_hwaddrdup = _B_TRUE; (void) phyint_offline(pi, 0); } } return (pii); } /* * Bind pii_probe_sock to the address associated with pii_probe_logint. * This socket will be used for sending and receiving ICMP/ICMPv6 probes to * targets. Do the common part in this function, and complete the * initializations by calling the protocol specific functions * phyint_inst_v{4,6}_sockinit() respectively. * * Return values: _B_TRUE/_B_FALSE for success or failure respectively. */ boolean_t phyint_inst_sockinit(struct phyint_instance *pii) { boolean_t success; struct phyint_group *pg; if (debug & D_PHYINT) { logdebug("phyint_inst_sockinit(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } assert(pii->pii_probe_logint != NULL); assert(pii->pii_probe_logint->li_flags & IFF_UP); assert(pii->pii_probe_logint->li_flags & IFF_NOFAILOVER); assert(pii->pii_af == AF_INET || pii->pii_af == AF_INET6); /* * If the socket is already bound, close pii_probe_sock */ if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); /* * If the phyint is not part of a named group and track_all_phyints is * false, simply return. */ pg = pii->pii_phyint->pi_group; if (pg == phyint_anongroup && !track_all_phyints) { if (debug & D_PHYINT) logdebug("phyint_inst_sockinit: no group\n"); return (_B_FALSE); } /* * Initialize the socket by calling the protocol specific function. * If it succeeds, add the socket to the poll list. */ if (pii->pii_af == AF_INET6) success = phyint_inst_v6_sockinit(pii); else success = phyint_inst_v4_sockinit(pii); if (success && (poll_add(pii->pii_probe_sock) == 0)) return (_B_TRUE); /* Something failed, cleanup and return false */ if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_FALSE); return (_B_FALSE); } /* * IPv6 specific part in initializing the pii_probe_sock. This socket is * used to send/receive ICMPv6 probe packets. */ static boolean_t phyint_inst_v6_sockinit(struct phyint_instance *pii) { icmp6_filter_t filter; int hopcount = 1; int off = 0; int on = 1; struct sockaddr_in6 testaddr; int flags; /* * Open a raw socket with ICMPv6 protocol. * * Use IPV6_BOUND_IF to make sure that probes are sent and received on * the specified phyint only. Bind to the test address to ensure that * the responses are sent to the specified phyint. * * Set the hopcount to 1 so that probe packets are not routed. * Disable multicast loopback. Set the receive filter to * receive only ICMPv6 echo replies. */ pii->pii_probe_sock = socket(pii->pii_af, SOCK_RAW, IPPROTO_ICMPV6); if (pii->pii_probe_sock < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: socket"); return (_B_FALSE); } /* * Probes must not block in case of lower layer issues. */ if ((flags = fcntl(pii->pii_probe_sock, F_GETFL, 0)) == -1) { logperror_pii(pii, "phyint_inst_v6_sockinit: fcntl" " F_GETFL"); return (_B_FALSE); } if (fcntl(pii->pii_probe_sock, F_SETFL, flags | O_NONBLOCK) == -1) { logperror_pii(pii, "phyint_inst_v6_sockinit: fcntl" " F_SETFL O_NONBLOCK"); return (_B_FALSE); } bzero(&testaddr, sizeof (testaddr)); testaddr.sin6_family = AF_INET6; testaddr.sin6_port = 0; testaddr.sin6_addr = pii->pii_probe_logint->li_addr; if (bind(pii->pii_probe_sock, (struct sockaddr *)&testaddr, sizeof (testaddr)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: IPv6 bind"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_IF, (char *)&pii->pii_ifindex, sizeof (uint_t)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_MULTICAST_IF"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_BOUND_IF, &pii->pii_ifindex, sizeof (uint_t)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_BOUND_IF"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_UNICAST_HOPS, (char *)&hopcount, sizeof (hopcount)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_UNICAST_HOPS"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, (char *)&hopcount, sizeof (hopcount)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_MULTICAST_HOPS"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_LOOP, (char *)&off, sizeof (off)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_MULTICAST_LOOP"); return (_B_FALSE); } /* * Filter out so that we only receive ICMP echo replies */ ICMP6_FILTER_SETBLOCKALL(&filter); ICMP6_FILTER_SETPASS(ICMP6_ECHO_REPLY, &filter); if (setsockopt(pii->pii_probe_sock, IPPROTO_ICMPV6, ICMP6_FILTER, (char *)&filter, sizeof (filter)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " ICMP6_FILTER"); return (_B_FALSE); } /* Enable receipt of hoplimit */ if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_RECVHOPLIMIT, &on, sizeof (on)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " IPV6_RECVHOPLIMIT"); return (_B_FALSE); } /* Enable receipt of timestamp */ if (setsockopt(pii->pii_probe_sock, SOL_SOCKET, SO_TIMESTAMP, &on, sizeof (on)) < 0) { logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt" " SO_TIMESTAMP"); return (_B_FALSE); } return (_B_TRUE); } /* * IPv4 specific part in initializing the pii_probe_sock. This socket is * used to send/receive ICMPv4 probe packets. */ static boolean_t phyint_inst_v4_sockinit(struct phyint_instance *pii) { struct sockaddr_in testaddr; char char_off = 0; int ttl = 1; char char_ttl = 1; int on = 1; int flags; /* * Open a raw socket with ICMPv4 protocol. * * Use IP_BOUND_IF to make sure that probes are sent and received on * the specified phyint only. Bind to the test address to ensure that * the responses are sent to the specified phyint. * * Set the ttl to 1 so that probe packets are not routed. * Disable multicast loopback. Enable receipt of timestamp. */ pii->pii_probe_sock = socket(pii->pii_af, SOCK_RAW, IPPROTO_ICMP); if (pii->pii_probe_sock < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: socket"); return (_B_FALSE); } /* * Probes must not block in case of lower layer issues. */ if ((flags = fcntl(pii->pii_probe_sock, F_GETFL, 0)) == -1) { logperror_pii(pii, "phyint_inst_v4_sockinit: fcntl" " F_GETFL"); return (_B_FALSE); } if (fcntl(pii->pii_probe_sock, F_SETFL, flags | O_NONBLOCK) == -1) { logperror_pii(pii, "phyint_inst_v4_sockinit: fcntl" " F_SETFL O_NONBLOCK"); return (_B_FALSE); } bzero(&testaddr, sizeof (testaddr)); testaddr.sin_family = AF_INET; testaddr.sin_port = 0; IN6_V4MAPPED_TO_INADDR(&pii->pii_probe_logint->li_addr, &testaddr.sin_addr); if (bind(pii->pii_probe_sock, (struct sockaddr *)&testaddr, sizeof (testaddr)) < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: IPv4 bind"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_BOUND_IF, &pii->pii_ifindex, sizeof (uint_t)) < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " IP_BOUND_IF"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_IF, (char *)&testaddr.sin_addr, sizeof (struct in_addr)) < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " IP_MULTICAST_IF"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_TTL, (char *)&ttl, sizeof (ttl)) < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " IP_TTL"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_LOOP, (char *)&char_off, sizeof (char_off)) == -1) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " IP_MULTICAST_LOOP"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_TTL, (char *)&char_ttl, sizeof (char_ttl)) == -1) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " IP_MULTICAST_TTL"); return (_B_FALSE); } if (setsockopt(pii->pii_probe_sock, SOL_SOCKET, SO_TIMESTAMP, &on, sizeof (on)) < 0) { logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt" " SO_TIMESTAMP"); return (_B_FALSE); } return (_B_TRUE); } /* * Remove the phyint group from the list of 'all phyint groups' * and free it. */ void phyint_group_delete(struct phyint_group *pg) { /* * The anonymous group always exists, even when empty. */ if (pg == phyint_anongroup) return; if (debug & D_PHYINT) logdebug("phyint_group_delete('%s')\n", pg->pg_name); /* * The phyint group must be empty, and must not have any phyints. * The phyint group must be in the list of all phyint groups */ assert(pg->pg_phyint == NULL); assert(phyint_groups == pg || pg->pg_prev != NULL); if (pg->pg_prev != NULL) pg->pg_prev->pg_next = pg->pg_next; else phyint_groups = pg->pg_next; if (pg->pg_next != NULL) pg->pg_next->pg_prev = pg->pg_prev; pg->pg_next = NULL; pg->pg_prev = NULL; phyint_grouplistsig++; (void) phyint_group_change_event(pg, IPMP_GROUP_REMOVE); addrlist_free(&pg->pg_addrs); free(pg); } /* * Refresh the state of `pg' based on its current members. */ void phyint_group_refresh_state(struct phyint_group *pg) { enum pg_state state; enum pg_state origstate = pg->pg_state; struct phyint *pi, *usablepi; uint_t nif = 0, nusable = 0; /* * Anonymous groups never change state. */ if (pg == phyint_anongroup) return; for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { nif++; if (phyint_is_usable(pi)) { nusable++; usablepi = pi; } } if (nusable == 0) state = PG_FAILED; else if (nif == nusable) state = PG_OK; else state = PG_DEGRADED; phyint_group_chstate(pg, state); /* * If we're shutting down, skip logging messages since otherwise our * shutdown housecleaning will make us report that groups are unusable. */ if (cleanup_started) return; /* * NOTE: We use pg_failmsg_printed rather than origstate since * otherwise at startup we'll log a "now usable" message when the * first usable phyint is added to an empty group. */ if (state != PG_FAILED && pg->pg_failmsg_printed) { assert(origstate == PG_FAILED); logerr("At least 1 IP interface (%s) in group %s is now " "usable\n", usablepi->pi_name, pg->pg_name); pg->pg_failmsg_printed = _B_FALSE; } else if (origstate != PG_FAILED && state == PG_FAILED) { logerr("All IP interfaces in group %s are now unusable\n", pg->pg_name); pg->pg_failmsg_printed = _B_TRUE; } } /* * Extract information from the kernel about the desired phyint. * Look only for properties of the phyint and not properties of logints. * Take appropriate action on the changes. * Return codes: * PI_OK * The phyint exists in the kernel and matches our knowledge * of the phyint. * PI_DELETED * The phyint has vanished in the kernel. * PI_IFINDEX_CHANGED * The phyint's interface index has changed. * Ask the caller to delete and recreate the phyint. * PI_IOCTL_ERROR * Some ioctl error. Don't change anything. * PI_GROUP_CHANGED * The phyint has changed group. */ int phyint_inst_update_from_k(struct phyint_instance *pii) { struct lifreq lifr; int ifsock; struct phyint *pi; pi = pii->pii_phyint; if (debug & D_PHYINT) { logdebug("phyint_inst_update_from_k(%s %s)\n", AF_STR(pii->pii_af), pi->pi_name); } /* * Get the ifindex from the kernel, for comparison with the * value in our tables. */ (void) strncpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name)); lifr.lifr_name[sizeof (lifr.lifr_name) - 1] = '\0'; ifsock = (pii->pii_af == AF_INET) ? ifsock_v4 : ifsock_v6; if (ioctl(ifsock, SIOCGLIFINDEX, &lifr) < 0) { if (errno == ENXIO) { return (PI_DELETED); } else { logperror_pii(pii, "phyint_inst_update_from_k:" " ioctl (get lifindex)"); return (PI_IOCTL_ERROR); } } if (lifr.lifr_index != pi->pi_ifindex) { /* * The index has changed. Most likely the interface has * been unplumbed and replumbed. Ask the caller to take * appropriate action. */ if (debug & D_PHYINT) { logdebug("phyint_inst_update_from_k:" " old index %d new index %d\n", pi->pi_ifindex, lifr.lifr_index); } return (PI_IFINDEX_CHANGED); } /* * Get the group name from the kernel, for comparison with * the value in our tables. */ if (ioctl(ifsock, SIOCGLIFGROUPNAME, &lifr) < 0) { if (errno == ENXIO) { return (PI_DELETED); } else { logperror_pii(pii, "phyint_inst_update_from_k:" " ioctl (get groupname)"); return (PI_IOCTL_ERROR); } } /* * If the phyint has changed group i.e. if the phyint group name * returned by the kernel is different, ask the caller to delete * and recreate the phyint in the right group */ if (strcmp(lifr.lifr_groupname, pi->pi_group->pg_name) != 0) { /* Groupname has changed */ if (debug & D_PHYINT) { logdebug("phyint_inst_update_from_k:" " groupname change\n"); } return (PI_GROUP_CHANGED); } /* * Get the current phyint flags from the kernel, and determine what * flags have changed by comparing against our tables. Note that the * IFF_INACTIVE processing in initifs() relies on this call to ensure * that IFF_INACTIVE is really still set on the interface. */ if (ioctl(ifsock, SIOCGLIFFLAGS, &lifr) < 0) { if (errno == ENXIO) { return (PI_DELETED); } else { logperror_pii(pii, "phyint_inst_update_from_k: " " ioctl (get flags)"); return (PI_IOCTL_ERROR); } } pi->pi_flags = PHYINT_FLAGS(lifr.lifr_flags); if (pi->pi_v4 != NULL) pi->pi_v4->pii_flags = pi->pi_flags; if (pi->pi_v6 != NULL) pi->pi_v6->pii_flags = pi->pi_flags; /* * Make sure the IFF_FAILED flag is set if and only if we think * the interface should be failed. */ if (pi->pi_flags & IFF_FAILED) { if (pi->pi_state == PI_RUNNING) (void) change_pif_flags(pi, 0, IFF_FAILED); } else { if (pi->pi_state == PI_FAILED) (void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE); } /* No change in phyint status */ return (PI_OK); } /* * Delete the phyint. Remove it from the list of all phyints, and the * list of phyint group members. */ static void phyint_delete(struct phyint *pi) { boolean_t active; struct phyint *pi2; struct phyint_group *pg = pi->pi_group; if (debug & D_PHYINT) logdebug("phyint_delete(%s)\n", pi->pi_name); /* Both IPv4 and IPv6 phyint instances must have been deleted. */ assert(pi->pi_v4 == NULL && pi->pi_v6 == NULL); /* * The phyint must belong to a group. */ assert(pg->pg_phyint == pi || pi->pi_pgprev != NULL); /* The phyint must be in the list of all phyints */ assert(phyints == pi || pi->pi_prev != NULL); /* Remove the phyint from the phyint group list */ pg->pg_sig++; (void) phyint_group_member_event(pg, pi, IPMP_IF_REMOVE); if (pi->pi_pgprev == NULL) { /* Phyint is the 1st in the phyint group list */ pg->pg_phyint = pi->pi_pgnext; } else { pi->pi_pgprev->pi_pgnext = pi->pi_pgnext; } if (pi->pi_pgnext != NULL) pi->pi_pgnext->pi_pgprev = pi->pi_pgprev; pi->pi_pgnext = NULL; pi->pi_pgprev = NULL; /* Refresh the group state now that this phyint has been removed */ phyint_group_refresh_state(pg); /* Remove the phyint from the global list of phyints */ if (pi->pi_prev == NULL) { /* Phyint is the 1st in the list */ phyints = pi->pi_next; } else { pi->pi_prev->pi_next = pi->pi_next; } if (pi->pi_next != NULL) pi->pi_next->pi_prev = pi->pi_prev; pi->pi_next = NULL; pi->pi_prev = NULL; /* * See if another phyint in the group had been offlined because * it was a dup of `pi' -- and if so, online it. */ if (!pi->pi_hwaddrdup && (pi2 = phyint_lookup_hwaddr(pi, _B_FALSE)) != NULL) { assert(pi2->pi_hwaddrdup); (void) phyint_undo_offline(pi2); } /* * If the interface was in a named group and was either an active * standby or the last active interface, try to activate another * interface to compensate. */ if (pg != phyint_anongroup) { active = _B_FALSE; for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (phyint_is_functioning(pi2) && !(pi2->pi_flags & IFF_INACTIVE)) { active = _B_TRUE; break; } } if (!active || (pi->pi_flags & (IFF_STANDBY|IFF_INACTIVE)) == IFF_STANDBY) phyint_activate_another(pi); } phyint_link_close(pi); free(pi); } /* * Offline phyint `pi' if at least `minred' usable interfaces remain in the * group. Returns an IPMP error code. */ int phyint_offline(struct phyint *pi, uint_t minred) { boolean_t was_active; unsigned int nusable = 0; struct phyint *pi2; struct phyint_group *pg = pi->pi_group; /* * Verify that enough usable interfaces in the group would remain. * As a special case, if the group has failed, allow any non-offline * phyints to be offlined. */ if (pg != phyint_anongroup) { for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (pi2 == pi) continue; if (phyint_is_usable(pi2) || (GROUP_FAILED(pg) && pi2->pi_state != PI_OFFLINE)) nusable++; } } if (nusable < minred) return (IPMP_EMINRED); was_active = ((pi->pi_flags & IFF_INACTIVE) == 0); if (!change_pif_flags(pi, IFF_OFFLINE, IFF_INACTIVE)) return (IPMP_FAILURE); /* * The interface is now offline, so stop probing it. Note that * if_mpadm(8) will down the test addresses, after receiving a * success reply from us. The routing socket message will then make us * close the socket used for sending probes. But it is more logical * that an offlined interface must not be probed, even if it has test * addresses. * * NOTE: stop_probing() also sets PI_OFFLINE. */ stop_probing(pi); /* * If we're offlining the phyint because it has a duplicate hardware * address, print a warning -- and leave the link open so that we can * be notified of hardware address changes that make it usable again. * Otherwise, close the link so that we won't prevent a detach. */ if (pi->pi_hwaddrdup) { logerr("IP interface %s has a hardware address which is not " "unique in group %s; offlining\n", pi->pi_name, pg->pg_name); } else { phyint_link_close(pi); } /* * If this phyint was preventing another phyint with a duplicate * hardware address from being online, bring that one online now. */ if (!pi->pi_hwaddrdup && (pi2 = phyint_lookup_hwaddr(pi, _B_FALSE)) != NULL) { assert(pi2->pi_hwaddrdup); (void) phyint_undo_offline(pi2); } /* * If this interface was active, try to activate another INACTIVE * interface in the group. */ if (was_active) phyint_activate_another(pi); return (IPMP_SUCCESS); } /* * Undo a previous offline of `pi'. Returns an IPMP error code. */ int phyint_undo_offline(struct phyint *pi) { if (pi->pi_state != PI_OFFLINE) { errno = EINVAL; return (IPMP_FAILURE); } /* * If necessary, reinitialize our link information and verify that its * hardware address is still unique across the group. */ if (pi->pi_dh == NULL && !phyint_link_init(pi)) { errno = EIO; return (IPMP_FAILURE); } if (phyint_lookup_hwaddr(pi, _B_TRUE) != NULL) { pi->pi_hwaddrdup = _B_TRUE; return (IPMP_EHWADDRDUP); } if (pi->pi_hwaddrdup) { logerr("IP interface %s now has a unique hardware address in " "group %s; onlining\n", pi->pi_name, pi->pi_group->pg_name); pi->pi_hwaddrdup = _B_FALSE; } if (!change_pif_flags(pi, 0, IFF_OFFLINE)) return (IPMP_FAILURE); /* * While the interface was offline, it may have failed (e.g. the link * may have gone down). phyint_inst_check_for_failure() will have * already set pi_flags with IFF_FAILED, so we can use that to decide * whether the phyint should transition to running. Note that after * we transition to running, we will start sending probes again (if * test addresses are configured), which may also reveal that the * interface is in fact failed. */ if (pi->pi_flags & IFF_FAILED) { phyint_chstate(pi, PI_FAILED); } else { /* calls phyint_chstate() */ phyint_transition_to_running(pi); } /* * Give the requestor time to configure test addresses before * complaining that they're missing. */ pi->pi_taddrthresh = getcurrentsec() + TESTADDR_CONF_TIME; return (IPMP_SUCCESS); } /* * Delete (unlink and free), the phyint instance. */ void phyint_inst_delete(struct phyint_instance *pii) { struct phyint *pi = pii->pii_phyint; assert(pi != NULL); if (debug & D_PHYINT) { logdebug("phyint_inst_delete(%s %s)\n", AF_STR(pii->pii_af), pi->pi_name); } /* * If the phyint instance has associated probe targets * delete all the targets */ while (pii->pii_targets != NULL) target_delete(pii->pii_targets); /* * Delete all the logints associated with this phyint * instance. */ while (pii->pii_logint != NULL) logint_delete(pii->pii_logint); /* * Close the socket used to send probes to targets from this phyint. */ if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); /* * Phyint instance must be in the list of all phyint instances. * Remove phyint instance from the global list of phyint instances. */ assert(phyint_instances == pii || pii->pii_prev != NULL); if (pii->pii_prev == NULL) { /* Phyint is the 1st in the list */ phyint_instances = pii->pii_next; } else { pii->pii_prev->pii_next = pii->pii_next; } if (pii->pii_next != NULL) pii->pii_next->pii_prev = pii->pii_prev; pii->pii_next = NULL; pii->pii_prev = NULL; /* * Reset the phyint instance pointer in the phyint. * If this is the last phyint instance (being deleted) on this * phyint, then delete the phyint. */ if (pii->pii_af == AF_INET) pi->pi_v4 = NULL; else pi->pi_v6 = NULL; if (pi->pi_v4 == NULL && pi->pi_v6 == NULL) phyint_delete(pi); free(pii); } static void phyint_inst_print(struct phyint_instance *pii) { struct logint *li; struct target *tg; char abuf[INET6_ADDRSTRLEN]; int most_recent; int i; if (pii->pii_phyint == NULL) { logdebug("pii->pi_phyint NULL can't print\n"); return; } logdebug("\nPhyint instance: %s %s index %u state %x flags %llx " "sock %x in_use %d\n", AF_STR(pii->pii_af), pii->pii_name, pii->pii_ifindex, pii->pii_state, pii->pii_phyint->pi_flags, pii->pii_probe_sock, pii->pii_in_use); for (li = pii->pii_logint; li != NULL; li = li->li_next) logint_print(li); logdebug("\n"); for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) target_print(tg); if (pii->pii_targets == NULL) logdebug("pi_targets NULL\n"); if (pii->pii_target_next != NULL) { logdebug("pi_target_next %s %s\n", AF_STR(pii->pii_af), pr_addr(pii->pii_af, pii->pii_target_next->tg_address, abuf, sizeof (abuf))); } else { logdebug("pi_target_next NULL\n"); } if (pii->pii_rtt_target_next != NULL) { logdebug("pi_rtt_target_next %s %s\n", AF_STR(pii->pii_af), pr_addr(pii->pii_af, pii->pii_rtt_target_next->tg_address, abuf, sizeof (abuf))); } else { logdebug("pi_rtt_target_next NULL\n"); } if (pii->pii_targets != NULL) { most_recent = PROBE_INDEX_PREV(pii->pii_probe_next); i = most_recent; do { if (pii->pii_probes[i].pr_target != NULL) { logdebug("#%d target %s ", i, pr_addr(pii->pii_af, pii->pii_probes[i].pr_target->tg_address, abuf, sizeof (abuf))); } else { logdebug("#%d target NULL ", i); } logdebug("time_start %lld status %d " "time_ackproc %lld time_lost %u", pii->pii_probes[i].pr_hrtime_start, pii->pii_probes[i].pr_status, pii->pii_probes[i].pr_hrtime_ackproc, pii->pii_probes[i].pr_time_lost); i = PROBE_INDEX_PREV(i); } while (i != most_recent); } } /* * Lookup a logint based on the logical interface name, on the given * phyint instance. */ static struct logint * logint_lookup(struct phyint_instance *pii, char *name) { struct logint *li; if (debug & D_LOGINT) { logdebug("logint_lookup(%s, %s)\n", AF_STR(pii->pii_af), name); } for (li = pii->pii_logint; li != NULL; li = li->li_next) { if (strncmp(name, li->li_name, sizeof (li->li_name)) == 0) break; } return (li); } /* * Insert a logint at the head of the list of logints of the given * phyint instance */ static void logint_insert(struct phyint_instance *pii, struct logint *li) { li->li_next = pii->pii_logint; li->li_prev = NULL; if (pii->pii_logint != NULL) pii->pii_logint->li_prev = li; pii->pii_logint = li; li->li_phyint_inst = pii; } /* * Create a new named logint, on the specified phyint instance. */ static struct logint * logint_create(struct phyint_instance *pii, char *name) { struct logint *li; if (debug & D_LOGINT) { logdebug("logint_create(%s %s %s)\n", AF_STR(pii->pii_af), pii->pii_name, name); } li = calloc(1, sizeof (struct logint)); if (li == NULL) { logperror("logint_create: calloc"); return (NULL); } (void) strncpy(li->li_name, name, sizeof (li->li_name)); li->li_name[sizeof (li->li_name) - 1] = '\0'; logint_insert(pii, li); return (li); } /* * Initialize the logint based on the data returned by the kernel. */ void logint_init_from_k(struct phyint_instance *pii, char *li_name) { int ifsock; uint64_t flags; uint64_t saved_flags; struct logint *li; struct lifreq lifr; struct in6_addr test_subnet; struct in6_addr testaddr; int test_subnet_len; struct sockaddr_in6 *sin6; struct sockaddr_in *sin; char abuf[INET6_ADDRSTRLEN]; boolean_t ptp = _B_FALSE; struct in6_addr tgaddr; if (debug & D_LOGINT) { logdebug("logint_init_from_k(%s %s)\n", AF_STR(pii->pii_af), li_name); } /* Get the socket for doing ioctls */ ifsock = (pii->pii_af == AF_INET) ? ifsock_v4 : ifsock_v6; /* * Get the flags from the kernel. Also serves as a check whether * the logical still exists. If it doesn't exist, no need to proceed * any further. li_in_use will make the caller clean up the logint */ (void) strncpy(lifr.lifr_name, li_name, sizeof (lifr.lifr_name)); lifr.lifr_name[sizeof (lifr.lifr_name) - 1] = '\0'; if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) { /* Interface may have vanished */ if (errno != ENXIO) { logperror_pii(pii, "logint_init_from_k: " "ioctl (get flags)"); } return; } flags = lifr.lifr_flags; /* * Verified the logint exists. Now lookup the logint in our tables. * If it does not exist, create a new logint. */ li = logint_lookup(pii, li_name); if (li == NULL) { li = logint_create(pii, li_name); if (li == NULL) { /* * Pretend the interface does not exist * in the kernel */ return; } } /* * Update li->li_flags with the new flags, after saving the old * value. This is used later to check what flags has changed and * take any action */ saved_flags = li->li_flags; li->li_flags = flags; /* * Get the address, prefix, prefixlength and update the logint. * Check if anything has changed. If the logint used for the * test address has changed, take suitable action. */ if (ioctl(ifsock, SIOCGLIFADDR, (char *)&lifr) < 0) { /* Interface may have vanished */ if (errno != ENXIO) { logperror_li(li, "logint_init_from_k: (get addr)"); } goto error; } if (pii->pii_af == AF_INET) { sin = (struct sockaddr_in *)&lifr.lifr_addr; IN6_INADDR_TO_V4MAPPED(&sin->sin_addr, &testaddr); } else { sin6 = (struct sockaddr_in6 *)&lifr.lifr_addr; testaddr = sin6->sin6_addr; } if (ioctl(ifsock, SIOCGLIFSUBNET, (char *)&lifr) < 0) { /* Interface may have vanished */ if (errno != ENXIO) logperror_li(li, "logint_init_from_k: (get subnet)"); goto error; } if (lifr.lifr_subnet.ss_family == AF_INET6) { sin6 = (struct sockaddr_in6 *)&lifr.lifr_subnet; test_subnet = sin6->sin6_addr; test_subnet_len = lifr.lifr_addrlen; } else { sin = (struct sockaddr_in *)&lifr.lifr_subnet; IN6_INADDR_TO_V4MAPPED(&sin->sin_addr, &test_subnet); test_subnet_len = lifr.lifr_addrlen + (IPV6_ABITS - IP_ABITS); } /* * If this is the logint corresponding to the test address used for * sending probes, then if anything significant has changed we need to * determine the test address again. We ignore changes to the * IFF_FAILED and IFF_RUNNING flags since those happen as a matter of * course. */ if (pii->pii_probe_logint == li) { if (((li->li_flags ^ saved_flags) & ~(IFF_FAILED | IFF_RUNNING)) != 0 || !IN6_ARE_ADDR_EQUAL(&testaddr, &li->li_addr) || (!ptp && !IN6_ARE_ADDR_EQUAL(&test_subnet, &li->li_subnet)) || (!ptp && test_subnet_len != li->li_subnet_len) || (ptp && !IN6_ARE_ADDR_EQUAL(&tgaddr, &li->li_dstaddr))) { /* * Something significant that affects the testaddress * has changed. Redo the testaddress selection later on * in select_test_ifs(). For now do the cleanup and * set pii_probe_logint to NULL. */ if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); pii->pii_probe_logint = NULL; } } /* Update the logint with the values obtained from the kernel. */ li->li_addr = testaddr; li->li_in_use = 1; if (ptp) { li->li_dstaddr = tgaddr; li->li_subnet_len = (pii->pii_af == AF_INET) ? IP_ABITS : IPV6_ABITS; } else { li->li_subnet = test_subnet; li->li_subnet_len = test_subnet_len; } if (debug & D_LOGINT) logint_print(li); return; error: logerr("logint_init_from_k: IGNORED %s %s %s addr %s\n", AF_STR(pii->pii_af), pii->pii_name, li->li_name, pr_addr(pii->pii_af, testaddr, abuf, sizeof (abuf))); logint_delete(li); } /* * Delete (unlink and free) a logint. */ void logint_delete(struct logint *li) { struct phyint_instance *pii; pii = li->li_phyint_inst; assert(pii != NULL); if (debug & D_LOGINT) { int af; char abuf[INET6_ADDRSTRLEN]; af = pii->pii_af; logdebug("logint_delete(%s %s %s/%u)\n", AF_STR(af), li->li_name, pr_addr(af, li->li_addr, abuf, sizeof (abuf)), li->li_subnet_len); } /* logint must be in the list of logints */ assert(pii->pii_logint == li || li->li_prev != NULL); /* Remove the logint from the list of logints */ if (li->li_prev == NULL) { /* logint is the 1st in the list */ pii->pii_logint = li->li_next; } else { li->li_prev->li_next = li->li_next; } if (li->li_next != NULL) li->li_next->li_prev = li->li_prev; li->li_next = NULL; li->li_prev = NULL; /* * If this logint is also being used for probing, then close the * associated socket, if it exists. */ if (pii->pii_probe_logint == li) { if (pii->pii_probe_sock != -1) close_probe_socket(pii, _B_TRUE); pii->pii_probe_logint = NULL; } free(li); } static void logint_print(struct logint *li) { char abuf[INET6_ADDRSTRLEN]; int af = li->li_phyint_inst->pii_af; logdebug("logint: %s %s addr %s/%u", AF_STR(af), li->li_name, pr_addr(af, li->li_addr, abuf, sizeof (abuf)), li->li_subnet_len); logdebug("\tFlags: %llx in_use %d\n", li->li_flags, li->li_in_use); } char * pr_addr(int af, struct in6_addr addr, char *abuf, int len) { struct in_addr addr_v4; if (af == AF_INET) { IN6_V4MAPPED_TO_INADDR(&addr, &addr_v4); (void) inet_ntop(AF_INET, (void *)&addr_v4, abuf, len); } else { (void) inet_ntop(AF_INET6, (void *)&addr, abuf, len); } return (abuf); } /* * Fill in the sockaddr_storage pointed to by `ssp' with the IP address * represented by the [`af',`addr'] pair. Needed because in.mpathd internally * stores all addresses as in6_addrs, but we don't want to expose that. */ void addr2storage(int af, const struct in6_addr *addr, struct sockaddr_storage *ssp) { struct sockaddr_in *sinp = (struct sockaddr_in *)ssp; struct sockaddr_in6 *sin6p = (struct sockaddr_in6 *)ssp; assert(af == AF_INET || af == AF_INET6); switch (af) { case AF_INET: (void) memset(sinp, 0, sizeof (*sinp)); sinp->sin_family = AF_INET; IN6_V4MAPPED_TO_INADDR(addr, &sinp->sin_addr); break; case AF_INET6: (void) memset(sin6p, 0, sizeof (*sin6p)); sin6p->sin6_family = AF_INET6; sin6p->sin6_addr = *addr; break; } } /* Lookup target on its address */ struct target * target_lookup(struct phyint_instance *pii, struct in6_addr addr) { struct target *tg; if (debug & D_TARGET) { char abuf[INET6_ADDRSTRLEN]; logdebug("target_lookup(%s %s): addr %s\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, addr, abuf, sizeof (abuf))); } for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { if (IN6_ARE_ADDR_EQUAL(&tg->tg_address, &addr)) break; } return (tg); } /* * Find and return the next active target, for the next probe. * If no active targets are available, return NULL. */ struct target * target_next(struct target *tg) { struct phyint_instance *pii = tg->tg_phyint_inst; struct target *marker = tg; hrtime_t now; now = gethrtime(); /* * Target must be in the list of targets for this phyint * instance. */ assert(pii->pii_targets == tg || tg->tg_prev != NULL); assert(pii->pii_targets != NULL); /* Return the next active target */ do { /* * Go to the next target. If we hit the end, * reset the ptr to the head */ tg = tg->tg_next; if (tg == NULL) tg = pii->pii_targets; assert(TG_STATUS_VALID(tg->tg_status)); switch (tg->tg_status) { case TG_ACTIVE: return (tg); case TG_UNUSED: assert(pii->pii_targets_are_routers); if (pii->pii_ntargets < MAX_PROBE_TARGETS) { /* * Bubble up the unused target to active */ tg->tg_status = TG_ACTIVE; pii->pii_ntargets++; return (tg); } break; case TG_SLOW: assert(pii->pii_targets_are_routers); if (tg->tg_latime + MIN_RECOVERY_TIME < now) { /* * Bubble up the slow target to unused */ tg->tg_status = TG_UNUSED; } break; case TG_DEAD: assert(pii->pii_targets_are_routers); if (tg->tg_latime + MIN_RECOVERY_TIME < now) { /* * Bubble up the dead target to slow */ tg->tg_status = TG_SLOW; tg->tg_latime = now; } break; } } while (tg != marker); return (NULL); } /* * Select the best available target, that is not already TG_ACTIVE, * for the caller. The caller will determine whether it wants to * make the returned target TG_ACTIVE. * The selection order is as follows. * 1. pick a TG_UNSED target, if it exists. * 2. else pick a TG_SLOW target that has recovered, if it exists * 3. else pick any TG_SLOW target, if it exists * 4. else pick a TG_DEAD target that has recovered, if it exists * 5. else pick any TG_DEAD target, if it exists * 6. else return null */ static struct target * target_select_best(struct phyint_instance *pii) { struct target *tg; struct target *slow = NULL; struct target *dead = NULL; struct target *slow_recovered = NULL; struct target *dead_recovered = NULL; hrtime_t now; now = gethrtime(); for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { assert(TG_STATUS_VALID(tg->tg_status)); switch (tg->tg_status) { case TG_UNUSED: return (tg); case TG_SLOW: if (tg->tg_latime + MIN_RECOVERY_TIME < now) { slow_recovered = tg; /* * Promote the slow_recovered to unused */ tg->tg_status = TG_UNUSED; } else { slow = tg; } break; case TG_DEAD: if (tg->tg_latime + MIN_RECOVERY_TIME < now) { dead_recovered = tg; /* * Promote the dead_recovered to slow */ tg->tg_status = TG_SLOW; tg->tg_latime = now; } else { dead = tg; } break; default: break; } } if (slow_recovered != NULL) return (slow_recovered); else if (slow != NULL) return (slow); else if (dead_recovered != NULL) return (dead_recovered); else return (dead); } /* * Some target was deleted. If we don't have even MIN_PROBE_TARGETS * that are active, pick the next best below. */ static void target_activate_all(struct phyint_instance *pii) { struct target *tg; assert(pii->pii_ntargets == 0); assert(pii->pii_target_next == NULL); assert(pii->pii_rtt_target_next == NULL); assert(pii->pii_targets_are_routers); while (pii->pii_ntargets < MIN_PROBE_TARGETS) { tg = target_select_best(pii); if (tg == NULL) { /* We are out of targets */ return; } assert(TG_STATUS_VALID(tg->tg_status)); assert(tg->tg_status != TG_ACTIVE); tg->tg_status = TG_ACTIVE; pii->pii_ntargets++; if (pii->pii_target_next == NULL) { pii->pii_target_next = tg; pii->pii_rtt_target_next = tg; } } } static struct target * target_first(struct phyint_instance *pii) { struct target *tg; for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { assert(TG_STATUS_VALID(tg->tg_status)); if (tg->tg_status == TG_ACTIVE) break; } return (tg); } /* * Create a default target entry. */ void target_create(struct phyint_instance *pii, struct in6_addr addr, boolean_t is_router) { struct target *tg; struct phyint *pi; struct logint *li; if (debug & D_TARGET) { char abuf[INET6_ADDRSTRLEN]; logdebug("target_create(%s %s, %s)\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, addr, abuf, sizeof (abuf))); } /* * If the test address is not yet initialized, do not add * any target, since we cannot determine whether the target * belongs to the same subnet as the test address. */ li = pii->pii_probe_logint; if (li == NULL) return; /* * If there are multiple subnets associated with an interface, then * add the target to this phyint instance only if it belongs to the * same subnet as the test address. This assures us that we will * be able to reach this target through our routing table. */ if (!prefix_equal(li->li_subnet, addr, li->li_subnet_len)) return; if (pii->pii_targets != NULL) { assert(pii->pii_ntargets <= MAX_PROBE_TARGETS); if (is_router) { if (!pii->pii_targets_are_routers) { /* * Prefer router over hosts. Using hosts is a * fallback mechanism, hence delete all host * targets. */ while (pii->pii_targets != NULL) target_delete(pii->pii_targets); } } else { /* * Routers take precedence over hosts. If this * is a router list and we are trying to add a * host, just return. If this is a host list * and if we have sufficient targets, just return */ if (pii->pii_targets_are_routers || pii->pii_ntargets == MAX_PROBE_TARGETS) return; } } tg = calloc(1, sizeof (struct target)); if (tg == NULL) { logperror("target_create: calloc"); return; } tg->tg_phyint_inst = pii; tg->tg_address = addr; tg->tg_in_use = 1; tg->tg_rtt_sa = -1; tg->tg_num_deferred = 0; /* * If this is the first target, set 'pii_targets_are_routers' * The list of targets is either a list of hosts or list or * routers, but not a mix. */ if (pii->pii_targets == NULL) { assert(pii->pii_ntargets == 0); assert(pii->pii_target_next == NULL); assert(pii->pii_rtt_target_next == NULL); pii->pii_targets_are_routers = is_router ? 1 : 0; } if (pii->pii_ntargets == MAX_PROBE_TARGETS) { assert(pii->pii_targets_are_routers); assert(pii->pii_target_next != NULL); assert(pii->pii_rtt_target_next != NULL); tg->tg_status = TG_UNUSED; } else { if (pii->pii_ntargets == 0) { assert(pii->pii_target_next == NULL); pii->pii_target_next = tg; pii->pii_rtt_target_next = tg; } pii->pii_ntargets++; tg->tg_status = TG_ACTIVE; } target_insert(pii, tg); /* * Change state to PI_RUNNING if this phyint instance is capable of * sending and receiving probes -- that is, if we know of at least 1 * target, and this phyint instance is probe-capable. For more * details, see the phyint state diagram in mpd_probe.c. */ pi = pii->pii_phyint; if (pi->pi_state == PI_NOTARGETS && PROBE_CAPABLE(pii)) { if (pi->pi_flags & IFF_FAILED) phyint_chstate(pi, PI_FAILED); else phyint_chstate(pi, PI_RUNNING); } } /* * Add the target address named by `addr' to phyint instance `pii' if it does * not already exist. If the target is a router, `is_router' should be set to * B_TRUE. */ void target_add(struct phyint_instance *pii, struct in6_addr addr, boolean_t is_router) { struct target *tg; if (pii == NULL) return; tg = target_lookup(pii, addr); /* * If the target does not exist, create it; target_create() will set * tg_in_use to true. Even if it exists already, if it's a router * target and we'd previously learned of it through multicast, then we * need to recreate it as a router target. Otherwise, just set * tg_in_use to to true so that init_router_targets() won't delete it. */ if (tg == NULL || (is_router && !pii->pii_targets_are_routers)) target_create(pii, addr, is_router); else if (is_router) tg->tg_in_use = 1; } /* * Insert target at head of linked list of targets for the associated * phyint instance */ static void target_insert(struct phyint_instance *pii, struct target *tg) { tg->tg_next = pii->pii_targets; tg->tg_prev = NULL; if (tg->tg_next != NULL) tg->tg_next->tg_prev = tg; pii->pii_targets = tg; } /* * Delete a target (unlink and free). */ void target_delete(struct target *tg) { int af; struct phyint_instance *pii; struct phyint_instance *pii_other; pii = tg->tg_phyint_inst; af = pii->pii_af; if (debug & D_TARGET) { char abuf[INET6_ADDRSTRLEN]; logdebug("target_delete(%s %s, %s)\n", AF_STR(af), pii->pii_name, pr_addr(af, tg->tg_address, abuf, sizeof (abuf))); } /* * Target must be in the list of targets for this phyint * instance. */ assert(pii->pii_targets == tg || tg->tg_prev != NULL); /* * Reset all references to 'tg' in the probe information * for this phyint. */ reset_pii_probes(pii, tg); /* * Remove this target from the list of targets of this * phyint instance. */ if (tg->tg_prev == NULL) { pii->pii_targets = tg->tg_next; } else { tg->tg_prev->tg_next = tg->tg_next; } if (tg->tg_next != NULL) tg->tg_next->tg_prev = tg->tg_prev; tg->tg_next = NULL; tg->tg_prev = NULL; if (tg->tg_status == TG_ACTIVE) pii->pii_ntargets--; /* * Adjust the next target to probe, if it points to * to the currently deleted target. */ if (pii->pii_target_next == tg) pii->pii_target_next = target_first(pii); if (pii->pii_rtt_target_next == tg) pii->pii_rtt_target_next = target_first(pii); free(tg); /* * The number of active targets pii_ntargets == 0 iff * the next active target pii->pii_target_next == NULL */ if (pii->pii_ntargets != 0) { assert(pii->pii_target_next != NULL); assert(pii->pii_rtt_target_next != NULL); assert(pii->pii_target_next->tg_status == TG_ACTIVE); assert(pii->pii_rtt_target_next->tg_status == TG_ACTIVE); return; } /* At this point, we don't have any active targets. */ assert(pii->pii_target_next == NULL); assert(pii->pii_rtt_target_next == NULL); if (pii->pii_targets_are_routers) { /* * Activate any TG_SLOW or TG_DEAD router targets, * since we don't have any other targets */ target_activate_all(pii); if (pii->pii_ntargets != 0) { assert(pii->pii_target_next != NULL); assert(pii->pii_rtt_target_next != NULL); assert(pii->pii_target_next->tg_status == TG_ACTIVE); assert(pii->pii_rtt_target_next->tg_status == TG_ACTIVE); return; } } /* * If we still don't have any active targets, the list must * must be really empty. There aren't even TG_SLOW or TG_DEAD * targets. Zero out the probe stats since it will not be * relevant any longer. */ assert(pii->pii_targets == NULL); pii->pii_targets_are_routers = _B_FALSE; clear_pii_probe_stats(pii); pii_other = phyint_inst_other(pii); /* * If there are no targets on both instances and the interface would * otherwise be considered PI_RUNNING, go back to PI_NOTARGETS state, * since we cannot probe this phyint any more. For more details, * please see phyint state diagram in mpd_probe.c. */ if (!PROBE_CAPABLE(pii_other) && LINK_UP(pii->pii_phyint) && pii->pii_phyint->pi_state != PI_OFFLINE) phyint_chstate(pii->pii_phyint, PI_NOTARGETS); } /* * Flush the target list of every phyint in the group, if the list * is a host target list. This is called if group failure is suspected. * If all targets have failed, multicast will subsequently discover new * targets. Else it is a group failure. * Note: This function is a no-op if the list is a router target list. */ static void target_flush_hosts(struct phyint_group *pg) { struct phyint *pi; struct phyint_instance *pii; if (debug & D_TARGET) logdebug("target_flush_hosts(%s)\n", pg->pg_name); for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { pii = pi->pi_v4; if (pii != NULL && !pii->pii_targets_are_routers) { /* * Delete all the targets. When the list becomes * empty, target_delete() will set pii->pii_targets * to NULL. */ while (pii->pii_targets != NULL) target_delete(pii->pii_targets); } pii = pi->pi_v6; if (pii != NULL && !pii->pii_targets_are_routers) { /* * Delete all the targets. When the list becomes * empty, target_delete() will set pii->pii_targets * to NULL. */ while (pii->pii_targets != NULL) target_delete(pii->pii_targets); } } } /* * Reset all references to 'target' in the probe info, as this target is * being deleted. The pr_target field is guaranteed to be non-null if * pr_status is PR_UNACKED. So we change the pr_status to PR_LOST, so that * pr_target will not be accessed unconditionally. */ static void reset_pii_probes(struct phyint_instance *pii, struct target *tg) { int i; for (i = 0; i < PROBE_STATS_COUNT; i++) { if (pii->pii_probes[i].pr_target == tg) { if (pii->pii_probes[i].pr_status == PR_UNACKED) { probe_chstate(&pii->pii_probes[i], pii, PR_LOST); } pii->pii_probes[i].pr_target = NULL; } } } /* * Clear the probe statistics array. */ void clear_pii_probe_stats(struct phyint_instance *pii) { bzero(pii->pii_probes, sizeof (struct probe_stats) * PROBE_STATS_COUNT); /* Reset the next probe index in the probe stats array */ pii->pii_probe_next = 0; } static void target_print(struct target *tg) { char abuf[INET6_ADDRSTRLEN]; char buf[128]; char buf2[128]; int af; int i; af = tg->tg_phyint_inst->pii_af; logdebug("Target on %s %s addr %s\n" "status %d rtt_sa %lld rtt_sd %lld crtt %d tg_in_use %d\n", AF_STR(af), tg->tg_phyint_inst->pii_name, pr_addr(af, tg->tg_address, abuf, sizeof (abuf)), tg->tg_status, tg->tg_rtt_sa, tg->tg_rtt_sd, tg->tg_crtt, tg->tg_in_use); buf[0] = '\0'; for (i = 0; i < tg->tg_num_deferred; i++) { (void) snprintf(buf2, sizeof (buf2), " %dms", tg->tg_deferred[i]); (void) strlcat(buf, buf2, sizeof (buf)); } logdebug("deferred rtts:%s\n", buf); } void phyint_inst_print_all(void) { struct phyint_instance *pii; for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { phyint_inst_print(pii); } } /* * Compare two prefixes that have the same prefix length. * Fails if the prefix length is unreasonable. */ boolean_t prefix_equal(struct in6_addr p1, struct in6_addr p2, uint_t prefix_len) { uchar_t mask; int j; if (prefix_len > IPV6_ABITS) return (_B_FALSE); for (j = 0; prefix_len > 8; prefix_len -= 8, j++) if (p1.s6_addr[j] != p2.s6_addr[j]) return (_B_FALSE); /* Make the N leftmost bits one */ mask = 0xff << (8 - prefix_len); if ((p1.s6_addr[j] & mask) != (p2.s6_addr[j] & mask)) return (_B_FALSE); return (_B_TRUE); } /* * Get the number of UP logints on phyint `pi'. */ static int logint_upcount(struct phyint *pi) { struct logint *li; int count = 0; if (pi->pi_v4 != NULL) { for (li = pi->pi_v4->pii_logint; li != NULL; li = li->li_next) { if (li->li_flags & IFF_UP) count++; } } if (pi->pi_v6 != NULL) { for (li = pi->pi_v6->pii_logint; li != NULL; li = li->li_next) { if (li->li_flags & IFF_UP) count++; } } return (count); } /* * Get the phyint instance with the other (IPv4 / IPv6) protocol */ struct phyint_instance * phyint_inst_other(struct phyint_instance *pii) { if (pii->pii_af == AF_INET) return (pii->pii_phyint->pi_v6); else return (pii->pii_phyint->pi_v4); } /* * Check whether a phyint is functioning. */ boolean_t phyint_is_functioning(struct phyint *pi) { if (pi->pi_state == PI_RUNNING) return (_B_TRUE); return (pi->pi_state == PI_NOTARGETS && !(pi->pi_flags & IFF_FAILED)); } /* * Check whether a phyint is usable. */ boolean_t phyint_is_usable(struct phyint *pi) { if (logint_upcount(pi) == 0) return (_B_FALSE); return (phyint_is_functioning(pi)); } /* * Post an EC_IPMP sysevent of subclass `subclass' and attributes `nvl'. * Before sending the event, it prepends the current version of the IPMP * sysevent API. Returns 0 on success, -1 on failure (in either case, * `nvl' is freed). */ static int post_event(const char *subclass, nvlist_t *nvl) { static evchan_t *evchp = NULL; /* * Initialize the event channel if we haven't already done so. */ if (evchp == NULL) { errno = sysevent_evc_bind(IPMP_EVENT_CHAN, &evchp, EVCH_CREAT); if (errno != 0) { logerr("cannot create event channel `%s': %s\n", IPMP_EVENT_CHAN, strerror(errno)); goto failed; } } errno = nvlist_add_uint32(nvl, IPMP_EVENT_VERSION, IPMP_EVENT_CUR_VERSION); if (errno != 0) { logerr("cannot create `%s' event: %s", subclass, strerror(errno)); goto failed; } errno = sysevent_evc_publish(evchp, EC_IPMP, subclass, "com.sun", "in.mpathd", nvl, EVCH_NOSLEEP); if (errno != 0) { logerr("cannot send `%s' event: %s\n", subclass, strerror(errno)); goto failed; } nvlist_free(nvl); return (0); failed: nvlist_free(nvl); return (-1); } /* * Return the external IPMP state associated with phyint `pi'. */ static ipmp_if_state_t ifstate(struct phyint *pi) { switch (pi->pi_state) { case PI_INIT: return (IPMP_IF_UNKNOWN); case PI_NOTARGETS: if (pi->pi_flags & IFF_FAILED) return (IPMP_IF_FAILED); return (IPMP_IF_UNKNOWN); case PI_OFFLINE: return (IPMP_IF_OFFLINE); case PI_FAILED: return (IPMP_IF_FAILED); case PI_RUNNING: return (IPMP_IF_OK); } logerr("ifstate: unknown state %d; aborting\n", pi->pi_state); abort(); /* NOTREACHED */ } /* * Return the external IPMP interface type associated with phyint `pi'. */ static ipmp_if_type_t iftype(struct phyint *pi) { if (pi->pi_flags & IFF_STANDBY) return (IPMP_IF_STANDBY); else return (IPMP_IF_NORMAL); } /* * Return the external IPMP link state associated with phyint `pi'. */ static ipmp_if_linkstate_t iflinkstate(struct phyint *pi) { if (!(pi->pi_notes & (DL_NOTE_LINK_UP|DL_NOTE_LINK_DOWN))) return (IPMP_LINK_UNKNOWN); return (LINK_DOWN(pi) ? IPMP_LINK_DOWN : IPMP_LINK_UP); } /* * Return the external IPMP probe state associated with phyint `pi'. */ static ipmp_if_probestate_t ifprobestate(struct phyint *pi) { if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) return (IPMP_PROBE_DISABLED); if (pi->pi_state == PI_FAILED) return (IPMP_PROBE_FAILED); if (!PROBE_CAPABLE(pi->pi_v4) && !PROBE_CAPABLE(pi->pi_v6)) return (IPMP_PROBE_UNKNOWN); return (IPMP_PROBE_OK); } /* * Return the external IPMP target mode associated with phyint instance `pii'. */ static ipmp_if_targmode_t iftargmode(struct phyint_instance *pii) { if (!PROBE_ENABLED(pii)) return (IPMP_TARG_DISABLED); else if (pii->pii_targets_are_routers) return (IPMP_TARG_ROUTES); else return (IPMP_TARG_MULTICAST); } /* * Return the external IPMP flags associated with phyint `pi'. */ static ipmp_if_flags_t ifflags(struct phyint *pi) { ipmp_if_flags_t flags = 0; if (logint_upcount(pi) == 0) flags |= IPMP_IFFLAG_DOWN; if (pi->pi_flags & IFF_INACTIVE) flags |= IPMP_IFFLAG_INACTIVE; if (pi->pi_hwaddrdup) flags |= IPMP_IFFLAG_HWADDRDUP; if (phyint_is_functioning(pi) && flags == 0) flags |= IPMP_IFFLAG_ACTIVE; return (flags); } /* * Store the test address used on phyint instance `pii' in `ssp'. If there's * no test address, 0.0.0.0 is stored. */ static struct sockaddr_storage * iftestaddr(struct phyint_instance *pii, struct sockaddr_storage *ssp) { if (PROBE_ENABLED(pii)) addr2storage(pii->pii_af, &pii->pii_probe_logint->li_addr, ssp); else addr2storage(AF_INET6, &in6addr_any, ssp); return (ssp); } /* * Return the external IPMP group state associated with phyint group `pg'. */ static ipmp_group_state_t groupstate(struct phyint_group *pg) { switch (pg->pg_state) { case PG_FAILED: return (IPMP_GROUP_FAILED); case PG_DEGRADED: return (IPMP_GROUP_DEGRADED); case PG_OK: return (IPMP_GROUP_OK); } logerr("groupstate: unknown state %d; aborting\n", pg->pg_state); abort(); /* NOTREACHED */ } /* * Return the external IPMP probe state associated with probe `ps'. */ static ipmp_probe_state_t probestate(struct probe_stats *ps) { switch (ps->pr_status) { case PR_UNUSED: case PR_LOST: return (IPMP_PROBE_LOST); case PR_UNACKED: return (IPMP_PROBE_SENT); case PR_ACKED: return (IPMP_PROBE_ACKED); } logerr("probestate: unknown state %d; aborting\n", ps->pr_status); abort(); /* NOTREACHED */ } /* * Generate an ESC_IPMP_PROBE_STATE sysevent for the probe described by `pr' * on phyint instance `pii'. Returns 0 on success, -1 on failure. */ int probe_state_event(struct probe_stats *pr, struct phyint_instance *pii) { nvlist_t *nvl; hrtime_t proc_time = 0, recv_time = 0; struct sockaddr_storage ss; struct target *tg = pr->pr_target; int64_t rttavg, rttdev; errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0); if (errno != 0) { logperror("cannot create `interface change' event"); return (-1); } errno = nvlist_add_uint32(nvl, IPMP_PROBE_ID, pr->pr_id); if (errno != 0) goto failed; errno = nvlist_add_string(nvl, IPMP_IF_NAME, pii->pii_phyint->pi_name); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_PROBE_STATE, probestate(pr)); if (errno != 0) goto failed; errno = nvlist_add_hrtime(nvl, IPMP_PROBE_START_TIME, pr->pr_hrtime_start); if (errno != 0) goto failed; errno = nvlist_add_hrtime(nvl, IPMP_PROBE_SENT_TIME, pr->pr_hrtime_sent); if (errno != 0) goto failed; if (pr->pr_status == PR_ACKED) { recv_time = pr->pr_hrtime_ackrecv; proc_time = pr->pr_hrtime_ackproc; } errno = nvlist_add_hrtime(nvl, IPMP_PROBE_ACKRECV_TIME, recv_time); if (errno != 0) goto failed; errno = nvlist_add_hrtime(nvl, IPMP_PROBE_ACKPROC_TIME, proc_time); if (errno != 0) goto failed; if (tg != NULL) addr2storage(pii->pii_af, &tg->tg_address, &ss); else addr2storage(pii->pii_af, &in6addr_any, &ss); errno = nvlist_add_byte_array(nvl, IPMP_PROBE_TARGET, (uchar_t *)&ss, sizeof (ss)); if (errno != 0) goto failed; rttavg = (tg != NULL) ? (tg->tg_rtt_sa / 8) : 0; errno = nvlist_add_int64(nvl, IPMP_PROBE_TARGET_RTTAVG, rttavg); if (errno != 0) goto failed; rttdev = (tg != NULL) ? (tg->tg_rtt_sd / 4) : 0; errno = nvlist_add_int64(nvl, IPMP_PROBE_TARGET_RTTDEV, rttdev); if (errno != 0) goto failed; return (post_event(ESC_IPMP_PROBE_STATE, nvl)); failed: logperror("cannot create `probe state' event"); nvlist_free(nvl); return (-1); } /* * Generate an ESC_IPMP_GROUP_STATE sysevent for phyint group `pg'. * Returns 0 on success, -1 on failure. */ static int phyint_group_state_event(struct phyint_group *pg) { nvlist_t *nvl; errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0); if (errno != 0) { logperror("cannot create `group state change' event"); return (-1); } errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name); if (errno != 0) goto failed; errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_GROUP_STATE, groupstate(pg)); if (errno != 0) goto failed; return (post_event(ESC_IPMP_GROUP_STATE, nvl)); failed: logperror("cannot create `group state change' event"); nvlist_free(nvl); return (-1); } /* * Generate an ESC_IPMP_GROUP_CHANGE sysevent of type `op' for phyint group * `pg'. Returns 0 on success, -1 on failure. */ static int phyint_group_change_event(struct phyint_group *pg, ipmp_group_op_t op) { nvlist_t *nvl; errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0); if (errno != 0) { logperror("cannot create `group change' event"); return (-1); } errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name); if (errno != 0) goto failed; errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig); if (errno != 0) goto failed; errno = nvlist_add_uint64(nvl, IPMP_GROUPLIST_SIGNATURE, phyint_grouplistsig); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_GROUP_OPERATION, op); if (errno != 0) goto failed; return (post_event(ESC_IPMP_GROUP_CHANGE, nvl)); failed: logperror("cannot create `group change' event"); nvlist_free(nvl); return (-1); } /* * Generate an ESC_IPMP_GROUP_MEMBER_CHANGE sysevent for phyint `pi' in * group `pg'. Returns 0 on success, -1 on failure. */ static int phyint_group_member_event(struct phyint_group *pg, struct phyint *pi, ipmp_if_op_t op) { nvlist_t *nvl; errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0); if (errno != 0) { logperror("cannot create `group member change' event"); return (-1); } errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name); if (errno != 0) goto failed; errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_IF_OPERATION, op); if (errno != 0) goto failed; errno = nvlist_add_string(nvl, IPMP_IF_NAME, pi->pi_name); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_IF_TYPE, iftype(pi)); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_IF_STATE, ifstate(pi)); if (errno != 0) goto failed; return (post_event(ESC_IPMP_GROUP_MEMBER_CHANGE, nvl)); failed: logperror("cannot create `group member change' event"); nvlist_free(nvl); return (-1); } /* * Generate an ESC_IPMP_IF_CHANGE sysevent for phyint `pi' in group `pg'. * Returns 0 on success, -1 on failure. */ static int phyint_state_event(struct phyint_group *pg, struct phyint *pi) { nvlist_t *nvl; errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0); if (errno != 0) { logperror("cannot create `interface change' event"); return (-1); } errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name); if (errno != 0) goto failed; errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig); if (errno != 0) goto failed; errno = nvlist_add_string(nvl, IPMP_IF_NAME, pi->pi_name); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_IF_TYPE, iftype(pi)); if (errno != 0) goto failed; errno = nvlist_add_uint32(nvl, IPMP_IF_STATE, ifstate(pi)); if (errno != 0) goto failed; return (post_event(ESC_IPMP_IF_CHANGE, nvl)); failed: logperror("cannot create `interface change' event"); nvlist_free(nvl); return (-1); } /* * Generate a signature for use. The signature is conceptually divided * into two pieces: a random 16-bit "generation number" and a 48-bit * monotonically increasing integer. The generation number protects * against stale updates to entities (e.g., IPMP groups) that have been * deleted and since recreated. */ static uint64_t gensig(void) { static int seeded = 0; if (seeded == 0) { srand48((long)gethrtime()); seeded++; } return ((uint64_t)lrand48() << 48 | 1); } /* * Store the information associated with group `grname' into a dynamically * allocated structure pointed to by `*grinfopp'. Returns an IPMP error code. */ unsigned int getgroupinfo(const char *grname, ipmp_groupinfo_t **grinfopp) { struct phyint *pi; struct phyint_group *pg; char (*ifs)[LIFNAMSIZ]; unsigned int i, j; unsigned int nif = 0, naddr = 0; lifgroupinfo_t lifgr; addrlist_t *addrp; struct sockaddr_storage *addrs; int fdt = 0; pg = phyint_group_lookup(grname); if (pg == NULL) return (IPMP_EUNKGROUP); /* * Tally up the number of interfaces, allocate an array to hold them, * and insert their names into the array. While we're at it, if any * interface is actually enabled to send probes, save the group fdt. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) nif++; ifs = alloca(nif * sizeof (*ifs)); for (i = 0, pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext, i++) { assert(i < nif); (void) strlcpy(ifs[i], pi->pi_name, LIFNAMSIZ); if (PROBE_ENABLED(pi->pi_v4) || PROBE_ENABLED(pi->pi_v6)) fdt = pg->pg_fdt; } assert(i == nif); /* * If this is the anonymous group, there's no other information to * collect (since there's no IPMP interface). */ if (pg == phyint_anongroup) { *grinfopp = ipmp_groupinfo_create(pg->pg_name, pg->pg_sig, fdt, groupstate(pg), nif, ifs, "", "", "", "", 0, NULL); return (*grinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); } /* * Grab some additional information about the group from the kernel. * (NOTE: since SIOCGLIFGROUPINFO does not look up by interface name, * we can use ifsock_v4 even for a V6-only group.) */ (void) strlcpy(lifgr.gi_grname, grname, LIFGRNAMSIZ); if (ioctl(ifsock_v4, SIOCGLIFGROUPINFO, &lifgr) == -1) { if (errno == ENOENT) return (IPMP_EUNKGROUP); logperror("getgroupinfo: SIOCGLIFGROUPINFO"); return (IPMP_FAILURE); } /* * Tally up the number of data addresses, allocate an array to hold * them, and insert their values into the array. */ for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next) naddr++; addrs = alloca(naddr * sizeof (*addrs)); i = 0; for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next) { /* * It's possible to have duplicate addresses (if some are * down). Weed the dups out to avoid confusing consumers. * (If groups start having tons of addresses, we'll need a * better algorithm here.) */ for (j = 0; j < i; j++) { if (sockaddrcmp(&addrs[j], &addrp->al_addr)) break; } if (j == i) { assert(i < naddr); addrs[i++] = addrp->al_addr; } } naddr = i; *grinfopp = ipmp_groupinfo_create(pg->pg_name, pg->pg_sig, fdt, groupstate(pg), nif, ifs, lifgr.gi_grifname, lifgr.gi_m4ifname, lifgr.gi_m6ifname, lifgr.gi_bcifname, naddr, addrs); return (*grinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); } /* * Store the target information associated with phyint instance `pii' into a * dynamically allocated structure pointed to by `*targinfopp'. Returns an * IPMP error code. */ unsigned int gettarginfo(struct phyint_instance *pii, const char *name, ipmp_targinfo_t **targinfopp) { uint_t ntarg = 0; struct target *tg; struct sockaddr_storage ss; struct sockaddr_storage *targs = NULL; if (PROBE_CAPABLE(pii)) { targs = alloca(pii->pii_ntargets * sizeof (*targs)); tg = pii->pii_target_next; do { if (tg->tg_status == TG_ACTIVE) { assert(ntarg < pii->pii_ntargets); addr2storage(pii->pii_af, &tg->tg_address, &targs[ntarg++]); } if ((tg = tg->tg_next) == NULL) tg = pii->pii_targets; } while (tg != pii->pii_target_next); assert(ntarg == pii->pii_ntargets); } *targinfopp = ipmp_targinfo_create(name, iftestaddr(pii, &ss), iftargmode(pii), ntarg, targs); return (*targinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); } /* * Store the information associated with interface `ifname' into a dynamically * allocated structure pointed to by `*ifinfopp'. Returns an IPMP error code. */ unsigned int getifinfo(const char *ifname, ipmp_ifinfo_t **ifinfopp) { int retval; struct phyint *pi; ipmp_targinfo_t *targinfo4; ipmp_targinfo_t *targinfo6; pi = phyint_lookup(ifname); if (pi == NULL) return (IPMP_EUNKIF); if ((retval = gettarginfo(pi->pi_v4, pi->pi_name, &targinfo4)) != 0 || (retval = gettarginfo(pi->pi_v6, pi->pi_name, &targinfo6)) != 0) goto out; *ifinfopp = ipmp_ifinfo_create(pi->pi_name, pi->pi_group->pg_name, ifstate(pi), iftype(pi), iflinkstate(pi), ifprobestate(pi), ifflags(pi), targinfo4, targinfo6); retval = (*ifinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); out: if (targinfo4 != NULL) ipmp_freetarginfo(targinfo4); if (targinfo6 != NULL) ipmp_freetarginfo(targinfo6); return (retval); } /* * Store the current list of IPMP groups into a dynamically allocated * structure pointed to by `*grlistpp'. Returns an IPMP error code. */ unsigned int getgrouplist(ipmp_grouplist_t **grlistpp) { struct phyint_group *pg; char (*groups)[LIFGRNAMSIZ]; unsigned int i, ngroup; /* * Tally up the number of groups, allocate an array to hold them, and * insert their names into the array. */ for (ngroup = 0, pg = phyint_groups; pg != NULL; pg = pg->pg_next) ngroup++; groups = alloca(ngroup * sizeof (*groups)); for (i = 0, pg = phyint_groups; pg != NULL; pg = pg->pg_next, i++) { assert(i < ngroup); (void) strlcpy(groups[i], pg->pg_name, LIFGRNAMSIZ); } assert(i == ngroup); *grlistpp = ipmp_grouplist_create(phyint_grouplistsig, ngroup, groups); return (*grlistpp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); } /* * Store the address information for `ssp' (in group `grname') into a * dynamically allocated structure pointed to by `*adinfopp'. Returns an IPMP * error code. (We'd call this function getaddrinfo(), but it would conflict * with getaddrinfo(3SOCKET)). */ unsigned int getgraddrinfo(const char *grname, struct sockaddr_storage *ssp, ipmp_addrinfo_t **adinfopp) { int ifsock; addrlist_t *addrp, *addrmatchp = NULL; ipmp_addr_state_t state; const char *binding = ""; struct lifreq lifr; struct phyint_group *pg; if ((pg = phyint_group_lookup(grname)) == NULL) return (IPMP_EUNKADDR); /* * Walk through the data addresses, and find a match. Note that since * some of the addresses may be down, more than one may match. We * prefer an up address (if one exists). */ for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next) { if (sockaddrcmp(ssp, &addrp->al_addr)) { addrmatchp = addrp; if (addrmatchp->al_flags & IFF_UP) break; } } if (addrmatchp == NULL) return (IPMP_EUNKADDR); state = (addrmatchp->al_flags & IFF_UP) ? IPMP_ADDR_UP : IPMP_ADDR_DOWN; if (state == IPMP_ADDR_UP) { ifsock = (ssp->ss_family == AF_INET) ? ifsock_v4 : ifsock_v6; (void) strlcpy(lifr.lifr_name, addrmatchp->al_name, LIFNAMSIZ); if (ioctl(ifsock, SIOCGLIFBINDING, &lifr) >= 0) binding = lifr.lifr_binding; } *adinfopp = ipmp_addrinfo_create(ssp, state, pg->pg_name, binding); return (*adinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS); } /* * Store a snapshot of the IPMP subsystem into a dynamically allocated * structure pointed to by `*snapp'. Returns an IPMP error code. */ unsigned int getsnap(ipmp_snap_t **snapp) { ipmp_grouplist_t *grlistp; ipmp_groupinfo_t *grinfop; ipmp_addrinfo_t *adinfop; ipmp_addrlist_t *adlistp; ipmp_ifinfo_t *ifinfop; ipmp_snap_t *snap; struct phyint *pi; unsigned int i, j; int retval; snap = ipmp_snap_create(); if (snap == NULL) return (IPMP_ENOMEM); /* * Add group list. */ retval = getgrouplist(&snap->sn_grlistp); if (retval != IPMP_SUCCESS) goto failed; /* * Add information for each group in the list, along with all of its * data addresses. */ grlistp = snap->sn_grlistp; for (i = 0; i < grlistp->gl_ngroup; i++) { retval = getgroupinfo(grlistp->gl_groups[i], &grinfop); if (retval != IPMP_SUCCESS) goto failed; retval = ipmp_snap_addgroupinfo(snap, grinfop); if (retval != IPMP_SUCCESS) { ipmp_freegroupinfo(grinfop); goto failed; } adlistp = grinfop->gr_adlistp; for (j = 0; j < adlistp->al_naddr; j++) { retval = getgraddrinfo(grinfop->gr_name, &adlistp->al_addrs[j], &adinfop); if (retval != IPMP_SUCCESS) goto failed; retval = ipmp_snap_addaddrinfo(snap, adinfop); if (retval != IPMP_SUCCESS) { ipmp_freeaddrinfo(adinfop); goto failed; } } } /* * Add information for each configured phyint. */ for (pi = phyints; pi != NULL; pi = pi->pi_next) { retval = getifinfo(pi->pi_name, &ifinfop); if (retval != IPMP_SUCCESS) goto failed; retval = ipmp_snap_addifinfo(snap, ifinfop); if (retval != IPMP_SUCCESS) { ipmp_freeifinfo(ifinfop); goto failed; } } *snapp = snap; return (IPMP_SUCCESS); failed: ipmp_snap_free(snap); return (retval); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _MPD_TABLES_H #define _MPD_TABLES_H #ifdef __cplusplus extern "C" { #endif /* * Terminology: * * phyint: A NIC eg. hme0. This is represented as 'struct phyint' * * phyint instance: A protocol instance of a phyint. Eg. the IPv4 instance of * hme0 or the IPv6 instance of hme0. (struct phyint_instance) * * logint: A logical interface eg. hme0:1 (struct logint) * * phyint_group: A group of phyints i.e. physical interfaces that are * (i) connected to the same level 2 topology e.g. the same ethernet * switch AND * (ii) share the same phyint group name. * Load spreading and failover occur across members of the same phyint group. * phyint group members must be homogeneous. i.e. if a phyint belonging to a * phyint group has a IPv6 protocol instance, then all members of the phyint * group, must have IPv6 protocol instances. (struct phyint_group) */ #define MAXDEFERREDRTT 1 /* Maximum number of deferred rtts */ /* * Status of the phyint, expressed by the return code of failure_state() */ #define PHYINT_OK 0 /* No failure detected */ #define PHYINT_FAILURE 1 /* NIC failure detected */ #define GROUP_FAILURE 2 /* All NICs have failed */ /* * Return values of phyint_inst_update_from_k() */ #define PI_OK 1 /* Phyint matches in the kernel */ #define PI_DELETED 2 /* Phyint has vanished in the kernel */ #define PI_IFINDEX_CHANGED 3 /* Phyint's ifindex has changed */ #define PI_IOCTL_ERROR 4 /* Some ioctl error */ #define PI_GROUP_CHANGED 5 /* The phyint has changed group. */ #define PHYINT_FLAGS(flags) \ (((flags) & (IFF_STANDBY | IFF_INACTIVE | IFF_FAILED | IFF_OFFLINE | \ IFF_RUNNING)) | (handle_link_notifications ? 0 : IFF_RUNNING)) /* A Phyint can have up to 2 instances, the IPv4 and the IPv6 instance */ #define PHYINT_INSTANCE(pi, af) \ ((af) == AF_INET ? (pi)->pi_v4 : (pi)->pi_v6) /* * A phyint instance is probe *enabled* if it has been configured with a * unique probe address (i.e., an IFF_NOFAILOVER address). It is probe * *capable* if it is also able to send probes (i.e., has one or more * targets available). */ #define PROBE_ENABLED(pii) \ (((pii) != NULL) && ((pii)->pii_probe_sock != -1) && \ ((pii)->pii_probe_logint != NULL) && \ (((pii)->pii_probe_logint->li_dupaddr == 0))) #define PROBE_CAPABLE(pii) \ (PROBE_ENABLED(pii) && ((pii)->pii_ntargets != 0)) /* Subtract b from a modulo n. i.e. (a - b) mod n */ #define MOD_SUB(a, b, n) \ ((((a) + (n)) - (b)) % (n)) /* Increment modulo n */ #define MOD_INCR(a, n) \ (((a) + 1) % (n)) /* Decrement modulo n */ #define MOD_DCR(a, n) \ MOD_SUB(a, 1, n) /* * 'index' represents an index into the circular probe stats array of * size PROBE_STATS_COUNT. 0 <= index < PROBE_STATS_COUNT. This is used * to access members of the pii_probes[] array defined in the phyint_instance * structure. */ #define PROBE_INDEX_PREV(index) \ MOD_DCR(index, PROBE_STATS_COUNT) #define PROBE_INDEX_NEXT(index) \ MOD_INCR(index, PROBE_STATS_COUNT) /* * If we receive more than LINK_UP_PERMIN "link up" notifications in a minute, * then don't actually perform the repair operation until we've dropped back * below the threshold (or we have a probe address and our probes indicate * that the link is functioning again). This is to prevent link flapping in * the case where we don't have a probe address. */ #define LINK_UP_PERMIN 2 #define LINK_DOWN(pi) ((pi)->pi_link_state == 0) #define LINK_UP(pi) (!LINK_DOWN(pi)) #define FLAGS_TO_LINK_STATE(pi) (((pi)->pi_flags & IFF_RUNNING) != 0) #define UPDATE_LINK_STATE(pi) ((pi)->pi_link_state = \ FLAGS_TO_LINK_STATE(pi) ? 1 : 0) #define INIT_LINK_STATE(pi) ((pi)->pi_link_state = 1) /* * Phyint group states; see below for the phyint group definition. */ enum pg_state { PG_OK = 1, /* all interfaces in the group are working */ PG_DEGRADED, /* some interfaces in the group are unusable */ PG_FAILED /* all interfaces in the group are unusable */ }; /* * Convenience macro to check if the whole group has failed. */ #define GROUP_FAILED(pg) ((pg)->pg_state == PG_FAILED) /* * A doubly linked list of all phyint groups in the system. * A phyint group is identified by its group name. */ struct phyint_group { char pg_name[LIFGRNAMSIZ]; /* Phyint group name */ struct phyint *pg_phyint; /* List of phyints in this group */ struct phyint_group *pg_next; /* Next phyint group */ struct phyint_group *pg_prev; /* Prev phyint group */ uint64_t pg_sig; /* Current signature of this group */ int pg_probeint; /* Interval between probes */ int pg_fdt; /* Time needed to detect failure */ enum pg_state pg_state; /* Current group state */ boolean_t pg_in_use; /* To detect removed groups */ struct addrlist *pg_addrs; /* Data addresses in this group */ boolean_t pg_failmsg_printed; /* Group failure msg printed */ }; /* * Phyint states; see below for the phyint definition. */ enum pi_state { PI_INIT = 0, /* Phyint is being initialized */ PI_NOTARGETS = 1, /* Phyint has no targets */ PI_RUNNING = 2, /* Phyint is functioning */ PI_FAILED = 3, /* Phyint is failed */ PI_OFFLINE = 4 /* Phyint is offline */ }; /* * Representation of a NIC or a phyint. There is a list of all known phyints. * There is also a list of phyints belonging to a phyint group, one list * per phyint group. */ struct phyint { char pi_name[LIFNAMSIZ + 1]; /* Phyint name eg. le0 */ struct phyint_instance *pi_v4; /* The IPv4 instance */ struct phyint_instance *pi_v6; /* The IPv6 instance */ struct phyint_group *pi_group; /* Pointer to the group */ struct phyint *pi_next; /* List of all phyints */ struct phyint *pi_prev; /* List of all phyints */ struct phyint *pi_pgnext; /* List of phyints in this group */ struct phyint *pi_pgprev; /* List of phyints in this group */ uint_t pi_ifindex; /* interface index */ enum pi_state pi_state; /* State of the phyint */ uint64_t pi_flags; /* Phyint flags from kernel */ uint16_t pi_icmpid; /* icmp id in icmp echo request */ uint64_t pi_taddrthresh; /* time (in secs) to delay logging */ /* about missing test addresses */ dlpi_handle_t pi_dh; /* DLPI handle to underlying link */ uint_t pi_notes; /* enabled DLPI notifications */ uchar_t pi_hwaddr[DLPI_PHYSADDR_MAX]; /* phyint's hw address */ size_t pi_hwaddrlen; /* phyint's hw address length */ /* * The pi_whenup array is a circular buffer of the most recent * times (in milliseconds since some arbitrary point of time in * the past) that the interface was brought up; pi_whendx identifies * the oldest element of the array. */ uint_t pi_whenup[LINK_UP_PERMIN]; unsigned int pi_whendx; uint_t pi_taddrmsg_printed : 1, /* testaddr msg printed */ pi_duptaddrmsg_printed : 1, /* dup testaddr msg printed */ pi_cfgmsg_printed : 1, /* bad config msg printed */ pi_lfmsg_printed : 1, /* link-flapping msg printed */ pi_link_state : 1, /* interface link state */ pi_hwaddrdup : 1; /* disabled due to dup hw address */ }; /* * A doubly linked list of all phyint_instances each of which contains a * doubly linked list of logical interfaces and targets. For eg. if both * IPv4 and IPv6 are used over hme0, we have 2 phyint instances, 1 for each * protocol. */ struct phyint_instance { struct phyint_instance *pii_next; /* List of all phyint insts */ struct phyint_instance *pii_prev; /* List of all phyint insts */ struct phyint *pii_phyint; /* Back pointer to the phyint */ struct target *pii_targets; /* List of targets on this link */ struct logint *pii_probe_logint; /* IFF_NOFAILOVER addr for probing */ struct logint *pii_logint; /* Doubly linked list of logical ifs */ int pii_probe_sock; /* Socket for ICMP Probe packets */ int pii_af; /* Address family */ uint16_t pii_rack; /* highest acknowledged seq number */ uint16_t pii_snxt; /* sequence number of next probe */ uint_t pii_snxt_time; /* actual next probe time that */ /* includes some randomness */ uint_t pii_snxt_basetime; /* strictly periodic base probe time */ /* for all periodic probes */ uint_t pii_fd_snxt_basetime; /* strictly periodic base probe time */ /* for failure detection probes */ hrtime_t pii_fd_hrtime; /* hrtime_t before which we should */ /* not send probes out this pii */ uint64_t pii_flags; /* Phyint flags from kernel */ struct probe_stats { uint_t pr_id; /* Full ID of probe */ struct target *pr_target; /* Probe Target */ uint_t pr_time_lost; /* Time probe declared lost */ struct timeval pr_tv_sent; /* Wall time probe was sent */ hrtime_t pr_hrtime_start; /* hrtime probe op started */ hrtime_t pr_hrtime_sent; /* hrtime probe was sent */ hrtime_t pr_hrtime_ackrecv; /* hrtime probe ack received */ hrtime_t pr_hrtime_ackproc; /* hrtime probe ack processed */ uint_t pr_status; /* probe status as below */ #define PR_UNUSED 0 /* Probe slot unused */ #define PR_UNACKED 1 /* Probe is unacknowledged */ #define PR_ACKED 2 /* Probe has been acknowledged */ #define PR_LOST 3 /* Probe is declared lost */ } pii_probes[PROBE_STATS_COUNT]; uint_t pii_in_use : 1, /* To detect removed phyints */ pii_basetime_inited : 1, /* probe time initialized */ pii_targets_are_routers : 1; /* routers or hosts ? */ uint_t pii_probe_next; /* next index to use in pii_probes[] */ struct target *pii_target_next; /* next target for probing */ struct target *pii_rtt_target_next; /* next target for rtt probes */ int pii_ntargets; /* Number of active targets */ struct stats { /* Cumulative statistics */ uint64_t lost; /* Number of probes lost */ uint64_t acked; /* Number of probes acked */ uint64_t sent; /* Number of probes sent */ uint64_t unknown; /* Number of ambiguous */ /* probe acks */ } pii_cum_stats; }; #define pii_name pii_phyint->pi_name #define pii_ifindex pii_phyint->pi_ifindex #define pii_state pii_phyint->pi_state #define pii_icmpid pii_phyint->pi_icmpid #define PR_STATUS_VALID(status) ((status) <= PR_LOST) /* * A doubly linked list of prefixes or logicals, hanging off the * phyint instance. */ struct logint { struct logint *li_next; /* Next logint of this phyint inst. */ struct logint *li_prev; /* Prev logint of this phyint inst. */ struct phyint_instance *li_phyint_inst; /* Back pointer to phyint inst. */ char li_name[LIFNAMSIZ + 1]; /* name Eg. hme0:1 */ struct in6_addr li_addr; /* IP address */ struct in6_addr li_dstaddr; /* Dst IP address for pointopoint */ struct in6_addr li_subnet; /* prefix / subnet */ uint_t li_subnet_len; /* prefix / subnet length */ uint64_t li_flags; /* IFF_* flags */ uint_t li_in_use : 1, /* flag to detect deleted logints */ li_dupaddr : 1; /* test address is not unique */ }; /* * Doubly-linked list of probe targets on a phyint instance. Probe targets are * usually onlink routers. If no onlink routers can be found, onlink hosts * are used. */ struct target { struct target *tg_next; /* Next target for this phyint inst. */ struct target *tg_prev; /* Prev target for this phyint inst. */ struct phyint_instance *tg_phyint_inst; /* Back pointer to phyint instance */ struct in6_addr tg_address; /* Target IP address */ int tg_status; /* Status of the target below */ #define TG_ACTIVE 1 /* active probe target */ #define TG_UNUSED 2 /* target not in use now */ #define TG_SLOW 3 /* rtt is high - Not in use now */ #define TG_DEAD 4 /* Target is not responding */ hrtime_t tg_latime; /* Target's last active time */ int64_t tg_rtt_sa; /* Scaled RTT average (in ns) */ int64_t tg_rtt_sd; /* Scaled RTT deviation (in ns) */ int tg_crtt; /* Conservative RTT = A + 4D (in ms) */ uint32_t tg_in_use : 1; /* In use flag */ int64_t tg_deferred[MAXDEFERREDRTT + 1]; /* Deferred rtt data points */ int tg_num_deferred; /* Number of deferred rtt data points */ }; #define TG_STATUS_VALID(status) \ (((status) >= TG_ACTIVE) && ((status) <= TG_DEAD)) /* * Statistics about consecutive probe failures are passed around between * functions in this structure. */ struct probe_fail_count { uint_t pf_tff; /* Earliest time of failure in a series */ int pf_nfail; /* Number of consecutive probe failures */ int pf_nfail_tg; /* Number of consecutive probe fails for */ /* some given target 'tg' */ }; /* * Statistics about consecutive probe successes is passed around between * functions in this structure. */ struct probe_success_count { uint_t ps_tls; /* Most recent time of probe success */ boolean_t ps_tls_valid; /* is ps_tls valid */ int ps_nsucc; /* Number of consecutive probe successes */ /* starting from the most recent */ int ps_nsucc_tg; /* Number of consecutive probe successes */ /* for some given target 'tg' */ }; /* * Statistics about missed probes that were never sent. * Happens due to scheduling delay. */ struct probes_missed { uint_t pm_nprobes; /* Cumulative number of missed probes */ uint_t pm_ntimes; /* Total number of occasions */ }; typedef struct addrlist { struct addrlist *al_next; /* next address */ char al_name[LIFNAMSIZ]; /* address lif name */ uint64_t al_flags; /* address flags */ struct sockaddr_storage al_addr; /* address */ } addrlist_t; /* * Globals */ extern addrlist_t *localaddrs; /* List of all local addresses, including local zones */ extern struct phyint *phyints; /* List of all phyints */ extern struct phyint_group *phyint_groups; /* List of all phyint groups */ extern struct phyint_group *phyint_anongroup; /* Pointer to the anon group */ extern struct phyint_instance *phyint_instances; /* List of all phyint instances */ extern struct probes_missed probes_missed; /* statistics about missed probes */ /* * Function prototypes */ extern int phyint_init(void); extern struct phyint *phyint_lookup(const char *name); extern struct phyint_instance *phyint_inst_lookup(int af, char *name); extern struct phyint_instance *phyint_inst_init_from_k(int af, char *name); extern struct phyint_instance *phyint_inst_other(struct phyint_instance *pii); extern int phyint_inst_update_from_k(struct phyint_instance *pii); extern void phyint_inst_delete(struct phyint_instance *pii); extern uint_t phyint_inst_timer(struct phyint_instance *pii); extern boolean_t phyint_inst_sockinit(struct phyint_instance *pii); extern void phyint_changed(struct phyint *pi); extern void phyint_chstate(struct phyint *pi, enum pi_state state); extern void phyint_group_chstate(struct phyint_group *pg, enum pg_state state); extern struct phyint_group *phyint_group_create(const char *pg_name); extern struct phyint_group *phyint_group_lookup(const char *pg_name); extern void phyint_group_insert(struct phyint_group *pg); extern void phyint_group_delete(struct phyint_group *pg); extern void phyint_group_refresh_state(struct phyint_group *pg); extern void phyint_standby_refresh_inactive(struct phyint *pi); extern void phyint_check_for_repair(struct phyint *pi); extern void phyint_transition_to_running(struct phyint *pi); extern void phyint_activate_another(struct phyint *pi); extern int phyint_offline(struct phyint *pi, unsigned int); extern int phyint_undo_offline(struct phyint *pi); extern boolean_t phyint_is_functioning(struct phyint *pi); extern void logint_init_from_k(struct phyint_instance *pii, char *li_name); extern void logint_delete(struct logint *li); extern struct target *target_lookup(struct phyint_instance *pii, struct in6_addr addr); extern void target_create(struct phyint_instance *pii, struct in6_addr addr, boolean_t is_router); extern void target_delete(struct target *tg); extern struct target *target_next(struct target *tg); extern void target_add(struct phyint_instance *pii, struct in6_addr addr, boolean_t is_router); extern void in_data(struct phyint_instance *pii); extern void in6_data(struct phyint_instance *pii); extern void logperror_pii(struct phyint_instance *pii, const char *str); extern void logperror_li(struct logint *li, const char *str); extern char *pr_addr(int af, struct in6_addr addr, char *abuf, int len); extern void addr2storage(int af, const struct in6_addr *addr, struct sockaddr_storage *ssp); extern void phyint_inst_print_all(void); extern boolean_t prefix_equal(struct in6_addr, struct in6_addr, uint_t); extern void reset_crtt_all(struct phyint *pi); extern int failure_state(struct phyint_instance *pii); extern void process_link_state_changes(void); extern void clear_pii_probe_stats(struct phyint_instance *pii); extern void start_timer(struct phyint_instance *pii); extern void stop_probing(struct phyint *pi); extern boolean_t own_address(struct in6_addr addr); extern boolean_t change_pif_flags(struct phyint *pi, uint64_t set, uint64_t clear); extern void close_probe_socket(struct phyint_instance *pii, boolean_t flag); extern int probe_state_event(struct probe_stats *, struct phyint_instance *); extern void probe_chstate(struct probe_stats *, struct phyint_instance *, int); extern unsigned int getgraddrinfo(const char *, struct sockaddr_storage *, ipmp_addrinfo_t **); extern unsigned int getifinfo(const char *, ipmp_ifinfo_t **); extern unsigned int getgroupinfo(const char *, ipmp_groupinfo_t **); extern unsigned int getgrouplist(ipmp_grouplist_t **); extern unsigned int getsnap(ipmp_snap_t **); extern boolean_t addrlist_add(addrlist_t **, const char *, uint64_t, struct sockaddr_storage *); extern void addrlist_free(addrlist_t **); #ifdef __cplusplus } #endif #endif /* _MPD_TABLES_H */ #!/sbin/sh # # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2021 Tintri by DDN, Inc. All rights reserved. # . /lib/svc/share/smf_include.sh if [ -z "$SMF_FMRI" ]; then echo "this script can only be invoked by smf(7)" exit $SMF_EXIT_ERR_NOSMF fi smf_configure_ip || exit $SMF_EXIT_NODAEMON if /lib/inet/in.mpathd; then exit $SMF_EXIT_OK else exit $SMF_EXIT_ERR_FATAL fi