/* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 1987 Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms are permitted * provided that the above copyright notice and this paragraph are * duplicated in all such forms and that any documentation, * advertising materials, and other materials related to such * distribution and use acknowledge that the software was developed * by the University of California, Berkeley. The name of the * University may not be used to endorse or promote products derived * from this software without specific prior written permission. * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. */ #include "mpd_defs.h" #include "mpd_tables.h" /* * Probe types for probe() */ #define PROBE_UNI 0x1234 /* Unicast probe packet */ #define PROBE_MULTI 0x5678 /* Multicast probe packet */ #define PROBE_RTT 0x9abc /* RTT only probe packet */ #define MSEC_PERMIN (60 * MILLISEC) /* Number of milliseconds in a minute */ /* * Format of probe / probe response packets. This is an ICMP Echo request * or ICMP Echo reply. Packet format is same for both IPv4 and IPv6 */ struct pr_icmp { uint8_t pr_icmp_type; /* type field */ uint8_t pr_icmp_code; /* code field */ uint16_t pr_icmp_cksum; /* checksum field */ uint16_t pr_icmp_id; /* Identification */ uint16_t pr_icmp_seq; /* sequence number */ uint64_t pr_icmp_timestamp; /* Time stamp (in ns) */ uint32_t pr_icmp_mtype; /* Message type */ }; static struct in6_addr all_nodes_mcast_v6 = { { 0xff, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1 } }; static struct in_addr all_nodes_mcast_v4 = { { { 0xe0, 0x0, 0x0, 0x1 } } }; static hrtime_t last_fdt_bumpup_time; /* When FDT was bumped up last */ static void *find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type); static void pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni); static void incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp); static void incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr); static void incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr); static boolean_t check_pg_crtt_improved(struct phyint_group *pg); static boolean_t check_pii_crtt_improved(struct phyint_instance *pii); static boolean_t check_exception_target(struct phyint_instance *pii, struct target *target); static void probe_fail_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_fail_count *pfinfo); static void probe_success_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_success_count *psinfo); static boolean_t phyint_repaired(struct phyint *pi); static boolean_t highest_ack_tg(uint16_t seq, struct target *tg); static int in_cksum(ushort_t *addr, int len); static void reset_snxt_basetimes(void); static int ns2ms(int64_t ns); static int64_t tv2ns(struct timeval *); /* * CRTT - Conservative Round Trip Time Estimate * Probe success - A matching probe reply received before CRTT ms has elapsed * after sending the probe. * Probe failure - No probe reply received and more than CRTT ms has elapsed * after sending the probe. * * TLS - Time last success. Most recent probe ack received at this time. * TFF - Time first fail. The time of the earliest probe failure in * a consecutive series of probe failures. * NUM_PROBE_REPAIRS - Number of consecutive successful probes required * before declaring phyint repair. * NUM_PROBE_FAILS - Number of consecutive probe failures required to * declare a phyint failure. * * Phyint state diagram * * The state of a phyint that is capable of being probed, is completely * specified by the 3-tuple . * * A phyint starts in either PI_RUNNING or PI_OFFLINE, depending on whether * IFF_OFFLINE is set. If the phyint is also configured with a test address * (the common case) and probe targets, then a phyint must also successfully * be able to send and receive probes in order to remain in the PI_RUNNING * state (otherwise, it transitions to PI_FAILED). * * Further, if a PI_RUNNING phyint is configured with a test address but is * unable to find any probe targets, it will transition to the PI_NOTARGETS * state, which indicates that the link is apparently functional but that * in.mpathd is unable to send probes to verify functionality (in this case, * in.mpathd makes the optimistic assumption that the interface is working * correctly and thus does not mark the interface FAILED, but reports it as * IPMP_IF_UNKNOWN through the async events and query interfaces). * * At any point, a phyint may be administratively marked offline via if_mpadm. * In this case, the interface always transitions to PI_OFFLINE, regardless * of its previous state. When the interface is later brought back online, * in.mpathd acts as if the interface is new (and thus it transitions to * PI_RUNNING or PI_FAILED based on the status of the link and the result of * its probes, if probes are sent). * * pi_state - PI_RUNNING or PI_FAILED * PI_RUNNING: The failure detection logic says the phyint is good. * PI_FAILED: The failure detection logic says the phyint has failed. * * pg_state - PG_OK, PG_DEGRADED, or PG_FAILED. * PG_OK: All interfaces in the group are OK. * PG_DEGRADED: Some interfaces in the group are unusable. * PG_FAILED: All interfaces in the group are unusable. * * In the case of router targets, we assume that the current list of * targets obtained from the routing table, is still valid, so the * phyint stat is PI_FAILED. In the case of host targets, we delete the * list of targets, and multicast to the all hosts, to reconstruct the * target list. So the phyints are in the PI_NOTARGETS state. * * I - value of (pi_flags & IFF_INACTIVE) * IFF_INACTIVE: This phyint will not send or receive packets. * Usually, inactive is tied to standby interfaces that are not yet * needed (e.g., no non-standby interfaces in the group have failed). * When failback has been disabled (FAILBACK=no configured), phyint can * also be a non-STANDBY. In this case IFF_INACTIVE is set when phyint * subsequently recovers after a failure. * * Not all 9 possible combinations of the above 3-tuple are possible. * * I is tracked by IP. pi_state is tracked by mpathd. * * pi_state state machine * --------------------------------------------------------------------------- * Event State New State * Action: * --------------------------------------------------------------------------- * IP interface failure (PI_RUNNING, I == 0) -> (PI_FAILED, I == 0) * detection : set IFF_FAILED on this phyint * * IP interface failure (PI_RUNNING, I == 1) -> (PI_FAILED, I == 0) * detection : set IFF_FAILED on this phyint * * IP interface repair (PI_FAILED, I == 0, FAILBACK=yes) * detection -> (PI_RUNNING, I == 0) * : clear IFF_FAILED on this phyint * * IP interface repair (PI_FAILED, I == 0, FAILBACK=no) * detection -> (PI_RUNNING, I == 1) * : clear IFF_FAILED on this phyint * : if failback is disabled set I == 1 * * Group failure (perform on all phyints in the group) * detection PI_RUNNING PI_FAILED * (Router targets) : set IFF_FAILED * * Group failure (perform on all phyints in the group) * detection PI_RUNNING PI_NOTARGETS * (Host targets) : set IFF_FAILED * : delete the target list on all phyints * --------------------------------------------------------------------------- */ struct probes_missed probes_missed; /* * Compose and transmit an ICMP ECHO REQUEST packet. The IP header * will be added on by the kernel. The id field identifies this phyint. * and the sequence number is an increasing (modulo 2^^16) integer. The data * portion holds the time value when the packet is sent. On echo this is * extracted to compute the round-trip time. Three different types of * probe packets are used. * * PROBE_UNI: This type is used to do failure detection / failure recovery * and RTT calculation. PROBE_UNI probes are spaced apart in time, * not less than the current CRTT. pii_probes[] stores data * about these probes. These packets consume sequence number space. * * PROBE_RTT: This type is used to make only rtt measurements. Normally these * are not used. Under heavy network load, the rtt may go up very high, * due to a spike, or may appear to go high, due to extreme scheduling * delays. Once the network stress is removed, mpathd takes long time to * recover, because the probe_interval is already high, and it takes * a long time to send out sufficient number of probes to bring down the * rtt. To avoid this problem, PROBE_RTT probes are sent out every * user_probe_interval ms. and will cause only rtt updates. These packets * do not consume sequence number space nor is information about these * packets stored in the pii_probes[] * * PROBE_MULTI: This type is only used to construct a list of targets, when * no targets are known. The packet is multicast to the all hosts addr. */ static void probe(struct phyint_instance *pii, uint_t probe_type, hrtime_t start_hrtime) { hrtime_t sent_hrtime; struct timeval sent_tv; struct pr_icmp probe_pkt; /* Probe packet */ struct sockaddr_storage targ; /* target address */ uint_t targaddrlen; /* targed address length */ int pr_ndx; /* probe index in pii->pii_probes[] */ boolean_t sent = _B_FALSE; int rval; if (debug & D_TARGET) { logdebug("probe(%s %s %d %lld)\n", AF_STR(pii->pii_af), pii->pii_name, probe_type, start_hrtime); } assert(pii->pii_probe_sock != -1); assert(probe_type == PROBE_UNI || probe_type == PROBE_MULTI || probe_type == PROBE_RTT); probe_pkt.pr_icmp_type = (pii->pii_af == AF_INET) ? ICMP_ECHO_REQUEST : ICMP6_ECHO_REQUEST; probe_pkt.pr_icmp_code = 0; probe_pkt.pr_icmp_cksum = 0; probe_pkt.pr_icmp_seq = htons(pii->pii_snxt); /* * Since there is no need to do arithmetic on the icmpid, * (only equality check is done) pii_icmpid is stored in * network byte order at initialization itself. */ probe_pkt.pr_icmp_id = pii->pii_icmpid; probe_pkt.pr_icmp_timestamp = htonll(start_hrtime); probe_pkt.pr_icmp_mtype = htonl(probe_type); /* * If probe_type is PROBE_MULTI, this packet will be multicast to * the all hosts address. Otherwise it is unicast to the next target. */ assert(probe_type == PROBE_MULTI || ((pii->pii_target_next != NULL) && pii->pii_rtt_target_next != NULL)); bzero(&targ, sizeof (targ)); targ.ss_family = pii->pii_af; if (pii->pii_af == AF_INET6) { struct in6_addr *addr6; addr6 = &((struct sockaddr_in6 *)&targ)->sin6_addr; targaddrlen = sizeof (struct sockaddr_in6); if (probe_type == PROBE_MULTI) { *addr6 = all_nodes_mcast_v6; } else if (probe_type == PROBE_UNI) { *addr6 = pii->pii_target_next->tg_address; } else { /* type is PROBE_RTT */ *addr6 = pii->pii_rtt_target_next->tg_address; } } else { struct in_addr *addr4; addr4 = &((struct sockaddr_in *)&targ)->sin_addr; targaddrlen = sizeof (struct sockaddr_in); if (probe_type == PROBE_MULTI) { *addr4 = all_nodes_mcast_v4; } else if (probe_type == PROBE_UNI) { IN6_V4MAPPED_TO_INADDR( &pii->pii_target_next->tg_address, addr4); } else { /* type is PROBE_RTT */ IN6_V4MAPPED_TO_INADDR( &pii->pii_rtt_target_next->tg_address, addr4); } /* * Compute the IPv4 icmp checksum. Does not cover the IP header. */ probe_pkt.pr_icmp_cksum = in_cksum((ushort_t *)&probe_pkt, (int)sizeof (probe_pkt)); } /* * Use the current time as the time we sent. Not atomic, but the best * we can do from here. */ sent_hrtime = gethrtime(); (void) gettimeofday(&sent_tv, NULL); rval = sendto(pii->pii_probe_sock, &probe_pkt, sizeof (probe_pkt), 0, (struct sockaddr *)&targ, targaddrlen); /* * If the send would block, this may either be transient or a hang in a * lower layer. We pretend the probe was actually sent, the daemon will * not see a reply to the probe and will fail the interface if normal * failure detection criteria are met. */ if (rval == sizeof (probe_pkt) || (rval == -1 && errno == EWOULDBLOCK)) { sent = _B_TRUE; } else { logperror_pii(pii, "probe: probe sendto"); } /* * If this is a PROBE_UNI probe packet being unicast to a target, then * update our tables. We will need this info in processing the probe * response. PROBE_MULTI and PROBE_RTT packets are not used for * the purpose of failure or recovery detection. PROBE_MULTI packets * are only used to construct a list of targets. PROBE_RTT packets are * used only for updating the rtt and not for failure detection. */ if (probe_type == PROBE_UNI && sent) { pr_ndx = pii->pii_probe_next; assert(pr_ndx >= 0 && pr_ndx < PROBE_STATS_COUNT); /* Collect statistics, before we reuse the last slot. */ if (pii->pii_probes[pr_ndx].pr_status == PR_LOST) pii->pii_cum_stats.lost++; else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) pii->pii_cum_stats.acked++; pii->pii_cum_stats.sent++; pii->pii_probes[pr_ndx].pr_id = pii->pii_snxt; pii->pii_probes[pr_ndx].pr_tv_sent = sent_tv; pii->pii_probes[pr_ndx].pr_hrtime_sent = sent_hrtime; pii->pii_probes[pr_ndx].pr_hrtime_start = start_hrtime; pii->pii_probes[pr_ndx].pr_target = pii->pii_target_next; probe_chstate(&pii->pii_probes[pr_ndx], pii, PR_UNACKED); pii->pii_probe_next = PROBE_INDEX_NEXT(pii->pii_probe_next); pii->pii_target_next = target_next(pii->pii_target_next); assert(pii->pii_target_next != NULL); /* * If we have a single variable to denote the next target to * probe for both rtt probes and failure detection probes, we * could end up with a situation where the failure detection * probe targets become disjoint from the rtt probe targets. * Eg. if 2 targets and the actual fdt is double the user * specified fdt. So we have 2 variables. In this scheme * we also reset pii_rtt_target_next for every fdt probe, * though that may not be necessary. */ pii->pii_rtt_target_next = pii->pii_target_next; pii->pii_snxt++; } else if (probe_type == PROBE_RTT) { pii->pii_rtt_target_next = target_next(pii->pii_rtt_target_next); assert(pii->pii_rtt_target_next != NULL); } } /* * Incoming IPv4 data from wire, is received here. Called from main. */ void in_data(struct phyint_instance *pii) { struct sockaddr_in from; struct in6_addr fromaddr; static uint64_t in_packet[(IP_MAXPACKET + 1)/8]; static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8]; struct ip *ip; int iphlen; int len; char abuf[INET_ADDRSTRLEN]; struct msghdr msg; struct iovec iov; struct pr_icmp *reply; struct timeval *recv_tvp; if (debug & D_PROBE) { logdebug("in_data(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } iov.iov_base = (char *)in_packet; iov.iov_len = sizeof (in_packet); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_name = (struct sockaddr *)&from; msg.msg_namelen = sizeof (from); msg.msg_control = ancillary_data; msg.msg_controllen = sizeof (ancillary_data); /* * Poll has already told us that a message is waiting, * on this socket. Read it now. We should not block. */ if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) { logperror_pii(pii, "in_data: recvmsg"); return; } /* * If the datalink has indicated the link is down, don't go * any further. */ if (LINK_DOWN(pii->pii_phyint)) return; /* Get the printable address for error reporting */ (void) inet_ntop(AF_INET, &from.sin_addr, abuf, sizeof (abuf)); /* Ignore packets > 64k or control buffers that don't fit */ if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } /* Make sure packet contains at least minimum ICMP header */ ip = (struct ip *)in_packet; iphlen = ip->ip_hl << 2; if (len < iphlen + ICMP_MINLEN) { if (debug & D_PKTBAD) { logdebug("in_data: packet too short (%d bytes)" " from %s\n", len, abuf); } return; } /* * Subtract the IP hdr length, 'len' will be length of the probe * reply, starting from the icmp hdr. */ len -= iphlen; /* LINTED */ reply = (struct pr_icmp *)((char *)in_packet + iphlen); /* Probe replies are icmp echo replies. Ignore anything else */ if (reply->pr_icmp_type != ICMP_ECHO_REPLY) return; /* * The icmp id should match what we sent, which is stored * in pi_icmpid. The icmp code for reply must be 0. * The reply content must be a struct pr_icmp */ if (reply->pr_icmp_id != pii->pii_icmpid) { /* Not in response to our probe */ return; } if (reply->pr_icmp_code != 0) { logtrace("probe reply code %d from %s on %s\n", reply->pr_icmp_code, abuf, pii->pii_name); return; } if (len < sizeof (struct pr_icmp)) { logtrace("probe reply too short: %d bytes from %s on %s\n", len, abuf, pii->pii_name); return; } recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP); if (recv_tvp == NULL) { logtrace("message without timestamp from %s on %s\n", abuf, pii->pii_name); return; } IN6_INADDR_TO_V4MAPPED(&from.sin_addr, &fromaddr); if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) /* Unicast probe reply */ incoming_echo_reply(pii, reply, fromaddr, recv_tvp); else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) { /* Multicast reply */ incoming_mcast_reply(pii, reply, fromaddr); } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) { incoming_rtt_reply(pii, reply, fromaddr); } else { /* Probably not in response to our probe */ logtrace("probe reply type: %d from %s on %s\n", reply->pr_icmp_mtype, abuf, pii->pii_name); return; } } /* * Incoming IPv6 data from wire is received here. Called from main. */ void in6_data(struct phyint_instance *pii) { struct sockaddr_in6 from; static uint64_t in_packet[(IP_MAXPACKET + 1)/8]; static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8]; int len; char abuf[INET6_ADDRSTRLEN]; struct msghdr msg; struct iovec iov; void *opt; struct pr_icmp *reply; struct timeval *recv_tvp; if (debug & D_PROBE) { logdebug("in6_data(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } iov.iov_base = (char *)in_packet; iov.iov_len = sizeof (in_packet); msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_name = (struct sockaddr *)&from; msg.msg_namelen = sizeof (from); msg.msg_control = ancillary_data; msg.msg_controllen = sizeof (ancillary_data); if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) { logperror_pii(pii, "in6_data: recvmsg"); return; } /* * If the datalink has indicated that the link is down, don't go * any further. */ if (LINK_DOWN(pii->pii_phyint)) return; /* Get the printable address for error reporting */ (void) inet_ntop(AF_INET6, &from.sin6_addr, abuf, sizeof (abuf)); if (len < ICMP_MINLEN) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } /* Ignore packets > 64k or control buffers that don't fit */ if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) { if (debug & D_PKTBAD) { logdebug("Truncated message: msg_flags 0x%x from %s\n", msg.msg_flags, abuf); } return; } reply = (struct pr_icmp *)in_packet; if (reply->pr_icmp_type != ICMP6_ECHO_REPLY) return; if (reply->pr_icmp_id != pii->pii_icmpid) { /* Not in response to our probe */ return; } /* * The kernel has already verified the the ICMP checksum. */ if (!IN6_IS_ADDR_LINKLOCAL(&from.sin6_addr)) { logtrace("ICMPv6 echo reply source address not linklocal from " "%s on %s\n", abuf, pii->pii_name); return; } opt = find_ancillary(&msg, IPPROTO_IPV6, IPV6_RTHDR); if (opt != NULL) { /* Can't allow routing headers in probe replies */ logtrace("message with routing header from %s on %s\n", abuf, pii->pii_name); return; } if (reply->pr_icmp_code != 0) { logtrace("probe reply code: %d from %s on %s\n", reply->pr_icmp_code, abuf, pii->pii_name); return; } if (len < (sizeof (struct pr_icmp))) { logtrace("probe reply too short: %d bytes from %s on %s\n", len, abuf, pii->pii_name); return; } recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP); if (recv_tvp == NULL) { logtrace("message without timestamp from %s on %s\n", abuf, pii->pii_name); return; } if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) { incoming_echo_reply(pii, reply, from.sin6_addr, recv_tvp); } else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) { incoming_mcast_reply(pii, reply, from.sin6_addr); } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) { incoming_rtt_reply(pii, reply, from.sin6_addr); } else { /* Probably not in response to our probe */ logtrace("probe reply type: %d from %s on %s\n", reply->pr_icmp_mtype, abuf, pii->pii_name); } } /* * Process the incoming rtt reply, in response to our rtt probe. * Common for both IPv4 and IPv6. Unlike incoming_echo_reply() we don't * have any stored information about the probe we sent. So we don't log * any errors if we receive bad replies. */ static void incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr) { int64_t m; /* rtt measurement in ns */ char abuf[INET6_ADDRSTRLEN]; struct target *target; struct phyint_group *pg; /* Get the printable address for error reporting */ (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)); if (debug & D_PROBE) { logdebug("incoming_rtt_reply: %s %s %s\n", AF_STR(pii->pii_af), pii->pii_name, abuf); } /* Do we know this target ? */ target = target_lookup(pii, fromaddr); if (target == NULL) return; m = (int64_t)(gethrtime() - ntohll(reply->pr_icmp_timestamp)); /* Invalid rtt. It has wrapped around */ if (m < 0) return; /* * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses * The initial few responses after the interface is repaired may * contain high rtt's because they could have been queued up waiting * for ARP/NDP resolution on a failed interface. */ pg = pii->pii_phyint->pi_group; if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg)) return; /* * Update rtt only if the new rtt is lower than the current rtt. * (specified by the 3rd parameter to pi_set_crtt). * If a spike has caused the current probe_interval to be > * user_probe_interval, then this mechanism is used to bring down * the rtt rapidly once the network stress is removed. * If the new rtt is higher than the current rtt, we don't want to * update the rtt. We are having more than 1 outstanding probe and * the increase in rtt we are seeing is being unnecessarily weighted * many times. The regular rtt update will be handled by * incoming_echo_reply() and will take care of any rtt increase. */ pi_set_crtt(target, m, _B_FALSE); if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) && (user_failure_detection_time < pg->pg_fdt) && (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) { /* * If the crtt has now dropped by a factor of LOWER_FT_TRIGGER, * investigate if we can improve the failure detection time to * meet whatever the user specified. */ if (check_pg_crtt_improved(pg)) { pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE, user_failure_detection_time); pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2); if (pii->pii_phyint->pi_group != phyint_anongroup) { logerr("Improved failure detection time %d ms " "on (%s %s) for group \"%s\"\n", pg->pg_fdt, AF_STR(pii->pii_af), pii->pii_name, pii->pii_phyint->pi_group->pg_name); } if (user_failure_detection_time == pg->pg_fdt) { /* Avoid any truncation or rounding errors */ pg->pg_probeint = user_probe_interval; /* * No more rtt probes will be sent. The actual * fdt has dropped to the user specified value. * pii_fd_snxt_basetime and pii_snxt_basetime * will be in sync henceforth. */ reset_snxt_basetimes(); } } } } /* * Process the incoming echo reply, in response to our unicast probe. * Common for both IPv4 and IPv6 */ static void incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp) { int64_t m; /* rtt measurement in ns */ hrtime_t cur_hrtime; /* in ns from some arbitrary point */ char abuf[INET6_ADDRSTRLEN]; int pr_ndx; struct target *target; boolean_t exception; uint64_t pr_icmp_timestamp; uint16_t pr_icmp_seq; struct probe_stats *pr_statp; struct phyint_group *pg = pii->pii_phyint->pi_group; /* Get the printable address for error reporting */ (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)); if (debug & D_PROBE) { logdebug("incoming_echo_reply: %s %s %s seq %u recv_tvp %lld\n", AF_STR(pii->pii_af), pii->pii_name, abuf, ntohs(reply->pr_icmp_seq), tv2ns(recv_tvp)); } pr_icmp_timestamp = ntohll(reply->pr_icmp_timestamp); pr_icmp_seq = ntohs(reply->pr_icmp_seq); /* Reject out of window probe replies */ if (SEQ_GE(pr_icmp_seq, pii->pii_snxt) || SEQ_LT(pr_icmp_seq, pii->pii_snxt - PROBE_STATS_COUNT)) { logtrace("out of window probe seq %u snxt %u on %s from %s\n", pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } cur_hrtime = gethrtime(); m = (int64_t)(cur_hrtime - pr_icmp_timestamp); if (m < 0) { /* * This is a ridiculously high value of rtt. rtt has wrapped * around. Log a message, and ignore the rtt. */ logerr("incoming_echo_reply: rtt wraparound cur_hrtime %lld " "reply timestamp %lld\n", cur_hrtime, pr_icmp_timestamp); } /* * Get the probe index pr_ndx corresponding to the received icmp seq. * number in our pii->pii_probes[] array. The icmp sequence number * pii_snxt corresponds to the probe index pii->pii_probe_next */ pr_ndx = MOD_SUB(pii->pii_probe_next, (uint16_t)(pii->pii_snxt - pr_icmp_seq), PROBE_STATS_COUNT); assert(PR_STATUS_VALID(pii->pii_probes[pr_ndx].pr_status)); target = pii->pii_probes[pr_ndx].pr_target; /* * Perform sanity checks, whether this probe reply that we * have received is genuine */ if (target != NULL) { /* * Compare the src. addr of the received ICMP or ICMPv6 * probe reply with the target address in our tables. */ if (!IN6_ARE_ADDR_EQUAL(&target->tg_address, &fromaddr)) { /* * We don't have any record of having sent a probe to * this target. This is a fake probe reply. Log an error */ logtrace("probe status %d Fake probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) { /* * The address matches, but our tables indicate that * this probe reply has been acked already. So this * is a duplicate probe reply. Log an error */ logtrace("probe status %d Duplicate probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); pii->pii_cum_stats.unknown++; return; } } else { /* * Target must not be NULL in the PR_UNACKED state */ assert(pii->pii_probes[pr_ndx].pr_status != PR_UNACKED); if (pii->pii_probes[pr_ndx].pr_status == PR_UNUSED) { /* * The probe stats slot is unused. So we didn't * send out any probe to this target. This is a fake. * Log an error. */ logtrace("probe status %d Fake probe reply seq %u " "snxt %u on %s from %s\n", pii->pii_probes[pr_ndx].pr_status, pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf); } pii->pii_cum_stats.unknown++; return; } /* * If the rtt does not appear to be right, don't update the * rtt stats. This can happen if the system dropped into the * debugger, or the system was hung or too busy for a * substantial time that we didn't get a chance to run. */ if ((m < 0) || (ns2ms(m) > PROBE_STATS_COUNT * pg->pg_probeint)) { /* * If the probe corresponding to this received response * was truly sent 'm' ns. ago, then this response must * have been rejected by the sequence number checks. The * fact that it has passed the sequence number checks * means that the measured rtt is wrong. We were probably * scheduled long after the packet was received. */ goto out; } /* * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses * The initial few responses after the interface is repaired may * contain high rtt's because they could have been queued up waiting * for ARP/NDP resolution on a failed interface. */ if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg)) goto out; /* * Don't update the Conservative Round Trip Time estimate for this * (phint, target) pair if this is the not the highest ack seq seen * thus far on this target. */ if (!highest_ack_tg(pr_icmp_seq, target)) goto out; /* * Always update the rtt. This is a failure detection probe * and we want to measure both increase / decrease in rtt. */ pi_set_crtt(target, m, _B_TRUE); /* * If the crtt exceeds the average time between probes, * investigate if this slow target is an exception. If so we * can avoid this target and still meet the failure detection * time. Otherwise we can't meet the failure detection time. */ if (target->tg_crtt > pg->pg_probeint) { exception = check_exception_target(pii, target); if (exception) { /* * This target is exceptionally slow. Don't use it * for future probes. check_exception_target() has * made sure that we have at least MIN_PROBE_TARGETS * other active targets */ if (pii->pii_targets_are_routers) { /* * This is a slow router, mark it as slow * and don't use it for further probes. We * don't delete it, since it will be populated * again when we do a router scan. Hence we * need to maintain extra state (unlike the * host case below). Mark it as TG_SLOW. */ if (target->tg_status == TG_ACTIVE) pii->pii_ntargets--; target->tg_status = TG_SLOW; target->tg_latime = gethrtime(); target->tg_rtt_sa = -1; target->tg_crtt = 0; target->tg_rtt_sd = 0; if (pii->pii_target_next == target) { pii->pii_target_next = target_next(target); } } else { /* * the slow target is not a router, we can * just delete it. Send an icmp multicast and * pick the fastest responder that is not * already an active target. target_delete() * adjusts pii->pii_target_next */ target_delete(target); probe(pii, PROBE_MULTI, cur_hrtime); } } else { /* * We can't meet the failure detection time. * Log a message, and update the detection time to * whatever we can achieve. */ pg->pg_probeint = target->tg_crtt * NEXT_FDT_MULTIPLE; pg->pg_fdt = pg->pg_probeint * (NUM_PROBE_FAILS + 2); last_fdt_bumpup_time = gethrtime(); if (pg != phyint_anongroup) { logtrace("Cannot meet requested failure" " detection time of %d ms on (%s %s) new" " failure detection time for group \"%s\"" " is %d ms\n", user_failure_detection_time, AF_STR(pii->pii_af), pii->pii_name, pg->pg_name, pg->pg_fdt); } } } else if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) && (user_failure_detection_time < pg->pg_fdt) && (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) { /* * If the crtt has now dropped by a factor of LOWER_FDT_TRIGGER * investigate if we can improve the failure detection time to * meet whatever the user specified. */ if (check_pg_crtt_improved(pg)) { pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE, user_failure_detection_time); pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2); if (pg != phyint_anongroup) { logtrace("Improved failure detection time %d ms" " on (%s %s) for group \"%s\"\n", pg->pg_fdt, AF_STR(pii->pii_af), pii->pii_name, pg->pg_name); } if (user_failure_detection_time == pg->pg_fdt) { /* Avoid any truncation or rounding errors */ pg->pg_probeint = user_probe_interval; /* * No more rtt probes will be sent. The actual * fdt has dropped to the user specified value. * pii_fd_snxt_basetime and pii_snxt_basetime * will be in sync henceforth. */ reset_snxt_basetimes(); } } } out: pr_statp = &pii->pii_probes[pr_ndx]; pr_statp->pr_hrtime_ackproc = cur_hrtime; pr_statp->pr_hrtime_ackrecv = pr_statp->pr_hrtime_sent + (tv2ns(recv_tvp) - tv2ns(&pr_statp->pr_tv_sent)); probe_chstate(pr_statp, pii, PR_ACKED); /* * Update pii->pii_rack, i.e. the sequence number of the last received * probe response, based on the echo reply we have received now, if * either of the following conditions are satisfied. * a. pii_rack is outside the current receive window of * [pii->pii_snxt - PROBE_STATS_COUNT, pii->pii_snxt). * This means we have not received probe responses for a * long time, and the sequence number has wrapped around. * b. pii_rack is within the current receive window and this echo * reply corresponds to the highest sequence number we have seen * so far. */ if (SEQ_GE(pii->pii_rack, pii->pii_snxt) || SEQ_LT(pii->pii_rack, pii->pii_snxt - PROBE_STATS_COUNT) || SEQ_GT(pr_icmp_seq, pii->pii_rack)) { pii->pii_rack = pr_icmp_seq; } } /* * Returns true if seq is the highest unacknowledged seq for target tg * else returns false */ static boolean_t highest_ack_tg(uint16_t seq, struct target *tg) { struct phyint_instance *pii; int pr_ndx; uint16_t pr_seq; pii = tg->tg_phyint_inst; /* * Get the seq number of the most recent probe sent so far, * and also get the corresponding probe index in the probe stats * array. */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); pr_seq = pii->pii_snxt; pr_seq--; /* * Start from the most recent probe and walk back, trying to find * an acked probe corresponding to target tg. */ for (; pr_ndx != pii->pii_probe_next; pr_ndx = PROBE_INDEX_PREV(pr_ndx), pr_seq--) { if (pii->pii_probes[pr_ndx].pr_target == tg && pii->pii_probes[pr_ndx].pr_status == PR_ACKED) { if (SEQ_GT(pr_seq, seq)) return (_B_FALSE); } } return (_B_TRUE); } /* * Check whether the crtt for the group has improved by a factor of * LOWER_FDT_TRIGGER. Small crtt improvements are ignored to avoid failure * detection time flapping in the face of small crtt changes. */ static boolean_t check_pg_crtt_improved(struct phyint_group *pg) { struct phyint *pi; if (debug & D_PROBE) logdebug("check_pg_crtt_improved()\n"); /* * The crtt for the group is only improved if each phyint_instance * for both ipv4 and ipv6 is improved. */ for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) { if (!check_pii_crtt_improved(pi->pi_v4) || !check_pii_crtt_improved(pi->pi_v6)) return (_B_FALSE); } return (_B_TRUE); } /* * Check whether the crtt has improved substantially on this phyint_instance. * Returns _B_TRUE if there's no crtt information available, because pii * is NULL or the phyint_instance is not capable of probing. */ boolean_t check_pii_crtt_improved(struct phyint_instance *pii) { struct target *tg; if (pii == NULL) return (_B_TRUE); if (!PROBE_CAPABLE(pii) || pii->pii_phyint->pi_state == PI_FAILED) return (_B_TRUE); for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { if (tg->tg_status != TG_ACTIVE) continue; if (tg->tg_crtt > (pii->pii_phyint->pi_group->pg_probeint / LOWER_FDT_TRIGGER)) { return (_B_FALSE); } } return (_B_TRUE); } /* * This target responds very slowly to probes. The target's crtt exceeds * the probe interval of its group. Compare against other targets * and determine if this target is an exception, if so return true, else false */ static boolean_t check_exception_target(struct phyint_instance *pii, struct target *target) { struct target *tg; char abuf[INET6_ADDRSTRLEN]; if (debug & D_PROBE) { logdebug("check_exception_target(%s %s target %s)\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, target->tg_address, abuf, sizeof (abuf))); } /* * We should have at least MIN_PROBE_TARGETS + 1 good targets now, * to make a good judgement. Otherwise don't drop this target. */ if (pii->pii_ntargets < MIN_PROBE_TARGETS + 1) return (_B_FALSE); /* * Determine whether only this particular target is slow. * We know that this target's crtt exceeds the group's probe interval. * If all other active targets have a * crtt < (this group's probe interval) / EXCEPTION_FACTOR, * then this target is considered slow. */ for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { if (tg != target && tg->tg_status == TG_ACTIVE) { if (tg->tg_crtt > pii->pii_phyint->pi_group->pg_probeint / EXCEPTION_FACTOR) { return (_B_FALSE); } } } return (_B_TRUE); } /* * Update the target list. The icmp all hosts multicast has given us * some host to which we can send probes. If we already have sufficient * targets, discard it. */ static void incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply, struct in6_addr fromaddr) /* ARGSUSED */ { int af; char abuf[INET6_ADDRSTRLEN]; struct phyint *pi; if (debug & D_PROBE) { logdebug("incoming_mcast_reply(%s %s %s)\n", AF_STR(pii->pii_af), pii->pii_name, pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf))); } /* * Using host targets is a fallback mechanism. If we have * found a router, don't add this host target. If we already * know MAX_PROBE_TARGETS, don't add another target. */ assert(pii->pii_ntargets <= MAX_PROBE_TARGETS); if (pii->pii_targets != NULL) { if (pii->pii_targets_are_routers || (pii->pii_ntargets == MAX_PROBE_TARGETS)) { return; } } if (IN6_IS_ADDR_UNSPECIFIED(&fromaddr) || IN6_IS_ADDR_V4MAPPED_ANY(&fromaddr)) { /* * Guard against response from 0.0.0.0 * and ::. Log a trace message */ logtrace("probe response from %s on %s\n", pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)), pii->pii_name); return; } /* * This address is one of our own, so reject this address as a * valid probe target. */ af = pii->pii_af; if (own_address(fromaddr)) return; /* * If the phyint is part a named group, then add the address to all * members of the group. Otherwise, add the address only to the * phyint itself, since other phyints in the anongroup may not be on * the same subnet. */ pi = pii->pii_phyint; if (pi->pi_group == phyint_anongroup) { target_add(pii, fromaddr, _B_FALSE); } else { pi = pi->pi_group->pg_phyint; for (; pi != NULL; pi = pi->pi_pgnext) target_add(PHYINT_INSTANCE(pi, af), fromaddr, _B_FALSE); } } /* * Compute CRTT given an existing scaled average, scaled deviation estimate * and a new rtt time. The formula is from Jacobson and Karels' * "Congestion Avoidance and Control" in SIGCOMM '88. The variable names * are the same as those in Appendix A.2 of that paper. * * m = new measurement * sa = scaled RTT average (8 * average estimates) * sv = scaled mean deviation (mdev) of RTT (4 * deviation estimates). * crtt = Conservative round trip time. Used to determine whether probe * has timed out. * * New scaled average and deviation are passed back via sap and svp */ static int64_t compute_crtt(int64_t *sap, int64_t *svp, int64_t m) { int64_t sa = *sap; int64_t sv = *svp; int64_t crtt; int64_t saved_m = m; assert(*sap >= -1); assert(*svp >= 0); if (sa != -1) { /* * Update average estimator: * new rtt = old rtt + 1/8 Error * where Error = m - old rtt * i.e. 8 * new rtt = 8 * old rtt + Error * i.e. new sa = old sa + Error */ m -= sa >> 3; /* m is now Error in estimate. */ if ((sa += m) < 0) { /* Don't allow the smoothed average to be negative. */ sa = 0; } /* * Update deviation estimator: * new mdev = old mdev + 1/4 (abs(Error) - old mdev) * i.e. 4 * new mdev = 4 * old mdev + * (abs(Error) - old mdev) * i.e. new sv = old sv + (abs(Error) - old mdev) */ if (m < 0) m = -m; m -= sv >> 2; sv += m; } else { /* Initialization. This is the first response received. */ sa = (m << 3); sv = (m << 1); } crtt = (sa >> 3) + sv; if (debug & D_PROBE) { logerr("compute_crtt: m = %lld sa = %lld, sv = %lld -> " "crtt = %lld\n", saved_m, sa, sv, crtt); } *sap = sa; *svp = sv; /* * CRTT = average estimates + 4 * deviation estimates * = sa / 8 + sv */ return (crtt); } static void pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni) { struct phyint_instance *pii = tg->tg_phyint_inst; int probe_interval = pii->pii_phyint->pi_group->pg_probeint; int64_t sa = tg->tg_rtt_sa; int64_t sv = tg->tg_rtt_sd; int new_crtt; int i; if (debug & D_PROBE) logdebug("pi_set_crtt: target - m %lld\n", m); /* store the round trip time, in case we need to defer computation */ tg->tg_deferred[tg->tg_num_deferred] = m; new_crtt = ns2ms(compute_crtt(&sa, &sv, m)); /* * If this probe's round trip time would singlehandedly cause an * increase in the group's probe interval consider it suspect. */ if ((new_crtt > probe_interval) && is_probe_uni) { if (debug & D_PROBE) { logdebug("Received a suspect probe on %s, new_crtt =" " %d, probe_interval = %d, num_deferred = %d\n", pii->pii_probe_logint->li_name, new_crtt, probe_interval, tg->tg_num_deferred); } /* * If we've deferred as many rtts as we plan on deferring, then * assume the link really did slow down and process all queued * rtts */ if (tg->tg_num_deferred == MAXDEFERREDRTT) { if (debug & D_PROBE) { logdebug("Received MAXDEFERREDRTT probes which " "would cause an increased probe_interval. " "Integrating queued rtt data points.\n"); } for (i = 0; i <= tg->tg_num_deferred; i++) { tg->tg_crtt = ns2ms(compute_crtt(&tg->tg_rtt_sa, &tg->tg_rtt_sd, tg->tg_deferred[i])); } tg->tg_num_deferred = 0; } else { tg->tg_num_deferred++; } return; } /* * If this is a normal probe, or an RTT probe that would lead to a * reduced CRTT, then update our CRTT data. Further, if this was * a normal probe, pitch any deferred probes since our probes are * again being answered within our CRTT estimates. */ if (is_probe_uni || new_crtt < tg->tg_crtt) { tg->tg_rtt_sa = sa; tg->tg_rtt_sd = sv; tg->tg_crtt = new_crtt; if (is_probe_uni) tg->tg_num_deferred = 0; } } /* * Return a pointer to the specified option buffer. * If not found return NULL. */ static void * find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type) { struct cmsghdr *cmsg; for (cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL; cmsg = CMSG_NXTHDR(msg, cmsg)) { if (cmsg->cmsg_level == cmsg_level && cmsg->cmsg_type == cmsg_type) { return (CMSG_DATA(cmsg)); } } return (NULL); } /* * Try to activate another INACTIVE interface in the same group as `pi'. * Prefer STANDBY INACTIVE to just INACTIVE. */ void phyint_activate_another(struct phyint *pi) { struct phyint *pi2; struct phyint *inactivepi = NULL; if (pi->pi_group == phyint_anongroup) return; for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (pi == pi2 || !phyint_is_functioning(pi2) || !(pi2->pi_flags & IFF_INACTIVE)) continue; inactivepi = pi2; if (pi2->pi_flags & IFF_STANDBY) break; } if (inactivepi != NULL) (void) change_pif_flags(inactivepi, 0, IFF_INACTIVE); } /* * Transition a phyint to PI_RUNNING. The caller must ensure that the * transition is appropriate. Clears IFF_OFFLINE or IFF_FAILED if * appropriate. Also sets IFF_INACTIVE on this or other interfaces as * appropriate (see comment below). Finally, also updates the phyint's group * state to account for the change. */ void phyint_transition_to_running(struct phyint *pi) { struct phyint *pi2; struct phyint *actstandbypi = NULL; uint_t nactive = 0, nnonstandby = 0; boolean_t onlining = (pi->pi_state == PI_OFFLINE); boolean_t initial = (pi->pi_state == PI_INIT); uint64_t set, clear; /* * The interface is running again, but should it or another interface * in the group end up INACTIVE? There are three cases: * * 1. If it's a STANDBY interface, it should be end up INACTIVE if * the group is operating at capacity (i.e., there are at least as * many active interfaces as non-STANDBY interfaces in the group). * No other interfaces should be changed. * * 2. If it's a non-STANDBY interface and we're onlining it or * FAILBACK is enabled, then it should *not* end up INACTIVE. * Further, if the group is above capacity as a result of this * interface, then an active STANDBY interface in the group should * end up INACTIVE. * * 3. If it's a non-STANDBY interface, we're repairing it, and * FAILBACK is disabled, then it should end up INACTIVE *unless* * the group was failed (in which case we have no choice but to * use it). No other interfaces should be changed. */ if (pi->pi_group != phyint_anongroup) { pi2 = pi->pi_group->pg_phyint; for (; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (!(pi2->pi_flags & IFF_STANDBY)) nnonstandby++; if (phyint_is_functioning(pi2) && !(pi2->pi_flags & IFF_INACTIVE)) { nactive++; if (pi2->pi_flags & IFF_STANDBY) actstandbypi = pi2; } } } set = 0; clear = (onlining ? IFF_OFFLINE : IFF_FAILED); if (pi->pi_flags & IFF_STANDBY) { /* case 1 */ if (nactive >= nnonstandby) set |= IFF_INACTIVE; else clear |= IFF_INACTIVE; } else if (onlining || failback_enabled) { /* case 2 */ if (nactive >= nnonstandby && actstandbypi != NULL) (void) change_pif_flags(actstandbypi, IFF_INACTIVE, 0); } else if (!initial && !GROUP_FAILED(pi->pi_group)) { /* case 3 */ set |= IFF_INACTIVE; } (void) change_pif_flags(pi, set, clear); phyint_chstate(pi, PI_RUNNING); /* * Update the group state to account for the change. */ phyint_group_refresh_state(pi->pi_group); } /* * Adjust IFF_INACTIVE on the provided `pi' to trend the group configuration * to have at least one active interface and as many active interfaces as * non-standby interfaces. */ void phyint_standby_refresh_inactive(struct phyint *pi) { struct phyint *pi2; uint_t nactive = 0, nnonstandby = 0; /* * All phyints in the anonymous group are effectively in their own * group and thus active regardless of whether they're marked standby. */ if (pi->pi_group == phyint_anongroup) { (void) change_pif_flags(pi, 0, IFF_INACTIVE); return; } /* * If the phyint isn't functioning we can't consider it. */ if (!phyint_is_functioning(pi)) return; for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { if (!(pi2->pi_flags & IFF_STANDBY)) nnonstandby++; if (phyint_is_functioning(pi2) && !(pi2->pi_flags & IFF_INACTIVE)) nactive++; } if (nactive == 0 || nactive < nnonstandby) (void) change_pif_flags(pi, 0, IFF_INACTIVE); else if (nactive > nnonstandby) (void) change_pif_flags(pi, IFF_INACTIVE, 0); } /* * See if a previously failed interface has started working again. */ void phyint_check_for_repair(struct phyint *pi) { if (!phyint_repaired(pi)) return; if (pi->pi_group == phyint_anongroup) { logerr("IP interface repair detected on %s\n", pi->pi_name); } else { logerr("IP interface repair detected on %s of group %s\n", pi->pi_name, pi->pi_group->pg_name); } /* * If the interface is PI_OFFLINE, it can't be made PI_RUNNING yet. * So just clear IFF_OFFLINE and defer phyint_transition_to_running() * until it is brought back online. */ if (pi->pi_state == PI_OFFLINE) { (void) change_pif_flags(pi, 0, IFF_FAILED); return; } phyint_transition_to_running(pi); /* calls phyint_chstate() */ } /* * See if an interface has failed, or if the whole group of interfaces has * failed. */ static void phyint_inst_check_for_failure(struct phyint_instance *pii) { struct phyint *pi = pii->pii_phyint; struct phyint *pi2; boolean_t was_active; switch (failure_state(pii)) { case PHYINT_FAILURE: was_active = ((pi->pi_flags & IFF_INACTIVE) == 0); (void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE); if (pi->pi_group == phyint_anongroup) { logerr("IP interface failure detected on %s\n", pii->pii_name); } else { logerr("IP interface failure detected on %s of group" " %s\n", pii->pii_name, pi->pi_group->pg_name); } /* * If the failed interface was active, activate another * INACTIVE interface in the group if possible. */ if (was_active) phyint_activate_another(pi); /* * If the interface is offline, the state change will be * noted when it comes back online. */ if (pi->pi_state != PI_OFFLINE) { phyint_chstate(pi, PI_FAILED); reset_crtt_all(pi); } break; case GROUP_FAILURE: pi2 = pi->pi_group->pg_phyint; for (; pi2 != NULL; pi2 = pi2->pi_pgnext) { (void) change_pif_flags(pi2, IFF_FAILED, IFF_INACTIVE); if (pi2->pi_state == PI_OFFLINE) /* see comment above */ continue; reset_crtt_all(pi2); /* * In the case of host targets, we would have flushed * the targets, and gone to PI_NOTARGETS state. */ if (pi2->pi_state == PI_RUNNING) phyint_chstate(pi2, PI_FAILED); } break; default: break; } } /* * Determines if any timeout event has occurred and returns the number of * milliseconds until the next timeout event for the phyint. Returns * TIMER_INFINITY for "never". */ uint_t phyint_inst_timer(struct phyint_instance *pii) { int pr_ndx; uint_t timeout; struct target *cur_tg; struct probe_stats *pr_statp; struct phyint_instance *pii_other; struct phyint *pi; int valid_unack_count; int i; int interval; uint_t check_time; uint_t cur_time; hrtime_t cur_hrtime; int probe_interval = pii->pii_phyint->pi_group->pg_probeint; cur_hrtime = gethrtime(); cur_time = ns2ms(cur_hrtime); if (debug & D_TIMER) { logdebug("phyint_inst_timer(%s %s)\n", AF_STR(pii->pii_af), pii->pii_name); } pii_other = phyint_inst_other(pii); if (!PROBE_ENABLED(pii) && !PROBE_ENABLED(pii_other)) { /* * Check to see if we're here due to link up/down flapping; If * enough time has passed, then try to bring the interface * back up; otherwise, schedule a timer to bring it back up * when enough time *has* elapsed. */ pi = pii->pii_phyint; if (pi->pi_state == PI_FAILED && LINK_UP(pi)) { check_time = pi->pi_whenup[pi->pi_whendx] + MSEC_PERMIN; if (check_time > cur_time) return (check_time - cur_time); phyint_check_for_repair(pi); } } /* * If probing is not enabled on this phyint instance, don't proceed. */ if (!PROBE_ENABLED(pii)) return (TIMER_INFINITY); /* * If the timer has fired too soon, probably triggered * by some other phyint instance, return the remaining * time */ if (TIME_LT(cur_time, pii->pii_snxt_time)) return (pii->pii_snxt_time - cur_time); /* * If the link is down, don't send any probes for now. */ if (LINK_DOWN(pii->pii_phyint)) return (TIMER_INFINITY); /* * Randomize the next probe time, between MIN_RANDOM_FACTOR * and MAX_RANDOM_FACTOR with respect to the base probe time. * Base probe time is strictly periodic. */ interval = GET_RANDOM( (int)(MIN_RANDOM_FACTOR * user_probe_interval), (int)(MAX_RANDOM_FACTOR * user_probe_interval)); pii->pii_snxt_time = pii->pii_snxt_basetime + interval; /* * Check if the current time > next time to probe. If so, we missed * sending 1 or more probes, probably due to heavy system load. At least * 'MIN_RANDOM_FACTOR * user_probe_interval' ms has elapsed since we * were scheduled. Make adjustments to the times, in multiples of * user_probe_interval. */ if (TIME_GT(cur_time, pii->pii_snxt_time)) { int n; n = (cur_time - pii->pii_snxt_time) / user_probe_interval; pii->pii_snxt_time += (n + 1) * user_probe_interval; pii->pii_snxt_basetime += (n + 1) * user_probe_interval; logtrace("missed sending %d probes cur_time %u snxt_time %u" " snxt_basetime %u\n", n + 1, cur_time, pii->pii_snxt_time, pii->pii_snxt_basetime); /* Collect statistics about missed probes */ probes_missed.pm_nprobes += n + 1; probes_missed.pm_ntimes++; } pii->pii_snxt_basetime += user_probe_interval; interval = pii->pii_snxt_time - cur_time; if (debug & D_TARGET) { logdebug("cur_time %u snxt_time %u snxt_basetime %u" " interval %u\n", cur_time, pii->pii_snxt_time, pii->pii_snxt_basetime, interval); } /* * If no targets are known, we need to send an ICMP multicast. The * probe type is PROBE_MULTI. We'll check back in 'interval' msec * to see if we found a target. */ if (pii->pii_target_next == NULL) { assert(pii->pii_ntargets == 0); pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; probe(pii, PROBE_MULTI, cur_time); return (interval); } if ((user_probe_interval != probe_interval) && TIME_LT(pii->pii_snxt_time, pii->pii_fd_snxt_basetime)) { /* * the failure detection (fd) probe timer has not yet fired. * Need to send only an rtt probe. The probe type is PROBE_RTT. */ probe(pii, PROBE_RTT, cur_hrtime); return (interval); } /* * the fd probe timer has fired. Need to do all failure * detection / recovery calculations, and then send an fd probe * of type PROBE_UNI. */ if (user_probe_interval == probe_interval) { /* * We could have missed some probes, and then adjusted * pii_snxt_basetime above. Otherwise we could have * blindly added probe_interval to pii_fd_snxt_basetime. */ pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; } else { pii->pii_fd_snxt_basetime += probe_interval; if (TIME_GT(cur_time, pii->pii_fd_snxt_basetime)) { int n; n = (cur_time - pii->pii_fd_snxt_basetime) / probe_interval; pii->pii_fd_snxt_basetime += (n + 1) * probe_interval; } } /* * We can have at most, the latest 2 probes that we sent, in * the PR_UNACKED state. All previous probes sent, are either * PR_LOST or PR_ACKED. An unacknowledged probe is considered * timed out if the probe's time_start + the CRTT < currenttime. * For each of the last 2 probes, examine whether it has timed * out. If so, mark it PR_LOST. The probe stats is a circular array. */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); valid_unack_count = 0; for (i = 0; i < 2; i++) { pr_statp = &pii->pii_probes[pr_ndx]; cur_tg = pii->pii_probes[pr_ndx].pr_target; switch (pr_statp->pr_status) { case PR_ACKED: /* * We received back an ACK, so the switch clearly * is not dropping our traffic, and thus we can * enable failure detection immediately. */ if (pii->pii_fd_hrtime > gethrtime()) { if (debug & D_PROBE) { logdebug("successful probe on %s; " "ending quiet period\n", pii->pii_phyint->pi_name); } pii->pii_fd_hrtime = gethrtime(); } break; case PR_UNACKED: assert(cur_tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use group's probe interval, * which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (cur_tg->tg_crtt != 0) { timeout += cur_tg->tg_crtt; } else { timeout += probe_interval; } if (TIME_LT(timeout, cur_time)) { pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); } else if (i == 1) { /* * We are forced to consider this probe * lost, as we can have at most 2 unack. * probes any time, and we will be sending a * probe at the end of this function. * Normally, we should not be here, but * this can happen if an incoming response * that was considered lost has increased * the crtt for this target, and also bumped * up the FDT. Note that we never cancel or * increase the current pii_time_left, so * when the timer fires, we find 2 valid * unacked probes, and they are yet to timeout */ pr_statp->pr_time_lost = cur_time; probe_chstate(pr_statp, pii, PR_LOST); } else { /* * Only the most recent probe can enter * this 'else' arm. The second most recent * probe must take either of the above arms, * if it is unacked. */ valid_unack_count++; } break; } pr_ndx = PROBE_INDEX_PREV(pr_ndx); } /* * We send out 1 probe randomly in the interval between one half * and one probe interval for the group. Given that the CRTT is always * less than the group's probe interval, we can have at most 1 * unacknowledged probe now. All previous probes are either lost or * acked. */ assert(valid_unack_count == 0 || valid_unack_count == 1); /* * The timer has fired. Take appropriate action depending * on the current state of the phyint. * * PI_RUNNING state - Failure detection * PI_FAILED state - Repair detection */ switch (pii->pii_phyint->pi_state) { case PI_FAILED: /* * If the most recent probe (excluding unacked probes that * are yet to time out) has been acked, check whether the * phyint is now repaired. */ if (pii->pii_rack + valid_unack_count + 1 == pii->pii_snxt) { phyint_check_for_repair(pii->pii_phyint); } break; case PI_RUNNING: /* * It's possible our probes have been lost because of a * spanning-tree mandated quiet period on the switch. If so, * ignore the lost probes. */ if (pii->pii_fd_hrtime - cur_hrtime > 0) break; if (pii->pii_rack + valid_unack_count + 1 != pii->pii_snxt) { /* * We have 1 or more failed probes (excluding unacked * probes that are yet to time out). Determine if the * phyint has failed. */ phyint_inst_check_for_failure(pii); } break; default: logerr("phyint_inst_timer: invalid state %d\n", pii->pii_phyint->pi_state); abort(); } /* * Start the next probe. probe() will also set pii->pii_probe_time_left * to the group's probe interval. If phyint_failed -> target_flush_hosts * was called, the target list may be empty. */ if (pii->pii_target_next != NULL) { probe(pii, PROBE_UNI, cur_hrtime); /* * If we have just the one probe target, and we're not using * router targets, try to find another as we presently have * no resilience. */ if (!pii->pii_targets_are_routers && pii->pii_ntargets == 1) probe(pii, PROBE_MULTI, cur_hrtime); } else { probe(pii, PROBE_MULTI, cur_hrtime); } return (interval); } /* * Start the probe timer for an interface instance. */ void start_timer(struct phyint_instance *pii) { uint32_t interval; /* * Spread the base probe times (pi_snxt_basetime) across phyints * uniformly over the (curtime..curtime + the group's probe_interval). * pi_snxt_basetime is strictly periodic with a frequency of * the group's probe interval. The actual probe time pi_snxt_time * adds some randomness to pi_snxt_basetime and happens in probe(). * For the 1st probe on each phyint after the timer is started, * pi_snxt_time and pi_snxt_basetime are the same. */ interval = GET_RANDOM(0, (int)pii->pii_phyint->pi_group->pg_probeint); pii->pii_snxt_basetime = getcurrenttime() + interval; pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; pii->pii_snxt_time = pii->pii_snxt_basetime; timer_schedule(interval); } /* * Restart the probe timer on an interface instance. */ static void restart_timer(struct phyint_instance *pii) { /* * We don't need to restart the timer if it was never started in * the first place (pii->pii_basetime_inited not set), as the timer * won't have gone off yet. */ if (pii->pii_basetime_inited != 0) { if (debug & D_LINKNOTE) logdebug("restart timer: restarting timer on %s, " "address family %s\n", pii->pii_phyint->pi_name, AF_STR(pii->pii_af)); start_timer(pii); } } static void process_link_state_down(struct phyint *pi) { logerr("The link has gone down on %s\n", pi->pi_name); /* * Clear the probe statistics arrays, we don't want the repair * detection logic relying on probes that were successful prior * to the link going down. */ if (PROBE_CAPABLE(pi->pi_v4)) clear_pii_probe_stats(pi->pi_v4); if (PROBE_CAPABLE(pi->pi_v6)) clear_pii_probe_stats(pi->pi_v6); /* * Check for interface failure. Although we know the interface * has failed, we don't know if all the other interfaces in the * group have failed as well. */ if ((pi->pi_state == PI_RUNNING) || (pi->pi_state != PI_FAILED && !GROUP_FAILED(pi->pi_group))) { if (debug & D_LINKNOTE) { logdebug("process_link_state_down:" " checking for failure on %s\n", pi->pi_name); } if (pi->pi_v4 != NULL) phyint_inst_check_for_failure(pi->pi_v4); else if (pi->pi_v6 != NULL) phyint_inst_check_for_failure(pi->pi_v6); } } static void process_link_state_up(struct phyint *pi) { logerr("The link has come up on %s\n", pi->pi_name); /* * We stopped any running timers on each instance when the link * went down, so restart them. */ if (pi->pi_v4) restart_timer(pi->pi_v4); if (pi->pi_v6) restart_timer(pi->pi_v6); phyint_check_for_repair(pi); pi->pi_whenup[pi->pi_whendx++] = getcurrenttime(); if (pi->pi_whendx == LINK_UP_PERMIN) pi->pi_whendx = 0; } /* * Process any changes in link state passed up from the interfaces. */ void process_link_state_changes(void) { struct phyint *pi; /* Look for interfaces where the link state has just changed */ for (pi = phyints; pi != NULL; pi = pi->pi_next) { boolean_t old_link_state_up = LINK_UP(pi); /* * Except when the "phyint" structure is created, this is * the only place the link state is updated. This allows * this routine to detect changes in link state, rather * than just the current state. */ UPDATE_LINK_STATE(pi); if (LINK_DOWN(pi)) { /* * Has link just gone down? */ if (old_link_state_up) process_link_state_down(pi); } else { /* * Has link just gone back up? */ if (!old_link_state_up) process_link_state_up(pi); } } } void reset_crtt_all(struct phyint *pi) { struct phyint_instance *pii; struct target *tg; pii = pi->pi_v4; if (pii != NULL) { for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { tg->tg_crtt = 0; tg->tg_rtt_sa = -1; tg->tg_rtt_sd = 0; } } pii = pi->pi_v6; if (pii != NULL) { for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) { tg->tg_crtt = 0; tg->tg_rtt_sa = -1; tg->tg_rtt_sd = 0; } } } /* * Check if the phyint has failed the last NUM_PROBE_FAILS consecutive * probes on both instances IPv4 and IPv6. * If the interface has failed, return the time of the first probe failure * in "tff". */ static int phyint_inst_probe_failure_state(struct phyint_instance *pii, uint_t *tff) { uint_t pi_tff; struct target *cur_tg; struct probe_fail_count pfinfo; struct phyint_instance *pii_other; int pr_ndx; /* * Get the number of consecutive failed probes on * this phyint across all targets. Also get the number * of consecutive failed probes on this target only */ pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_fail_info(pii, cur_tg, &pfinfo); /* Get the time of first failure, for later use */ pi_tff = pfinfo.pf_tff; /* * If the current target has not responded to the * last NUM_PROBE_FAILS probes, and other targets are * responding delete this target. Dead gateway detection * will eventually remove this target (if router) from the * routing tables. If that does not occur, we may end * up adding this to our list again. */ if (pfinfo.pf_nfail < NUM_PROBE_FAILS && pfinfo.pf_nfail_tg >= NUM_PROBE_FAILS) { if (pii->pii_targets_are_routers) { if (cur_tg->tg_status == TG_ACTIVE) pii->pii_ntargets--; cur_tg->tg_status = TG_DEAD; cur_tg->tg_crtt = 0; cur_tg->tg_rtt_sa = -1; cur_tg->tg_rtt_sd = 0; if (pii->pii_target_next == cur_tg) pii->pii_target_next = target_next(cur_tg); } else { target_delete(cur_tg); probe(pii, PROBE_MULTI, gethrtime()); } return (PHYINT_OK); } /* * If the phyint has lost NUM_PROBE_FAILS or more * consecutive probes, on both IPv4 and IPv6 protocol * instances of the phyint, then trigger failure * detection, else return false */ if (pfinfo.pf_nfail < NUM_PROBE_FAILS) return (PHYINT_OK); pii_other = phyint_inst_other(pii); if (PROBE_CAPABLE(pii_other)) { probe_fail_info(pii_other, NULL, &pfinfo); if (pfinfo.pf_nfail >= NUM_PROBE_FAILS) { /* * We have NUM_PROBE_FAILS or more failures * on both IPv4 and IPv6. Get the earliest * time when failure was detected on this * phyint across IPv4 and IPv6. */ if (TIME_LT(pfinfo.pf_tff, pi_tff)) pi_tff = pfinfo.pf_tff; } else { /* * This instance has < NUM_PROBE_FAILS failure. * So return false */ return (PHYINT_OK); } } *tff = pi_tff; return (PHYINT_FAILURE); } /* * Check if the link has gone down on this phyint, or it has failed the * last NUM_PROBE_FAILS consecutive probes on both instances IPv4 and IPv6. * Also look at other phyints of this group, for group failures. */ int failure_state(struct phyint_instance *pii) { struct probe_success_count psinfo; uint_t pi2_tls; /* time last success */ uint_t pi_tff; /* time first fail */ struct phyint *pi2; struct phyint *pi; struct phyint_instance *pii2; struct phyint_group *pg; int retval; if (debug & D_FAILREP) logdebug("phyint_failed(%s)\n", pii->pii_name); pi = pii->pii_phyint; pg = pi->pi_group; if (LINK_UP(pi) && phyint_inst_probe_failure_state(pii, &pi_tff) == PHYINT_OK) return (PHYINT_OK); /* * At this point, the link is down, or the phyint is suspect, as it * has lost NUM_PROBE_FAILS or more probes. If the phyint does not * belong to any group, this is a PHYINT_FAILURE. Otherwise, continue * on to determine whether this should be considered a PHYINT_FAILURE * or GROUP_FAILURE. */ if (pg == phyint_anongroup) return (PHYINT_FAILURE); /* * Need to compare against other phyints of the same group * to exclude group failures. If the failure was detected via * probing, then if the time of last success (tls) of any * phyint is more recent than the time of first fail (tff) of the * phyint in question, and the link is up on the phyint, * then it is a phyint failure. Otherwise it is a group failure. * If failure was detected via a link down notification sent from * the driver to IP, we see if any phyints in the group are still * running and haven't received a link down notification. We * will usually be processing the link down notification shortly * after it was received, so there is no point looking at the tls * of other phyints. */ retval = GROUP_FAILURE; for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) { /* Exclude ourself from comparison */ if (pi2 == pi) continue; if (LINK_DOWN(pi)) { /* * We use FLAGS_TO_LINK_STATE() to test the flags * directly, rather then LINK_UP() or LINK_DOWN(), as * we may not have got round to processing the link * state for the other phyints in the group yet. * * The check for PI_RUNNING and group failure handles * the case when the group begins to recover. * PI_RUNNING will be set, and group failure cleared * only after receipt of NUM_PROBE_REPAIRS, by which * time the other phyints should have received at * least 1 packet, and so will not have NUM_PROBE_FAILS. */ if ((pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } continue; } if (LINK_DOWN(pi2)) continue; /* * If there's no probe-based failure detection on this * interface, and its link is still up, then it's still * working and thus the group has not failed. */ if (!PROBE_ENABLED(pi2->pi_v4) && !PROBE_ENABLED(pi2->pi_v6)) { retval = PHYINT_FAILURE; break; } /* * Need to compare against both IPv4 and IPv6 instances. */ pii2 = pi2->pi_v4; if (pii2 != NULL) { probe_success_info(pii2, NULL, &psinfo); if (psinfo.ps_tls_valid) { pi2_tls = psinfo.ps_tls; /* * See comment above regarding check * for PI_RUNNING and group failure. */ if (TIME_GT(pi2_tls, pi_tff) && (pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } } } pii2 = pi2->pi_v6; if (pii2 != NULL) { probe_success_info(pii2, NULL, &psinfo); if (psinfo.ps_tls_valid) { pi2_tls = psinfo.ps_tls; /* * See comment above regarding check * for PI_RUNNING and group failure. */ if (TIME_GT(pi2_tls, pi_tff) && (pi2->pi_state == PI_RUNNING) && !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) { retval = PHYINT_FAILURE; break; } } } } /* * Update the group state to account for the changes. */ phyint_group_refresh_state(pg); return (retval); } /* * Return the information associated with consecutive probe successes * starting with the most recent probe. At most the last 2 probes can be * in the unacknowledged state. All previous probes have either failed * or succeeded. */ static void probe_success_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_success_count *psinfo) { uint_t i; struct probe_stats *pr_statp; uint_t most_recent; uint_t second_most_recent; boolean_t pi_found_failure = _B_FALSE; boolean_t tg_found_failure = _B_FALSE; uint_t now; uint_t timeout; struct target *tg; if (debug & D_FAILREP) logdebug("probe_success_info(%s)\n", pii->pii_name); bzero(psinfo, sizeof (*psinfo)); now = getcurrenttime(); /* * Start with the most recent probe, and count the number * of consecutive probe successes. Latch the number of successes * on hitting a failure. */ most_recent = PROBE_INDEX_PREV(pii->pii_probe_next); second_most_recent = PROBE_INDEX_PREV(most_recent); for (i = most_recent; i != pii->pii_probe_next; i = PROBE_INDEX_PREV(i)) { pr_statp = &pii->pii_probes[i]; switch (pr_statp->pr_status) { case PR_UNACKED: /* * Only the most recent 2 probes can be unacknowledged */ assert(i == most_recent || i == second_most_recent); tg = pr_statp->pr_target; assert(tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use the value of the group's probe * interval which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (tg->tg_crtt != 0) { timeout += tg->tg_crtt; } else { timeout += pii->pii_phyint->pi_group->pg_probeint; } if (TIME_LT(timeout, now)) { /* * We hit a failure. Latch the total number of * recent consecutive successes. */ pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); pi_found_failure = _B_TRUE; if (cur_tg != NULL && tg == cur_tg) { /* * We hit a failure for the desired * target. Latch the number of recent * consecutive successes for this target */ tg_found_failure = _B_TRUE; } } break; case PR_ACKED: /* * Bump up the count of probe successes, if we * have not seen any failure so far. */ if (!pi_found_failure) psinfo->ps_nsucc++; if (cur_tg != NULL && pr_statp->pr_target == cur_tg && !tg_found_failure) { psinfo->ps_nsucc_tg++; } /* * Record the time of last success, if this is * the most recent probe success. */ if (!psinfo->ps_tls_valid) { psinfo->ps_tls = ns2ms(pr_statp->pr_hrtime_ackproc); psinfo->ps_tls_valid = _B_TRUE; } break; case PR_LOST: /* * We hit a failure. Latch the total number of * recent consecutive successes. */ pi_found_failure = _B_TRUE; if (cur_tg != NULL && pr_statp->pr_target == cur_tg) { /* * We hit a failure for the desired target. * Latch the number of recent consecutive * successes for this target */ tg_found_failure = _B_TRUE; } break; default: return; } } } /* * Return the information associated with consecutive probe failures * starting with the most recent probe. Only the last 2 probes can be in the * unacknowledged state. All previous probes have either failed or succeeded. */ static void probe_fail_info(struct phyint_instance *pii, struct target *cur_tg, struct probe_fail_count *pfinfo) { int i; struct probe_stats *pr_statp; boolean_t tg_found_success = _B_FALSE; boolean_t pi_found_success = _B_FALSE; int most_recent; int second_most_recent; uint_t now; uint_t timeout; struct target *tg; if (debug & D_FAILREP) logdebug("probe_fail_info(%s)\n", pii->pii_name); bzero(pfinfo, sizeof (*pfinfo)); now = getcurrenttime(); /* * Start with the most recent probe, and count the number * of consecutive probe failures. Latch the number of failures * on hitting a probe success. */ most_recent = PROBE_INDEX_PREV(pii->pii_probe_next); second_most_recent = PROBE_INDEX_PREV(most_recent); for (i = most_recent; i != pii->pii_probe_next; i = PROBE_INDEX_PREV(i)) { pr_statp = &pii->pii_probes[i]; assert(PR_STATUS_VALID(pr_statp->pr_status)); switch (pr_statp->pr_status) { case PR_UNACKED: /* * Only the most recent 2 probes can be unacknowledged */ assert(i == most_recent || i == second_most_recent); tg = pr_statp->pr_target; /* * Target is guaranteed to exist in the unack. state */ assert(tg != NULL); /* * The crtt could be zero for some reason, * Eg. the phyint could be failed. If the crtt is * not available use the group's probe interval, * which is a worst case estimate. */ timeout = ns2ms(pr_statp->pr_hrtime_start); if (tg->tg_crtt != 0) { timeout += tg->tg_crtt; } else { timeout += pii->pii_phyint->pi_group->pg_probeint; } if (TIME_GT(timeout, now)) break; pr_statp->pr_time_lost = timeout; probe_chstate(pr_statp, pii, PR_LOST); /* FALLTHRU */ case PR_LOST: if (!pi_found_success) { pfinfo->pf_nfail++; pfinfo->pf_tff = pr_statp->pr_time_lost; } if (cur_tg != NULL && pr_statp->pr_target == cur_tg && !tg_found_success) { pfinfo->pf_nfail_tg++; } break; default: /* * We hit a success or unused slot. Latch the * total number of recent consecutive failures. */ pi_found_success = _B_TRUE; if (cur_tg != NULL && pr_statp->pr_target == cur_tg) { /* * We hit a success for the desired target. * Latch the number of recent consecutive * failures for this target */ tg_found_success = _B_TRUE; } } } } /* * Change the state of probe `pr' on phyint_instance `pii' to state `state'. */ void probe_chstate(struct probe_stats *pr, struct phyint_instance *pii, int state) { if (pr->pr_status == state) return; pr->pr_status = state; (void) probe_state_event(pr, pii); } /* * Check if the phyint has been repaired. If no test address has been * configured, then consider the interface repaired if the link is up (unless * the link is flapping; see below). Otherwise, look for proof of probes * being sent and received. If last NUM_PROBE_REPAIRS probes are fine on * either IPv4 or IPv6 instance, the phyint can be considered repaired. */ static boolean_t phyint_repaired(struct phyint *pi) { struct probe_success_count psinfo; struct phyint_instance *pii; struct target *cur_tg; int pr_ndx; uint_t cur_time; if (debug & D_FAILREP) logdebug("phyint_repaired(%s)\n", pi->pi_name); if (LINK_DOWN(pi)) return (_B_FALSE); /* * If we don't have any test addresses and the link is up, then * consider the interface repaired, unless we've received more than * LINK_UP_PERMIN link up notifications in the last minute, in * which case we keep the link down until we drop back below * the threshold. */ if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) { cur_time = getcurrenttime(); if ((pi->pi_whenup[pi->pi_whendx] == 0 || (cur_time - pi->pi_whenup[pi->pi_whendx]) > MSEC_PERMIN)) { pi->pi_lfmsg_printed = 0; return (_B_TRUE); } if (!pi->pi_lfmsg_printed) { logerr("The link has come up on %s more than %d times " "in the last minute; disabling repair until it " "stabilizes\n", pi->pi_name, LINK_UP_PERMIN); pi->pi_lfmsg_printed = 1; } return (_B_FALSE); } pii = pi->pi_v4; if (PROBE_CAPABLE(pii)) { pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_success_info(pii, cur_tg, &psinfo); if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS || psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS) return (_B_TRUE); } pii = pi->pi_v6; if (PROBE_CAPABLE(pii)) { pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next); cur_tg = pii->pii_probes[pr_ndx].pr_target; probe_success_info(pii, cur_tg, &psinfo); if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS || psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS) return (_B_TRUE); } return (_B_FALSE); } /* * Used to set/clear phyint flags, by making a SIOCSLIFFLAGS call. */ boolean_t change_pif_flags(struct phyint *pi, uint64_t set, uint64_t clear) { int ifsock; struct lifreq lifr; uint64_t old_flags; if (debug & D_FAILREP) { logdebug("change_pif_flags(%s): set %llx clear %llx\n", pi->pi_name, set, clear); } if (pi->pi_v4 != NULL) ifsock = ifsock_v4; else ifsock = ifsock_v6; /* * Get the current flags from the kernel, and set/clear the * desired phyint flags. Since we set only phyint flags, we can * do it on either IPv4 or IPv6 instance. */ (void) strlcpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name)); if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) logperror("change_pif_flags: ioctl (get flags)"); return (_B_FALSE); } old_flags = lifr.lifr_flags; lifr.lifr_flags |= set; lifr.lifr_flags &= ~clear; if (old_flags == lifr.lifr_flags) { /* No change in the flags. No need to send ioctl */ return (_B_TRUE); } if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) { if (errno != ENXIO) logperror("change_pif_flags: ioctl (set flags)"); return (_B_FALSE); } /* * Keep pi_flags in synch. with actual flags. Assumes flags are * phyint flags. */ pi->pi_flags |= set; pi->pi_flags &= ~clear; if (pi->pi_v4 != NULL) pi->pi_v4->pii_flags = pi->pi_flags; if (pi->pi_v6 != NULL) pi->pi_v6->pii_flags = pi->pi_flags; return (_B_TRUE); } /* * icmp cksum computation for IPv4. */ static int in_cksum(ushort_t *addr, int len) { register int nleft = len; register ushort_t *w = addr; register ushort_t answer; ushort_t odd_byte = 0; register int sum = 0; /* * Our algorithm is simple, using a 32 bit accumulator (sum), * we add sequential 16 bit words to it, and at the end, fold * back all the carry bits from the top 16 bits into the lower * 16 bits. */ while (nleft > 1) { sum += *w++; nleft -= 2; } /* mop up an odd byte, if necessary */ if (nleft == 1) { *(uchar_t *)(&odd_byte) = *(uchar_t *)w; sum += odd_byte; } /* * add back carry outs from top 16 bits to low 16 bits */ sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */ sum += (sum >> 16); /* add carry */ answer = ~sum; /* truncate to 16 bits */ return (answer); } static void reset_snxt_basetimes(void) { struct phyint_instance *pii; for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) { pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime; } } /* * Is the address one of our own addresses? Unfortunately, * we cannot check our phyint tables to determine if the address * is our own. This is because, we don't track interfaces that * are not part of any group. We have to either use a 'bind' or * get the complete list of all interfaces using SIOCGLIFCONF, * to do this check. We could also use SIOCTMYADDR. * Bind fails for the local zone address, so we might include local zone * address as target address. If local zone address is a target address * and it is up, it is not possible to detect the interface failure. * SIOCTMYADDR also doesn't consider local zone address as own address. * So, we choose to use SIOCGLIFCONF to collect the local addresses, and they * are stored in `localaddrs' */ boolean_t own_address(struct in6_addr addr) { addrlist_t *addrp; struct sockaddr_storage ss; int af = IN6_IS_ADDR_V4MAPPED(&addr) ? AF_INET : AF_INET6; addr2storage(af, &addr, &ss); for (addrp = localaddrs; addrp != NULL; addrp = addrp->al_next) { if (sockaddrcmp(&ss, &addrp->al_addr)) return (_B_TRUE); } return (_B_FALSE); } static int ns2ms(int64_t ns) { return (NSEC2MSEC(ns)); } static int64_t tv2ns(struct timeval *tvp) { return (tvp->tv_sec * NANOSEC + tvp->tv_usec * 1000); }