diff options
Diffstat (limited to 'drivers/net/ethernet/sfc')
| -rw-r--r-- | drivers/net/ethernet/sfc/ef10.c | 14 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/efx.c | 67 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/efx.h | 5 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/ethtool.c | 59 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/falcon.c | 9 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/mcdi_port.c | 11 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/net_driver.h | 154 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/nic.h | 13 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/rx.c | 12 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/selftest.c | 5 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/siena.c | 9 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/tx.c | 28 |
12 files changed, 350 insertions, 36 deletions
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index b5ed30a39144..002d4cdc319f 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -755,6 +755,8 @@ static int efx_ef10_reset(struct efx_nic *efx, enum reset_type reset_type) { NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name } #define EF10_OTHER_STAT(ext_name) \ [EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 } +#define GENERIC_SW_STAT(ext_name) \ + [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 } static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = { EF10_DMA_STAT(tx_bytes, TX_BYTES), @@ -798,6 +800,8 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = { EF10_DMA_STAT(rx_align_error, RX_ALIGN_ERROR_PKTS), EF10_DMA_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS), EF10_DMA_STAT(rx_nodesc_drops, RX_NODESC_DROPS), + GENERIC_SW_STAT(rx_nodesc_trunc), + GENERIC_SW_STAT(rx_noskb_drops), EF10_DMA_STAT(rx_pm_trunc_bb_overflow, PM_TRUNC_BB_OVERFLOW), EF10_DMA_STAT(rx_pm_discard_bb_overflow, PM_DISCARD_BB_OVERFLOW), EF10_DMA_STAT(rx_pm_trunc_vfifo_full, PM_TRUNC_VFIFO_FULL), @@ -841,7 +845,9 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = { (1ULL << EF10_STAT_rx_gtjumbo) | \ (1ULL << EF10_STAT_rx_bad_gtjumbo) | \ (1ULL << EF10_STAT_rx_overflow) | \ - (1ULL << EF10_STAT_rx_nodesc_drops)) + (1ULL << EF10_STAT_rx_nodesc_drops) | \ + (1ULL << GENERIC_STAT_rx_nodesc_trunc) | \ + (1ULL << GENERIC_STAT_rx_noskb_drops)) /* These statistics are only provided by the 10G MAC. For a 10G/40G * switchable port we do not expose these because they might not @@ -951,7 +957,7 @@ static int efx_ef10_try_update_nic_stats(struct efx_nic *efx) stats[EF10_STAT_rx_bytes_minus_good_bytes]; efx_update_diff_stat(&stats[EF10_STAT_rx_bad_bytes], stats[EF10_STAT_rx_bytes_minus_good_bytes]); - + efx_update_sw_stats(efx, stats); return 0; } @@ -990,7 +996,9 @@ static size_t efx_ef10_update_stats(struct efx_nic *efx, u64 *full_stats, core_stats->tx_packets = stats[EF10_STAT_tx_packets]; core_stats->rx_bytes = stats[EF10_STAT_rx_bytes]; core_stats->tx_bytes = stats[EF10_STAT_tx_bytes]; - core_stats->rx_dropped = stats[EF10_STAT_rx_nodesc_drops]; + core_stats->rx_dropped = stats[EF10_STAT_rx_nodesc_drops] + + stats[GENERIC_STAT_rx_nodesc_trunc] + + stats[GENERIC_STAT_rx_noskb_drops]; core_stats->multicast = stats[EF10_STAT_rx_multicast]; core_stats->rx_length_errors = stats[EF10_STAT_rx_gtjumbo] + diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 1e274045970f..4cebe9d37816 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -272,6 +272,9 @@ static int efx_poll(struct napi_struct *napi, int budget) struct efx_nic *efx = channel->efx; int spent; + if (!efx_channel_lock_napi(channel)) + return budget; + netif_vdbg(efx, intr, efx->net_dev, "channel %d NAPI poll executing on CPU %d\n", channel->channel, raw_smp_processor_id()); @@ -311,6 +314,7 @@ static int efx_poll(struct napi_struct *napi, int budget) efx_nic_eventq_read_ack(channel); } + efx_channel_unlock_napi(channel); return spent; } @@ -357,7 +361,7 @@ static int efx_init_eventq(struct efx_channel *channel) } /* Enable event queue processing and NAPI */ -static void efx_start_eventq(struct efx_channel *channel) +void efx_start_eventq(struct efx_channel *channel) { netif_dbg(channel->efx, ifup, channel->efx->net_dev, "chan %d start event queue\n", channel->channel); @@ -366,17 +370,20 @@ static void efx_start_eventq(struct efx_channel *channel) channel->enabled = true; smp_wmb(); + efx_channel_enable(channel); napi_enable(&channel->napi_str); efx_nic_eventq_read_ack(channel); } /* Disable event queue processing and NAPI */ -static void efx_stop_eventq(struct efx_channel *channel) +void efx_stop_eventq(struct efx_channel *channel) { if (!channel->enabled) return; napi_disable(&channel->napi_str); + while (!efx_channel_disable(channel)) + usleep_range(1000, 20000); channel->enabled = false; } @@ -1960,6 +1967,8 @@ static void efx_init_napi_channel(struct efx_channel *channel) channel->napi_dev = efx->net_dev; netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, napi_weight); + napi_hash_add(&channel->napi_str); + efx_channel_init_lock(channel); } static void efx_init_napi(struct efx_nic *efx) @@ -1972,8 +1981,10 @@ static void efx_init_napi(struct efx_nic *efx) static void efx_fini_napi_channel(struct efx_channel *channel) { - if (channel->napi_dev) + if (channel->napi_dev) { netif_napi_del(&channel->napi_str); + napi_hash_del(&channel->napi_str); + } channel->napi_dev = NULL; } @@ -2008,6 +2019,37 @@ static void efx_netpoll(struct net_device *net_dev) #endif +#ifdef CONFIG_NET_RX_BUSY_POLL +static int efx_busy_poll(struct napi_struct *napi) +{ + struct efx_channel *channel = + container_of(napi, struct efx_channel, napi_str); + struct efx_nic *efx = channel->efx; + int budget = 4; + int old_rx_packets, rx_packets; + + if (!netif_running(efx->net_dev)) + return LL_FLUSH_FAILED; + + if (!efx_channel_lock_poll(channel)) + return LL_FLUSH_BUSY; + + old_rx_packets = channel->rx_queue.rx_packets; + efx_process_channel(channel, budget); + + rx_packets = channel->rx_queue.rx_packets - old_rx_packets; + + /* There is no race condition with NAPI here. + * NAPI will automatically be rescheduled if it yielded during busy + * polling, because it was not able to take the lock and thus returned + * the full budget. + */ + efx_channel_unlock_poll(channel); + + return rx_packets; +} +#endif + /************************************************************************** * * Kernel net device interface @@ -2177,6 +2219,9 @@ static const struct net_device_ops efx_farch_netdev_ops = { .ndo_poll_controller = efx_netpoll, #endif .ndo_setup_tc = efx_setup_tc, +#ifdef CONFIG_NET_RX_BUSY_POLL + .ndo_busy_poll = efx_busy_poll, +#endif #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = efx_filter_rfs, #endif @@ -2197,6 +2242,9 @@ static const struct net_device_ops efx_ef10_netdev_ops = { #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = efx_netpoll, #endif +#ifdef CONFIG_NET_RX_BUSY_POLL + .ndo_busy_poll = efx_busy_poll, +#endif #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = efx_filter_rfs, #endif @@ -2607,6 +2655,8 @@ static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = { .driver_data = (unsigned long) &siena_a0_nic_type}, {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, + {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ + .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, {0} /* end of list */ }; @@ -2722,6 +2772,17 @@ static void efx_fini_struct(struct efx_nic *efx) } } +void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) +{ + u64 n_rx_nodesc_trunc = 0; + struct efx_channel *channel; + + efx_for_each_channel(channel, efx) + n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; + stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; + stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); +} + /************************************************************************** * * PCI interface diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h index 99032581336f..2587c582a821 100644 --- a/drivers/net/ethernet/sfc/efx.h +++ b/drivers/net/ethernet/sfc/efx.h @@ -194,11 +194,16 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, bool rx_may_override_tx); void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, unsigned int *rx_usecs, bool *rx_adaptive); +void efx_stop_eventq(struct efx_channel *channel); +void efx_start_eventq(struct efx_channel *channel); /* Dummy PHY ops for PHY drivers */ int efx_port_dummy_op_int(struct efx_nic *efx); void efx_port_dummy_op_void(struct efx_nic *efx); +/* Update the generic software stats in the passed stats array */ +void efx_update_sw_stats(struct efx_nic *efx, u64 *stats); + /* MTD */ #ifdef CONFIG_SFC_MTD int efx_mtd_add(struct efx_nic *efx, struct efx_mtd_partition *parts, diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c index 74739c4b9997..cad258a78708 100644 --- a/drivers/net/ethernet/sfc/ethtool.c +++ b/drivers/net/ethernet/sfc/ethtool.c @@ -77,7 +77,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_tcp_udp_chksum_err), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_mcast_mismatch), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_frm_trunc), - EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_nodesc_trunc), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_events), EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_merge_packets), }; @@ -360,6 +359,37 @@ static int efx_ethtool_fill_self_tests(struct efx_nic *efx, return n; } +static size_t efx_describe_per_queue_stats(struct efx_nic *efx, u8 *strings) +{ + size_t n_stats = 0; + struct efx_channel *channel; + + efx_for_each_channel(channel, efx) { + if (efx_channel_has_tx_queues(channel)) { + n_stats++; + if (strings != NULL) { + snprintf(strings, ETH_GSTRING_LEN, + "tx-%u.tx_packets", + channel->tx_queue[0].queue / + EFX_TXQ_TYPES); + + strings += ETH_GSTRING_LEN; + } + } + } + efx_for_each_channel(channel, efx) { + if (efx_channel_has_rx_queue(channel)) { + n_stats++; + if (strings != NULL) { + snprintf(strings, ETH_GSTRING_LEN, + "rx-%d.rx_packets", channel->channel); + strings += ETH_GSTRING_LEN; + } + } + } + return n_stats; +} + static int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set) { @@ -368,8 +398,9 @@ static int efx_ethtool_get_sset_count(struct net_device *net_dev, switch (string_set) { case ETH_SS_STATS: return efx->type->describe_stats(efx, NULL) + - EFX_ETHTOOL_SW_STAT_COUNT + - efx_ptp_describe_stats(efx, NULL); + EFX_ETHTOOL_SW_STAT_COUNT + + efx_describe_per_queue_stats(efx, NULL) + + efx_ptp_describe_stats(efx, NULL); case ETH_SS_TEST: return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL); default: @@ -391,6 +422,8 @@ static void efx_ethtool_get_strings(struct net_device *net_dev, strlcpy(strings + i * ETH_GSTRING_LEN, efx_sw_stat_desc[i].name, ETH_GSTRING_LEN); strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN; + strings += (efx_describe_per_queue_stats(efx, strings) * + ETH_GSTRING_LEN); efx_ptp_describe_stats(efx, strings); break; case ETH_SS_TEST: @@ -410,6 +443,7 @@ static void efx_ethtool_get_stats(struct net_device *net_dev, const struct efx_sw_stat_desc *stat; struct efx_channel *channel; struct efx_tx_queue *tx_queue; + struct efx_rx_queue *rx_queue; int i; spin_lock_bh(&efx->stats_lock); @@ -445,6 +479,25 @@ static void efx_ethtool_get_stats(struct net_device *net_dev, spin_unlock_bh(&efx->stats_lock); + efx_for_each_channel(channel, efx) { + if (efx_channel_has_tx_queues(channel)) { + *data = 0; + efx_for_each_channel_tx_queue(tx_queue, channel) { + *data += tx_queue->tx_packets; + } + data++; + } + } + efx_for_each_channel(channel, efx) { + if (efx_channel_has_rx_queue(channel)) { + *data = 0; + efx_for_each_channel_rx_queue(rx_queue, channel) { + *data += rx_queue->rx_packets; + } + data++; + } + } + efx_ptp_update_stats(efx, data); } diff --git a/drivers/net/ethernet/sfc/falcon.c b/drivers/net/ethernet/sfc/falcon.c index fae25a418647..157037546d30 100644 --- a/drivers/net/ethernet/sfc/falcon.c +++ b/drivers/net/ethernet/sfc/falcon.c @@ -142,6 +142,8 @@ hw_name ## _ ## offset } #define FALCON_OTHER_STAT(ext_name) \ [FALCON_STAT_ ## ext_name] = { #ext_name, 0, 0 } +#define GENERIC_SW_STAT(ext_name) \ + [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 } static const struct efx_hw_stat_desc falcon_stat_desc[FALCON_STAT_COUNT] = { FALCON_DMA_STAT(tx_bytes, XgTxOctets), @@ -191,6 +193,8 @@ static const struct efx_hw_stat_desc falcon_stat_desc[FALCON_STAT_COUNT] = { FALCON_DMA_STAT(rx_length_error, XgRxLengthError), FALCON_DMA_STAT(rx_internal_error, XgRxInternalMACError), FALCON_OTHER_STAT(rx_nodesc_drop_cnt), + GENERIC_SW_STAT(rx_nodesc_trunc), + GENERIC_SW_STAT(rx_noskb_drops), }; static const unsigned long falcon_stat_mask[] = { [0 ... BITS_TO_LONGS(FALCON_STAT_COUNT) - 1] = ~0UL, @@ -2574,6 +2578,7 @@ static size_t falcon_update_nic_stats(struct efx_nic *efx, u64 *full_stats, stats[FALCON_STAT_rx_bytes] - stats[FALCON_STAT_rx_good_bytes] - stats[FALCON_STAT_rx_control] * 64); + efx_update_sw_stats(efx, stats); } if (full_stats) @@ -2584,7 +2589,9 @@ static size_t falcon_update_nic_stats(struct efx_nic *efx, u64 *full_stats, core_stats->tx_packets = stats[FALCON_STAT_tx_packets]; core_stats->rx_bytes = stats[FALCON_STAT_rx_bytes]; core_stats->tx_bytes = stats[FALCON_STAT_tx_bytes]; - core_stats->rx_dropped = stats[FALCON_STAT_rx_nodesc_drop_cnt]; + core_stats->rx_dropped = stats[FALCON_STAT_rx_nodesc_drop_cnt] + + stats[GENERIC_STAT_rx_nodesc_trunc] + + stats[GENERIC_STAT_rx_noskb_drops]; core_stats->multicast = stats[FALCON_STAT_rx_multicast]; core_stats->rx_length_errors = stats[FALCON_STAT_rx_gtjumbo] + diff --git a/drivers/net/ethernet/sfc/mcdi_port.c b/drivers/net/ethernet/sfc/mcdi_port.c index e5fc4e1574b5..fb19b70eac01 100644 --- a/drivers/net/ethernet/sfc/mcdi_port.c +++ b/drivers/net/ethernet/sfc/mcdi_port.c @@ -183,6 +183,8 @@ static u32 mcdi_to_ethtool_cap(u32 media, u32 cap) result |= SUPPORTED_1000baseKX_Full; if (cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN)) result |= SUPPORTED_10000baseKX4_Full; + if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) + result |= SUPPORTED_40000baseKR4_Full; break; case MC_CMD_MEDIA_XFP: @@ -190,6 +192,12 @@ static u32 mcdi_to_ethtool_cap(u32 media, u32 cap) result |= SUPPORTED_FIBRE; break; + case MC_CMD_MEDIA_QSFP_PLUS: + result |= SUPPORTED_FIBRE; + if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) + result |= SUPPORTED_40000baseCR4_Full; + break; + case MC_CMD_MEDIA_BASE_T: result |= SUPPORTED_TP; if (cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN)) @@ -237,6 +245,8 @@ static u32 ethtool_to_mcdi_cap(u32 cap) result |= (1 << MC_CMD_PHY_CAP_1000FDX_LBN); if (cap & (SUPPORTED_10000baseT_Full | SUPPORTED_10000baseKX4_Full)) result |= (1 << MC_CMD_PHY_CAP_10000FDX_LBN); + if (cap & (SUPPORTED_40000baseCR4_Full | SUPPORTED_40000baseKR4_Full)) + result |= (1 << MC_CMD_PHY_CAP_40000FDX_LBN); if (cap & SUPPORTED_Pause) result |= (1 << MC_CMD_PHY_CAP_PAUSE_LBN); if (cap & SUPPORTED_Asym_Pause) @@ -285,6 +295,7 @@ static u32 mcdi_to_ethtool_media(u32 media) case MC_CMD_MEDIA_XFP: case MC_CMD_MEDIA_SFP_PLUS: + case MC_CMD_MEDIA_QSFP_PLUS: return PORT_FIBRE; case MC_CMD_MEDIA_BASE_T: diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 5bdae8ed7c57..9ede32064685 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -28,6 +28,7 @@ #include <linux/vmalloc.h> #include <linux/i2c.h> #include <linux/mtd/mtd.h> +#include <net/busy_poll.h> #include "enum.h" #include "bitfield.h" @@ -249,6 +250,8 @@ struct efx_tx_queue { unsigned int tso_packets; unsigned int pushes; unsigned int pio_packets; + /* Statistics to supplement MAC stats */ + unsigned long tx_packets; /* Members shared between paths and sometimes updated */ unsigned int empty_read_count ____cacheline_aligned_in_smp; @@ -358,6 +361,8 @@ struct efx_rx_queue { unsigned int recycle_count; struct timer_list slow_fill; unsigned int slow_fill_count; + /* Statistics to supplement MAC stats */ + unsigned long rx_packets; }; enum efx_sync_events_state { @@ -383,6 +388,8 @@ enum efx_sync_events_state { * @irq_moderation: IRQ moderation value (in hardware ticks) * @napi_dev: Net device used with NAPI * @napi_str: NAPI control structure + * @state: state for NAPI vs busy polling + * @state_lock: lock protecting @state * @eventq: Event queue buffer * @eventq_mask: Event queue pointer mask * @eventq_read_ptr: Event queue read pointer @@ -420,6 +427,22 @@ struct efx_channel { unsigned int irq_moderation; struct net_device *napi_dev; struct napi_struct napi_str; +#ifdef CONFIG_NET_RX_BUSY_POLL + unsigned int state; + spinlock_t state_lock; +#define EFX_CHANNEL_STATE_IDLE 0 +#define EFX_CHANNEL_STATE_NAPI (1 << 0) /* NAPI owns this channel */ +#define EFX_CHANNEL_STATE_POLL (1 << 1) /* poll owns this channel */ +#define EFX_CHANNEL_STATE_DISABLED (1 << 2) /* channel is disabled */ +#define EFX_CHANNEL_STATE_NAPI_YIELD (1 << 3) /* NAPI yielded this channel */ +#define EFX_CHANNEL_STATE_POLL_YIELD (1 << 4) /* poll yielded this channel */ +#define EFX_CHANNEL_OWNED \ + (EFX_CHANNEL_STATE_NAPI | EFX_CHANNEL_STATE_POLL) +#define EFX_CHANNEL_LOCKED \ + (EFX_CHANNEL_OWNED | EFX_CHANNEL_STATE_DISABLED) +#define EFX_CHANNEL_USER_PEND \ + (EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_POLL_YIELD) +#endif /* CONFIG_NET_RX_BUSY_POLL */ struct efx_special_buffer eventq; unsigned int eventq_mask; unsigned int eventq_read_ptr; @@ -453,6 +476,135 @@ struct efx_channel { u32 sync_timestamp_minor; }; +#ifdef CONFIG_NET_RX_BUSY_POLL +static inline void efx_channel_init_lock(struct efx_channel *channel) +{ + spin_lock_init(&channel->state_lock); +} + +/* Called from the device poll routine to get ownership of a channel. */ +static inline bool efx_channel_lock_napi(struct efx_channel *channel) +{ + bool rc = true; + + spin_lock_bh(&channel->state_lock); + if (channel->state & EFX_CHANNEL_LOCKED) { + WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI); + channel->state |= EFX_CHANNEL_STATE_NAPI_YIELD; + rc = false; + } else { + /* we don't care if someone yielded */ + channel->state = EFX_CHANNEL_STATE_NAPI; + } + spin_unlock_bh(&channel->state_lock); + return rc; +} + +static inline void efx_channel_unlock_napi(struct efx_channel *channel) +{ + spin_lock_bh(&channel->state_lock); + WARN_ON(channel->state & + (EFX_CHANNEL_STATE_POLL | EFX_CHANNEL_STATE_NAPI_YIELD)); + + channel->state &= EFX_CHANNEL_STATE_DISABLED; + spin_unlock_bh(&channel->state_lock); +} + +/* Called from efx_busy_poll(). */ +static inline bool efx_channel_lock_poll(struct efx_channel *channel) +{ + bool rc = true; + + spin_lock_bh(&channel->state_lock); + if ((channel->state & EFX_CHANNEL_LOCKED)) { + channel->state |= EFX_CHANNEL_STATE_POLL_YIELD; + rc = false; + } else { + /* preserve yield marks */ + channel->state |= EFX_CHANNEL_STATE_POLL; + } + spin_unlock_bh(&channel->state_lock); + return rc; +} + +/* Returns true if NAPI tried to get the channel while it was locked. */ +static inline void efx_channel_unlock_poll(struct efx_channel *channel) +{ + spin_lock_bh(&channel->state_lock); + WARN_ON(channel->state & EFX_CHANNEL_STATE_NAPI); + + /* will reset state to idle, unless channel is disabled */ + channel->state &= EFX_CHANNEL_STATE_DISABLED; + spin_unlock_bh(&channel->state_lock); +} + +/* True if a socket is polling, even if it did not get the lock. */ +static inline bool efx_channel_busy_polling(struct efx_channel *channel) +{ + WARN_ON(!(channel->state & EFX_CHANNEL_OWNED)); + return channel->state & EFX_CHANNEL_USER_PEND; +} + +static inline void efx_channel_enable(struct efx_channel *channel) +{ + spin_lock_bh(&channel->state_lock); + channel->state = EFX_CHANNEL_STATE_IDLE; + spin_unlock_bh(&channel->state_lock); +} + +/* False if the channel is currently owned. */ +static inline bool efx_channel_disable(struct efx_channel *channel) +{ + bool rc = true; + + spin_lock_bh(&channel->state_lock); + if (channel->state & EFX_CHANNEL_OWNED) + rc = false; + channel->state |= EFX_CHANNEL_STATE_DISABLED; + spin_unlock_bh(&channel->state_lock); + + return rc; +} + +#else /* CONFIG_NET_RX_BUSY_POLL */ + +static inline void efx_channel_init_lock(struct efx_channel *channel) +{ +} + +static inline bool efx_channel_lock_napi(struct efx_channel *channel) +{ + return true; +} + +static inline void efx_channel_unlock_napi(struct efx_channel *channel) +{ +} + +static inline bool efx_channel_lock_poll(struct efx_channel *channel) +{ + return false; +} + +static inline void efx_channel_unlock_poll(struct efx_channel *channel) +{ +} + +static inline bool efx_channel_busy_polling(struct efx_channel *channel) +{ + return false; +} + +static inline void efx_channel_enable(struct efx_channel *channel) +{ +} + +static inline bool efx_channel_disable(struct efx_channel *channel) +{ + return true; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + /** * struct efx_msi_context - Context for each MSI * @efx: The associated NIC @@ -777,6 +929,7 @@ struct vfdi_status; * interrupt has occurred. * @stats_lock: Statistics update lock. Must be held when calling * efx_nic_type::{update,start,stop}_stats. + * @n_rx_noskb_drops: Count of RX packets dropped due to failure to allocate an skb * * This is stored in the private area of the &struct net_device. */ @@ -930,6 +1083,7 @@ struct efx_nic { spinlock_t biu_lock; int last_irq_cpu; spinlock_t stats_lock; + atomic_t n_rx_noskb_drops; }; static inline int efx_dev_registered(struct efx_nic *efx) diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h index d3ad8ed8d901..60f85149fc4c 100644 --- a/drivers/net/ethernet/sfc/nic.h +++ b/drivers/net/ethernet/sfc/nic.h @@ -135,6 +135,13 @@ enum { /* Size and alignment of buffer table entries (same) */ #define EFX_BUF_SIZE EFX_PAGE_SIZE +/* NIC-generic software stats */ +enum { + GENERIC_STAT_rx_noskb_drops, + GENERIC_STAT_rx_nodesc_trunc, + GENERIC_STAT_COUNT +}; + /** * struct falcon_board_type - board operations and type information * @id: Board type id, as found in NVRAM @@ -205,7 +212,7 @@ static inline bool falcon_spi_present(const struct falcon_spi_device *spi) } enum { - FALCON_STAT_tx_bytes, + FALCON_STAT_tx_bytes = GENERIC_STAT_COUNT, FALCON_STAT_tx_packets, FALCON_STAT_tx_pause, FALCON_STAT_tx_control, @@ -290,7 +297,7 @@ static inline struct falcon_board *falcon_board(struct efx_nic *efx) } enum { - SIENA_STAT_tx_bytes, + SIENA_STAT_tx_bytes = GENERIC_STAT_COUNT, SIENA_STAT_tx_good_bytes, SIENA_STAT_tx_bad_bytes, SIENA_STAT_tx_packets, @@ -361,7 +368,7 @@ struct siena_nic_data { }; enum { - EF10_STAT_tx_bytes, + EF10_STAT_tx_bytes = GENERIC_STAT_COUNT, EF10_STAT_tx_packets, EF10_STAT_tx_pause, EF10_STAT_tx_control, diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index 48588ddf81b0..c0ad95d2f63d 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -462,6 +462,7 @@ efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, skb_record_rx_queue(skb, channel->rx_queue.core_index); + skb_mark_napi_id(skb, &channel->napi_str); gro_result = napi_gro_frags(napi); if (gro_result != GRO_DROP) channel->irq_mod_score += 2; @@ -480,8 +481,10 @@ static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, skb = netdev_alloc_skb(efx->net_dev, efx->rx_ip_align + efx->rx_prefix_size + hdr_len); - if (unlikely(skb == NULL)) + if (unlikely(skb == NULL)) { + atomic_inc(&efx->n_rx_noskb_drops); return NULL; + } EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); @@ -518,6 +521,8 @@ static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, /* Move past the ethernet header */ skb->protocol = eth_type_trans(skb, efx->net_dev); + skb_mark_napi_id(skb, &channel->napi_str); + return skb; } @@ -528,6 +533,8 @@ void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, struct efx_channel *channel = efx_rx_queue_channel(rx_queue); struct efx_rx_buffer *rx_buf; + rx_queue->rx_packets++; + rx_buf = efx_rx_buffer(rx_queue, index); rx_buf->flags |= flags; @@ -662,7 +669,8 @@ void __efx_rx_packet(struct efx_channel *channel) if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; - if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb) + if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb && + !efx_channel_busy_polling(channel)) efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); else efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); diff --git a/drivers/net/ethernet/sfc/selftest.c b/drivers/net/ethernet/sfc/selftest.c index 0fc5baef45b1..10b6173d557d 100644 --- a/drivers/net/ethernet/sfc/selftest.c +++ b/drivers/net/ethernet/sfc/selftest.c @@ -188,7 +188,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx, schedule_timeout_uninterruptible(wait); efx_for_each_channel(channel, efx) { - napi_disable(&channel->napi_str); + efx_stop_eventq(channel); if (channel->eventq_read_ptr != read_ptr[channel->channel]) { set_bit(channel->channel, &napi_ran); @@ -200,8 +200,7 @@ static int efx_test_eventq_irq(struct efx_nic *efx, if (efx_nic_event_test_irq_cpu(channel) >= 0) clear_bit(channel->channel, &int_pend); } - napi_enable(&channel->napi_str); - efx_nic_eventq_read_ack(channel); + efx_start_eventq(channel); } wait *= 2; diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c index 50ffefed492c..ae696855f21a 100644 --- a/drivers/net/ethernet/sfc/siena.c +++ b/drivers/net/ethernet/sfc/siena.c @@ -424,6 +424,8 @@ static void siena_remove_nic(struct efx_nic *efx) { #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name } #define SIENA_OTHER_STAT(ext_name) \ [SIENA_STAT_ ## ext_name] = { #ext_name, 0, 0 } +#define GENERIC_SW_STAT(ext_name) \ + [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 } static const struct efx_hw_stat_desc siena_stat_desc[SIENA_STAT_COUNT] = { SIENA_DMA_STAT(tx_bytes, TX_BYTES), @@ -483,6 +485,8 @@ static const struct efx_hw_stat_desc siena_stat_desc[SIENA_STAT_COUNT] = { SIENA_DMA_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS), SIENA_DMA_STAT(rx_internal_error, RX_INTERNAL_ERROR_PKTS), SIENA_DMA_STAT(rx_nodesc_drop_cnt, RX_NODESC_DROPS), + GENERIC_SW_STAT(rx_nodesc_trunc), + GENERIC_SW_STAT(rx_noskb_drops), }; static const unsigned long siena_stat_mask[] = { [0 ... BITS_TO_LONGS(SIENA_STAT_COUNT) - 1] = ~0UL, @@ -528,6 +532,7 @@ static int siena_try_update_nic_stats(struct efx_nic *efx) efx_update_diff_stat(&stats[SIENA_STAT_rx_good_bytes], stats[SIENA_STAT_rx_bytes] - stats[SIENA_STAT_rx_bad_bytes]); + efx_update_sw_stats(efx, stats); return 0; } @@ -554,7 +559,9 @@ static size_t siena_update_nic_stats(struct efx_nic *efx, u64 *full_stats, core_stats->tx_packets = stats[SIENA_STAT_tx_packets]; core_stats->rx_bytes = stats[SIENA_STAT_rx_bytes]; core_stats->tx_bytes = stats[SIENA_STAT_tx_bytes]; - core_stats->rx_dropped = stats[SIENA_STAT_rx_nodesc_drop_cnt]; + core_stats->rx_dropped = stats[SIENA_STAT_rx_nodesc_drop_cnt] + + stats[GENERIC_STAT_rx_nodesc_trunc] + + stats[GENERIC_STAT_rx_noskb_drops]; core_stats->multicast = stats[SIENA_STAT_rx_multicast]; core_stats->collisions = stats[SIENA_STAT_tx_collision]; core_stats->rx_length_errors = diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index ede8dcca0ff3..65c220f8661d 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -189,18 +189,6 @@ struct efx_short_copy_buffer { u8 buf[L1_CACHE_BYTES]; }; -/* Copy in explicit 64-bit writes. */ -static void efx_memcpy_64(void __iomem *dest, void *src, size_t len) -{ - u64 *src64 = src; - u64 __iomem *dest64 = dest; - size_t l64 = len / 8; - size_t i; - - for (i = 0; i < l64; i++) - writeq(src64[i], &dest64[i]); -} - /* Copy to PIO, respecting that writes to PIO buffers must be dword aligned. * Advances piobuf pointer. Leaves additional data in the copy buffer. */ @@ -210,7 +198,7 @@ static void efx_memcpy_toio_aligned(struct efx_nic *efx, u8 __iomem **piobuf, { int block_len = len & ~(sizeof(copy_buf->buf) - 1); - efx_memcpy_64(*piobuf, data, block_len); + __iowrite64_copy(*piobuf, data, block_len >> 3); *piobuf += block_len; len -= block_len; @@ -242,7 +230,8 @@ static void efx_memcpy_toio_aligned_cb(struct efx_nic *efx, u8 __iomem **piobuf, if (copy_buf->used < sizeof(copy_buf->buf)) return; - efx_memcpy_64(*piobuf, copy_buf->buf, sizeof(copy_buf->buf)); + __iowrite64_copy(*piobuf, copy_buf->buf, + sizeof(copy_buf->buf) >> 3); *piobuf += sizeof(copy_buf->buf); data += copy_to_buf; len -= copy_to_buf; @@ -257,7 +246,8 @@ static void efx_flush_copy_buffer(struct efx_nic *efx, u8 __iomem *piobuf, { /* if there's anything in it, write the whole buffer, including junk */ if (copy_buf->used) - efx_memcpy_64(piobuf, copy_buf->buf, sizeof(copy_buf->buf)); + __iowrite64_copy(piobuf, copy_buf->buf, + sizeof(copy_buf->buf) >> 3); } /* Traverse skb structure and copy fragments in to PIO buffer. @@ -316,8 +306,8 @@ efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue, struct sk_buff *skb) */ BUILD_BUG_ON(L1_CACHE_BYTES > SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); - efx_memcpy_64(tx_queue->piobuf, skb->data, - ALIGN(skb->len, L1_CACHE_BYTES)); + __iowrite64_copy(tx_queue->piobuf, skb->data, + ALIGN(skb->len, L1_CACHE_BYTES) >> 3); } EFX_POPULATE_QWORD_5(buffer->option, @@ -452,6 +442,8 @@ finish_packet: /* Pass off to hardware */ efx_nic_push_buffers(tx_queue); + tx_queue->tx_packets++; + efx_tx_maybe_stop_queue(tx_queue); return NETDEV_TX_OK; @@ -1245,6 +1237,8 @@ static int tso_start_new_packet(struct efx_tx_queue *tx_queue, ++tx_queue->tso_packets; + ++tx_queue->tx_packets; + return 0; } |