From 2f40d128e1f1b19471031ba9bc5ac0112e7f53e2 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 4 Aug 2022 13:20:06 +0200 Subject: [PATCH 01/16] bbr2 Signed-off-by: Peter Jung --- include/linux/tcp.h | 3 +- include/net/inet_connection_sock.h | 3 +- include/net/tcp.h | 41 +- include/uapi/linux/inet_diag.h | 33 + net/ipv4/Kconfig | 22 + net/ipv4/Makefile | 1 + net/ipv4/bpf_tcp_ca.c | 2 +- net/ipv4/tcp.c | 1 + net/ipv4/tcp_bbr.c | 38 +- net/ipv4/tcp_bbr2.c | 2674 ++++++++++++++++++++++++++++ net/ipv4/tcp_cong.c | 1 + net/ipv4/tcp_input.c | 27 +- net/ipv4/tcp_output.c | 26 +- net/ipv4/tcp_rate.c | 30 +- net/ipv4/tcp_timer.c | 1 + 15 files changed, 2868 insertions(+), 35 deletions(-) create mode 100644 net/ipv4/tcp_bbr2.c diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 1168302b79274..138c1e6f161a9 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -225,7 +225,8 @@ struct tcp_sock { u8 compressed_ack; u8 dup_ack_counter:2, tlp_retrans:1, /* TLP is a retransmission */ - unused:5; + fast_ack_mode:2, /* which fast ack mode ? */ + unused:3; u32 chrono_start; /* Start time in jiffies of a TCP chrono */ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ u8 chrono_type:2, /* current chronograph type */ diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index ee88f0f1350f4..1412edffa9a0c 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -132,7 +132,8 @@ struct inet_connection_sock { u32 icsk_probes_tstamp; u32 icsk_user_timeout; - u64 icsk_ca_priv[104 / sizeof(u64)]; +/* XXX inflated by temporary internal debugging info */ + u64 icsk_ca_priv[216 / sizeof(u64)]; #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) }; diff --git a/include/net/tcp.h b/include/net/tcp.h index 78a64e1b33a7e..21775ebe90cab 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -370,6 +370,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, #define TCP_ECN_QUEUE_CWR 2 #define TCP_ECN_DEMAND_CWR 4 #define TCP_ECN_SEEN 8 +#define TCP_ECN_ECT_PERMANENT 16 enum tcp_tw_status { TCP_TW_SUCCESS = 0, @@ -810,6 +811,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) return max_t(s64, t1 - t0, 0); } +static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) +{ + return max_t(s32, t1 - t0, 0); +} + static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) { return tcp_ns_to_ts(skb->skb_mstamp_ns); @@ -885,9 +891,14 @@ struct tcp_skb_cb { /* pkts S/ACKed so far upon tx of skb, incl retrans: */ __u32 delivered; /* start of send pipeline phase */ - u64 first_tx_mstamp; + u32 first_tx_mstamp; /* when we reached the "delivered" count */ - u64 delivered_mstamp; + u32 delivered_mstamp; +#define TCPCB_IN_FLIGHT_BITS 20 +#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) + u32 in_flight:20, /* packets in flight at transmit */ + unused2:12; + u32 lost; /* packets lost so far upon tx of skb */ } tx; /* only used for outgoing skbs */ union { struct inet_skb_parm h4; @@ -1013,7 +1024,11 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED 0x1 /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN 0x2 -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ +#define TCP_CONG_WANTS_CE_EVENTS 0x4 +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ + TCP_CONG_NEEDS_ECN | \ + TCP_CONG_WANTS_CE_EVENTS) union tcp_cc_info; @@ -1033,8 +1048,11 @@ struct ack_sample { */ struct rate_sample { u64 prior_mstamp; /* starting timestamp for interval */ + u32 prior_lost; /* tp->lost at "prior_mstamp" */ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ + u32 tx_in_flight; /* packets in flight at starting timestamp */ + s32 lost; /* number of packets lost over interval */ s32 delivered; /* number of packets delivered over interval */ s32 delivered_ce; /* number of packets delivered w/ CE marks*/ long interval_us; /* time for tp->delivered to incr "delivered" */ @@ -1048,6 +1066,7 @@ struct rate_sample { bool is_app_limited; /* is sample from packet with bubble in pipe? */ bool is_retrans; /* is sample from retransmission? */ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + bool is_ece; /* did this ACK have ECN marked? */ }; struct tcp_congestion_ops { @@ -1071,8 +1090,11 @@ struct tcp_congestion_ops { /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs */ - u32 (*min_tso_segs)(struct sock *sk); + /* pick target number of segments per TSO/GSO skb (optional): */ + u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); + + /* react to a specific lost skb (optional) */ + void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); /* call when packets are delivered to update cwnd and pacing rate, * after all the ca_state processing. (optional) @@ -1135,6 +1157,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) } #endif +static inline bool tcp_ca_wants_ce_events(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | + TCP_CONG_WANTS_CE_EVENTS); +} + static inline bool tcp_ca_needs_ecn(const struct sock *sk) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1154,6 +1184,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) void tcp_set_ca_state(struct sock *sk, const u8 ca_state); /* From tcp_rate.c */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, struct rate_sample *rs); diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h index 20ee93f0f8761..96d52dd9c48ac 100644 --- a/include/uapi/linux/inet_diag.h +++ b/include/uapi/linux/inet_diag.h @@ -231,9 +231,42 @@ struct tcp_bbr_info { __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ }; +/* Phase as reported in netlink/ss stats. */ +enum tcp_bbr2_phase { + BBR2_PHASE_INVALID = 0, + BBR2_PHASE_STARTUP = 1, + BBR2_PHASE_DRAIN = 2, + BBR2_PHASE_PROBE_RTT = 3, + BBR2_PHASE_PROBE_BW_UP = 4, + BBR2_PHASE_PROBE_BW_DOWN = 5, + BBR2_PHASE_PROBE_BW_CRUISE = 6, + BBR2_PHASE_PROBE_BW_REFILL = 7 +}; + +struct tcp_bbr2_info { + /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ + __u32 bbr_bw_lsb; /* lower 32 bits of bw */ + __u32 bbr_bw_msb; /* upper 32 bits of bw */ + __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ + __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ + __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ + __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ + __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ + __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ + __u8 bbr_mode; /* current bbr_mode in state machine */ + __u8 bbr_phase; /* current state machine phase */ + __u8 unused1; /* alignment padding; not used yet */ + __u8 bbr_version; /* MUST be at this offset in struct */ + __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ + __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ + __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ +}; + union tcp_cc_info { struct tcpvegas_info vegas; struct tcp_dctcp_info dctcp; struct tcp_bbr_info bbr; + struct tcp_bbr2_info bbr2; }; #endif /* _UAPI_INET_DIAG_H_ */ diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index e983bb0c50127..d172ffbdf27f7 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -668,6 +668,24 @@ config TCP_CONG_BBR AQM schemes that do not provide a delay signal. It requires the fq ("Fair Queue") pacing packet scheduler. +config TCP_CONG_BBR2 + tristate "BBR2 TCP" + default n + help + + BBR2 TCP congestion control is a model-based congestion control + algorithm that aims to maximize network utilization, keep queues and + retransmit rates low, and to be able to coexist with Reno/CUBIC in + common scenarios. It builds an explicit model of the network path. It + tolerates a targeted degree of random packet loss and delay that are + unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, + or cable modem links, and can use DCTCP-L4S-style ECN signals. It can + coexist with flows that use loss-based congestion control, and can + operate with shallow buffers, deep buffers, bufferbloat, policers, or + AQM schemes that do not provide a delay signal. It requires pacing, + using either TCP internal pacing or the fq ("Fair Queue") pacing packet + scheduler. + choice prompt "Default TCP congestion control" default DEFAULT_CUBIC @@ -705,6 +723,9 @@ choice config DEFAULT_BBR bool "BBR" if TCP_CONG_BBR=y + config DEFAULT_BBR2 + bool "BBR2" if TCP_CONG_BBR2=y + config DEFAULT_RENO bool "Reno" endchoice @@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG default "dctcp" if DEFAULT_DCTCP default "cdg" if DEFAULT_CDG default "bbr" if DEFAULT_BBR + default "bbr2" if DEFAULT_BBR2 default "cubic" config TCP_MD5SIG diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index bbdd9c44f14e3..8dee1547d8202 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o +obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index f79ab942f03b6..389f4c874f870 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -21,7 +21,7 @@ static u32 optional_ops[] = { offsetof(struct tcp_congestion_ops, cwnd_event), offsetof(struct tcp_congestion_ops, in_ack_event), offsetof(struct tcp_congestion_ops, pkts_acked), - offsetof(struct tcp_congestion_ops, min_tso_segs), + offsetof(struct tcp_congestion_ops, tso_segs), offsetof(struct tcp_congestion_ops, sndbuf_expand), offsetof(struct tcp_congestion_ops, cong_control), }; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 766881775abb7..dd41c2c6459db 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3067,6 +3067,7 @@ int tcp_disconnect(struct sock *sk, int flags) tp->rx_opt.dsack = 0; tp->rx_opt.num_sacks = 0; tp->rcv_ooopack = 0; + tp->fast_ack_mode = 0; /* Clean up fastopen related fields */ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 075e744bfb482..8ab6e8f713f87 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) sk->sk_pacing_rate = rate; } -/* override sysctl_tcp_min_tso_segs */ static u32 bbr_min_tso_segs(struct sock *sk) { return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; } +/* Return the number of segments BBR would like in a TSO/GSO skb, given + * a particular max gso size as a constraint. + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + u32 segs; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); +} + +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ static u32 bbr_tso_segs_goal(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); - u32 segs, bytes; - - /* Sort of tcp_tso_autosize() but ignoring - * driver provided sk_gso_max_size. - */ - bytes = min_t(unsigned long, - sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), - GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); - segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - return min(segs, 0x7FU); + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); } /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ @@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { .undo_cwnd = bbr_undo_cwnd, .cwnd_event = bbr_cwnd_event, .ssthresh = bbr_ssthresh, - .min_tso_segs = bbr_min_tso_segs, + .tso_segs = bbr_tso_segs, .get_info = bbr_get_info, .set_state = bbr_set_state, }; diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c new file mode 100644 index 0000000000000..488429f0f3d0a --- /dev/null +++ b/net/ipv4/tcp_bbr2.c @@ -0,0 +1,2674 @@ +/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 + * + * BBRv2 is a model-based congestion control algorithm that aims for low + * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model + * of the network path, it uses measurements of bandwidth and RTT, as well as + * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that + * although it can use ECN or loss signals explicitly, it does not require + * either; it can bound its in-flight data based on its estimate of the BDP. + * + * The model has both higher and lower bounds for the operating range: + * lo: bw_lo, inflight_lo: conservative short-term lower bound + * hi: bw_hi, inflight_hi: robust long-term upper bound + * The bandwidth-probing time scale is (a) extended dynamically based on + * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by + * an interactive wall-clock time-scale to be more scalable and responsive + * than Reno and CUBIC. + * + * Here is a state transition diagram for BBR: + * + * | + * V + * +---> STARTUP ----+ + * | | | + * | V | + * | DRAIN ----+ + * | | | + * | V | + * +---> PROBE_BW ----+ + * | ^ | | + * | | | | + * | +----+ | + * | | + * +---- PROBE_RTT <--+ + * + * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. + * When it estimates the pipe is full, it enters DRAIN to drain the queue. + * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. + * A long-lived BBR flow spends the vast majority of its time remaining + * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth + * in a fair manner, with a small, bounded queue. *If* a flow has been + * continuously sending for the entire min_rtt window, and hasn't seen an RTT + * sample that matches or decreases its min_rtt estimate for 10 seconds, then + * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe + * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if + * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; + * otherwise we enter STARTUP to try to fill the pipe. + * + * BBR is described in detail in: + * "BBR: Congestion-Based Congestion Control", + * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, + * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. + * + * There is a public e-mail list for discussing BBR development and testing: + * https://groups.google.com/forum/#!forum/bbr-dev + * + * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, + * otherwise TCP stack falls back to an internal pacing using one high + * resolution timer per TCP socket and may use more resources. + */ +#include +#include +#include +#include +#include + +#include "tcp_dctcp.h" + +/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth + * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. + * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. + * Since the minimum window is >=4 packets, the lower bound isn't + * an issue. The upper bound isn't an issue with existing technologies. + */ +#define BW_SCALE 24 +#define BW_UNIT (1 << BW_SCALE) + +#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ +#define BBR_UNIT (1 << BBR_SCALE) + +#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ +#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ + +#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ + +/* BBR has the following modes for deciding how fast to send: */ +enum bbr_mode { + BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ + BBR_DRAIN, /* drain any queue created during startup */ + BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ + BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ +}; + +/* How does the incoming ACK stream relate to our bandwidth probing? */ +enum bbr_ack_phase { + BBR_ACKS_INIT, /* not probing; not getting probe feedback */ + BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ + BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ + BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ + BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ +}; + +/* BBR congestion control block */ +struct bbr { + u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ + u32 min_rtt_stamp; /* timestamp of min_rtt_us */ + u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ + u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ + u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ + u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ + u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ + u64 cycle_mstamp; /* time of this cycle phase start */ + u32 mode:3, /* current bbr_mode in state machine */ + prev_ca_state:3, /* CA state on previous ACK */ + packet_conservation:1, /* use packet conservation? */ + round_start:1, /* start of packet-timed tx->ack round? */ + ce_state:1, /* If most recent data has CE bit set */ + bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ + try_fast_path:1, /* can we take fast path? */ + unused2:11, + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ + cycle_idx:3, /* current index in pacing_gain cycle array */ + has_seen_rtt:1; /* have we seen an RTT sample yet? */ + u32 pacing_gain:11, /* current gain for setting pacing rate */ + cwnd_gain:11, /* current gain for setting cwnd */ + full_bw_reached:1, /* reached full bw in Startup? */ + full_bw_cnt:2, /* number of rounds without large bw gains */ + init_cwnd:7; /* initial cwnd */ + u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ + u32 full_bw; /* recent bw, to estimate if pipe is full */ + + /* For tracking ACK aggregation: */ + u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ + u16 extra_acked[2]; /* max excess data ACKed in epoch */ + u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ + extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ + extra_acked_win_idx:1, /* current index in extra_acked array */ + /* BBR v2 state: */ + unused1:2, + startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ + loss_in_cycle:1, /* packet loss in this cycle? */ + ecn_in_cycle:1; /* ECN in this cycle? */ + u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ + u32 undo_bw_lo; /* bw_lo before latest losses */ + u32 undo_inflight_lo; /* inflight_lo before latest losses */ + u32 undo_inflight_hi; /* inflight_hi before latest losses */ + u32 bw_latest; /* max delivered bw in last round trip */ + u32 bw_lo; /* lower bound on sending bandwidth */ + u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ + u32 inflight_latest; /* max delivered data in last round trip */ + u32 inflight_lo; /* lower bound of inflight data range */ + u32 inflight_hi; /* upper bound of inflight data range */ + u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ + u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ + u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ + u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ + ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ + bw_probe_samples:1, /* rate samples reflect bw probing? */ + prev_probe_too_high:1, /* did last PROBE_UP go too high? */ + stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ + rounds_since_probe:8, /* packet-timed rounds since probed bw */ + loss_round_start:1, /* loss_round_delivered round trip? */ + loss_in_round:1, /* loss marked in this round trip? */ + ecn_in_round:1, /* ECN marked in this round trip? */ + ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ + loss_events_in_round:4,/* losses in STARTUP round */ + initialized:1; /* has bbr_init() been called? */ + u32 alpha_last_delivered; /* tp->delivered at alpha update */ + u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ + + /* Params configurable using setsockopt. Refer to correspoding + * module param for detailed description of params. + */ + struct bbr_params { + u32 high_gain:11, /* max allowed value: 2047 */ + drain_gain:10, /* max allowed value: 1023 */ + cwnd_gain:11; /* max allowed value: 2047 */ + u32 cwnd_min_target:4, /* max allowed value: 15 */ + min_rtt_win_sec:5, /* max allowed value: 31 */ + probe_rtt_mode_ms:9, /* max allowed value: 511 */ + full_bw_cnt:3, /* max allowed value: 7 */ + cwnd_tso_budget:1, /* allowed values: {0, 1} */ + unused3:6, + drain_to_target:1, /* boolean */ + precise_ece_ack:1, /* boolean */ + extra_acked_in_startup:1, /* allowed values: {0, 1} */ + fast_path:1; /* boolean */ + u32 full_bw_thresh:10, /* max allowed value: 1023 */ + startup_cwnd_gain:11, /* max allowed value: 2047 */ + bw_probe_pif_gain:9, /* max allowed value: 511 */ + usage_based_cwnd:1, /* boolean */ + unused2:1; + u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ + refill_add_inc:2; /* max allowed value: 3 */ + u16 extra_acked_gain:11, /* max allowed value: 2047 */ + extra_acked_win_rtts:5; /* max allowed value: 31*/ + u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ + /* Mostly BBR v2 parameters below here: */ + u32 ecn_alpha_gain:8, /* max allowed value: 255 */ + ecn_factor:8, /* max allowed value: 255 */ + ecn_thresh:8, /* max allowed value: 255 */ + beta:8; /* max allowed value: 255 */ + u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ + bw_probe_reno_gain:9, /* max allowed value: 511 */ + full_loss_cnt:4; /* max allowed value: 15 */ + u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ + inflight_headroom:8, /* max allowed value: 255 */ + loss_thresh:8, /* max allowed value: 255 */ + bw_probe_max_rounds:8; /* max allowed value: 255 */ + u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ + bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ + full_ecn_cnt:2; /* max allowed value: 3 */ + u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ + undo:1, /* boolean */ + tso_rtt_shift:4, /* max allowed value: 15 */ + unused5:1; + u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ + unused1:14, + ecn_alpha_init:9; /* max allowed value: 256 */ + } params; + + struct { + u32 snd_isn; /* Initial sequence number */ + u32 rs_bw; /* last valid rate sample bw */ + u32 target_cwnd; /* target cwnd, based on BDP */ + u8 undo:1, /* Undo even happened but not yet logged */ + unused:7; + char event; /* single-letter event debug codes */ + u16 unused2; + } debug; +}; + +struct bbr_context { + u32 sample_bw; + u32 target_cwnd; + u32 log:1; +}; + +/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ +static u32 bbr_min_rtt_win_sec = 10; +/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. + * Max allowed value is 511 (0x1FF). + */ +static u32 bbr_probe_rtt_mode_ms = 200; +/* Window length of probe_rtt_min_us filter (in ms), and consequently the + * typical interval between PROBE_RTT mode entries. + * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC + */ +static u32 bbr_probe_rtt_win_ms = 5000; +/* Skip TSO below the following bandwidth (bits/sec): */ +static int bbr_min_tso_rate = 1200000; + +/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting + * in bigger TSO bursts. By default we cut the RTT-based allowance in half + * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance + * is below 1500 bytes after 6 * ~500 usec = 3ms. + */ +static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ + +/* Select cwnd TSO budget approach: + * 0: padding + * 1: flooring + */ +static uint bbr_cwnd_tso_budget = 1; + +/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. + * In order to help drive the network toward lower queues and low latency while + * maintaining high utilization, the average pacing rate aims to be slightly + * lower than the estimated bandwidth. This is an important aspect of the + * design. + */ +static const int bbr_pacing_margin_percent = 1; + +/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain + * that will allow a smoothly increasing pacing rate that will double each RTT + * and send the same number of packets per RTT that an un-paced, slow-starting + * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). + */ +static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; +/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ +static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; +/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain + * the queue created in BBR_STARTUP in a single round. Max allowed value + * is 1023 (0x3FF). + */ +static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; +/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. + * Max allowed value is 2047 (0x7FF). + */ +static int bbr_cwnd_gain = BBR_UNIT * 2; +/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. + * Max allowed value for each element is 1023 (0x3FF). + */ +enum bbr_pacing_gain_phase { + BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ + BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ + BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ + BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ +}; +static int bbr_pacing_gain[] = { + BBR_UNIT * 5 / 4, /* probe for more available bw */ + BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ + BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ + BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ +}; + +/* Try to keep at least this many packets in flight, if things go smoothly. For + * smooth functioning, a sliding window protocol ACKing every other packet + * needs at least 4 packets in flight. Max allowed value is 15 (0xF). + */ +static u32 bbr_cwnd_min_target = 4; + +/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. + * Use 0 to disable. Max allowed value is 255. + */ +static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; + +/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ +/* If bw has increased significantly (1.25x), there may be more bw available. + * Max allowed value is 1023 (0x3FF). + */ +static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; +/* But after 3 rounds w/o significant bw growth, estimate pipe is full. + * Max allowed value is 7 (0x7). + */ +static u32 bbr_full_bw_cnt = 3; + +static u32 bbr_flags; /* Debugging related stuff */ + +/* Whether to debug using printk. + */ +static bool bbr_debug_with_printk; + +/* Whether to debug using ftrace event tcp:tcp_bbr_event. + * Ignored when bbr_debug_with_printk is set. + */ +static bool bbr_debug_ftrace; + +/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ +static bool bbr_drain_to_target = true; /* default: enabled */ + +/* Experiment: Flags to control BBR with ECN behavior. + */ +static bool bbr_precise_ece_ack = true; /* default: enabled */ + +/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is + * (2^(16+14) B)/(1024 B/packet) = 1M packets. + */ +static u32 bbr_cwnd_warn_val = 1U << 20; + +static u16 bbr_debug_port_mask; + +/* BBR module parameters. These are module parameters only in Google prod. + * Upstream these are intentionally not module parameters. + */ +static int bbr_pacing_gain_size = CYCLE_LEN; + +/* Gain factor for adding extra_acked to target cwnd: */ +static int bbr_extra_acked_gain = 256; + +/* Window length of extra_acked window. Max allowed val is 31. */ +static u32 bbr_extra_acked_win_rtts = 5; + +/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ +static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; + +/* Time period for clamping cwnd increment due to ack aggregation */ +static u32 bbr_extra_acked_max_us = 100 * 1000; + +/* Use extra acked in startup ? + * 0: disabled + * 1: use latest extra_acked value from 1-2 rtt in startup + */ +static int bbr_extra_acked_in_startup = 1; /* default: enabled */ + +/* Experiment: don't grow cwnd beyond twice of what we just probed. */ +static bool bbr_usage_based_cwnd; /* default: disabled */ + +/* For lab testing, researchers can enable BBRv2 ECN support with this flag, + * when they know that any ECN marks that the connections experience will be + * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. + * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on + * negotiation or configuration that is outside the scope of the BBRv2 + * alpha release. + */ +static bool bbr_ecn_enable = false; + +module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); +module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); +module_param_named(high_gain, bbr_high_gain, int, 0644); +module_param_named(drain_gain, bbr_drain_gain, int, 0644); +module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); +module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); +module_param_array_named(pacing_gain, bbr_pacing_gain, int, + &bbr_pacing_gain_size, 0644); +module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); +module_param_named(probe_rtt_cwnd_gain, + bbr_probe_rtt_cwnd_gain, uint, 0664); +module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); +module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); +module_param_named(flags, bbr_flags, uint, 0644); +module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); +module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); +module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); +module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); +module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); +module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); +module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); +module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); +module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); +module_param_named(extra_acked_win_rtts, + bbr_extra_acked_win_rtts, uint, 0664); +module_param_named(extra_acked_max_us, + bbr_extra_acked_max_us, uint, 0664); +module_param_named(ack_epoch_acked_reset_thresh, + bbr_ack_epoch_acked_reset_thresh, uint, 0664); +module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); +module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); +module_param_named(extra_acked_in_startup, + bbr_extra_acked_in_startup, int, 0664); +module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); +module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); + +static void bbr2_exit_probe_rtt(struct sock *sk); +static void bbr2_reset_congestion_signals(struct sock *sk); + +static void bbr_check_probe_rtt_done(struct sock *sk); + +/* Do we estimate that STARTUP filled the pipe? */ +static bool bbr_full_bw_reached(const struct sock *sk) +{ + const struct bbr *bbr = inet_csk_ca(sk); + + return bbr->full_bw_reached; +} + +/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ +static u32 bbr_max_bw(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->bw_hi[0], bbr->bw_hi[1]); +} + +/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ +static u32 bbr_bw(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return min(bbr_max_bw(sk), bbr->bw_lo); +} + +/* Return maximum extra acked in past k-2k round trips, + * where k = bbr_extra_acked_win_rtts. + */ +static u16 bbr_extra_acked(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->extra_acked[0], bbr->extra_acked[1]); +} + +/* Return rate in bytes per second, optionally with a gain. + * The order here is chosen carefully to avoid overflow of u64. This should + * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. + */ +static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, + int margin) +{ + unsigned int mss = tcp_sk(sk)->mss_cache; + + rate *= mss; + rate *= gain; + rate >>= BBR_SCALE; + rate *= USEC_PER_SEC / 100 * (100 - margin); + rate >>= BW_SCALE; + rate = max(rate, 1ULL); + return rate; +} + +static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) +{ + return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); +} + +static u64 bbr_rate_kbps(struct sock *sk, u64 rate) +{ + rate = bbr_bw_bytes_per_sec(sk, rate); + rate *= 8; + do_div(rate, 1000); + return rate; +} + +static u32 bbr_tso_segs_goal(struct sock *sk); +static void bbr_debug(struct sock *sk, u32 acked, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + static const char ca_states[] = { + [TCP_CA_Open] = 'O', + [TCP_CA_Disorder] = 'D', + [TCP_CA_CWR] = 'C', + [TCP_CA_Recovery] = 'R', + [TCP_CA_Loss] = 'L', + }; + static const char mode[] = { + 'G', /* Growing - BBR_STARTUP */ + 'D', /* Drain - BBR_DRAIN */ + 'W', /* Window - BBR_PROBE_BW */ + 'M', /* Min RTT - BBR_PROBE_RTT */ + }; + static const char ack_phase[] = { /* bbr_ack_phase strings */ + 'I', /* BBR_ACKS_INIT - 'Init' */ + 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ + 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ + 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ + 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ + }; + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + const u32 una = tp->snd_una - bbr->debug.snd_isn; + const u32 fack = tcp_highest_sack_seq(tp); + const u16 dport = ntohs(inet_sk(sk)->inet_dport); + bool is_port_match = (bbr_debug_port_mask && + ((dport & bbr_debug_port_mask) == 0)); + char debugmsg[320]; + + if (sk->sk_state == TCP_SYN_SENT) + return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ + + if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { + char addr[INET6_ADDRSTRLEN + 10] = { 0 }; + + if (sk->sk_family == AF_INET) + snprintf(addr, sizeof(addr), "%pI4:%u", + &inet_sk(sk)->inet_daddr, dport); + else if (sk->sk_family == AF_INET6) + snprintf(addr, sizeof(addr), "%pI6:%u", + &sk->sk_v6_daddr, dport); + + WARN_ONCE(1, + "BBR %s cwnd alert: %u " + "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " + "bw: %u rtt: %u min_rtt: %u " + "acked: %u tso_segs: %u " + "bw: %d %ld %d pif: %u\n", + addr, tp->snd_cwnd, + una, inet_csk(sk)->icsk_ca_state, + bbr->pacing_gain, bbr->cwnd_gain, + bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, + acked, bbr_tso_segs_goal(sk), + rs->delivered, rs->interval_us, rs->is_retrans, + tcp_packets_in_flight(tp)); + } + + if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) + return; + + if (!sock_flag(sk, SOCK_DBG) && !is_port_match) + return; + + if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) + return; + + if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && + !(bbr_flags & FLAG_DEBUG_LOOPBACK)) + return; + + snprintf(debugmsg, sizeof(debugmsg) - 1, + "BBR %pI4:%-5u %5u,%03u:%-7u %c " + "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " + "bw %llu lb %llu ib %llu qb %llu " + "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " + "lr %d er %d ea %d bwl %lld il %d ih %d c %d " + "v %d %c %u %c %s\n", + &inet_sk(sk)->inet_daddr, dport, + una / 1000, una % 1000, fack - tp->snd_una, + ca_states[inet_csk(sk)->icsk_ca_state], + bbr->debug.undo ? '@' : mode[bbr->mode], + tp->snd_cwnd, + bbr_extra_acked(sk), /* br (legacy): extra_acked */ + rs->tx_in_flight, /* cr (legacy): tx_inflight */ + rs->rtt_us, + rs->delivered, + rs->interval_us, + bbr->min_rtt_us, + rs->is_app_limited ? '_' : 'l', + bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ + bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ + 0ULL, /* lb: [obsolete] */ + 0ULL, /* ib: [obsolete] */ + div_u64((u64)sk->sk_pacing_rate * 8, 1000), + acked, + tcp_packets_in_flight(tp), + rs->is_ack_delayed ? 'd' : '.', + bbr->round_start ? '*' : '.', + tp->delivered, tp->lost, + tp->app_limited, + 0, /* #: [obsolete] */ + ctx->target_cwnd, + tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ + ca_states[bbr->prev_ca_state], + (rs->lost + rs->delivered) > 0 ? + (1000 * rs->lost / + (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ + (rs->delivered) > 0 ? + (1000 * rs->delivered_ce / + (rs->delivered)) : 0, /* er: ECN rate x1000 */ + 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ + bbr->bw_lo == ~0U ? + -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ + bbr->inflight_lo, /* il */ + bbr->inflight_hi, /* ih */ + bbr->bw_probe_up_cnt, /* c */ + 2, /* v: version */ + bbr->debug.event, + bbr->cycle_idx, + ack_phase[bbr->ack_phase], + bbr->bw_probe_samples ? "Y" : "N"); + debugmsg[sizeof(debugmsg) - 1] = 0; + + /* printk takes a higher precedence. */ + if (bbr_debug_with_printk) + printk(KERN_DEBUG "%s", debugmsg); + + if (unlikely(bbr->debug.undo)) + bbr->debug.undo = 0; +} + +/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ +static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + u64 rate = bw; + + rate = bbr_rate_bytes_per_sec(sk, rate, gain, + bbr_pacing_margin_percent); + rate = min_t(u64, rate, sk->sk_max_pacing_rate); + return rate; +} + +/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ +static void bbr_init_pacing_rate_from_rtt(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + u32 rtt_us; + + if (tp->srtt_us) { /* any RTT sample yet? */ + rtt_us = max(tp->srtt_us >> 3, 1U); + bbr->has_seen_rtt = 1; + } else { /* no RTT sample yet */ + rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ + } + bw = (u64)tp->snd_cwnd * BW_UNIT; + do_div(bw, rtt_us); + sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); +} + +/* Pace using current bw estimate and a gain factor. */ +static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); + + if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) + bbr_init_pacing_rate_from_rtt(sk); + if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) + sk->sk_pacing_rate = rate; +} + +static u32 bbr_min_tso_segs(struct sock *sk) +{ + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; +} + +/* Return the number of segments BBR would like in a TSO/GSO skb, given + * a particular max gso size as a constraint. + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 segs, r; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; + + /* Budget a TSO/GSO burst size allowance based on min_rtt. For every + * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. + * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) + */ + if (bbr->params.tso_rtt_shift) { + r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; + if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ + bytes += GSO_MAX_SIZE >> r; + } + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); +} + +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ +static u32 bbr_tso_segs_goal(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); +} + +/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +static void bbr_save_cwnd(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) + bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ + else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ + bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); +} + +static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (event == CA_EVENT_TX_START && tp->app_limited) { + bbr->idle_restart = 1; + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + /* Avoid pointless buffer overflows: pace at est. bw if we don't + * need more speed (we're restarting from idle and app-limited). + */ + if (bbr->mode == BBR_PROBE_BW) + bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); + else if (bbr->mode == BBR_PROBE_RTT) + bbr_check_probe_rtt_done(sk); + } else if ((event == CA_EVENT_ECN_IS_CE || + event == CA_EVENT_ECN_NO_CE) && + bbr_ecn_enable && + bbr->params.precise_ece_ack) { + u32 state = bbr->ce_state; + dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); + bbr->ce_state = state; + if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) + tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); + } +} + +/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * + * bdp = ceil(bw * min_rtt * gain) + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT + * measurements (e.g., delayed ACKs or other ACK compression effects). This + * noise may cause BBR to under-estimate the rate. + */ +static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; + u64 w; + + /* If we've never had a valid RTT sample, cap cwnd at the initial + * default. This should only happen when the connection is not using TCP + * timestamps and has retransmitted all of the SYN/SYNACK/data packets + * ACKed so far. In this case, an RTO can cut cwnd to 1, in which + * case we need to slow-start up toward something safe: initial cwnd. + */ + if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ + return bbr->init_cwnd; /* be safe: cap at initial cwnd */ + + w = (u64)bw * bbr->min_rtt_us; + + /* Apply a gain to the given value, remove the BW_SCALE shift, and + * round the value up to avoid a negative feedback loop. + */ + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + return bdp; +} + +/* To achieve full performance in high-speed paths, we budget enough cwnd to + * fit full-sized skbs in-flight on both end hosts to fully utilize the path: + * - one skb in sending host Qdisc, + * - one skb in sending host TSO/GSO engine + * - one skb being received by receiver host LRO/GRO/delayed-ACK engine + * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because + * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ +static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 tso_segs_goal; + + tso_segs_goal = 3 * bbr_tso_segs_goal(sk); + + /* Allow enough full-sized skbs in flight to utilize end systems. */ + if (bbr->params.cwnd_tso_budget == 1) { + cwnd = max_t(u32, cwnd, tso_segs_goal); + cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); + } else { + cwnd += tso_segs_goal; + cwnd = (cwnd + 1) & ~1U; + } + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + cwnd += 2; + + return cwnd; +} + +/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ +static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) +{ + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); + inflight = bbr_quantization_budget(sk, inflight); + + return inflight; +} + +/* With pacing at lower layers, there's often less data "in the network" than + * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), + * we often have several skbs queued in the pacing layer with a pre-scheduled + * earliest departure time (EDT). BBR adapts its pacing rate based on the + * inflight level that it estimates has already been "baked in" by previous + * departure time decisions. We calculate a rough estimate of the number of our + * packets that might be in the network at the earliest departure time for the + * next skb scheduled: + * in_network_at_edt = inflight_at_edt - (EDT - now) * bw + * If we're increasing inflight, then we want to know if the transmit of the + * EDT skb will push inflight above the target, so inflight_at_edt includes + * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, + * then estimate if inflight will sink too low just before the EDT transmit. + */ +static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 now_ns, edt_ns, interval_us; + u32 interval_delivered, inflight_at_edt; + + now_ns = tp->tcp_clock_cache; + edt_ns = max(tp->tcp_wstamp_ns, now_ns); + interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); + interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; + inflight_at_edt = inflight_now; + if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ + inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ + if (interval_delivered >= inflight_at_edt) + return 0; + return inflight_at_edt - interval_delivered; +} + +/* Find the cwnd increment based on estimate of ack aggregation */ +static u32 bbr_ack_aggregation_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 max_aggr_cwnd, aggr_cwnd = 0; + + if (bbr->params.extra_acked_gain && + (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { + max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) + / BW_UNIT; + aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) + >> BBR_SCALE; + aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); + } + + return aggr_cwnd; +} + +/* Returns the cwnd for PROBE_RTT mode. */ +static u32 bbr_probe_rtt_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->params.probe_rtt_cwnd_gain == 0) + return bbr->params.cwnd_min_target; + return max_t(u32, bbr->params.cwnd_min_target, + bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); +} + +/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss + * has drawn us down below target), or snap down to target if we're above it. + */ +static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, + u32 acked, u32 bw, int gain, u32 cwnd, + struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; + + if (!acked) + goto done; /* no packet fully ACKed; just apply caps */ + + target_cwnd = bbr_bdp(sk, bw, gain); + + /* Increment the cwnd to account for excess ACKed data that seems + * due to aggregation (of data and/or ACKs) visible in the ACK stream. + */ + target_cwnd += bbr_ack_aggregation_cwnd(sk); + target_cwnd = bbr_quantization_budget(sk, target_cwnd); + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ + bbr->debug.target_cwnd = target_cwnd; + + /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ + bbr->try_fast_path = 0; + if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ + cwnd += acked; + if (cwnd >= target_cwnd) { + cwnd = target_cwnd; + bbr->try_fast_path = 1; + } + } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { + cwnd += acked; + } else { + bbr->try_fast_path = 1; + } + + /* When growing cwnd, don't grow beyond twice what we just probed. */ + if (bbr->params.usage_based_cwnd) { + max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); + cwnd = min(cwnd, max_probe); + } + + cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); +done: + tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ + if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ + tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); + + ctx->target_cwnd = target_cwnd; + ctx->log = (tp->snd_cwnd != prev_cwnd); +} + +/* See if we have reached next round trip */ +static void bbr_update_round_start(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->round_start = 0; + + /* See if we've reached the next RTT */ + if (rs->interval_us > 0 && + !before(rs->prior_delivered, bbr->next_rtt_delivered)) { + bbr->next_rtt_delivered = tp->delivered; + bbr->round_start = 1; + } +} + +/* Calculate the bandwidth based on how fast packets are delivered */ +static void bbr_calculate_bw_sample(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u64 bw = 0; + + /* Divide delivered by the interval to find a (lower bound) bottleneck + * bandwidth sample. Delivered is in packets and interval_us in uS and + * ratio will be <<1 for most connections. So delivered is first scaled. + * Round up to allow growth at low rates, even with integer division. + */ + if (rs->interval_us > 0) { + if (WARN_ONCE(rs->delivered < 0, + "negative delivered: %d interval_us: %ld\n", + rs->delivered, rs->interval_us)) + return; + + bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); + } + + ctx->sample_bw = bw; + bbr->debug.rs_bw = bw; +} + +/* Estimates the windowed max degree of ack aggregation. + * This is used to provision extra in-flight data to keep sending during + * inter-ACK silences. + * + * Degree of ack aggregation is estimated as extra data acked beyond expected. + * + * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" + * cwnd += max_extra_acked + * + * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). + * Max filter is an approximate sliding window of 5-10 (packet timed) round + * trips for non-startup phase, and 1-2 round trips for startup. + */ +static void bbr_update_ack_aggregation(struct sock *sk, + const struct rate_sample *rs) +{ + u32 epoch_us, expected_acked, extra_acked; + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_sock *tp = tcp_sk(sk); + u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; + + if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || + rs->delivered < 0 || rs->interval_us <= 0) + return; + + if (bbr->round_start) { + bbr->extra_acked_win_rtts = min(0x1F, + bbr->extra_acked_win_rtts + 1); + if (bbr->params.extra_acked_in_startup && + !bbr_full_bw_reached(sk)) + extra_acked_win_rtts_thresh = 1; + if (bbr->extra_acked_win_rtts >= + extra_acked_win_rtts_thresh) { + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? + 0 : 1; + bbr->extra_acked[bbr->extra_acked_win_idx] = 0; + } + } + + /* Compute how many packets we expected to be delivered over epoch. */ + epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, + bbr->ack_epoch_mstamp); + expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; + + /* Reset the aggregation epoch if ACK rate is below expected rate or + * significantly large no. of ack received since epoch (potentially + * quite old epoch). + */ + if (bbr->ack_epoch_acked <= expected_acked || + (bbr->ack_epoch_acked + rs->acked_sacked >= + bbr_ack_epoch_acked_reset_thresh)) { + bbr->ack_epoch_acked = 0; + bbr->ack_epoch_mstamp = tp->delivered_mstamp; + expected_acked = 0; + } + + /* Compute excess data delivered, beyond what was expected. */ + bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, + bbr->ack_epoch_acked + rs->acked_sacked); + extra_acked = bbr->ack_epoch_acked - expected_acked; + extra_acked = min(extra_acked, tp->snd_cwnd); + if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) + bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; +} + +/* Estimate when the pipe is full, using the change in delivery rate: BBR + * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by + * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited + * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the + * higher rwin, 3: we get higher delivery rate samples. Or transient + * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar + * design goal, but uses delay and inter-ACK spacing instead of bandwidth. + */ +static void bbr_check_full_bw_reached(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bw_thresh; + + if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) + return; + + bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; + if (bbr_max_bw(sk) >= bw_thresh) { + bbr->full_bw = bbr_max_bw(sk); + bbr->full_bw_cnt = 0; + return; + } + ++bbr->full_bw_cnt; + bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; +} + +/* If pipe is probably full, drain the queue and then enter steady-state. */ +static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr2_reset_congestion_signals(sk); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) + return true; /* exiting DRAIN now */ + return false; +} + +static void bbr_check_probe_rtt_done(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (!(bbr->probe_rtt_done_stamp && + after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) + return; + + bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ + tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); + bbr2_exit_probe_rtt(sk); +} + +/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and + * periodically drain the bottleneck queue, to converge to measure the true + * min_rtt (unloaded propagation delay). This allows the flows to keep queues + * small (reducing queuing delay and packet loss) and achieve fairness among + * BBR flows. + * + * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, + * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. + * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed + * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and + * re-enter the previous mode. BBR uses 200ms to approximately bound the + * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). + * + * Note that flows need only pay 2% if they are busy sending over the last 10 + * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have + * natural silences or low-rate periods within 10 seconds where the rate is low + * enough for long enough to drain its queue in the bottleneck. We pick up + * these min RTT measurements opportunistically with our min_rtt filter. :-) + */ +static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + bool probe_rtt_expired, min_rtt_expired; + u32 expire; + + /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ + expire = bbr->probe_rtt_min_stamp + + msecs_to_jiffies(bbr->params.probe_rtt_win_ms); + probe_rtt_expired = after(tcp_jiffies32, expire); + if (rs->rtt_us >= 0 && + (rs->rtt_us <= bbr->probe_rtt_min_us || + (probe_rtt_expired && !rs->is_ack_delayed))) { + bbr->probe_rtt_min_us = rs->rtt_us; + bbr->probe_rtt_min_stamp = tcp_jiffies32; + } + /* Track min RTT seen in the min_rtt_win_sec filter window: */ + expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; + min_rtt_expired = after(tcp_jiffies32, expire); + if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || + min_rtt_expired) { + bbr->min_rtt_us = bbr->probe_rtt_min_us; + bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; + } + + if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && + !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { + bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ + bbr_save_cwnd(sk); /* note cwnd so we can restore it */ + bbr->probe_rtt_done_stamp = 0; + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + } + + if (bbr->mode == BBR_PROBE_RTT) { + /* Ignore low rate samples during this mode. */ + tp->app_limited = + (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; + /* Maintain min packets in flight for max(200 ms, 1 round). */ + if (!bbr->probe_rtt_done_stamp && + tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { + bbr->probe_rtt_done_stamp = tcp_jiffies32 + + msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); + bbr->probe_rtt_round_done = 0; + bbr->next_rtt_delivered = tp->delivered; + } else if (bbr->probe_rtt_done_stamp) { + if (bbr->round_start) + bbr->probe_rtt_round_done = 1; + if (bbr->probe_rtt_round_done) + bbr_check_probe_rtt_done(sk); + } + } + /* Restart after idle ends only once we process a new S/ACK for data */ + if (rs->delivered > 0) + bbr->idle_restart = 0; +} + +static void bbr_update_gains(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + switch (bbr->mode) { + case BBR_STARTUP: + bbr->pacing_gain = bbr->params.high_gain; + bbr->cwnd_gain = bbr->params.startup_cwnd_gain; + break; + case BBR_DRAIN: + bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ + bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ + break; + case BBR_PROBE_BW: + bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; + bbr->cwnd_gain = bbr->params.cwnd_gain; + break; + case BBR_PROBE_RTT: + bbr->pacing_gain = BBR_UNIT; + bbr->cwnd_gain = BBR_UNIT; + break; + default: + WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); + break; + } +} + +static void bbr_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + int i; + + WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); + + bbr->initialized = 1; + bbr->params.high_gain = min(0x7FF, bbr_high_gain); + bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); + bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); + bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); + bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); + bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); + bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); + bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); + bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); + bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); + bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); + bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); + bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; + bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; + bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; + bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); + bbr->params.probe_rtt_win_ms = + min(0x3FFFU, + min_t(u32, bbr_probe_rtt_win_ms, + bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); + for (i = 0; i < CYCLE_LEN; i++) + bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); + bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; + bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); + + bbr->debug.snd_isn = tp->snd_una; + bbr->debug.target_cwnd = 0; + bbr->debug.undo = 0; + + bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); + bbr->prior_cwnd = tp->prior_cwnd; + tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; + bbr->packet_conservation = 0; + + bbr->probe_rtt_done_stamp = 0; + bbr->probe_rtt_round_done = 0; + bbr->probe_rtt_min_us = tcp_min_rtt(tp); + bbr->probe_rtt_min_stamp = tcp_jiffies32; + bbr->min_rtt_us = tcp_min_rtt(tp); + bbr->min_rtt_stamp = tcp_jiffies32; + + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; + bbr->full_bw = 0; + bbr->full_bw_cnt = 0; + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; + bbr->mode = BBR_STARTUP; + bbr->debug.rs_bw = 0; + + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = 0; + bbr->extra_acked[0] = 0; + bbr->extra_acked[1] = 0; + + bbr->ce_state = 0; + bbr->prior_rcv_nxt = tp->rcv_nxt; + bbr->try_fast_path = 0; + + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); +} + +static u32 bbr_sndbuf_expand(struct sock *sk) +{ + /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ + return 3; +} + +/* __________________________________________________________________________ + * + * Functions new to BBR v2 ("bbr") congestion control are below here. + * __________________________________________________________________________ + */ + +/* Incorporate a new bw sample into the current window of our max filter. */ +static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); +} + +/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ +static void bbr2_advance_bw_hi_filter(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (!bbr->bw_hi[1]) + return; /* no samples in this window; remember old window */ + bbr->bw_hi[0] = bbr->bw_hi[1]; + bbr->bw_hi[1] = 0; +} + +/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ +static u32 bbr2_target_inflight(struct sock *sk) +{ + u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); + + return min(bdp, tcp_sk(sk)->snd_cwnd); +} + +static bool bbr2_is_probing_bandwidth(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return (bbr->mode == BBR_STARTUP) || + (bbr->mode == BBR_PROBE_BW && + (bbr->cycle_idx == BBR_BW_PROBE_REFILL || + bbr->cycle_idx == BBR_BW_PROBE_UP)); +} + +/* Has the given amount of time elapsed since we marked the phase start? */ +static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) +{ + const struct tcp_sock *tp = tcp_sk(sk); + const struct bbr *bbr = inet_csk_ca(sk); + + return tcp_stamp_us_delta(tp->tcp_mstamp, + bbr->cycle_mstamp + interval_us) > 0; +} + +static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->full_bw_reached = 1; + bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); +} + +/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ +static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || + !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) + return; + + if (ce_ratio >= bbr->params.ecn_thresh) + bbr->startup_ecn_rounds++; + else + bbr->startup_ecn_rounds = 0; + + if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { + bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ + bbr2_handle_queue_too_high_in_startup(sk); + return; + } +} + +static void bbr2_update_ecn_alpha(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + s32 delivered, delivered_ce; + u64 alpha, ce_ratio; + u32 gain; + + if (bbr->params.ecn_factor == 0) + return; + + delivered = tp->delivered - bbr->alpha_last_delivered; + delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; + + if (delivered == 0 || /* avoid divide by zero */ + WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ + return; + + /* See if we should use ECN sender logic for this connection. */ + if (!bbr->ecn_eligible && bbr_ecn_enable && + (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || + !bbr->params.ecn_max_rtt_us)) + bbr->ecn_eligible = 1; + + ce_ratio = (u64)delivered_ce << BBR_SCALE; + do_div(ce_ratio, delivered); + gain = bbr->params.ecn_alpha_gain; + alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; + alpha += (gain * ce_ratio) >> BBR_SCALE; + bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); + + bbr->alpha_last_delivered = tp->delivered; + bbr->alpha_last_delivered_ce = tp->delivered_ce; + + bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); +} + +/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ +static void bbr2_raise_inflight_hi_slope(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 growth_this_round, cnt; + + /* Calculate "slope": packets S/Acked per inflight_hi increment. */ + growth_this_round = 1 << bbr->bw_probe_up_rounds; + bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); + cnt = tp->snd_cwnd / growth_this_round; + cnt = max(cnt, 1U); + bbr->bw_probe_up_cnt = cnt; + bbr->debug.event = 'G'; /* Grow inflight_hi slope */ +} + +/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ +static void bbr2_probe_inflight_hi_upward(struct sock *sk, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 delta; + + if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { + bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ + return; /* not fully using inflight_hi, so don't grow it */ + } + + /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ + bbr->bw_probe_up_acks += rs->acked_sacked; + if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { + delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; + bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; + bbr->inflight_hi += delta; + bbr->debug.event = 'I'; /* Increment inflight_hi */ + } + + if (bbr->round_start) + bbr2_raise_inflight_hi_slope(sk); +} + +/* Does loss/ECN rate for this sample say inflight is "too high"? + * This is used by both the bbr_check_loss_too_high_in_startup() function, + * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which + * uses it to notice when loss/ECN rates suggest inflight is too high. + */ +static bool bbr2_is_inflight_too_high(const struct sock *sk, + const struct rate_sample *rs) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh, ecn_thresh; + + if (rs->lost > 0 && rs->tx_in_flight) { + loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> + BBR_SCALE; + if (rs->lost > loss_thresh) + return true; + } + + if (rs->delivered_ce > 0 && rs->delivered > 0 && + bbr->ecn_eligible && bbr->params.ecn_thresh) { + ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> + BBR_SCALE; + if (rs->delivered_ce >= ecn_thresh) + return true; + } + + return false; +} + +/* Calculate the tx_in_flight level that corresponded to excessive loss. + * We find "lost_prefix" segs of the skb where loss rate went too high, + * by solving for "lost_prefix" in the following equation: + * lost / inflight >= loss_thresh + * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh + * Then we take that equation, convert it to fixed point, and + * round up to the nearest packet. + */ +static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, + const struct rate_sample *rs, + const struct sk_buff *skb) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh = bbr->params.loss_thresh; + u32 pcount, divisor, inflight_hi; + s32 inflight_prev, lost_prev; + u64 loss_budget, lost_prefix; + + pcount = tcp_skb_pcount(skb); + + /* How much data was in flight before this skb? */ + inflight_prev = rs->tx_in_flight - pcount; + if (WARN_ONCE(inflight_prev < 0, + "tx_in_flight: %u pcount: %u reneg: %u", + rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) + return ~0U; + + /* How much inflight data was marked lost before this skb? */ + lost_prev = rs->lost - pcount; + if (WARN_ON_ONCE(lost_prev < 0)) + return ~0U; + + /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ + loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; + loss_budget >>= BBR_SCALE; + if (lost_prev >= loss_budget) { + lost_prefix = 0; /* previous losses crossed loss_thresh */ + } else { + lost_prefix = loss_budget - lost_prev; + lost_prefix <<= BBR_SCALE; + divisor = BBR_UNIT - loss_thresh; + if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ + return ~0U; + do_div(lost_prefix, divisor); + } + + inflight_hi = inflight_prev + lost_prefix; + return inflight_hi; +} + +/* If loss/ECN rates during probing indicated we may have overfilled a + * buffer, return an operating point that tries to leave unutilized headroom in + * the path for other flows, for fairness convergence and lower RTTs and loss. + */ +static u32 bbr2_inflight_with_headroom(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 headroom, headroom_fraction; + + if (bbr->inflight_hi == ~0U) + return ~0U; + + headroom_fraction = bbr->params.inflight_headroom; + headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; + headroom = max(headroom, 1U); + return max_t(s32, bbr->inflight_hi - headroom, + bbr->params.cwnd_min_target); +} + +/* Bound cwnd to a sensible level, based on our current probing state + * machine phase and model of a good inflight level (inflight_lo, inflight_hi). + */ +static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 cap; + + /* tcp_rcv_synsent_state_process() currently calls tcp_ack() + * and thus cong_control() without first initializing us(!). + */ + if (!bbr->initialized) + return; + + cap = ~0U; + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { + /* Probe to see if more packets fit in the path. */ + cap = bbr->inflight_hi; + } else { + if (bbr->mode == BBR_PROBE_RTT || + (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) + cap = bbr2_inflight_with_headroom(sk); + } + /* Adapt to any loss/ECN since our last bw probe. */ + cap = min(cap, bbr->inflight_lo); + + cap = max_t(u32, cap, bbr->params.cwnd_min_target); + tp->snd_cwnd = min(cap, tp->snd_cwnd); +} + +/* Estimate a short-term lower bound on the capacity available now, based + * on measurements of the current delivery process and recent history. When we + * are seeing loss/ECN at times when we are not probing bw, then conservatively + * move toward flow balance by multiplicatively cutting our short-term + * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a + * multiplicative decrease in order to converge to a lower capacity in time + * logarithmic in the magnitude of the decrease. + * + * However, we do not cut our short-term estimates lower than the current rate + * and volume of delivered data from this round trip, since from the current + * delivery process we can estimate the measured capacity available now. + * + * Anything faster than that approach would knowingly risk high loss, which can + * cause low bw for Reno/CUBIC and high loss recovery latency for + * request/response flows using any congestion control. + */ +static void bbr2_adapt_lower_bounds(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_cut, ecn_inflight_lo, beta; + + /* We only use lower-bound estimates when not probing bw. + * When probing we need to push inflight higher to probe bw. + */ + if (bbr2_is_probing_bandwidth(sk)) + return; + + /* ECN response. */ + if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { + /* Reduce inflight to (1 - alpha*ecn_factor). */ + ecn_cut = (BBR_UNIT - + ((bbr->ecn_alpha * bbr->params.ecn_factor) >> + BBR_SCALE)); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tp->snd_cwnd; + ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; + } else { + ecn_inflight_lo = ~0U; + } + + /* Loss response. */ + if (bbr->loss_in_round) { + /* Reduce bw and inflight to (1 - beta). */ + if (bbr->bw_lo == ~0U) + bbr->bw_lo = bbr_max_bw(sk); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tp->snd_cwnd; + beta = bbr->params.beta; + bbr->bw_lo = + max_t(u32, bbr->bw_latest, + (u64)bbr->bw_lo * + (BBR_UNIT - beta) >> BBR_SCALE); + bbr->inflight_lo = + max_t(u32, bbr->inflight_latest, + (u64)bbr->inflight_lo * + (BBR_UNIT - beta) >> BBR_SCALE); + } + + /* Adjust to the lower of the levels implied by loss or ECN. */ + bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); +} + +/* Reset any short-term lower-bound adaptation to congestion, so that we can + * push our inflight up. + */ +static void bbr2_reset_lower_bounds(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_lo = ~0U; + bbr->inflight_lo = ~0U; +} + +/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state + * machine phase where we adapt our lower bound based on congestion signals. + */ +static void bbr2_reset_congestion_signals(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->loss_in_cycle = 0; + bbr->ecn_in_cycle = 0; + bbr->bw_latest = 0; + bbr->inflight_latest = 0; +} + +/* Update (most of) our congestion signals: track the recent rate and volume of + * delivered data, presence of loss, and EWMA degree of ECN marking. + */ +static void bbr2_update_congestion_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + + bbr->loss_round_start = 0; + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + bw = ctx->sample_bw; + + if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) + bbr2_take_bw_hi_sample(sk, bw); + + bbr->loss_in_round |= (rs->losses > 0); + + /* Update rate and volume of delivered data from latest round trip: */ + bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); + bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); + + if (before(rs->prior_delivered, bbr->loss_round_delivered)) + return; /* skip the per-round-trip updates */ + /* Now do per-round-trip updates. */ + bbr->loss_round_delivered = tp->delivered; /* mark round trip */ + bbr->loss_round_start = 1; + bbr2_adapt_lower_bounds(sk); + + /* Update windowed "latest" (single-round-trip) filters. */ + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->bw_latest = ctx->sample_bw; + bbr->inflight_latest = rs->delivered; +} + +/* Bandwidth probing can cause loss. To help coexistence with loss-based + * congestion control we spread out our probing in a Reno-conscious way. Due to + * the shape of the Reno sawtooth, the time required between loss epochs for an + * idealized Reno flow is a number of round trips that is the BDP of that + * flow. We count packet-timed round trips directly, since measured RTT can + * vary widely, and Reno is driven by packet-timed round trips. + */ +static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 inflight, rounds, reno_gain, reno_rounds; + + /* Random loss can shave some small percentage off of our inflight + * in each round. To survive this, flows need robust periodic probes. + */ + rounds = bbr->params.bw_probe_max_rounds; + + reno_gain = bbr->params.bw_probe_reno_gain; + if (reno_gain) { + inflight = bbr2_target_inflight(sk); + reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; + rounds = min(rounds, reno_rounds); + } + return bbr->rounds_since_probe >= rounds; +} + +/* How long do we want to wait before probing for bandwidth (and risking + * loss)? We randomize the wait, for better mixing and fairness convergence. + * + * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. + * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, + * (eg 4K video to a broadband user): + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + * + * We bound the BBR-native inter-bw-probe wall clock time to be: + * (a) higher than 2 sec: to try to avoid causing loss for a long enough time + * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must + * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs + * (b) lower than 3 sec: to ensure flows can start probing in a reasonable + * amount of time to discover unutilized bw on human-scale interactive + * time-scales (e.g. perhaps traffic from a web page download that we + * were competing with is now complete). + */ +static void bbr2_pick_probe_wait(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Decide the random round-trip bound for wait until probe: */ + bbr->rounds_since_probe = + prandom_u32_max(bbr->params.bw_probe_rand_rounds); + /* Decide the random wall clock bound for wait until probe: */ + bbr->probe_wait_us = bbr->params.bw_probe_base_us + + prandom_u32_max(bbr->params.bw_probe_rand_us); +} + +static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->cycle_idx = cycle_idx; + /* New phase, so need to update cwnd and pacing rate. */ + bbr->try_fast_path = 0; +} + +/* Send at estimated bw to fill the pipe, but not queue. We need this phase + * before PROBE_UP, because as soon as we send faster than the available bw + * we will start building a queue, and if the buffer is shallow we can cause + * loss. If we do not fill the pipe before we cause this loss, our bw_hi and + * inflight_hi estimates will underestimate. + */ +static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_lower_bounds(sk); + if (bbr->inflight_hi != ~0U) + bbr->inflight_hi += bbr->params.refill_add_inc; + bbr->bw_probe_up_rounds = bw_probe_up_rounds; + bbr->bw_probe_up_acks = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_REFILLING; + bbr->next_rtt_delivered = tp->delivered; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); +} + +/* Now probe max deliverable data rate and volume. */ +static void bbr2_start_bw_probe_up(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->ack_phase = BBR_ACKS_PROBE_STARTING; + bbr->next_rtt_delivered = tp->delivered; + bbr->cycle_mstamp = tp->tcp_mstamp; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); + bbr2_raise_inflight_hi_slope(sk); +} + +/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall + * clock time at which to probe beyond an inflight that we think to be + * safe. This will knowingly risk packet loss, so we want to do this rarely, to + * keep packet loss rates low. Also start a round-trip counter, to probe faster + * if we estimate a Reno flow at our BDP would probe faster. + */ +static void bbr2_start_bw_probe_down(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_congestion_signals(sk); + bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ + bbr2_pick_probe_wait(sk); + bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); +} + +/* Cruise: maintain what we estimate to be a neutral, conservative + * operating point, without attempting to probe up for bandwidth or down for + * RTT, and only reducing inflight in response to loss/ECN signals. + */ +static void bbr2_start_bw_probe_cruise(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->inflight_lo != ~0U) + bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); + + bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); +} + +/* Loss and/or ECN rate is too high while probing. + * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. + */ +static void bbr2_handle_inflight_too_high(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + const u32 beta = bbr->params.beta; + + bbr->prev_probe_too_high = 1; + bbr->bw_probe_samples = 0; /* only react once per probe */ + bbr->debug.event = 'L'; /* Loss/ECN too high */ + /* If we are app-limited then we are not robustly + * probing the max volume of inflight data we think + * might be safe (analogous to how app-limited bw + * samples are not known to be robustly probing bw). + */ + if (!rs->is_app_limited) + bbr->inflight_hi = max_t(u32, rs->tx_in_flight, + (u64)bbr2_target_inflight(sk) * + (BBR_UNIT - beta) >> BBR_SCALE); + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr2_start_bw_probe_down(sk); +} + +/* If we're seeing bw and loss samples reflecting our bw probing, adapt + * using the signals we see. If loss or ECN mark rate gets too high, then adapt + * inflight_hi downward. If we're able to push inflight higher without such + * signals, push higher: adapt inflight_hi upward. + */ +static bool bbr2_adapt_upper_bounds(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Track when we'll see bw/loss samples resulting from our bw probes. */ + if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) + bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; + if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { + /* End of samples from bw probing phase. */ + bbr->bw_probe_samples = 0; + bbr->ack_phase = BBR_ACKS_INIT; + /* At this point in the cycle, our current bw sample is also + * our best recent chance at finding the highest available bw + * for this flow. So now is the best time to forget the bw + * samples from the previous cycle, by advancing the window. + */ + if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) + bbr2_advance_bw_hi_filter(sk); + /* If we had an inflight_hi, then probed and pushed inflight all + * the way up to hit that inflight_hi without seeing any + * high loss/ECN in all the resulting ACKs from that probing, + * then probe up again, this time letting inflight persist at + * inflight_hi for a round trip, then accelerating beyond. + */ + if (bbr->mode == BBR_PROBE_BW && + bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { + bbr->debug.event = 'R'; /* reprobe */ + bbr2_start_bw_probe_refill(sk, 0); + return true; /* yes, decided state transition */ + } + } + + if (bbr2_is_inflight_too_high(sk, rs)) { + if (bbr->bw_probe_samples) /* sample is from bw probing? */ + bbr2_handle_inflight_too_high(sk, rs); + } else { + /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ + if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ + return false; + + /* To be resilient to random loss, we must raise inflight_hi + * if we observe in any phase that a higher level is safe. + */ + if (rs->tx_in_flight > bbr->inflight_hi) { + bbr->inflight_hi = rs->tx_in_flight; + bbr->debug.event = 'U'; /* raise up inflight_hi */ + } + + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr2_probe_inflight_hi_upward(sk, rs); + } + + return false; +} + +/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ +static bool bbr2_check_time_to_probe_bw(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 n; + + /* If we seem to be at an operating point where we are not seeing loss + * but we are seeing ECN marks, then when the ECN marks cease we reprobe + * quickly (in case a burst of cross-traffic has ceased and freed up bw, + * or in case we are sharing with multiplicatively probing traffic). + */ + if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && + bbr->ecn_in_cycle && !bbr->loss_in_cycle && + inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { + bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ + /* Calculate n so that when bbr2_raise_inflight_hi_slope() + * computes growth_this_round as 2^n it will be roughly the + * desired volume of data (inflight_hi*ecn_reprobe_gain). + */ + n = ilog2((((u64)bbr->inflight_hi * + bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); + bbr2_start_bw_probe_refill(sk, n); + return true; + } + + if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || + bbr2_is_reno_coexistence_probe_time(sk)) { + bbr2_start_bw_probe_refill(sk, 0); + return true; + } + return false; +} + +/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ +static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) +{ + struct bbr *bbr = inet_csk_ca(sk); + bool is_under_bdp, is_long_enough; + + /* Always need to pull inflight down to leave headroom in queue. */ + if (inflight > bbr2_inflight_with_headroom(sk)) + return false; + + is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); + if (bbr->params.drain_to_target) + return is_under_bdp; + + is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); + return is_under_bdp || is_long_enough; +} + +/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ +static void bbr2_update_cycle_phase(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + bool is_risky = false, is_queuing = false; + u32 inflight, bw; + + if (!bbr_full_bw_reached(sk)) + return; + + /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ + if (bbr2_adapt_upper_bounds(sk, rs)) + return; /* already decided state transition */ + + if (bbr->mode != BBR_PROBE_BW) + return; + + inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); + bw = bbr_max_bw(sk); + + switch (bbr->cycle_idx) { + /* First we spend most of our time cruising with a pacing_gain of 1.0, + * which paces at the estimated bw, to try to fully use the pipe + * without building queue. If we encounter loss/ECN marks, we adapt + * by slowing down. + */ + case BBR_BW_PROBE_CRUISE: + if (bbr2_check_time_to_probe_bw(sk)) + return; /* already decided state transition */ + break; + + /* After cruising, when it's time to probe, we first "refill": we send + * at the estimated bw to fill the pipe, before probing higher and + * knowingly risking overflowing the bottleneck buffer (causing loss). + */ + case BBR_BW_PROBE_REFILL: + if (bbr->round_start) { + /* After one full round trip of sending in REFILL, we + * start to see bw samples reflecting our REFILL, which + * may be putting too much data in flight. + */ + bbr->bw_probe_samples = 1; + bbr2_start_bw_probe_up(sk); + } + break; + + /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to + * probe for bw. If we have not seen loss/ECN, we try to raise inflight + * to at least pacing_gain*BDP; note that this may take more than + * min_rtt if min_rtt is small (e.g. on a LAN). + * + * We terminate PROBE_UP bandwidth probing upon any of the following: + * + * (1) We've pushed inflight up to hit the inflight_hi target set in the + * most recent previous bw probe phase. Thus we want to start + * draining the queue immediately because it's very likely the most + * recently sent packets will fill the queue and cause drops. + * (checked here) + * (2) We have probed for at least 1*min_rtt_us, and the + * estimated queue is high enough (inflight > 1.25 * estimated_bdp). + * (checked here) + * (3) Loss filter says loss rate is "too high". + * (checked in bbr_is_inflight_too_high()) + * (4) ECN filter says ECN mark rate is "too high". + * (checked in bbr_is_inflight_too_high()) + */ + case BBR_BW_PROBE_UP: + if (bbr->prev_probe_too_high && + inflight >= bbr->inflight_hi) { + bbr->stopped_risky_probe = 1; + is_risky = true; + bbr->debug.event = 'D'; /* D for danger */ + } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && + inflight >= + bbr_inflight(sk, bw, + bbr->params.bw_probe_pif_gain)) { + is_queuing = true; + bbr->debug.event = 'Q'; /* building Queue */ + } + if (is_risky || is_queuing) { + bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ + bbr2_start_bw_probe_down(sk); /* restart w/ down */ + } + break; + + /* After probing in PROBE_UP, we have usually accumulated some data in + * the bottleneck buffer (if bw probing didn't find more bw). We next + * enter PROBE_DOWN to try to drain any excess data from the queue. To + * do this, we use a pacing_gain < 1.0. We hold this pacing gain until + * our inflight is less then that target cruising point, which is the + * minimum of (a) the amount needed to leave headroom, and (b) the + * estimated BDP. Once inflight falls to match the target, we estimate + * the queue is drained; persisting would underutilize the pipe. + */ + case BBR_BW_PROBE_DOWN: + if (bbr2_check_time_to_probe_bw(sk)) + return; /* already decided state transition */ + if (bbr2_check_time_to_cruise(sk, inflight, bw)) + bbr2_start_bw_probe_cruise(sk); + break; + + default: + WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); + } +} + +/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ +static void bbr2_exit_probe_rtt(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr2_reset_lower_bounds(sk); + if (bbr_full_bw_reached(sk)) { + bbr->mode = BBR_PROBE_BW; + /* Raising inflight after PROBE_RTT may cause loss, so reset + * the PROBE_BW clock and schedule the next bandwidth probe for + * a friendly and randomized future point in time. + */ + bbr2_start_bw_probe_down(sk); + /* Since we are exiting PROBE_RTT, we know inflight is + * below our estimated BDP, so it is reasonable to cruise. + */ + bbr2_start_bw_probe_cruise(sk); + } else { + bbr->mode = BBR_STARTUP; + } +} + +/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until + * the end of the round in recovery to get a good estimate of how many packets + * have been lost, and how many we need to drain with a low pacing rate. + */ +static void bbr2_check_loss_too_high_in_startup(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk)) + return; + + /* For STARTUP exit, check the loss rate at the end of each round trip + * of Recovery episodes in STARTUP. We check the loss rate at the end + * of the round trip to filter out noisy/low loss and have a better + * sense of inflight (extent of loss), so we can drain more accurately. + */ + if (rs->losses && bbr->loss_events_in_round < 0xf) + bbr->loss_events_in_round++; /* update saturating counter */ + if (bbr->params.full_loss_cnt && bbr->loss_round_start && + inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && + bbr->loss_events_in_round >= bbr->params.full_loss_cnt && + bbr2_is_inflight_too_high(sk, rs)) { + bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ + bbr2_handle_queue_too_high_in_startup(sk); + return; + } + if (bbr->loss_round_start) + bbr->loss_events_in_round = 0; +} + +/* If we are done draining, advance into steady state operation in PROBE_BW. */ +static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_check_drain(sk, rs, ctx)) { + bbr->mode = BBR_PROBE_BW; + bbr2_start_bw_probe_down(sk); + } +} + +static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + bbr2_update_congestion_signals(sk, rs, ctx); + bbr_update_ack_aggregation(sk, rs); + bbr2_check_loss_too_high_in_startup(sk, rs); + bbr_check_full_bw_reached(sk, rs); + bbr2_check_drain(sk, rs, ctx); + bbr2_update_cycle_phase(sk, rs); + bbr_update_min_rtt(sk, rs); +} + +/* Fast path for app-limited case. + * + * On each ack, we execute bbr state machine, which primarily consists of: + * 1) update model based on new rate sample, and + * 2) update control based on updated model or state change. + * + * There are certain workload/scenarios, e.g. app-limited case, where + * either we can skip updating model or we can skip update of both model + * as well as control. This provides signifcant softirq cpu savings for + * processing incoming acks. + * + * In case of app-limited, if there is no congestion (loss/ecn) and + * if observed bw sample is less than current estimated bw, then we can + * skip some of the computation in bbr state processing: + * + * - if there is no rtt/mode/phase change: In this case, since all the + * parameters of the network model are constant, we can skip model + * as well control update. + * + * - else we can skip rest of the model update. But we still need to + * update the control to account for the new rtt/mode/phase. + * + * Returns whether we can take fast path or not. + */ +static bool bbr2_fast_path(struct sock *sk, bool *update_model, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 prev_min_rtt_us, prev_mode; + + if (bbr->params.fast_path && bbr->try_fast_path && + rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && + !bbr->loss_in_round && !bbr->ecn_in_round) { + prev_mode = bbr->mode; + prev_min_rtt_us = bbr->min_rtt_us; + bbr2_check_drain(sk, rs, ctx); + bbr2_update_cycle_phase(sk, rs); + bbr_update_min_rtt(sk, rs); + + if (bbr->mode == prev_mode && + bbr->min_rtt_us == prev_min_rtt_us && + bbr->try_fast_path) + return true; + + /* Skip model update, but control still needs to be updated */ + *update_model = false; + } + return false; +} + +static void bbr2_main(struct sock *sk, const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct bbr_context ctx = { 0 }; + bool update_model = true; + u32 bw; + + bbr->debug.event = '.'; /* init to default NOP (no event yet) */ + + bbr_update_round_start(sk, rs, &ctx); + if (bbr->round_start) { + bbr->rounds_since_probe = + min_t(s32, bbr->rounds_since_probe + 1, 0xFF); + bbr2_update_ecn_alpha(sk); + } + + bbr->ecn_in_round |= rs->is_ece; + bbr_calculate_bw_sample(sk, rs, &ctx); + + if (bbr2_fast_path(sk, &update_model, rs, &ctx)) + goto out; + + if (update_model) + bbr2_update_model(sk, rs, &ctx); + + bbr_update_gains(sk); + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, + tp->snd_cwnd, &ctx); + bbr2_bound_cwnd_for_inflight_model(sk); + +out: + bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; + bbr->loss_in_cycle |= rs->lost > 0; + bbr->ecn_in_cycle |= rs->delivered_ce > 0; + + bbr_debug(sk, rs->acked_sacked, rs, &ctx); +} + +/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared + * down here, so that the algorithm functions that use the parameters must use + * the per-socket parameters; if they accidentally use the global version + * then there will be a compile error. + * TODO(ncardwell): move all per-socket parameters down to this section. + */ + +/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. + * No loss response when 0. Max allwed value is 255. + */ +static u32 bbr_beta = BBR_UNIT * 30 / 100; + +/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. + * Max allowed value is 255. + */ +static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ + +/* The initial value for the ecn_alpha state variable. Default and max + * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly + * to congestion if the bottleneck is congested when the flow starts up. + */ +static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ + +/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. + * No ECN based bounding when 0. Max allwed value is 255. + */ +static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ + +/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. + * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. + */ +static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ + +/* Max RTT (in usec) at which to use sender-side ECN logic. + * Disabled when 0 (ECN allowed at any RTT). + * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. + */ +static u32 bbr_ecn_max_rtt_us = 5000; + +/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN + * clears then use a multiplicative increase to quickly reprobe bw by + * starting inflight probing at the given multiple of inflight_hi. + * Default for this experimental knob is 0 (disabled). + * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. + */ +static u32 bbr_ecn_reprobe_gain; + +/* Estimate bw probing has gone too far if loss rate exceeds this level. */ +static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ + +/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, + * and loss rate is higher than bbr_loss_thresh. + * Disabled if 0. Max allowed value is 15 (0xF). + */ +static u32 bbr_full_loss_cnt = 8; + +/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh + * meets this count. Max allowed value is 3. + */ +static u32 bbr_full_ecn_cnt = 2; + +/* Fraction of unutilized headroom to try to leave in path upon high loss. */ +static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + +/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. + * Default is 1.25x, as in BBR v1. Max allowed is 511. + */ +static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; + +/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. + * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. + * Max allowed is 511. + */ +static u32 bbr_bw_probe_reno_gain = BBR_UNIT; + +/* Max number of packet-timed rounds to wait before probing for bandwidth. If + * we want to tolerate 1% random loss per round, and not have this cut our + * inflight too much, we must probe for bw periodically on roughly this scale. + * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. + * We aim to be fair with Reno/CUBIC up to a BDP of at least: + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + */ +static u32 bbr_bw_probe_max_rounds = 63; + +/* Max amount of randomness to inject in round counting for Reno-coexistence. + * Max value is 15. + */ +static u32 bbr_bw_probe_rand_rounds = 2; + +/* Use BBR-native probe time scale starting at this many usec. + * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: + * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs + */ +static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ + +/* Use BBR-native probes spread over this many usec: */ +static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + +/* Undo the model changes made in loss recovery if recovery was spurious? */ +static bool bbr_undo = true; + +/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ +static bool bbr_fast_path = true; /* default: enabled */ + +/* Use fast ack mode ? */ +static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ + +/* How much to additively increase inflight_hi when entering REFILL? */ +static u32 bbr_refill_add_inc; /* default: disabled */ + +module_param_named(beta, bbr_beta, uint, 0644); +module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); +module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); +module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); +module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); +module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); +module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); +module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); +module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); +module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); +module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); +module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664); +module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664); +module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664); +module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664); +module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664); +module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664); +module_param_named(undo, bbr_undo, bool, 0664); +module_param_named(fast_path, bbr_fast_path, bool, 0664); +module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); +module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); + +static void bbr2_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_init(sk); /* run shared init code for v1 and v2 */ + + /* BBR v2 parameters: */ + bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); + bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); + bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); + bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); + bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); + bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); + bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); + bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); + bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); + bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); + bbr->params.inflight_headroom = + min_t(u32, 0xFFU, bbr_inflight_headroom); + bbr->params.bw_probe_pif_gain = + min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); + bbr->params.bw_probe_reno_gain = + min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); + bbr->params.bw_probe_max_rounds = + min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); + bbr->params.bw_probe_rand_rounds = + min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); + bbr->params.bw_probe_base_us = + min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); + bbr->params.bw_probe_rand_us = + min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); + bbr->params.undo = bbr_undo; + bbr->params.fast_path = bbr_fast_path ? 1 : 0; + bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); + + /* BBR v2 state: */ + bbr->initialized = 1; + /* Start sampling ECN mark rate after first full flight is ACKed: */ + bbr->loss_round_delivered = tp->delivered + 1; + bbr->loss_round_start = 0; + bbr->undo_bw_lo = 0; + bbr->undo_inflight_lo = 0; + bbr->undo_inflight_hi = 0; + bbr->loss_events_in_round = 0; + bbr->startup_ecn_rounds = 0; + bbr2_reset_congestion_signals(sk); + bbr->bw_lo = ~0U; + bbr->bw_hi[0] = 0; + bbr->bw_hi[1] = 0; + bbr->inflight_lo = ~0U; + bbr->inflight_hi = ~0U; + bbr->bw_probe_up_cnt = ~0U; + bbr->bw_probe_up_acks = 0; + bbr->bw_probe_up_rounds = 0; + bbr->probe_wait_us = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_INIT; + bbr->rounds_since_probe = 0; + bbr->bw_probe_samples = 0; + bbr->prev_probe_too_high = 0; + bbr->ecn_eligible = 0; + bbr->ecn_alpha = bbr->params.ecn_alpha_init; + bbr->alpha_last_delivered = 0; + bbr->alpha_last_delivered_ce = 0; + + tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); + + if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) + tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; +} + +/* Core TCP stack informs us that the given skb was just marked lost. */ +static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_skb_cb *scb = TCP_SKB_CB(skb); + struct rate_sample rs; + + /* Capture "current" data over the full round trip of loss, + * to have a better chance to see the full capacity of the path. + */ + if (!bbr->loss_in_round) /* first loss in this round trip? */ + bbr->loss_round_delivered = tp->delivered; /* set round trip */ + bbr->loss_in_round = 1; + bbr->loss_in_cycle = 1; + + if (!bbr->bw_probe_samples) + return; /* not an skb sent while probing for bandwidth */ + if (unlikely(!scb->tx.delivered_mstamp)) + return; /* skb was SACKed, reneged, marked lost; ignore it */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this lost skb, + * then see if the loss rate went too high, and if so at which packet. + */ + memset(&rs, 0, sizeof(rs)); + rs.tx_in_flight = scb->tx.in_flight; + rs.lost = tp->lost - scb->tx.lost; + rs.is_app_limited = scb->tx.is_app_limited; + if (bbr2_is_inflight_too_high(sk, &rs)) { + rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); + bbr2_handle_inflight_too_high(sk, &rs); + } +} + +/* Revert short-term model if current loss recovery event was spurious. */ +static u32 bbr2_undo_cwnd(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->debug.undo = 1; + bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ + bbr->full_bw_cnt = 0; + bbr->loss_in_round = 0; + + if (!bbr->params.undo) + return tp->snd_cwnd; + + /* Revert to cwnd and other state saved before loss episode. */ + bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); + bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); + bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); + return bbr->prior_cwnd; +} + +/* Entering loss recovery, so save state for when we undo recovery. */ +static u32 bbr2_ssthresh(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_save_cwnd(sk); + /* For undo, save state that adapts based on loss signal. */ + bbr->undo_bw_lo = bbr->bw_lo; + bbr->undo_inflight_lo = bbr->inflight_lo; + bbr->undo_inflight_hi = bbr->inflight_hi; + return tcp_sk(sk)->snd_ssthresh; +} + +static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) +{ + switch (bbr->mode) { + case BBR_STARTUP: + return BBR2_PHASE_STARTUP; + case BBR_DRAIN: + return BBR2_PHASE_DRAIN; + case BBR_PROBE_BW: + break; + case BBR_PROBE_RTT: + return BBR2_PHASE_PROBE_RTT; + default: + return BBR2_PHASE_INVALID; + } + switch (bbr->cycle_idx) { + case BBR_BW_PROBE_UP: + return BBR2_PHASE_PROBE_BW_UP; + case BBR_BW_PROBE_DOWN: + return BBR2_PHASE_PROBE_BW_DOWN; + case BBR_BW_PROBE_CRUISE: + return BBR2_PHASE_PROBE_BW_CRUISE; + case BBR_BW_PROBE_REFILL: + return BBR2_PHASE_PROBE_BW_REFILL; + default: + return BBR2_PHASE_INVALID; + } +} + +static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, + union tcp_cc_info *info) +{ + if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || + ext & (1 << (INET_DIAG_VEGASINFO - 1))) { + struct bbr *bbr = inet_csk_ca(sk); + u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); + u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); + u64 bw_lo = bbr->bw_lo == ~0U ? + ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); + + memset(&info->bbr2, 0, sizeof(info->bbr2)); + info->bbr2.bbr_bw_lsb = (u32)bw; + info->bbr2.bbr_bw_msb = (u32)(bw >> 32); + info->bbr2.bbr_min_rtt = bbr->min_rtt_us; + info->bbr2.bbr_pacing_gain = bbr->pacing_gain; + info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; + info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; + info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); + info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; + info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); + info->bbr2.bbr_mode = bbr->mode; + info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); + info->bbr2.bbr_version = (__u8)2; + info->bbr2.bbr_inflight_lo = bbr->inflight_lo; + info->bbr2.bbr_inflight_hi = bbr->inflight_hi; + info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); + *attr = INET_DIAG_BBRINFO; + return sizeof(info->bbr2); + } + return 0; +} + +static void bbr2_set_state(struct sock *sk, u8 new_state) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (new_state == TCP_CA_Loss) { + struct rate_sample rs = { .losses = 1 }; + struct bbr_context ctx = { 0 }; + + bbr->prev_ca_state = TCP_CA_Loss; + bbr->full_bw = 0; + if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { + /* bbr_adapt_lower_bounds() needs cwnd before + * we suffered an RTO, to update inflight_lo: + */ + bbr->inflight_lo = + max(tp->snd_cwnd, bbr->prior_cwnd); + } + bbr_debug(sk, 0, &rs, &ctx); + } else if (bbr->prev_ca_state == TCP_CA_Loss && + new_state != TCP_CA_Loss) { + tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); + bbr->try_fast_path = 0; /* bound cwnd using latest model */ + } +} + +static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { + .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, + .name = "bbr2", + .owner = THIS_MODULE, + .init = bbr2_init, + .cong_control = bbr2_main, + .sndbuf_expand = bbr_sndbuf_expand, + .skb_marked_lost = bbr2_skb_marked_lost, + .undo_cwnd = bbr2_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr2_ssthresh, + .tso_segs = bbr_tso_segs, + .get_info = bbr2_get_info, + .set_state = bbr2_set_state, +}; + +static int __init bbr_register(void) +{ + BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); + return tcp_register_congestion_control(&tcp_bbr2_cong_ops); +} + +static void __exit bbr_unregister(void) +{ + tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); +} + +module_init(bbr_register); +module_exit(bbr_unregister); + +MODULE_AUTHOR("Van Jacobson "); +MODULE_AUTHOR("Neal Cardwell "); +MODULE_AUTHOR("Yuchung Cheng "); +MODULE_AUTHOR("Soheil Hassas Yeganeh "); +MODULE_AUTHOR("Priyaranjan Jha "); +MODULE_AUTHOR("Yousuk Seung "); +MODULE_AUTHOR("Kevin Yang "); +MODULE_AUTHOR("Arjun Roy "); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index d3cae40749e87..0f268f2ff2e9a 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) struct inet_connection_sock *icsk = inet_csk(sk); tcp_sk(sk)->prior_ssthresh = 0; + tcp_sk(sk)->fast_ack_mode = 0; if (icsk->icsk_ca_ops->init) icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index b1637990d5708..21c2922d937b2 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { @@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); tp->ecn_flags |= TCP_ECN_SEEN; break; @@ -1080,7 +1080,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) */ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) { + struct sock *sk = (struct sock *)tp; + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; + tp->lost += tcp_skb_pcount(skb); + if (ca_ops->skb_marked_lost) + ca_ops->skb_marked_lost(sk, skb); } void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) @@ -1461,6 +1466,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); tcp_skb_pcount_add(skb, -pcount); + /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ + if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, + "prev in_flight: %u skb in_flight: %u pcount: %u", + TCP_SKB_CB(prev)->tx.in_flight, + TCP_SKB_CB(skb)->tx.in_flight, + pcount)) + TCP_SKB_CB(skb)->tx.in_flight = 0; + else + TCP_SKB_CB(skb)->tx.in_flight -= pcount; + TCP_SKB_CB(prev)->tx.in_flight += pcount; + /* When we're adding to gso_segs == 1, gso_size will be zero, * in theory this shouldn't be necessary but as long as DSACK * code can come after this skb later on it's better to keep @@ -3800,6 +3816,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; rs.prior_in_flight = tcp_packets_in_flight(tp); + tcp_rate_check_app_limited(sk); /* ts_recent update must be made after we are sure that the packet * is in window. @@ -3898,6 +3915,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) delivered = tcp_newly_delivered(sk, delivered, flag); lost = tp->lost - lost; /* freshly marked lost */ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + rs.is_ece = !!(flag & FLAG_ECE); tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); tcp_xmit_recovery(sk, rexmit); @@ -5499,13 +5517,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && + (tp->fast_ack_mode == 1 || /* ... and right edge of window advances far enough. * (tcp_recvmsg() will send ACK otherwise). * If application uses SO_RCVLOWAT, we want send ack now if * we have not received enough bytes to satisfy the condition. */ - (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || - __tcp_select_window(sk) >= tp->rcv_wnd)) || + (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || + __tcp_select_window(sk) >= tp->rcv_wnd))) || /* We ACK each frame or... */ tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 4c376b6d87649..9860adfce22ae 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, th->cwr = 1; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; } - } else if (!tcp_ca_needs_ecn(sk)) { + } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && + !tcp_ca_needs_ecn(sk)) { /* ACK or retransmitted segment: clear ECT|CE */ INET_ECN_dontxmit(sk); } @@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; - int nsize, old_factor; + int nsize, old_factor, inflight_prev; long limit; int nlen; u8 flags; @@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, if (diff) tcp_adjust_pcount(sk, skb, diff); + + /* Set buff tx.in_flight as if buff were sent by itself. */ + inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; + if (WARN_ONCE(inflight_prev < 0, + "inconsistent: tx.in_flight: %u old_factor: %d", + TCP_SKB_CB(skb)->tx.in_flight, old_factor)) + inflight_prev = 0; + TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + + tcp_skb_pcount(buff); } /* Link BUFF into the send queue. */ @@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; - u32 min_tso, tso_segs; - - min_tso = ca_ops->min_tso_segs ? - ca_ops->min_tso_segs(sk) : - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + u32 tso_segs; - tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + tso_segs = ca_ops->tso_segs ? + ca_ops->tso_segs(sk, mss_now) : + tcp_tso_autosize(sk, mss_now, + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); return min_t(u32, tso_segs, sk->sk_gso_max_segs); } @@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); + tcp_set_tx_in_flight(sk, skb); goto repair; /* Skip network transmission */ } diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c index a8f6d9d06f2eb..a8b4c95045701 100644 --- a/net/ipv4/tcp_rate.c +++ b/net/ipv4/tcp_rate.c @@ -34,6 +34,24 @@ * ready to send in the write queue. */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 in_flight; + + /* Check, sanitize, and record packets in flight after skb was sent. */ + in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); + if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, + "insane in_flight %u cc %s mss %u " + "cwnd %u pif %u %u %u %u\n", + in_flight, inet_csk(sk)->icsk_ca_ops->name, + tp->mss_cache, tp->snd_cwnd, + tp->packets_out, tp->retrans_out, + tp->sacked_out, tp->lost_out)) + in_flight = TCPCB_IN_FLIGHT_MAX; + TCP_SKB_CB(skb)->tx.in_flight = in_flight; +} + /* Snapshot the current delivery information in the skb, to generate * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). */ @@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; TCP_SKB_CB(skb)->tx.delivered = tp->delivered; TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; + TCP_SKB_CB(skb)->tx.lost = tp->lost; TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; + tcp_set_tx_in_flight(sk, skb); } /* When an skb is sacked or acked, we fill in the rate sample with the (prior) @@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, if (!rs->prior_delivered || tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, scb->end_seq, rs->last_end_seq)) { + rs->prior_lost = scb->tx.lost; rs->prior_delivered_ce = scb->tx.delivered_ce; rs->prior_delivered = scb->tx.delivered; rs->prior_mstamp = scb->tx.delivered_mstamp; rs->is_app_limited = scb->tx.is_app_limited; rs->is_retrans = scb->sacked & TCPCB_RETRANS; rs->last_end_seq = scb->end_seq; + rs->tx_in_flight = scb->tx.in_flight; /* Record send time of most recently ACKed packet: */ tp->first_tx_mstamp = tx_tstamp; /* Find the duration of the "send phase" of this window: */ - rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, - scb->tx.first_tx_mstamp); + rs->interval_us = tcp_stamp32_us_delta( + tp->first_tx_mstamp, + scb->tx.first_tx_mstamp); } /* Mark off the skb delivered once it's sacked to avoid being @@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, return; } rs->delivered = tp->delivered - rs->prior_delivered; + rs->lost = tp->lost - rs->prior_lost; rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; /* delivered_ce occupies less than 32 bits in the skb control block */ @@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, * longer phase. */ snd_us = rs->interval_us; /* send phase */ - ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, + ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, rs->prior_mstamp); /* ack phase */ rs->interval_us = max(snd_us, ack_us); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 50bba370486e8..e873a768f9e82 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -611,6 +611,7 @@ void tcp_write_timer_handler(struct sock *sk) goto out; } + tcp_rate_check_app_limited(sk); tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending; -- 2.37.1 From c7b7a949bb3d9a135eb82c53f7c971cc608d3dc7 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:31:23 +0200 Subject: [PATCH 02/16] cachy Signed-off-by: Peter Jung --- Documentation/admin-guide/pm/cpuidle.rst | 15 +- Documentation/core-api/mm-api.rst | 8 +- Documentation/core-api/protection-keys.rst | 44 +- MAINTAINERS | 1 + arch/alpha/include/asm/bitops.h | 32 +- arch/alpha/kernel/syscalls/syscall.tbl | 1 + arch/arm/include/asm/bitops.h | 18 +- arch/arm/tools/syscall.tbl | 1 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 2 + arch/hexagon/include/asm/bitops.h | 24 +- arch/ia64/include/asm/bitops.h | 42 +- arch/ia64/include/asm/processor.h | 2 +- arch/ia64/kernel/syscalls/syscall.tbl | 1 + arch/m68k/include/asm/bitops.h | 49 ++- arch/m68k/kernel/syscalls/syscall.tbl | 1 + arch/microblaze/kernel/syscalls/syscall.tbl | 1 + arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + arch/parisc/kernel/syscalls/syscall.tbl | 1 + arch/powerpc/include/asm/archrandom.h | 9 +- arch/powerpc/kernel/setup-common.c | 12 + arch/powerpc/kernel/syscalls/syscall.tbl | 1 + arch/s390/include/asm/bitops.h | 61 +-- arch/s390/kernel/syscalls/syscall.tbl | 1 + arch/sh/include/asm/bitops-op32.h | 34 +- arch/sh/kernel/syscalls/syscall.tbl | 1 + arch/sparc/include/asm/bitops_32.h | 18 +- arch/sparc/kernel/syscalls/syscall.tbl | 1 + arch/sparc/lib/atomic32.c | 12 +- arch/x86/Kconfig.cpu | 332 ++++++++++++++- arch/x86/Makefile | 40 +- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + arch/x86/include/asm/bitops.h | 22 +- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/include/asm/msr-index.h | 3 + arch/x86/include/asm/mwait.h | 1 + arch/x86/include/asm/topology.h | 1 + arch/x86/include/asm/vdso/processor.h | 2 +- arch/x86/include/asm/vermagic.h | 66 +++ arch/x86/kernel/alternative.c | 2 + arch/x86/kernel/cpu/intel.c | 4 +- arch/x86/kernel/cpu/intel_epb.c | 4 + arch/x86/kernel/cpu/microcode/core.c | 40 +- arch/x86/kernel/cpu/microcode/intel.c | 14 +- arch/x86/kernel/itmt.c | 29 +- arch/x86/kernel/msr.c | 2 +- arch/x86/kernel/process.c | 44 +- arch/x86/kernel/tsc.c | 3 + arch/x86/mm/fault.c | 4 +- arch/x86/mm/pkeys.c | 15 +- arch/x86/mm/tlb.c | 18 +- arch/x86/platform/olpc/olpc-xo1-sci.c | 2 +- arch/xtensa/kernel/syscalls/syscall.tbl | 1 + block/blk-core.c | 3 + block/elevator.c | 7 +- drivers/Makefile | 15 +- drivers/acpi/cppc_acpi.c | 3 + drivers/ata/libahci.c | 4 +- drivers/base/arch_topology.c | 2 +- drivers/base/firmware_loader/main.c | 2 + drivers/block/zram/Kconfig | 18 + drivers/block/zram/zram_drv.c | 39 ++ drivers/cpufreq/amd-pstate.c | 62 +-- drivers/cpufreq/cppc_cpufreq.c | 2 +- drivers/cpufreq/intel_pstate.c | 7 + drivers/dma/ti/k3-udma.c | 6 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- .../drm/i915/display/intel_display_power.c | 2 +- drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c | 2 +- drivers/idle/intel_idle.c | 44 +- drivers/input/serio/i8042.c | 10 +- drivers/iommu/intel/dmar.c | 2 +- drivers/iommu/intel/iommu.c | 2 +- drivers/md/dm-crypt.c | 5 + drivers/net/dummy.c | 2 +- drivers/net/ethernet/intel/ice/ice_switch.c | 2 +- drivers/net/ethernet/mellanox/mlx4/fw.c | 2 +- .../net/wireless/ath/ath9k/htc_drv_debug.c | 2 +- drivers/net/wireless/ath/carl9170/debug.c | 2 +- drivers/nvme/host/core.c | 2 +- drivers/pci/pci.c | 2 +- drivers/powercap/intel_rapl_common.c | 2 +- drivers/soundwire/bus.h | 2 +- drivers/thermal/intel/intel_powerclamp.c | 10 + fs/xattr.c | 15 +- .../asm-generic/bitops/generic-non-atomic.h | 161 ++++++++ .../bitops/instrumented-non-atomic.h | 35 +- include/asm-generic/bitops/non-atomic.h | 121 +----- .../bitops/non-instrumented-non-atomic.h | 16 + include/linux/bitmap.h | 37 +- include/linux/bitops.h | 50 +++ include/linux/cpumask.h | 125 ++++-- include/linux/find.h | 45 +- include/linux/gfp.h | 348 +--------------- include/linux/gfp_types.h | 348 ++++++++++++++++ include/linux/ipc_namespace.h | 5 +- include/linux/jbd2.h | 2 +- include/linux/ksm.h | 4 + include/linux/mm_types.h | 5 + include/linux/mmzone.h | 12 +- include/linux/page_counter.h | 1 + include/linux/pagemap.h | 2 +- include/linux/sched.h | 1 + include/linux/syscalls.h | 1 + include/linux/user_namespace.h | 4 + include/linux/wait.h | 2 + include/uapi/asm-generic/unistd.h | 5 +- include/uapi/linux/if_bonding.h | 2 +- init/Kconfig | 24 +- init/do_mounts.c | 16 +- kernel/Kconfig.hz | 24 ++ kernel/fork.c | 14 + kernel/locking/rwsem.c | 4 +- kernel/module/internal.h | 2 + kernel/module/main.c | 1 + kernel/module/procfs.c | 13 + kernel/module/signing.c | 4 + kernel/sched/debug.c | 1 + kernel/sched/fair.c | 20 +- kernel/sched/wait.c | 24 ++ kernel/sys_ni.c | 1 + kernel/sysctl.c | 12 + kernel/user_namespace.c | 7 + kernel/watchdog.c | 2 +- lib/bitmap.c | 11 +- lib/cpumask.c | 97 +---- lib/find_bit.c | 155 ++++--- lib/raid6/algos.c | 4 +- lib/string.c | 62 ++- lib/test_bitmap.c | 68 +++ lib/zstd/compress/zstd_double_fast.c | 61 ++- lib/zstd/compress/zstd_fast.c | 69 +++- lib/zstd/compress/zstd_lazy.c | 223 ++++------ mm/compaction.c | 6 +- mm/ksm.c | 311 ++++++++++++-- mm/madvise.c | 115 ++++++ mm/memcontrol.c | 2 +- mm/page-writeback.c | 8 + mm/page_alloc.c | 391 +++++++++++------- mm/vmpressure.c | 4 + mm/vmscan.c | 4 + net/ipv4/inet_connection_sock.c | 2 +- net/ipv4/tcp.c | 4 +- tools/include/asm-generic/bitops/non-atomic.h | 34 +- tools/include/linux/bitmap.h | 12 +- tools/include/linux/bitops.h | 16 + tools/lib/bitmap.c | 6 +- 150 files changed, 3107 insertions(+), 1392 deletions(-) create mode 100644 include/asm-generic/bitops/generic-non-atomic.h create mode 100644 include/asm-generic/bitops/non-instrumented-non-atomic.h create mode 100644 include/linux/gfp_types.h diff --git a/Documentation/admin-guide/pm/cpuidle.rst b/Documentation/admin-guide/pm/cpuidle.rst index aec2cd2aaea73..19754beb5a4e6 100644 --- a/Documentation/admin-guide/pm/cpuidle.rst +++ b/Documentation/admin-guide/pm/cpuidle.rst @@ -612,8 +612,8 @@ the ``menu`` governor to be used on the systems that use the ``ladder`` governor by default this way, for example. The other kernel command line parameters controlling CPU idle time management -described below are only relevant for the *x86* architecture and some of -them affect Intel processors only. +described below are only relevant for the *x86* architecture and references +to ``intel_idle`` affect Intel processors only. The *x86* architecture support code recognizes three kernel command line options related to CPU idle time management: ``idle=poll``, ``idle=halt``, @@ -635,10 +635,13 @@ idle, so it very well may hurt single-thread computations performance as well as energy-efficiency. Thus using it for performance reasons may not be a good idea at all.] -The ``idle=nomwait`` option disables the ``intel_idle`` driver and causes -``acpi_idle`` to be used (as long as all of the information needed by it is -there in the system's ACPI tables), but it is not allowed to use the -``MWAIT`` instruction of the CPUs to ask the hardware to enter idle states. +The ``idle=nomwait`` option prevents the use of ``MWAIT`` instruction of +the CPU to enter idle states. When this option is used, the ``acpi_idle`` +driver will use the ``HLT`` instruction instead of ``MWAIT``. On systems +running Intel processors, this option disables the ``intel_idle`` driver +and forces the use of the ``acpi_idle`` driver instead. Note that in either +case, ``acpi_idle`` driver will function only if all the information needed +by it is in the system's ACPI tables. In addition to the architecture-level kernel command line options affecting CPU idle time management, there are parameters affecting individual ``CPUIdle`` diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index f5b2f92822c86..1ebcc6c3fafe7 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -22,16 +22,16 @@ Memory Allocation Controls .. kernel-doc:: include/linux/gfp.h :internal: -.. kernel-doc:: include/linux/gfp.h +.. kernel-doc:: include/linux/gfp_types.h :doc: Page mobility and placement hints -.. kernel-doc:: include/linux/gfp.h +.. kernel-doc:: include/linux/gfp_types.h :doc: Watermark modifiers -.. kernel-doc:: include/linux/gfp.h +.. kernel-doc:: include/linux/gfp_types.h :doc: Reclaim modifiers -.. kernel-doc:: include/linux/gfp.h +.. kernel-doc:: include/linux/gfp_types.h :doc: Useful GFP flag combinations The Slab Cache diff --git a/Documentation/core-api/protection-keys.rst b/Documentation/core-api/protection-keys.rst index ec575e72d0b2a..bf28ac0401f38 100644 --- a/Documentation/core-api/protection-keys.rst +++ b/Documentation/core-api/protection-keys.rst @@ -4,31 +4,29 @@ Memory Protection Keys ====================== -Memory Protection Keys for Userspace (PKU aka PKEYs) is a feature -which is found on Intel's Skylake (and later) "Scalable Processor" -Server CPUs. It will be available in future non-server Intel parts -and future AMD processors. - -For anyone wishing to test or use this feature, it is available in -Amazon's EC2 C5 instances and is known to work there using an Ubuntu -17.04 image. - -Memory Protection Keys provides a mechanism for enforcing page-based -protections, but without requiring modification of the page tables -when an application changes protection domains. It works by -dedicating 4 previously ignored bits in each page table entry to a -"protection key", giving 16 possible keys. - -There is also a new user-accessible register (PKRU) with two separate -bits (Access Disable and Write Disable) for each key. Being a CPU -register, PKRU is inherently thread-local, potentially giving each +Memory Protection Keys provide a mechanism for enforcing page-based +protections, but without requiring modification of the page tables when an +application changes protection domains. + +Pkeys Userspace (PKU) is a feature which can be found on: + * Intel server CPUs, Skylake and later + * Intel client CPUs, Tiger Lake (11th Gen Core) and later + * Future AMD CPUs + +Pkeys work by dedicating 4 previously Reserved bits in each page table entry to +a "protection key", giving 16 possible keys. + +Protections for each key are defined with a per-CPU user-accessible register +(PKRU). Each of these is a 32-bit register storing two bits (Access Disable +and Write Disable) for each of 16 keys. + +Being a CPU register, PKRU is inherently thread-local, potentially giving each thread a different set of protections from every other thread. -There are two new instructions (RDPKRU/WRPKRU) for reading and writing -to the new register. The feature is only available in 64-bit mode, -even though there is theoretically space in the PAE PTEs. These -permissions are enforced on data access only and have no effect on -instruction fetches. +There are two instructions (RDPKRU/WRPKRU) for reading and writing to the +register. The feature is only available in 64-bit mode, even though there is +theoretically space in the PAE PTEs. These permissions are enforced on data +access only and have no effect on instruction fetches. Syscalls ======== diff --git a/MAINTAINERS b/MAINTAINERS index 64379c699903b..5da47941f0998 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12956,6 +12956,7 @@ W: http://www.linux-mm.org T: git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm T: quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new F: include/linux/gfp.h +F: include/linux/gfp_types.h F: include/linux/memory_hotplug.h F: include/linux/mm.h F: include/linux/mmzone.h diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index e1d8483a45f2e..492c7713ddae6 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -46,8 +46,8 @@ set_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static inline void -__set_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -82,8 +82,8 @@ clear_bit_unlock(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ void -__clear_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -94,7 +94,7 @@ static inline void __clear_bit_unlock(unsigned long nr, volatile void * addr) { smp_mb(); - __clear_bit(nr, addr); + arch___clear_bit(nr, addr); } static inline void @@ -118,8 +118,8 @@ change_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ void -__change_bit(unsigned long nr, volatile void * addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { int *m = ((int *) addr) + (nr >> 5); @@ -186,8 +186,8 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) /* * WARNING: non atomic version. */ -static inline int -__test_and_set_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -230,8 +230,8 @@ test_and_clear_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static inline int -__test_and_clear_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -272,8 +272,8 @@ test_and_change_bit(unsigned long nr, volatile void * addr) /* * WARNING: non atomic version. */ -static __inline__ int -__test_and_change_bit(unsigned long nr, volatile void * addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = 1 << (nr & 0x1f); int *m = ((int *) addr) + (nr >> 5); @@ -283,8 +283,8 @@ __test_and_change_bit(unsigned long nr, volatile void * addr) return (old & mask) != 0; } -static inline int -test_bit(int nr, const volatile void * addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; } @@ -450,6 +450,8 @@ sched_find_first_bit(const unsigned long b[2]) return __ffs(tmp) + ofs; } +#include + #include #include diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index 3515bc4f16a4f..00ff721da300e 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -490,3 +490,4 @@ 558 common process_mrelease sys_process_mrelease 559 common futex_waitv sys_futex_waitv 560 common set_mempolicy_home_node sys_ni_syscall +561 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 8e94fe7ab5ebc..714440fa2fc6e 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -160,18 +160,20 @@ extern int _test_and_change_bit(int nr, volatile unsigned long * p); /* * Little endian assembly bitops. nr = 0 -> byte 0 bit 0. */ -extern int _find_first_zero_bit_le(const unsigned long *p, unsigned size); -extern int _find_next_zero_bit_le(const unsigned long *p, int size, int offset); -extern int _find_first_bit_le(const unsigned long *p, unsigned size); -extern int _find_next_bit_le(const unsigned long *p, int size, int offset); +unsigned long _find_first_zero_bit_le(const unsigned long *p, unsigned long size); +unsigned long _find_next_zero_bit_le(const unsigned long *p, + unsigned long size, unsigned long offset); +unsigned long _find_first_bit_le(const unsigned long *p, unsigned long size); +unsigned long _find_next_bit_le(const unsigned long *p, unsigned long size, unsigned long offset); /* * Big endian assembly bitops. nr = 0 -> byte 3 bit 0. */ -extern int _find_first_zero_bit_be(const unsigned long *p, unsigned size); -extern int _find_next_zero_bit_be(const unsigned long *p, int size, int offset); -extern int _find_first_bit_be(const unsigned long *p, unsigned size); -extern int _find_next_bit_be(const unsigned long *p, int size, int offset); +unsigned long _find_first_zero_bit_be(const unsigned long *p, unsigned long size); +unsigned long _find_next_zero_bit_be(const unsigned long *p, + unsigned long size, unsigned long offset); +unsigned long _find_first_bit_be(const unsigned long *p, unsigned long size); +unsigned long _find_next_bit_be(const unsigned long *p, unsigned long size, unsigned long offset); #ifndef CONFIG_SMP /* diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index ac964612d8b07..90933eabe1156 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -464,3 +464,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 037feba03a51d..64a514f90131b 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -39,7 +39,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 451 +#define __NR_compat_syscalls 452 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 604a2053d0067..91f2bb7199af9 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -907,6 +907,8 @@ __SYSCALL(__NR_process_mrelease, sys_process_mrelease) __SYSCALL(__NR_futex_waitv, sys_futex_waitv) #define __NR_set_mempolicy_home_node 450 __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) +#define __NR_pmadv_ksm 451 +__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) /* * Please add new compat syscalls above this comment and update diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index 75d6ba3643b8d..da500471ac73c 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -127,38 +127,45 @@ static inline void change_bit(int nr, volatile void *addr) * be atomic, particularly for things like slab_lock and slab_unlock. * */ -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { test_and_clear_bit(nr, addr); } -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { test_and_set_bit(nr, addr); } -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { test_and_change_bit(nr, addr); } /* Apparently, at least some of these are allowed to be non-atomic */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_clear_bit(nr, addr); } -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_set_bit(nr, addr); } -static inline int __test_and_change_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { return test_and_change_bit(nr, addr); } -static inline int __test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { int retval; @@ -172,8 +179,6 @@ static inline int __test_bit(int nr, const volatile unsigned long *addr) return retval; } -#define test_bit(nr, addr) __test_bit(nr, addr) - /* * ffz - find first zero in word. * @word: The word to search @@ -271,6 +276,7 @@ static inline unsigned long __fls(unsigned long word) } #include +#include #include #include diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index 577be93c0818c..9f62af7fd7c42 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -53,7 +53,7 @@ set_bit (int nr, volatile void *addr) } /** - * __set_bit - Set a bit in memory + * arch___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -61,8 +61,8 @@ set_bit (int nr, volatile void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__set_bit (int nr, volatile void *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) |= (1 << (nr & 31)); } @@ -135,7 +135,7 @@ __clear_bit_unlock(int nr, void *addr) } /** - * __clear_bit - Clears a bit in memory (non-atomic version) + * arch___clear_bit - Clears a bit in memory (non-atomic version) * @nr: the bit to clear * @addr: the address to start counting from * @@ -143,8 +143,8 @@ __clear_bit_unlock(int nr, void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__clear_bit (int nr, volatile void *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) &= ~(1 << (nr & 31)); } @@ -175,7 +175,7 @@ change_bit (int nr, volatile void *addr) } /** - * __change_bit - Toggle a bit in memory + * arch___change_bit - Toggle a bit in memory * @nr: the bit to toggle * @addr: the address to start counting from * @@ -183,8 +183,8 @@ change_bit (int nr, volatile void *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static __inline__ void -__change_bit (int nr, volatile void *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { *((__u32 *) addr + (nr >> 5)) ^= (1 << (nr & 31)); } @@ -224,7 +224,7 @@ test_and_set_bit (int nr, volatile void *addr) #define test_and_set_bit_lock test_and_set_bit /** - * __test_and_set_bit - Set a bit and return its old value + * arch___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -232,8 +232,8 @@ test_and_set_bit (int nr, volatile void *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __inline__ int -__test_and_set_bit (int nr, volatile void *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { __u32 *p = (__u32 *) addr + (nr >> 5); __u32 m = 1 << (nr & 31); @@ -269,7 +269,7 @@ test_and_clear_bit (int nr, volatile void *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * arch___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -277,8 +277,8 @@ test_and_clear_bit (int nr, volatile void *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __inline__ int -__test_and_clear_bit(int nr, volatile void * addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { __u32 *p = (__u32 *) addr + (nr >> 5); __u32 m = 1 << (nr & 31); @@ -314,14 +314,14 @@ test_and_change_bit (int nr, volatile void *addr) } /** - * __test_and_change_bit - Change a bit and return its old value + * arch___test_and_change_bit - Change a bit and return its old value * @nr: Bit to change * @addr: Address to count from * * This operation is non-atomic and can be reordered. */ -static __inline__ int -__test_and_change_bit (int nr, void *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { __u32 old, bit = (1 << (nr & 31)); __u32 *m = (__u32 *) addr + (nr >> 5); @@ -331,8 +331,8 @@ __test_and_change_bit (int nr, void *addr) return (old & bit) != 0; } -static __inline__ int -test_bit (int nr, const volatile void *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); } @@ -443,6 +443,8 @@ static __inline__ unsigned long __arch_hweight64(unsigned long x) #ifdef __KERNEL__ +#include + #include #include diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 7cbce290f4e5a..757c2f6d8d4b8 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -538,7 +538,7 @@ ia64_get_irr(unsigned int vector) { unsigned int reg = vector / 64; unsigned int bit = vector % 64; - u64 irr; + unsigned long irr; switch (reg) { case 0: irr = ia64_getreg(_IA64_REG_CR_IRR0); break; diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index 78b1d03e86e1d..79ad5a5682b30 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -371,3 +371,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 51283db53667f..71495faf2a904 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -65,8 +65,11 @@ static inline void bfset_mem_set_bit(int nr, volatile unsigned long *vaddr) bfset_mem_set_bit(nr, vaddr)) #endif -#define __set_bit(nr, vaddr) set_bit(nr, vaddr) - +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + set_bit(nr, addr); +} static inline void bclr_reg_clear_bit(int nr, volatile unsigned long *vaddr) { @@ -105,8 +108,11 @@ static inline void bfclr_mem_clear_bit(int nr, volatile unsigned long *vaddr) bfclr_mem_clear_bit(nr, vaddr)) #endif -#define __clear_bit(nr, vaddr) clear_bit(nr, vaddr) - +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + clear_bit(nr, addr); +} static inline void bchg_reg_change_bit(int nr, volatile unsigned long *vaddr) { @@ -145,14 +151,17 @@ static inline void bfchg_mem_change_bit(int nr, volatile unsigned long *vaddr) bfchg_mem_change_bit(nr, vaddr)) #endif -#define __change_bit(nr, vaddr) change_bit(nr, vaddr) - - -static inline int test_bit(int nr, const volatile unsigned long *vaddr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { - return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0; + change_bit(nr, addr); } +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + return (addr[nr >> 5] & (1UL << (nr & 31))) != 0; +} static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) @@ -201,8 +210,11 @@ static inline int bfset_mem_test_and_set_bit(int nr, bfset_mem_test_and_set_bit(nr, vaddr)) #endif -#define __test_and_set_bit(nr, vaddr) test_and_set_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_set_bit(nr, addr); +} static inline int bclr_reg_test_and_clear_bit(int nr, volatile unsigned long *vaddr) @@ -251,8 +263,11 @@ static inline int bfclr_mem_test_and_clear_bit(int nr, bfclr_mem_test_and_clear_bit(nr, vaddr)) #endif -#define __test_and_clear_bit(nr, vaddr) test_and_clear_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_clear_bit(nr, addr); +} static inline int bchg_reg_test_and_change_bit(int nr, volatile unsigned long *vaddr) @@ -301,8 +316,11 @@ static inline int bfchg_mem_test_and_change_bit(int nr, bfchg_mem_test_and_change_bit(nr, vaddr)) #endif -#define __test_and_change_bit(nr, vaddr) test_and_change_bit(nr, vaddr) - +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + return test_and_change_bit(nr, addr); +} /* * The true 68020 and more advanced processors support the "bfffo" @@ -522,6 +540,7 @@ static inline int __fls(int x) #define clear_bit_unlock clear_bit #define __clear_bit_unlock clear_bit_unlock +#include #include #include #include diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index b1f3940bc2981..5ccf925567da0 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -450,3 +450,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index 820145e473501..6b76208597f3c 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -456,3 +456,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 253ff994ed2ec..e4aeedb17c383 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -389,3 +389,4 @@ 448 n32 process_mrelease sys_process_mrelease 449 n32 futex_waitv sys_futex_waitv 450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node +451 n32 pmadv_ksm sys_pmadv_ksm diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index 3f1886ad9d806..fe88db51efa00 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -365,3 +365,4 @@ 448 n64 process_mrelease sys_process_mrelease 449 n64 futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 n64 pmadv_ksm sys_pmadv_ksm diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 8f243e35a7b20..674cb940bd153 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -438,3 +438,4 @@ 448 o32 process_mrelease sys_process_mrelease 449 o32 futex_waitv sys_futex_waitv 450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node +451 o32 pmadv_ksm sys_pmadv_ksm diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 68b46fe2f17c5..e19e8deb4c322 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -448,3 +448,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 9a53e29680f41..21def59ef1a68 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -4,7 +4,7 @@ #ifdef CONFIG_ARCH_RANDOM -#include +bool __must_check arch_get_random_seed_long(unsigned long *v); static inline bool __must_check arch_get_random_long(unsigned long *v) { @@ -16,13 +16,6 @@ static inline bool __must_check arch_get_random_int(unsigned int *v) return false; } -static inline bool __must_check arch_get_random_seed_long(unsigned long *v) -{ - if (ppc_md.get_random_seed) - return ppc_md.get_random_seed(v); - - return false; -} static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 1a02629ec70b0..3b1cf9ca4814b 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -171,6 +171,18 @@ EXPORT_SYMBOL_GPL(machine_power_off); void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); +#ifdef CONFIG_ARCH_RANDOM +bool __must_check arch_get_random_seed_long(unsigned long *v) +{ + if (ppc_md.get_random_seed) + return ppc_md.get_random_seed(v); + + return false; +} +EXPORT_SYMBOL(arch_get_random_seed_long); + +#endif + void machine_halt(void) { machine_shutdown(); diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index 2600b4237292c..bb2f71a369415 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -530,3 +530,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 191dc7898b0f7..9a7d15da966e3 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -113,75 +113,76 @@ static inline bool arch_test_and_change_bit(unsigned long nr, return old & mask; } -static inline void arch___set_bit(unsigned long nr, volatile unsigned long *ptr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr |= mask; + *p |= mask; } -static inline void arch___clear_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr &= ~mask; + *p &= ~mask; } -static inline void arch___change_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - *addr ^= mask; + *p ^= mask; } -static inline bool arch___test_and_set_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr |= mask; + old = *p; + *p |= mask; return old & mask; } -static inline bool arch___test_and_clear_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr &= ~mask; + old = *p; + *p &= ~mask; return old & mask; } -static inline bool arch___test_and_change_bit(unsigned long nr, - volatile unsigned long *ptr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { - unsigned long *addr = __bitops_word(nr, ptr); + unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); unsigned long old; - old = *addr; - *addr ^= mask; + old = *p; + *p ^= mask; return old & mask; } -static inline bool arch_test_bit(unsigned long nr, - const volatile unsigned long *ptr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { - const volatile unsigned long *addr = __bitops_word(nr, ptr); + const volatile unsigned long *p = __bitops_word(nr, addr); unsigned long mask = __bitops_mask(nr); - return *addr & mask; + return *p & mask; } static inline bool arch_test_and_set_bit_lock(unsigned long nr, diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index 799147658dee2..1cd523748bd2e 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -453,3 +453,4 @@ 448 common process_mrelease sys_process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm sys_pmadv_ksm diff --git a/arch/sh/include/asm/bitops-op32.h b/arch/sh/include/asm/bitops-op32.h index cfe5465acce71..565a85d8b7fb0 100644 --- a/arch/sh/include/asm/bitops-op32.h +++ b/arch/sh/include/asm/bitops-op32.h @@ -2,6 +2,8 @@ #ifndef __ASM_SH_BITOPS_OP32_H #define __ASM_SH_BITOPS_OP32_H +#include + /* * The bit modifying instructions on SH-2A are only capable of working * with a 3-bit immediate, which signifies the shift position for the bit @@ -16,7 +18,8 @@ #define BYTE_OFFSET(nr) ((nr) % BITS_PER_BYTE) #endif -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -33,7 +36,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr) } } -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -52,7 +56,7 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) } /** - * __change_bit - Toggle a bit in memory + * arch___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -60,7 +64,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { if (__builtin_constant_p(nr)) { __asm__ __volatile__ ( @@ -79,7 +84,7 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_set_bit - Set a bit and return its old value + * arch___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -87,7 +92,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -98,7 +104,7 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * arch___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -106,7 +112,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -117,8 +124,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -129,13 +136,16 @@ static inline int __test_and_change_bit(int nr, } /** - * test_bit - Determine whether a bit is set + * arch_test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ -static inline int test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +#include + #endif /* __ASM_SH_BITOPS_OP32_H */ diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index 2de85c977f54f..cfc75fa43eae4 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -453,3 +453,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 889afa9f990ff..3448c191b4845 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -19,9 +19,9 @@ #error only can be included directly #endif -unsigned long ___set_bit(unsigned long *addr, unsigned long mask); -unsigned long ___clear_bit(unsigned long *addr, unsigned long mask); -unsigned long ___change_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask); /* * Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0' @@ -36,7 +36,7 @@ static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *add ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___set_bit(ADDR, mask) != 0; + return sp32___set_bit(ADDR, mask) != 0; } static inline void set_bit(unsigned long nr, volatile unsigned long *addr) @@ -46,7 +46,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___set_bit(ADDR, mask); + (void) sp32___set_bit(ADDR, mask); } static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -56,7 +56,7 @@ static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *a ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___clear_bit(ADDR, mask) != 0; + return sp32___clear_bit(ADDR, mask) != 0; } static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -66,7 +66,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___clear_bit(ADDR, mask); + (void) sp32___clear_bit(ADDR, mask); } static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr) @@ -76,7 +76,7 @@ static inline int test_and_change_bit(unsigned long nr, volatile unsigned long * ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___change_bit(ADDR, mask) != 0; + return sp32___change_bit(ADDR, mask) != 0; } static inline void change_bit(unsigned long nr, volatile unsigned long *addr) @@ -86,7 +86,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___change_bit(ADDR, mask); + (void) sp32___change_bit(ADDR, mask); } #include diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 4398cc6fb68dd..d2c0a6426f6b8 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -496,3 +496,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 8b81d0f00c971..cf80d1ae352be 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -120,7 +120,7 @@ void arch_atomic_set(atomic_t *v, int i) } EXPORT_SYMBOL(arch_atomic_set); -unsigned long ___set_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -131,9 +131,9 @@ unsigned long ___set_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___set_bit); +EXPORT_SYMBOL(sp32___set_bit); -unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -144,9 +144,9 @@ unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___clear_bit); +EXPORT_SYMBOL(sp32___clear_bit); -unsigned long ___change_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -157,7 +157,7 @@ unsigned long ___change_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___change_bit); +EXPORT_SYMBOL(sp32___change_bit); unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new) { diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 542377cd419d7..22b919cdb6d19 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -157,7 +157,7 @@ config MPENTIUM4 config MK6 - bool "K6/K6-II/K6-III" + bool "AMD K6/K6-II/K6-III" depends on X86_32 help Select this for an AMD K6-family processor. Enables use of @@ -165,7 +165,7 @@ config MK6 flags to GCC. config MK7 - bool "Athlon/Duron/K7" + bool "AMD Athlon/Duron/K7" depends on X86_32 help Select this for an AMD Athlon K7-family processor. Enables use of @@ -173,12 +173,98 @@ config MK7 flags to GCC. config MK8 - bool "Opteron/Athlon64/Hammer/K8" + bool "AMD Opteron/Athlon64/Hammer/K8" help Select this for an AMD Opteron or Athlon64 Hammer-family processor. Enables use of some extended instructions, and passes appropriate optimization flags to GCC. +config MK8SSE3 + bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" + help + Select this for improved AMD Opteron or Athlon64 Hammer-family processors. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + +config MK10 + bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" + help + Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, + Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + +config MBARCELONA + bool "AMD Barcelona" + help + Select this for AMD Family 10h Barcelona processors. + + Enables -march=barcelona + +config MBOBCAT + bool "AMD Bobcat" + help + Select this for AMD Family 14h Bobcat processors. + + Enables -march=btver1 + +config MJAGUAR + bool "AMD Jaguar" + help + Select this for AMD Family 16h Jaguar processors. + + Enables -march=btver2 + +config MBULLDOZER + bool "AMD Bulldozer" + help + Select this for AMD Family 15h Bulldozer processors. + + Enables -march=bdver1 + +config MPILEDRIVER + bool "AMD Piledriver" + help + Select this for AMD Family 15h Piledriver processors. + + Enables -march=bdver2 + +config MSTEAMROLLER + bool "AMD Steamroller" + help + Select this for AMD Family 15h Steamroller processors. + + Enables -march=bdver3 + +config MEXCAVATOR + bool "AMD Excavator" + help + Select this for AMD Family 15h Excavator processors. + + Enables -march=bdver4 + +config MZEN + bool "AMD Zen" + help + Select this for AMD Family 17h Zen processors. + + Enables -march=znver1 + +config MZEN2 + bool "AMD Zen 2" + help + Select this for AMD Family 17h Zen 2 processors. + + Enables -march=znver2 + +config MZEN3 + bool "AMD Zen 3" + depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + help + Select this for AMD Family 19h Zen 3 processors. + + Enables -march=znver3 + config MCRUSOE bool "Crusoe" depends on X86_32 @@ -270,7 +356,7 @@ config MPSC in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. config MCORE2 - bool "Core 2/newer Xeon" + bool "Intel Core 2" help Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and @@ -278,6 +364,8 @@ config MCORE2 family in /proc/cpuinfo. Newer ones have 6 and older ones 15 (not a typo) + Enables -march=core2 + config MATOM bool "Intel Atom" help @@ -287,6 +375,182 @@ config MATOM accordingly optimized code. Use a recent GCC with specific Atom support in order to fully benefit from selecting this option. +config MNEHALEM + bool "Intel Nehalem" + select X86_P6_NOP + help + + Select this for 1st Gen Core processors in the Nehalem family. + + Enables -march=nehalem + +config MWESTMERE + bool "Intel Westmere" + select X86_P6_NOP + help + + Select this for the Intel Westmere formerly Nehalem-C family. + + Enables -march=westmere + +config MSILVERMONT + bool "Intel Silvermont" + select X86_P6_NOP + help + + Select this for the Intel Silvermont platform. + + Enables -march=silvermont + +config MGOLDMONT + bool "Intel Goldmont" + select X86_P6_NOP + help + + Select this for the Intel Goldmont platform including Apollo Lake and Denverton. + + Enables -march=goldmont + +config MGOLDMONTPLUS + bool "Intel Goldmont Plus" + select X86_P6_NOP + help + + Select this for the Intel Goldmont Plus platform including Gemini Lake. + + Enables -march=goldmont-plus + +config MSANDYBRIDGE + bool "Intel Sandy Bridge" + select X86_P6_NOP + help + + Select this for 2nd Gen Core processors in the Sandy Bridge family. + + Enables -march=sandybridge + +config MIVYBRIDGE + bool "Intel Ivy Bridge" + select X86_P6_NOP + help + + Select this for 3rd Gen Core processors in the Ivy Bridge family. + + Enables -march=ivybridge + +config MHASWELL + bool "Intel Haswell" + select X86_P6_NOP + help + + Select this for 4th Gen Core processors in the Haswell family. + + Enables -march=haswell + +config MBROADWELL + bool "Intel Broadwell" + select X86_P6_NOP + help + + Select this for 5th Gen Core processors in the Broadwell family. + + Enables -march=broadwell + +config MSKYLAKE + bool "Intel Skylake" + select X86_P6_NOP + help + + Select this for 6th Gen Core processors in the Skylake family. + + Enables -march=skylake + +config MSKYLAKEX + bool "Intel Skylake X" + select X86_P6_NOP + help + + Select this for 6th Gen Core processors in the Skylake X family. + + Enables -march=skylake-avx512 + +config MCANNONLAKE + bool "Intel Cannon Lake" + select X86_P6_NOP + help + + Select this for 8th Gen Core processors + + Enables -march=cannonlake + +config MICELAKE + bool "Intel Ice Lake" + select X86_P6_NOP + help + + Select this for 10th Gen Core processors in the Ice Lake family. + + Enables -march=icelake-client + +config MCASCADELAKE + bool "Intel Cascade Lake" + select X86_P6_NOP + help + + Select this for Xeon processors in the Cascade Lake family. + + Enables -march=cascadelake + +config MCOOPERLAKE + bool "Intel Cooper Lake" + depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) + select X86_P6_NOP + help + + Select this for Xeon processors in the Cooper Lake family. + + Enables -march=cooperlake + +config MTIGERLAKE + bool "Intel Tiger Lake" + depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) + select X86_P6_NOP + help + + Select this for third-generation 10 nm process processors in the Tiger Lake family. + + Enables -march=tigerlake + +config MSAPPHIRERAPIDS + bool "Intel Sapphire Rapids" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + select X86_P6_NOP + help + + Select this for third-generation 10 nm process processors in the Sapphire Rapids family. + + Enables -march=sapphirerapids + +config MROCKETLAKE + bool "Intel Rocket Lake" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + select X86_P6_NOP + help + + Select this for eleventh-generation processors in the Rocket Lake family. + + Enables -march=rocketlake + +config MALDERLAKE + bool "Intel Alder Lake" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + select X86_P6_NOP + help + + Select this for twelfth-generation processors in the Alder Lake family. + + Enables -march=alderlake + config GENERIC_CPU bool "Generic-x86-64" depends on X86_64 @@ -294,6 +558,50 @@ config GENERIC_CPU Generic x86-64 CPU. Run equally well on all x86-64 CPUs. +config GENERIC_CPU2 + bool "Generic-x86-64-v2" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 + help + Generic x86-64 CPU. + Run equally well on all x86-64 CPUs with min support of x86-64-v2. + +config GENERIC_CPU3 + bool "Generic-x86-64-v3" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 + help + Generic x86-64-v3 CPU with v3 instructions. + Run equally well on all x86-64 CPUs with min support of x86-64-v3. + +config GENERIC_CPU4 + bool "Generic-x86-64-v4" + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 + help + Generic x86-64 CPU with v4 instructions. + Run equally well on all x86-64 CPUs with min support of x86-64-v4. + +config MNATIVE_INTEL + bool "Intel-Native optimizations autodetected by the compiler" + help + + Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects + the optimum settings to use based on your processor. Do NOT use this + for AMD CPUs. Intel Only! + + Enables -march=native + +config MNATIVE_AMD + bool "AMD-Native optimizations autodetected by the compiler" + help + + Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects + the optimum settings to use based on your processor. Do NOT use this + for Intel CPUs. AMD Only! + + Enables -march=native + endchoice config X86_GENERIC @@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT config X86_L1_CACHE_SHIFT int default "7" if MPENTIUM4 || MPSC - default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU + default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 default "4" if MELAN || M486SX || M486 || MGEODEGX1 default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX @@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 config X86_INTEL_USERCOPY def_bool y - depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 + depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL config X86_USE_PPRO_CHECKSUM def_bool y - depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM + depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD # # P6_NOPs are a relatively minor optimization that require a family >= @@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM config X86_P6_NOP def_bool y depends on X86_64 - depends on (MCORE2 || MPENTIUM4 || MPSC) + depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) config X86_TSC def_bool y - depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 + depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 config X86_CMPXCHG64 def_bool y - depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 + depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD # this should be set for all -march=.. options where the compiler # generates cmov. config X86_CMOV def_bool y - depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) + depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) config X86_MINIMUM_CPU_FAMILY int default "64" if X86_64 - default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) + default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) default "5" if X86_32 && X86_CMPXCHG64 default "4" diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 7854685c5f25b..ad47a4698b95c 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -150,8 +150,44 @@ else # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) cflags-$(CONFIG_MK8) += -march=k8 cflags-$(CONFIG_MPSC) += -march=nocona - cflags-$(CONFIG_MCORE2) += -march=core2 - cflags-$(CONFIG_MATOM) += -march=atom + cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 + cflags-$(CONFIG_MK10) += -march=amdfam10 + cflags-$(CONFIG_MBARCELONA) += -march=barcelona + cflags-$(CONFIG_MBOBCAT) += -march=btver1 + cflags-$(CONFIG_MJAGUAR) += -march=btver2 + cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 + cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm + cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm + cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm + cflags-$(CONFIG_MZEN) += -march=znver1 + cflags-$(CONFIG_MZEN2) += -march=znver2 + cflags-$(CONFIG_MZEN3) += -march=znver3 + cflags-$(CONFIG_MNATIVE_INTEL) += -march=native + cflags-$(CONFIG_MNATIVE_AMD) += -march=native + cflags-$(CONFIG_MATOM) += -march=bonnell + cflags-$(CONFIG_MCORE2) += -march=core2 + cflags-$(CONFIG_MNEHALEM) += -march=nehalem + cflags-$(CONFIG_MWESTMERE) += -march=westmere + cflags-$(CONFIG_MSILVERMONT) += -march=silvermont + cflags-$(CONFIG_MGOLDMONT) += -march=goldmont + cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus + cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge + cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge + cflags-$(CONFIG_MHASWELL) += -march=haswell + cflags-$(CONFIG_MBROADWELL) += -march=broadwell + cflags-$(CONFIG_MSKYLAKE) += -march=skylake + cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 + cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake + cflags-$(CONFIG_MICELAKE) += -march=icelake-client + cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake + cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake + cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake + cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids + cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake + cflags-$(CONFIG_MALDERLAKE) += -march=alderlake + cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 + cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 + cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic KBUILD_CFLAGS += $(cflags-y) diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 320480a8db4f8..331aaf1a782ff 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -455,3 +455,4 @@ 448 i386 process_mrelease sys_process_mrelease 449 i386 futex_waitv sys_futex_waitv 450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node +451 i386 pmadv_ksm sys_pmadv_ksm diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index c84d12608cd2d..14902db4c01fc 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -372,6 +372,7 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index a288ecd230ab0..973c6bd17f98e 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -63,7 +63,7 @@ arch_set_bit(long nr, volatile unsigned long *addr) } static __always_inline void -arch___set_bit(long nr, volatile unsigned long *addr) +arch___set_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -89,7 +89,7 @@ arch_clear_bit_unlock(long nr, volatile unsigned long *addr) } static __always_inline void -arch___clear_bit(long nr, volatile unsigned long *addr) +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -114,7 +114,7 @@ arch___clear_bit_unlock(long nr, volatile unsigned long *addr) } static __always_inline void -arch___change_bit(long nr, volatile unsigned long *addr) +arch___change_bit(unsigned long nr, volatile unsigned long *addr) { asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); } @@ -145,7 +145,7 @@ arch_test_and_set_bit_lock(long nr, volatile unsigned long *addr) } static __always_inline bool -arch___test_and_set_bit(long nr, volatile unsigned long *addr) +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -171,7 +171,7 @@ arch_test_and_clear_bit(long nr, volatile unsigned long *addr) * this without also updating arch/x86/kernel/kvm.c */ static __always_inline bool -arch___test_and_clear_bit(long nr, volatile unsigned long *addr) +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -183,7 +183,7 @@ arch___test_and_clear_bit(long nr, volatile unsigned long *addr) } static __always_inline bool -arch___test_and_change_bit(long nr, volatile unsigned long *addr) +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { bool oldbit; @@ -219,10 +219,12 @@ static __always_inline bool variable_test_bit(long nr, volatile const unsigned l return oldbit; } -#define arch_test_bit(nr, addr) \ - (__builtin_constant_p((nr)) \ - ? constant_test_bit((nr), (addr)) \ - : variable_test_bit((nr), (addr))) +static __always_inline bool +arch_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit(nr, addr) : + variable_test_bit(nr, addr); +} /** * __ffs - find first set bit in word diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index a77b915d36a8e..5fe7f6c8a7a41 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -219,7 +219,7 @@ #define X86_FEATURE_IBRS ( 7*32+25) /* Indirect Branch Restricted Speculation */ #define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */ #define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */ -#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 or above (Zen) */ +#define X86_FEATURE_ZEN (7*32+28) /* "" CPU based on Zen microarchitecture */ #define X86_FEATURE_L1TF_PTEINV ( 7*32+29) /* "" L1TF workaround PTE inversion */ #define X86_FEATURE_IBRS_ENHANCED ( 7*32+30) /* Enhanced IBRS */ #define X86_FEATURE_MSR_IA32_FEAT_CTL ( 7*32+31) /* "" MSR IA32_FEAT_CTL configured */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index cc615be27a54b..43a3d8e4eb9a7 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -558,6 +558,7 @@ #define MSR_AMD_CPPC_CAP2 0xc00102b2 #define MSR_AMD_CPPC_REQ 0xc00102b3 #define MSR_AMD_CPPC_STATUS 0xc00102b4 +#define MSR_AMD_CPPC_HW_CTL 0xc0010015 #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) @@ -568,6 +569,8 @@ #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) +#define AMD_CPPC_PRECISION_BOOST_BIT 25 +#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) /* AMD Performance Counter Global Status and Control MSRs */ #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 29dd27b5a339d..3a8fdf881313d 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -13,6 +13,7 @@ #define MWAIT_SUBSTATE_SIZE 4 #define MWAIT_HINT2CSTATE(hint) (((hint) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) #define MWAIT_HINT2SUBSTATE(hint) ((hint) & MWAIT_CSTATE_MASK) +#define MWAIT_C1_SUBSTATE_MASK 0xf0 #define CPUID_MWAIT_LEAF 5 #define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1 diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 458c891a82736..d86eb1ebf59f5 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; /* Interface to set priority of a cpu */ void sched_set_itmt_core_prio(int prio, int core_cpu); +void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); /* Interface to notify scheduler that system supports ITMT */ int sched_set_itmt_support(void); diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index 57b1a7034c640..e2c45674f9897 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -10,7 +10,7 @@ /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ static __always_inline void rep_nop(void) { - asm volatile("rep; nop" ::: "memory"); + asm volatile("lfence" ::: "memory"); } static __always_inline void cpu_relax(void) diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h index 75884d2cdec37..4e6a08d4c7e53 100644 --- a/arch/x86/include/asm/vermagic.h +++ b/arch/x86/include/asm/vermagic.h @@ -17,6 +17,48 @@ #define MODULE_PROC_FAMILY "586MMX " #elif defined CONFIG_MCORE2 #define MODULE_PROC_FAMILY "CORE2 " +#elif defined CONFIG_MNATIVE_INTEL +#define MODULE_PROC_FAMILY "NATIVE_INTEL " +#elif defined CONFIG_MNATIVE_AMD +#define MODULE_PROC_FAMILY "NATIVE_AMD " +#elif defined CONFIG_MNEHALEM +#define MODULE_PROC_FAMILY "NEHALEM " +#elif defined CONFIG_MWESTMERE +#define MODULE_PROC_FAMILY "WESTMERE " +#elif defined CONFIG_MSILVERMONT +#define MODULE_PROC_FAMILY "SILVERMONT " +#elif defined CONFIG_MGOLDMONT +#define MODULE_PROC_FAMILY "GOLDMONT " +#elif defined CONFIG_MGOLDMONTPLUS +#define MODULE_PROC_FAMILY "GOLDMONTPLUS " +#elif defined CONFIG_MSANDYBRIDGE +#define MODULE_PROC_FAMILY "SANDYBRIDGE " +#elif defined CONFIG_MIVYBRIDGE +#define MODULE_PROC_FAMILY "IVYBRIDGE " +#elif defined CONFIG_MHASWELL +#define MODULE_PROC_FAMILY "HASWELL " +#elif defined CONFIG_MBROADWELL +#define MODULE_PROC_FAMILY "BROADWELL " +#elif defined CONFIG_MSKYLAKE +#define MODULE_PROC_FAMILY "SKYLAKE " +#elif defined CONFIG_MSKYLAKEX +#define MODULE_PROC_FAMILY "SKYLAKEX " +#elif defined CONFIG_MCANNONLAKE +#define MODULE_PROC_FAMILY "CANNONLAKE " +#elif defined CONFIG_MICELAKE +#define MODULE_PROC_FAMILY "ICELAKE " +#elif defined CONFIG_MCASCADELAKE +#define MODULE_PROC_FAMILY "CASCADELAKE " +#elif defined CONFIG_MCOOPERLAKE +#define MODULE_PROC_FAMILY "COOPERLAKE " +#elif defined CONFIG_MTIGERLAKE +#define MODULE_PROC_FAMILY "TIGERLAKE " +#elif defined CONFIG_MSAPPHIRERAPIDS +#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " +#elif defined CONFIG_ROCKETLAKE +#define MODULE_PROC_FAMILY "ROCKETLAKE " +#elif defined CONFIG_MALDERLAKE +#define MODULE_PROC_FAMILY "ALDERLAKE " #elif defined CONFIG_MATOM #define MODULE_PROC_FAMILY "ATOM " #elif defined CONFIG_M686 @@ -35,6 +77,30 @@ #define MODULE_PROC_FAMILY "K7 " #elif defined CONFIG_MK8 #define MODULE_PROC_FAMILY "K8 " +#elif defined CONFIG_MK8SSE3 +#define MODULE_PROC_FAMILY "K8SSE3 " +#elif defined CONFIG_MK10 +#define MODULE_PROC_FAMILY "K10 " +#elif defined CONFIG_MBARCELONA +#define MODULE_PROC_FAMILY "BARCELONA " +#elif defined CONFIG_MBOBCAT +#define MODULE_PROC_FAMILY "BOBCAT " +#elif defined CONFIG_MBULLDOZER +#define MODULE_PROC_FAMILY "BULLDOZER " +#elif defined CONFIG_MPILEDRIVER +#define MODULE_PROC_FAMILY "PILEDRIVER " +#elif defined CONFIG_MSTEAMROLLER +#define MODULE_PROC_FAMILY "STEAMROLLER " +#elif defined CONFIG_MJAGUAR +#define MODULE_PROC_FAMILY "JAGUAR " +#elif defined CONFIG_MEXCAVATOR +#define MODULE_PROC_FAMILY "EXCAVATOR " +#elif defined CONFIG_MZEN +#define MODULE_PROC_FAMILY "ZEN " +#elif defined CONFIG_MZEN2 +#define MODULE_PROC_FAMILY "ZEN2 " +#elif defined CONFIG_MZEN3 +#define MODULE_PROC_FAMILY "ZEN3 " #elif defined CONFIG_MELAN #define MODULE_PROC_FAMILY "ELAN " #elif defined CONFIG_MCRUSOE diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62f6b8b7c4a52..f9c9b5850847d 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -936,7 +936,9 @@ void __init alternative_instructions(void) * Then patch alternatives, such that those paravirt calls that are in * alternatives can be overwritten by their immediate fragments. */ + printk("clr: Applying alternatives\n"); apply_alternatives(__alt_instructions, __alt_instructions_end); + printk("clr: Applying alternatives done\n"); apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index fd5dead8371cc..663f6e6dd2886 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -682,9 +682,9 @@ static void init_intel(struct cpuinfo_x86 *c) unsigned int l1, l2; rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); - if (!(l1 & (1<<11))) + if (!(l1 & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_BTS); - if (!(l1 & (1<<12))) + if (!(l1 & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_PEBS); } diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index fbaf12e43f416..c8c2d6f1a8aca 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, if (ret < 0) return ret; + /* update the ITMT scheduler logic to use the power policy data */ + /* scale the val up by 2 so the range is 224 - 256 */ + sched_set_itmt_power_ratio(256 - val * 2, cpu); + return count; } diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index ad57e0e4d6742..e2ed33c2bd4da 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -44,6 +44,8 @@ static struct microcode_ops *microcode_ops; static bool dis_ucode_ldr = true; +bool ucode_rollback = false; +int enable_rollback = 0; bool initrd_gone; @@ -80,6 +82,26 @@ static u32 final_levels[] = { 0, /* T-101 terminator */ }; +static int __init ucode_setup(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "rollback", 8)) { + enable_rollback = 1; + pr_info("Microcode Rollback Enabled\n"); + } + str += strcspn(str, ","); + while (*str == ',') + str++; + } + return 0; +} + +__setup("ucode=", ucode_setup); + + /* * Check the current patch level on this CPU. * @@ -512,6 +534,7 @@ static ssize_t reload_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t size) { + struct cpuinfo_x86 *c = &boot_cpu_data; enum ucode_state tmp_ret = UCODE_OK; int bsp = boot_cpu_data.cpu_index; unsigned long val; @@ -521,7 +544,7 @@ static ssize_t reload_store(struct device *dev, if (ret) return ret; - if (val != 1) + if (!val || val > 2) return size; cpus_read_lock(); @@ -529,6 +552,20 @@ static ssize_t reload_store(struct device *dev, ret = check_online_cpus(); if (ret) goto put; + /* + * Check if the vendor is Intel to permit reloading + * microcode even if the revision is unchanged. + * This is typically used during development of microcode + * and changing rev is a pain. + */ + if ((val == 2) && ((c->x86_vendor != X86_VENDOR_INTEL) || + !enable_rollback)) + return size; + else if (val == 2) { + mutex_lock(µcode_mutex); + ucode_rollback = true; + mutex_unlock(µcode_mutex); + } tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); if (tmp_ret != UCODE_NEW) @@ -539,6 +576,7 @@ static ssize_t reload_store(struct device *dev, mutex_unlock(µcode_mutex); put: + ucode_rollback = false; cpus_read_unlock(); if (ret == 0) diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 025c8f0cd948c..60473f577a251 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -44,6 +44,7 @@ static struct microcode_intel *intel_ucode_patch; /* last level cache size per core */ static int llc_size_per_core; +extern bool ucode_rollback; /* * Returns 1 if update has been found, 0 otherwise. @@ -80,7 +81,7 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev { struct microcode_header_intel *mc_hdr = mc; - if (mc_hdr->rev <= new_rev) + if (!ucode_rollback && mc_hdr->rev <= new_rev) return 0; return find_matching_signature(mc, csig, cpf); @@ -120,7 +121,7 @@ static void save_microcode_patch(struct ucode_cpu_info *uci, void *data, unsigne if (find_matching_signature(data, sig, pf)) { prev_found = true; - if (mc_hdr->rev <= mc_saved_hdr->rev) + if (!ucode_rollback && mc_hdr->rev <= mc_saved_hdr->rev) continue; p = memdup_patch(data, size); @@ -649,7 +650,7 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci) phdr = (struct microcode_header_intel *)iter->data; - if (phdr->rev <= uci->cpu_sig.rev) + if (!ucode_rollback && phdr->rev <= uci->cpu_sig.rev) continue; if (!find_matching_signature(phdr, @@ -734,10 +735,11 @@ static enum ucode_state apply_microcode_intel(int cpu) * already. */ rev = intel_get_microcode_revision(); - if (rev >= mc->hdr.rev) { + if (!ucode_rollback && rev >= mc->hdr.rev) { ret = UCODE_OK; goto out; - } + } else if (ucode_rollback) + ret = UCODE_OK; /* * Writeback and invalidate caches before updating microcode to avoid @@ -756,7 +758,7 @@ static enum ucode_state apply_microcode_intel(int cpu) return UCODE_ERROR; } - if (bsp && rev != prev_rev) { + if (bsp && ((rev != prev_rev) || ucode_rollback)) { pr_info("updated to revision 0x%x, date = %04x-%02x-%02x\n", rev, mc->hdr.date & 0xffff, diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 9ff480e94511b..d4326e050fb70 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -25,6 +25,7 @@ static DEFINE_MUTEX(itmt_update_mutex); DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); +DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); /* Boolean to track if system has ITMT capabilities */ static bool __read_mostly sched_itmt_capable; @@ -169,7 +170,12 @@ void sched_clear_itmt_support(void) int arch_asym_cpu_priority(int cpu) { - return per_cpu(sched_core_priority, cpu); + int power_ratio = per_cpu(sched_power_ratio, cpu); + + /* a power ratio of 0 (uninitialized) is assumed to be maximum */ + if (power_ratio == 0) + power_ratio = 256 - 2 * 6; + return per_cpu(sched_core_priority, cpu) * power_ratio / 256; } /** @@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) i++; } } + +/** + * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT + * @power_ratio: The power scaling ratio [1..256] for the core + * @core_cpu: The cpu number associated with the core + * + * Set a scaling to the cpu performance based on long term power + * settings (like EPB). + * + * Note this is for the policy not for the actual dynamic frequency; + * the frequency will increase itself as workloads run on a core. + */ + +void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) +{ + int cpu; + + for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { + per_cpu(sched_power_ratio, cpu) = power_ratio; + } +} diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index ed8ac6bcbafb2..d6fc5bdb0246a 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -48,7 +48,7 @@ enum allow_write_msrs { MSR_WRITES_DEFAULT, }; -static enum allow_write_msrs allow_writes = MSR_WRITES_DEFAULT; +static enum allow_write_msrs allow_writes = MSR_WRITES_ON; static ssize_t msr_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index d456ce21c2552..58a6ea472db92 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -810,24 +810,43 @@ static void amd_e400_idle(void) } /* - * Intel Core2 and older machines prefer MWAIT over HALT for C1. - * We can't rely on cpuidle installing MWAIT, because it will not load - * on systems that support only C1 -- so the boot default must be MWAIT. + * Prefer MWAIT over HALT if MWAIT is supported, MWAIT_CPUID leaf + * exists and whenever MONITOR/MWAIT extensions are present there is at + * least one C1 substate. * - * Some AMD machines are the opposite, they depend on using HALT. - * - * So for default C1, which is used during boot until cpuidle loads, - * use MWAIT-C1 on Intel HW that has it, else use HALT. + * Do not prefer MWAIT if MONITOR instruction has a bug or idle=nomwait + * is passed to kernel commandline parameter. */ static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c) { - if (c->x86_vendor != X86_VENDOR_INTEL) + u32 eax, ebx, ecx, edx; + + /* User has disallowed the use of MWAIT. Fallback to HALT */ + if (boot_option_idle_override == IDLE_NOMWAIT) return 0; - if (!cpu_has(c, X86_FEATURE_MWAIT) || boot_cpu_has_bug(X86_BUG_MONITOR)) + /* MWAIT is not supported on this platform. Fallback to HALT */ + if (!cpu_has(c, X86_FEATURE_MWAIT)) return 0; - return 1; + /* Monitor has a bug. Fallback to HALT */ + if (boot_cpu_has_bug(X86_BUG_MONITOR)) + return 0; + + cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx); + + /* + * If MWAIT extensions are not available, it is safe to use MWAIT + * with EAX=0, ECX=0. + */ + if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) + return 1; + + /* + * If MWAIT extensions are available, there should be at least one + * MWAIT C1 substate present. + */ + return (edx & MWAIT_C1_SUBSTATE_MASK); } /* @@ -932,9 +951,8 @@ static int __init idle_setup(char *str) } else if (!strcmp(str, "nomwait")) { /* * If the boot option of "idle=nomwait" is added, - * it means that mwait will be disabled for CPU C2/C3 - * states. In such case it won't touch the variable - * of boot_option_idle_override. + * it means that mwait will be disabled for CPU C1/C2/C3 + * states. */ boot_option_idle_override = IDLE_NOMWAIT; } else diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index cafacb2e58cce..c2f80184fd332 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) if (!constant_tsc || !mask) return 0; + if (cpu != 0) + return cpu_data(0).loops_per_jiffy; + sibling = cpumask_any_but(mask, cpu); if (sibling < nr_cpu_ids) return cpu_data(sibling).loops_per_jiffy; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index fad8faa29d042..3bcf302f84640 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, if (!printk_ratelimit()) return; - printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", + printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", loglvl, tsk->comm, task_pid_nr(tsk), address, - (void *)regs->ip, (void *)regs->sp, error_code); + (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); print_vma_addr(KERN_CONT " in ", regs->ip); diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index e44e938885b70..7418c367e3281 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -110,7 +110,7 @@ int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, int pkey return vma_pkey(vma); } -#define PKRU_AD_KEY(pkey) (PKRU_AD_BIT << ((pkey) * PKRU_BITS_PER_PKEY)) +#define PKRU_AD_MASK(pkey) (PKRU_AD_BIT << ((pkey) * PKRU_BITS_PER_PKEY)) /* * Make the default PKRU value (at execve() time) as restrictive @@ -118,11 +118,14 @@ int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, int pkey * in the process's lifetime will not accidentally get access * to data which is pkey-protected later on. */ -u32 init_pkru_value = PKRU_AD_KEY( 1) | PKRU_AD_KEY( 2) | PKRU_AD_KEY( 3) | - PKRU_AD_KEY( 4) | PKRU_AD_KEY( 5) | PKRU_AD_KEY( 6) | - PKRU_AD_KEY( 7) | PKRU_AD_KEY( 8) | PKRU_AD_KEY( 9) | - PKRU_AD_KEY(10) | PKRU_AD_KEY(11) | PKRU_AD_KEY(12) | - PKRU_AD_KEY(13) | PKRU_AD_KEY(14) | PKRU_AD_KEY(15); +u32 init_pkru_value = PKRU_AD_MASK( 1) | PKRU_AD_MASK( 2) | + PKRU_AD_MASK( 3) | PKRU_AD_MASK( 4) | + PKRU_AD_MASK( 5) | PKRU_AD_MASK( 6) | + PKRU_AD_MASK( 7) | PKRU_AD_MASK( 8) | + PKRU_AD_MASK( 9) | PKRU_AD_MASK(10) | + PKRU_AD_MASK(11) | PKRU_AD_MASK(12) | + PKRU_AD_MASK(13) | PKRU_AD_MASK(14) | + PKRU_AD_MASK(15); static ssize_t init_pkru_read_file(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index d400b6d9d246b..d9314cc8b81f7 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -734,10 +734,10 @@ static void flush_tlb_func(void *info) const struct flush_tlb_info *f = info; struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); - u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen); u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen); bool local = smp_processor_id() == f->initiating_cpu; unsigned long nr_invalidate = 0; + u64 mm_tlb_gen; /* This code cannot presently handle being reentered. */ VM_WARN_ON(!irqs_disabled()); @@ -771,6 +771,22 @@ static void flush_tlb_func(void *info) return; } + if (f->new_tlb_gen <= local_tlb_gen) { + /* + * The TLB is already up to date in respect to f->new_tlb_gen. + * While the core might be still behind mm_tlb_gen, checking + * mm_tlb_gen unnecessarily would have negative caching effects + * so avoid it. + */ + return; + } + + /* + * Defer mm_tlb_gen reading as long as possible to avoid cache + * contention. + */ + mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen); + if (unlikely(local_tlb_gen == mm_tlb_gen)) { /* * There's nothing to do: we're already up to date. This can diff --git a/arch/x86/platform/olpc/olpc-xo1-sci.c b/arch/x86/platform/olpc/olpc-xo1-sci.c index f03a6883dcc6d..89f25af4b3c33 100644 --- a/arch/x86/platform/olpc/olpc-xo1-sci.c +++ b/arch/x86/platform/olpc/olpc-xo1-sci.c @@ -80,7 +80,7 @@ static void send_ebook_state(void) return; } - if (!!test_bit(SW_TABLET_MODE, ebook_switch_idev->sw) == state) + if (test_bit(SW_TABLET_MODE, ebook_switch_idev->sw) == !!state) return; /* Nothing new to report. */ input_report_switch(ebook_switch_idev, SW_TABLET_MODE, state); diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 52c94ab5c2058..1518e261d882e 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -421,3 +421,4 @@ 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common pmadv_ksm sys_pmadv_ksm diff --git a/block/blk-core.c b/block/blk-core.c index 27fb1357ad4b8..6260bfacd7226 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -795,6 +795,9 @@ void submit_bio_noacct(struct bio *bio) status = BLK_STS_OK; goto end_io; } + + if (bio->bi_opf & REQ_PREFLUSH) + current->fsync_count++; } if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) diff --git a/block/elevator.c b/block/elevator.c index c319765892bb9..b2687032a0e07 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) if (q->nr_hw_queues != 1 && !blk_mq_is_shared_tags(q->tag_set->flags)) +#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) + return elevator_get(q, "kyber", false); +#elif defined(CONFIG_CACHY) + return elevator_get(q, "mq-deadline", false); +#else return NULL; - +#endif return elevator_get(q, "mq-deadline", false); } diff --git a/drivers/Makefile b/drivers/Makefile index 9a30842b22c54..32afd317d95ec 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -59,15 +59,8 @@ obj-y += char/ # iommu/ comes before gpu as gpu are using iommu controllers obj-y += iommu/ -# gpu/ comes after char for AGP vs DRM startup and after iommu -obj-y += gpu/ - obj-$(CONFIG_CONNECTOR) += connector/ -# i810fb and intelfb depend on char/agp/ -obj-$(CONFIG_FB_I810) += video/fbdev/i810/ -obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ - obj-$(CONFIG_PARPORT) += parport/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ @@ -79,6 +72,14 @@ obj-y += macintosh/ obj-y += scsi/ obj-y += nvme/ obj-$(CONFIG_ATA) += ata/ + +# gpu/ comes after char for AGP vs DRM startup and after iommu +obj-y += gpu/ + +# i810fb and intelfb depend on char/agp/ +obj-$(CONFIG_FB_I810) += video/fbdev/i810/ +obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ + obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 3c6d4ef87be0f..59c0e3410f97f 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) struct cpc_desc *cpc_ptr; int cpu; + if (acpi_disabled) + return false; + for_each_present_cpu(cpu) { cpc_ptr = per_cpu(cpc_desc_ptr, cpu); if (!cpc_ptr) diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index cf8c7fd59ada5..ad9bb5353dd3d 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -33,14 +33,14 @@ #include "libata.h" static int ahci_skip_host_reset; -int ahci_ignore_sss; +int ahci_ignore_sss=1; EXPORT_SYMBOL_GPL(ahci_ignore_sss); module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444); MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)"); module_param_named(ignore_sss, ahci_ignore_sss, int, 0444); -MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)"); +MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])"); static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, unsigned hints); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 579c851a2bd74..73a8cb31529d7 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -352,7 +352,7 @@ void topology_init_cpu_capacity_cppc(void) struct cppc_perf_caps perf_caps; int cpu; - if (likely(acpi_disabled || !acpi_cpc_valid())) + if (likely(!acpi_cpc_valid())) return; raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index ac3f34e801940..1d13577560ae1 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, static char fw_path_para[256]; static const char * const fw_path[] = { fw_path_para, + "/etc/firmware/" UTS_RELEASE, + "/etc/firmware", "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig index d4100b0c083ec..8739ea13161f7 100644 --- a/drivers/block/zram/Kconfig +++ b/drivers/block/zram/Kconfig @@ -78,3 +78,21 @@ config ZRAM_MEMORY_TRACKING /sys/kernel/debug/zram/zramX/block_state. See Documentation/admin-guide/blockdev/zram.rst for more information. + +config ZRAM_ENTROPY + bool "Use entropy optimization for zram" + depends on ZRAM && ZRAM_DEF_COMP_ZSTD + help + With this feature, entropy will be calculated for each page. + Pages above ZRAM_ENTROPY_THRESHOLD entropy will be + stored uncompressed. Use this feature if you need a performance + boost and a small loss in compression. + +config ZRAM_ENTROPY_THRESHOLD + int + depends on ZRAM && ZRAM_ENTROPY + default 100000 if ZRAM_DEF_COMP_ZSTD + help + Pages with entropy above ZRAM_ENTROPY_THRESHOLD will be stored + uncompressed. The default value was chosen as a result a lot of + experiments. You can try set your own value. diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index b8549c61ff2ce..1140bc12a4414 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1344,6 +1344,35 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, return ret; } + +#ifdef CONFIG_ZRAM_ENTROPY +static inline u32 ilog2_w(u64 n) +{ + return ilog2(n * n * n * n); +} + +static inline s32 shannon_entropy(const u8 *src) +{ + s32 entropy_sum = 0; + u32 sz_base, i; + u16 entropy_count[256] = { 0 }; + + for (i = 0; i < PAGE_SIZE; ++i) + entropy_count[src[i]]++; + + sz_base = ilog2_w(PAGE_SIZE); + for (i = 0; i < ARRAY_SIZE(entropy_count); ++i) { + if (entropy_count[i] > 0) { + s32 p = entropy_count[i]; + + entropy_sum += p * (sz_base - ilog2_w((u64)p)); + } + } + + return entropy_sum; +} +#endif + static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, struct bio *bio) { @@ -1369,7 +1398,17 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, zstrm = zcomp_stream_get(zram->comp); src = kmap_atomic(page); + +#ifdef CONFIG_ZRAM_ENTROPY + /* Just save this page uncompressible */ + if (shannon_entropy((const u8 *)src) > CONFIG_ZRAM_ENTROPY_THRESHOLD) + comp_len = PAGE_SIZE; + else + ret = zcomp_compress(zstrm, src, &comp_len); +#else ret = zcomp_compress(zstrm, src, &comp_len); +#endif + kunmap_atomic(src); if (unlikely(ret)) { diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9ac75c1cde9c2..d3bc441b39233 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -31,23 +31,18 @@ #include #include #include -#include #include #include #include #include -#include #include #include -#include -#include -#include #include "amd-pstate-trace.h" -#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 -#define AMD_PSTATE_TRANSITION_DELAY 500 +#define AMD_PSTATE_TRANSITION_LATENCY 20000 +#define AMD_PSTATE_TRANSITION_DELAY 1000 /* * TODO: We need more time to fine tune processors with shared memory solution @@ -97,6 +92,8 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @precision_boost_off: the core performance boost disabled state + * @cppc_hw_conf_cached: the cached hardware configuration register * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -120,8 +117,10 @@ struct amd_cpudata { struct amd_aperf_mperf cur; struct amd_aperf_mperf prev; - u64 freq; + u64 freq; bool boost_supported; + bool precision_boost_off; + u64 cppc_hw_conf_cached; }; static inline int pstate_enable(bool enable) @@ -269,6 +268,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, u64 prev = READ_ONCE(cpudata->cppc_req_cached); u64 value = prev; + des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); value &= ~AMD_CPPC_MIN_PERF(~0L); value |= AMD_CPPC_MIN_PERF(min_perf); @@ -312,7 +312,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, return -ENODEV; cap_perf = READ_ONCE(cpudata->highest_perf); - min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); + min_perf = READ_ONCE(cpudata->lowest_perf); max_perf = cap_perf; freqs.old = policy->cur; @@ -357,8 +357,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (max_perf < min_perf) max_perf = min_perf; - des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); } @@ -438,18 +436,27 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) { struct amd_cpudata *cpudata = policy->driver_data; int ret; + u64 value; if (!cpudata->boost_supported) { pr_err("Boost mode is not supported by this processor or SBIOS\n"); return -EINVAL; } - if (state) + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); + if (ret) + return ret; + + if (state) { + value |= AMD_CPPC_PRECISION_BOOST_ENABLED; policy->cpuinfo.max_freq = cpudata->max_freq; - else + } else { + value &= ~AMD_CPPC_PRECISION_BOOST_ENABLED; policy->cpuinfo.max_freq = cpudata->nominal_freq; - + } policy->max = policy->cpuinfo.max_freq; + WRITE_ONCE(cpudata->cppc_hw_conf_cached, value); + wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, value); ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); @@ -478,6 +485,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; struct device *dev; struct amd_cpudata *cpudata; + u64 value; dev = get_cpu_device(policy->cpu); if (!dev) @@ -541,7 +549,17 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; policy->driver_data = cpudata; + if (!shared_mem) { + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); + if (ret) + return ret; + cpudata->precision_boost_off = value & AMD_CPPC_PRECISION_BOOST_ENABLED; + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + if (ret) + return ret; + WRITE_ONCE(cpudata->cppc_req_cached, value); + } amd_pstate_boost_init(cpudata); return 0; @@ -555,9 +573,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); @@ -599,9 +615,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { int max_freq; - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; max_freq = amd_get_max_freq(cpudata); if (max_freq < 0) @@ -614,9 +628,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli char *buf) { int freq; - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; freq = amd_get_lowest_nonlinear_freq(cpudata); if (freq < 0) @@ -662,7 +674,7 @@ static struct cpufreq_driver amd_pstate_driver = { .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, .name = "amd-pstate", - .attr = amd_pstate_attr, + .attr = amd_pstate_attr, }; static int __init amd_pstate_init(void) @@ -673,7 +685,7 @@ static int __init amd_pstate_init(void) return -ENODEV; if (!acpi_cpc_valid()) { - pr_debug("the _CPC object is not present in SBIOS\n"); + pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); return -ENODEV; } diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 24eaf0ec344d5..9adb7612993e0 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -947,7 +947,7 @@ static int __init cppc_cpufreq_init(void) { int ret; - if ((acpi_disabled) || !acpi_cpc_valid()) + if (!acpi_cpc_valid()) return -ENODEV; cppc_check_hisi_workaround(); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 57cdb36798854..50881662b45b6 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) * update them at any time after it has been called. */ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); + /* + * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. + * In this case we can't use CPPC.highest_perf to enable ITMT. + * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. + */ + if (cppc_perf.highest_perf == 0xff) + cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); if (max_highest_perf <= min_highest_perf) { if (cppc_perf.highest_perf > max_highest_perf) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 2f0d2c68c93c6..07cb48db76baf 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -4997,7 +4997,7 @@ static int setup_resources(struct udma_dev *ud) switch (ud->match_data->type) { case DMA_TYPE_UDMA: dev_info(dev, - "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", + "Channels: %d (tchan: %lu, rchan: %lu, gp-rflow: %lu)\n", ch_count, ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), @@ -5008,7 +5008,7 @@ static int setup_resources(struct udma_dev *ud) break; case DMA_TYPE_BCDMA: dev_info(dev, - "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n", + "Channels: %d (bchan: %lu, tchan: %lu, rchan: %lu)\n", ch_count, ud->bchan_cnt - bitmap_weight(ud->bchan_map, ud->bchan_cnt), @@ -5019,7 +5019,7 @@ static int setup_resources(struct udma_dev *ud) break; case DMA_TYPE_PKTDMA: dev_info(dev, - "Channels: %d (tchan: %u, rchan: %u)\n", + "Channels: %d (tchan: %lu, rchan: %lu)\n", ch_count, ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 16699158e00d8..1098506ba1aaf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -195,7 +195,7 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) set_bit(i, adev->gfx.mec.queue_bitmap); } - dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); + dev_dbg(adev->dev, "mec queue bitmap weight=%ld\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); } void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 949edc983a169..4204e8a0e1a60 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -378,7 +378,7 @@ static void print_power_domains(struct i915_power_domains *power_domains, power_domains); enum intel_display_power_domain domain; - drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM)); + drm_dbg(&i915->drm, "%s (%ld):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM)); for_each_power_domain(domain, mask) drm_dbg(&i915->drm, "%s use_count %d\n", intel_display_power_domain_str(domain), diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c index 56a3063545ec4..3d35b3982a60c 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c @@ -363,7 +363,7 @@ void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p) } drm_printf(p, "TOTAL:\t%d\t(of %d)\n", total, smp->blk_cnt); - drm_printf(p, "AVAIL:\t%d\n", smp->blk_cnt - + drm_printf(p, "AVAIL:\t%ld\n", smp->blk_cnt - bitmap_weight(state->state, smp->blk_cnt)); if (drm_can_sleep()) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 907700d1e78eb..d0d4ed509b48b 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -561,7 +561,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x01", .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, .exit_latency = 10, - .target_residency = 20, + .target_residency = 120, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -569,7 +569,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x10", .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 33, - .target_residency = 100, + .target_residency = 900, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -577,7 +577,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x20", .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 133, - .target_residency = 400, + .target_residency = 1000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -585,7 +585,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x32", .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 166, - .target_residency = 500, + .target_residency = 1500, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -593,7 +593,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x40", .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 300, - .target_residency = 900, + .target_residency = 2000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -601,7 +601,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x50", .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 600, - .target_residency = 1800, + .target_residency = 5000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -609,7 +609,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { .desc = "MWAIT 0x60", .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 2600, - .target_residency = 7700, + .target_residency = 9000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -629,7 +629,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x01", .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, .exit_latency = 10, - .target_residency = 20, + .target_residency = 120, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -637,7 +637,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x10", .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 40, - .target_residency = 100, + .target_residency = 1000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -645,7 +645,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x20", .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 133, - .target_residency = 400, + .target_residency = 1000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -653,7 +653,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x32", .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 166, - .target_residency = 500, + .target_residency = 2000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -661,7 +661,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x40", .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 300, - .target_residency = 900, + .target_residency = 4000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -669,7 +669,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x50", .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 600, - .target_residency = 1800, + .target_residency = 7000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -677,7 +677,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { .desc = "MWAIT 0x60", .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 2600, - .target_residency = 7700, + .target_residency = 9000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -698,7 +698,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x01", .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, .exit_latency = 10, - .target_residency = 20, + .target_residency = 120, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -706,7 +706,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x10", .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 70, - .target_residency = 100, + .target_residency = 1000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -714,7 +714,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x20", .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, .exit_latency = 85, - .target_residency = 200, + .target_residency = 600, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -722,7 +722,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x33", .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, .exit_latency = 124, - .target_residency = 800, + .target_residency = 3000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -730,7 +730,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x40", .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, .exit_latency = 200, - .target_residency = 800, + .target_residency = 3200, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -738,7 +738,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x50", .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, .exit_latency = 480, - .target_residency = 5000, + .target_residency = 9000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -746,7 +746,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { .desc = "MWAIT 0x60", .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, .exit_latency = 890, - .target_residency = 5000, + .target_residency = 9000, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { @@ -767,7 +767,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { .desc = "MWAIT 0x01", .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, .exit_latency = 10, - .target_residency = 20, + .target_residency = 300, .enter = &intel_idle, .enter_s2idle = intel_idle_s2idle, }, { diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c index 3fc0a89cc785c..a7c103f9dfd37 100644 --- a/drivers/input/serio/i8042.c +++ b/drivers/input/serio/i8042.c @@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { i8042_ctr &= ~I8042_CTR_KBDINT; i8042_ctr |= I8042_CTR_KBDDIS; - pr_err("Failed to enable KBD port\n"); + pr_info("Failed to enable KBD port\n"); return -EIO; } @@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { i8042_ctr &= ~I8042_CTR_AUXINT; i8042_ctr |= I8042_CTR_AUXDIS; - pr_err("Failed to enable AUX port\n"); + pr_info("Failed to enable AUX port\n"); return -EIO; } @@ -732,7 +732,7 @@ static int i8042_check_mux(void) i8042_ctr &= ~I8042_CTR_AUXINT; if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - pr_err("Failed to disable AUX port, can't use MUX\n"); + pr_info("Failed to disable AUX port, can't use MUX\n"); return -EIO; } @@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) do { if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { - pr_err("i8042 controller selftest timeout\n"); + pr_info("i8042 controller selftest timeout\n"); return -ENODEV; } @@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) pr_info("giving up on controller selftest, continuing anyway...\n"); return 0; #else - pr_err("i8042 controller selftest failed\n"); + pr_info("i8042 controller selftest failed\n"); return -EIO; #endif } diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 9699ca101c624..64b14ac4c7b02 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -494,7 +494,7 @@ static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg) if (drhd->reg_base_addr == rhsa->base_address) { int node = pxm_to_node(rhsa->proximity_domain); - if (!node_online(node)) + if (node != NUMA_NO_NODE && !node_online(node)) node = NUMA_NO_NODE; drhd->iommu->node = node; return 0; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 5c0dce78586aa..64f6877c0b555 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3922,7 +3922,7 @@ static ssize_t domains_used_show(struct device *dev, struct device_attribute *attr, char *buf) { struct intel_iommu *iommu = dev_to_intel_iommu(dev); - return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids, + return sprintf(buf, "%ld\n", bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))); } static DEVICE_ATTR_RO(domains_used); diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 159c6806c19b8..a201f1c2fce5c 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar } } +#ifdef CONFIG_CACHY + set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); + set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); +#endif + return 0; } diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c index f82ad74195082..5e8faa70aad6a 100644 --- a/drivers/net/dummy.c +++ b/drivers/net/dummy.c @@ -43,7 +43,7 @@ #define DRV_NAME "dummy" -static int numdummies = 1; +static int numdummies = 0; /* fake multicast ability */ static void set_multicast_list(struct net_device *dev) diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 8d8f3eec79eeb..9b2872e891518 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -4934,7 +4934,7 @@ ice_find_free_recp_res_idx(struct ice_hw *hw, const unsigned long *profiles, bitmap_zero(recipes, ICE_MAX_NUM_RECIPES); bitmap_zero(used_idx, ICE_MAX_FV_WORDS); - bitmap_set(possible_idx, 0, ICE_MAX_FV_WORDS); + bitmap_fill(possible_idx, ICE_MAX_FV_WORDS); /* For each profile we are going to associate the recipe with, add the * recipes that are associated with that profile. This will give us diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c index 42c96c9d7fb16..af054a3808cae 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.c +++ b/drivers/net/ethernet/mellanox/mlx4/fw.c @@ -463,7 +463,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, field = min( bitmap_weight(actv_ports.ports, dev->caps.num_ports), - dev->caps.num_ports); + (unsigned long)dev->caps.num_ports); MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); size = dev->caps.function_caps; /* set PF behaviours */ diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c index b3ed65e5c4da8..d32056c4ee1d8 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_debug.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_debug.c @@ -296,7 +296,7 @@ static ssize_t read_file_slot(struct file *file, char __user *user_buf, spin_lock_bh(&priv->tx.tx_lock); len = scnprintf(buf, sizeof(buf), "TX slot bitmap : %*pb\n" - "Used slots : %d\n", + "Used slots : %ld\n", MAX_TX_BUF_NUM, priv->tx.tx_slot, bitmap_weight(priv->tx.tx_slot, MAX_TX_BUF_NUM)); spin_unlock_bh(&priv->tx.tx_lock); diff --git a/drivers/net/wireless/ath/carl9170/debug.c b/drivers/net/wireless/ath/carl9170/debug.c index bb40889d7c720..7e88882b87d9c 100644 --- a/drivers/net/wireless/ath/carl9170/debug.c +++ b/drivers/net/wireless/ath/carl9170/debug.c @@ -221,7 +221,7 @@ static char *carl9170_debugfs_mem_usage_read(struct ar9170 *ar, char *buf, ADD(buf, *len, bufsize, "jar: [%*pb]\n", ar->fw.mem_blocks, ar->mem_bitmap); - ADD(buf, *len, bufsize, "cookies: used:%3d / total:%3d, allocs:%d\n", + ADD(buf, *len, bufsize, "cookies: used:%3ld / total:%3d, allocs:%d\n", bitmap_weight(ar->mem_bitmap, ar->fw.mem_blocks), ar->fw.mem_blocks, atomic_read(&ar->mem_allocs)); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 6a12a906a11e4..ad5b92b23ea04 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -48,7 +48,7 @@ static u8 nvme_max_retries = 5; module_param_named(max_retries, nvme_max_retries, byte, 0644); MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); -static unsigned long default_ps_max_latency_us = 100000; +static unsigned long default_ps_max_latency_us = 200; module_param(default_ps_max_latency_us, ulong, 0644); MODULE_PARM_DESC(default_ps_max_latency_us, "max power saving latency for new devices; use PM QOS to change per device"); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index cfaf40a540a82..edf4bf6be8bbf 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -60,7 +60,7 @@ struct pci_pme_device { struct pci_dev *dev; }; -#define PME_TIMEOUT 1000 /* How long between PME checks */ +#define PME_TIMEOUT 4000 /* How long between PME checks */ static void pci_dev_d3_sleep(struct pci_dev *dev) { diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index a9c99d9e8b428..8b288a554a9ae 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1514,7 +1514,7 @@ static int __init rapl_init(void) id = x86_match_cpu(rapl_ids); if (!id) { - pr_err("driver does not support CPU family %d model %d\n", + pr_info("driver does not support CPU family %d model %d\n", boot_cpu_data.x86, boot_cpu_data.x86_model); return -ENODEV; diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h index 7631ef5e71fb0..d3ed828daac0a 100644 --- a/drivers/soundwire/bus.h +++ b/drivers/soundwire/bus.h @@ -5,7 +5,7 @@ #define __SDW_BUS_H #define DEFAULT_BANK_SWITCH_TIMEOUT 3000 -#define DEFAULT_PROBE_TIMEOUT 2000 +#define DEFAULT_PROBE_TIMEOUT 10000 u64 sdw_dmi_override_adr(struct sdw_bus *bus, u64 addr); diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c index c841ab37e7c6d..fe3d190ab12b4 100644 --- a/drivers/thermal/intel/intel_powerclamp.c +++ b/drivers/thermal/intel/intel_powerclamp.c @@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { .set_cur_state = powerclamp_set_cur_state, }; +static const struct x86_cpu_id amd_cpu[] = { + { X86_VENDOR_AMD }, + {}, +}; + static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), {} @@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); static int __init powerclamp_probe(void) { + if (x86_match_cpu(amd_cpu)){ + pr_info("Intel PowerClamp does not support AMD CPUs\n"); + return -ENODEV; + } + if (!x86_match_cpu(intel_powerclamp_ids)) { pr_err("CPU does not support MWAIT\n"); return -ENODEV; diff --git a/fs/xattr.c b/fs/xattr.c index e8dd03e4561e8..e0c120426802b 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, } /* - * In the user.* namespace, only regular files and directories can have - * extended attributes. For sticky directories, only the owner and - * privileged users can write attributes. + * In the user.* namespace, only regular files, symbolic links, and + * directories can have extended attributes. For symbolic links and + * sticky directories, only the owner and privileged users can write + * attributes. */ if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { - if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) + if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) return (mask & MAY_WRITE) ? -EPERM : -ENODATA; - if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && - (mask & MAY_WRITE) && - !inode_owner_or_capable(mnt_userns, inode)) + if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) + || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) + && !inode_owner_or_capable(mnt_userns, inode)) return -EPERM; } diff --git a/include/asm-generic/bitops/generic-non-atomic.h b/include/asm-generic/bitops/generic-non-atomic.h new file mode 100644 index 0000000000000..3d5ebd24652b9 --- /dev/null +++ b/include/asm-generic/bitops/generic-non-atomic.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H + +#include + +#ifndef _LINUX_BITOPS_H +#error only can be included directly +#endif + +/* + * Generic definitions for bit operations, should not be used in regular code + * directly. + */ + +/** + * generic___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static __always_inline void +generic___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * generic___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * generic___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline bool +generic___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * generic___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline bool +generic___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static __always_inline bool +generic___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * generic_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +/* + * const_*() definitions provide good compile-time optimizations when + * the passed arguments can be resolved at compile time. + */ +#define const___set_bit generic___set_bit +#define const___clear_bit generic___clear_bit +#define const___change_bit generic___change_bit +#define const___test_and_set_bit generic___test_and_set_bit +#define const___test_and_clear_bit generic___test_and_clear_bit +#define const___test_and_change_bit generic___test_and_change_bit + +/** + * const_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + * + * A version of generic_test_bit() which discards the `volatile` qualifier to + * allow a compiler to optimize code harder. Non-atomic and to be called only + * for testing compile-time constants, e.g. by the corresponding macros, not + * directly from "regular" code. + */ +static __always_inline bool +const_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + const unsigned long *p = (const unsigned long *)addr + BIT_WORD(nr); + unsigned long mask = BIT_MASK(nr); + unsigned long val = *p; + + return !!(val & mask); +} + +#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h index 7ab1ecc377824..988a3bbfba34e 100644 --- a/include/asm-generic/bitops/instrumented-non-atomic.h +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -14,7 +14,7 @@ #include /** - * __set_bit - Set a bit in memory + * ___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -22,14 +22,15 @@ * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __set_bit(long nr, volatile unsigned long *addr) +static __always_inline void +___set_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___set_bit(nr, addr); } /** - * __clear_bit - Clears a bit in memory + * ___clear_bit - Clears a bit in memory * @nr: the bit to clear * @addr: the address to start counting from * @@ -37,14 +38,15 @@ static __always_inline void __set_bit(long nr, volatile unsigned long *addr) * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __clear_bit(long nr, volatile unsigned long *addr) +static __always_inline void +___clear_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___clear_bit(nr, addr); } /** - * __change_bit - Toggle a bit in memory + * ___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -52,7 +54,8 @@ static __always_inline void __clear_bit(long nr, volatile unsigned long *addr) * region of memory concurrently, the effect may be that only one operation * succeeds. */ -static __always_inline void __change_bit(long nr, volatile unsigned long *addr) +static __always_inline void +___change_bit(unsigned long nr, volatile unsigned long *addr) { instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___change_bit(nr, addr); @@ -83,53 +86,57 @@ static __always_inline void __instrument_read_write_bitop(long nr, volatile unsi } /** - * __test_and_set_bit - Set a bit and return its old value + * ___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_set_bit(nr, addr); } /** - * __test_and_clear_bit - Clear a bit and return its old value + * ___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_clear_bit(nr, addr); } /** - * __test_and_change_bit - Change a bit and return its old value + * ___test_and_change_bit - Change a bit and return its old value * @nr: Bit to change * @addr: Address to count from * * This operation is non-atomic. If two instances of this operation race, one * can appear to succeed but actually fail. */ -static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { __instrument_read_write_bitop(nr, addr); return arch___test_and_change_bit(nr, addr); } /** - * test_bit - Determine whether a bit is set + * _test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ -static __always_inline bool test_bit(long nr, const volatile unsigned long *addr) +static __always_inline bool +_test_bit(unsigned long nr, const volatile unsigned long *addr) { instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); return arch_test_bit(nr, addr); diff --git a/include/asm-generic/bitops/non-atomic.h b/include/asm-generic/bitops/non-atomic.h index 078cc68be2f12..5c37ced343aed 100644 --- a/include/asm-generic/bitops/non-atomic.h +++ b/include/asm-generic/bitops/non-atomic.h @@ -2,121 +2,18 @@ #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ -#include +#include -/** - * arch___set_bit - Set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * Unlike set_bit(), this function is non-atomic and may be reordered. - * If it's called on the same region of memory simultaneously, the effect - * may be that only one operation succeeds. - */ -static __always_inline void -arch___set_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); +#define arch___set_bit generic___set_bit +#define arch___clear_bit generic___clear_bit +#define arch___change_bit generic___change_bit - *p |= mask; -} -#define __set_bit arch___set_bit +#define arch___test_and_set_bit generic___test_and_set_bit +#define arch___test_and_clear_bit generic___test_and_clear_bit +#define arch___test_and_change_bit generic___test_and_change_bit -static __always_inline void -arch___clear_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); +#define arch_test_bit generic_test_bit - *p &= ~mask; -} -#define __clear_bit arch___clear_bit - -/** - * arch___change_bit - Toggle a bit in memory - * @nr: the bit to change - * @addr: the address to start counting from - * - * Unlike change_bit(), this function is non-atomic and may be reordered. - * If it's called on the same region of memory simultaneously, the effect - * may be that only one operation succeeds. - */ -static __always_inline -void arch___change_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - - *p ^= mask; -} -#define __change_bit arch___change_bit - -/** - * arch___test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is non-atomic and can be reordered. - * If two examples of this operation race, one can appear to succeed - * but actually fail. You must protect multiple accesses with a lock. - */ -static __always_inline int -arch___test_and_set_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old | mask; - return (old & mask) != 0; -} -#define __test_and_set_bit arch___test_and_set_bit - -/** - * arch___test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This operation is non-atomic and can be reordered. - * If two examples of this operation race, one can appear to succeed - * but actually fail. You must protect multiple accesses with a lock. - */ -static __always_inline int -arch___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old & ~mask; - return (old & mask) != 0; -} -#define __test_and_clear_bit arch___test_and_clear_bit - -/* WARNING: non atomic and it can be reordered! */ -static __always_inline int -arch___test_and_change_bit(unsigned int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old ^ mask; - return (old & mask) != 0; -} -#define __test_and_change_bit arch___test_and_change_bit - -/** - * arch_test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static __always_inline int -arch_test_bit(unsigned int nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} -#define test_bit arch_test_bit +#include #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ diff --git a/include/asm-generic/bitops/non-instrumented-non-atomic.h b/include/asm-generic/bitops/non-instrumented-non-atomic.h new file mode 100644 index 0000000000000..bdb9b1ffaee90 --- /dev/null +++ b/include/asm-generic/bitops/non-instrumented-non-atomic.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H + +#define ___set_bit arch___set_bit +#define ___clear_bit arch___clear_bit +#define ___change_bit arch___change_bit + +#define ___test_and_set_bit arch___test_and_set_bit +#define ___test_and_clear_bit arch___test_and_clear_bit +#define ___test_and_change_bit arch___test_and_change_bit + +#define _test_bit arch_test_bit + +#endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 2e6cd56810409..035d4ac666419 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -71,9 +71,9 @@ struct device; * bitmap_release_region(bitmap, pos, order) Free specified bit region * bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region * bitmap_from_arr32(dst, buf, nbits) Copy nbits from u32[] buf to dst + * bitmap_from_arr64(dst, buf, nbits) Copy nbits from u64[] buf to dst * bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst * bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst - * bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst * bitmap_get_value8(map, start) Get 8bit value from map at start * bitmap_set_value8(map, value, start) Set 8bit value to map at start * @@ -148,13 +148,13 @@ void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, unsigned int shift, unsigned int nbits); void bitmap_cut(unsigned long *dst, const unsigned long *src, unsigned int first, unsigned int cut, unsigned int nbits); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); -int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); void __bitmap_replace(unsigned long *dst, const unsigned long *old, const unsigned long *new, @@ -163,7 +163,7 @@ bool __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); -int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); void __bitmap_set(unsigned long *map, unsigned int start, int len); void __bitmap_clear(unsigned long *map, unsigned int start, int len); @@ -238,20 +238,32 @@ extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memset(dst, 0, len); + + if (small_const_nbits(nbits)) + *dst = 0; + else + memset(dst, 0, len); } static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memset(dst, 0xff, len); + + if (small_const_nbits(nbits)) + *dst = ~0UL; + else + memset(dst, 0xff, len); } static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits) { unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long); - memcpy(dst, src, len); + + if (small_const_nbits(nbits)) + *dst = *src; + else + memcpy(dst, src, len); } /* @@ -303,7 +315,7 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits)) #endif -static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) @@ -329,7 +341,7 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, __bitmap_xor(dst, src1, src2, nbits); } -static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) @@ -419,7 +431,8 @@ static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static __always_inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static __always_inline +unsigned long bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -431,6 +444,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, { if (__builtin_constant_p(nbits) && nbits == 1) __set_bit(start, map); + else if (small_const_nbits(start + nbits)) + *map |= GENMASK(start + nbits - 1, start); else if (__builtin_constant_p(start & BITMAP_MEM_MASK) && IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) && __builtin_constant_p(nbits & BITMAP_MEM_MASK) && @@ -445,6 +460,8 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, { if (__builtin_constant_p(nbits) && nbits == 1) __clear_bit(start, map); + else if (small_const_nbits(start + nbits)) + *map &= ~GENMASK(start + nbits - 1, start); else if (__builtin_constant_p(start & BITMAP_MEM_MASK) && IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) && __builtin_constant_p(nbits & BITMAP_MEM_MASK) && diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 7aaed501f7686..cf9bf65039f22 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -26,12 +26,62 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +/* + * Defined here because those may be needed by architecture-specific static + * inlines. + */ + +#include + +/* + * Many architecture-specific non-atomic bitops contain inline asm code and due + * to that the compiler can't optimize them to compile-time expressions or + * constants. In contrary, generic_*() helpers are defined in pure C and + * compilers optimize them just well. + * Therefore, to make `unsigned long foo = 0; __set_bit(BAR, &foo)` effectively + * equal to `unsigned long foo = BIT(BAR)`, pick the generic C alternative when + * the arguments can be resolved at compile time. That expression itself is a + * constant and doesn't bring any functional changes to the rest of cases. + * The casts to `uintptr_t` are needed to mitigate `-Waddress` warnings when + * passing a bitmap from .bss or .data (-> `!!addr` is always true). + */ +#define bitop(op, nr, addr) \ + ((__builtin_constant_p(nr) && \ + __builtin_constant_p((uintptr_t)(addr) != (uintptr_t)NULL) && \ + (uintptr_t)(addr) != (uintptr_t)NULL && \ + __builtin_constant_p(*(const unsigned long *)(addr))) ? \ + const##op(nr, addr) : op(nr, addr)) + +#define __set_bit(nr, addr) bitop(___set_bit, nr, addr) +#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) +#define __change_bit(nr, addr) bitop(___change_bit, nr, addr) +#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr) +#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) +#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) +#define test_bit(nr, addr) bitop(_test_bit, nr, addr) + /* * Include this here because some architectures need generic_ffs/fls in * scope */ #include +/* Check that the bitops prototypes are sane */ +#define __check_bitop_pr(name) \ + static_assert(__same_type(arch_##name, generic_##name) && \ + __same_type(const_##name, generic_##name) && \ + __same_type(_##name, generic_##name)) + +__check_bitop_pr(__set_bit); +__check_bitop_pr(__clear_bit); +__check_bitop_pr(__change_bit); +__check_bitop_pr(__test_and_set_bit); +__check_bitop_pr(__test_and_clear_bit); +__check_bitop_pr(__test_and_change_bit); +__check_bitop_pr(test_bit); + +#undef __check_bitop_pr + static inline int get_bitmask_order(unsigned int count) { int order; diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index fe29ac7cc469c..80627362c7746 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include /* Don't assign or return these: may not be this big! */ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; @@ -176,12 +178,12 @@ static inline unsigned int cpumask_local_spread(unsigned int i, int node) return 0; } -static inline int cpumask_any_and_distribute(const struct cpumask *src1p, +static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) { return cpumask_first_and(src1p, src2p); } -static inline int cpumask_any_distribute(const struct cpumask *srcp) +static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp) { return cpumask_first(srcp); } @@ -241,7 +243,21 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits); } -unsigned int __pure cpumask_next(int n, const struct cpumask *srcp); +/** + * cpumask_next - get the next cpu in a cpumask + * @n: the cpu prior to the place to search (ie. return will be > @n) + * @srcp: the cpumask pointer + * + * Returns >= nr_cpu_ids if no further cpus set. + */ +static inline +unsigned int cpumask_next(int n, const struct cpumask *srcp) +{ + /* -1 is a legal arg here. */ + if (n != -1) + cpumask_check(n); + return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); +} /** * cpumask_next_zero - get the next unset cpu in a cpumask @@ -258,12 +274,29 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); } -int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); -int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu); +/** + * cpumask_next_and - get the next cpu in *src1p & *src2p + * @n: the cpu prior to the place to search (ie. return will be > @n) + * @src1p: the first cpumask pointer + * @src2p: the second cpumask pointer + * + * Returns >= nr_cpu_ids if no further cpus set in both. + */ +static inline +unsigned int cpumask_next_and(int n, const struct cpumask *src1p, + const struct cpumask *src2p) +{ + /* -1 is a legal arg here. */ + if (n != -1) + cpumask_check(n); + return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), + nr_cpumask_bits, n + 1); +} + unsigned int cpumask_local_spread(unsigned int i, int node); -int cpumask_any_and_distribute(const struct cpumask *src1p, +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p); -int cpumask_any_distribute(const struct cpumask *srcp); +unsigned int cpumask_any_distribute(const struct cpumask *srcp); /** * for_each_cpu - iterate over every cpu in a mask @@ -289,7 +322,7 @@ int cpumask_any_distribute(const struct cpumask *srcp); (cpu) = cpumask_next_zero((cpu), (mask)), \ (cpu) < nr_cpu_ids;) -extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); +unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); /** * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location @@ -324,6 +357,26 @@ extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool for ((cpu) = -1; \ (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ (cpu) < nr_cpu_ids;) + +/** + * cpumask_any_but - return a "random" in a cpumask, but not this one. + * @mask: the cpumask to search + * @cpu: the cpu to ignore. + * + * Often used to find any cpu but smp_processor_id() in a mask. + * Returns >= nr_cpu_ids if no cpus set. + */ +static inline +unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) +{ + unsigned int i; + + cpumask_check(cpu); + for_each_cpu(i, mask) + if (i != cpu) + break; + return i; +} #endif /* SMP */ #define CPU_BITS_NONE \ @@ -372,9 +425,9 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in @cpumask, else returns 0 + * Returns true if @cpu is set in @cpumask, else returns false */ -static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) +static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) { return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); } @@ -384,11 +437,11 @@ static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpuma * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 + * Returns true if @cpu is set in old bitmap of @cpumask, else returns false * * test_and_set_bit wrapper for cpumasks. */ -static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) +static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -398,11 +451,11 @@ static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpu * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 + * Returns true if @cpu is set in old bitmap of @cpumask, else returns false * * test_and_clear_bit wrapper for cpumasks. */ -static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) +static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -431,9 +484,9 @@ static inline void cpumask_clear(struct cpumask *dstp) * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns 0, else returns 1 + * If *@dstp is empty, returns false, else returns true */ -static inline int cpumask_and(struct cpumask *dstp, +static inline bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { @@ -474,9 +527,9 @@ static inline void cpumask_xor(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns 0, else returns 1 + * If *@dstp is empty, returns false, else returns true */ -static inline int cpumask_andnot(struct cpumask *dstp, +static inline bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { @@ -539,9 +592,9 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * - * Returns 1 if *@src1p is a subset of *@src2p, else returns 0 + * Returns true if *@src1p is a subset of *@src2p, else returns false */ -static inline int cpumask_subset(const struct cpumask *src1p, +static inline bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), @@ -743,9 +796,35 @@ typedef struct cpumask *cpumask_var_t; #define __cpumask_var_read_mostly __read_mostly bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); -bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); + +static inline +bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) +{ + return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); +} + +/** + * alloc_cpumask_var - allocate a struct cpumask + * @mask: pointer to cpumask_var_t where the cpumask is returned + * @flags: GFP_ flags + * + * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is + * a nop returning a constant 1 (in ). + * + * See alloc_cpumask_var_node. + */ +static inline +bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); +} + +static inline +bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +{ + return alloc_cpumask_var(mask, flags | __GFP_ZERO); +} + void alloc_bootmem_cpumask_var(cpumask_var_t *mask); void free_cpumask_var(cpumask_var_t mask); void free_bootmem_cpumask_var(cpumask_var_t mask); diff --git a/include/linux/find.h b/include/linux/find.h index 424ef67d4a426..8d326d1518f4d 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -8,9 +8,19 @@ #include -extern unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le); +unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, + unsigned long start); +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start); +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start); +#ifdef __BIG_ENDIAN +unsigned long _find_next_zero_bit_le(const void *addr, unsigned + long size, unsigned long offset); +unsigned long _find_next_bit_le(const void *addr, unsigned + long size, unsigned long offset); +unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size); +#endif extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); extern unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size); @@ -41,7 +51,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, return val ? __ffs(val) : size; } - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); + return _find_next_bit(addr, size, offset); } #endif @@ -71,7 +81,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, return val ? __ffs(val) : size; } - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); + return _find_next_and_bit(addr1, addr2, size, offset); } #endif @@ -99,7 +109,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, return val == ~0UL ? size : ffz(val); } - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); + return _find_next_zero_bit(addr, size, offset); } #endif @@ -247,7 +257,21 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned return val == ~0UL ? size : ffz(val); } - return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); + return _find_next_zero_bit_le(addr, size, offset); +} +#endif + +#ifndef find_first_zero_bit_le +static inline +unsigned long find_first_zero_bit_le(const unsigned long *addr, unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = swab(*addr) | ~GENMASK(size - 1, 0); + + return val == ~0UL ? size : ffz(val); + } + + return _find_first_zero_bit_le(addr, size); } #endif @@ -266,15 +290,10 @@ unsigned long find_next_bit_le(const void *addr, unsigned return val ? __ffs(val) : size; } - return _find_next_bit(addr, NULL, size, offset, 0UL, 1); + return _find_next_bit_le(addr, size, offset); } #endif -#ifndef find_first_zero_bit_le -#define find_first_zero_bit_le(addr, size) \ - find_next_zero_bit_le((addr), (size), 0) -#endif - #else #error "Please fix " #endif diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 2d2ccae933c20..f314be58fa776 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -2,357 +2,13 @@ #ifndef __LINUX_GFP_H #define __LINUX_GFP_H -#include +#include + #include -#include -#include #include -/* The typedef is in types.h but we want the documentation here */ -#if 0 -/** - * typedef gfp_t - Memory allocation flags. - * - * GFP flags are commonly used throughout Linux to indicate how memory - * should be allocated. The GFP acronym stands for get_free_pages(), - * the underlying memory allocation function. Not every GFP flag is - * supported by every function which may allocate memory. Most users - * will want to use a plain ``GFP_KERNEL``. - */ -typedef unsigned int __bitwise gfp_t; -#endif - struct vm_area_struct; -/* - * In case of changes, please don't forget to update - * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c - */ - -/* Plain integer GFP bitmasks. Do not use this directly. */ -#define ___GFP_DMA 0x01u -#define ___GFP_HIGHMEM 0x02u -#define ___GFP_DMA32 0x04u -#define ___GFP_MOVABLE 0x08u -#define ___GFP_RECLAIMABLE 0x10u -#define ___GFP_HIGH 0x20u -#define ___GFP_IO 0x40u -#define ___GFP_FS 0x80u -#define ___GFP_ZERO 0x100u -#define ___GFP_ATOMIC 0x200u -#define ___GFP_DIRECT_RECLAIM 0x400u -#define ___GFP_KSWAPD_RECLAIM 0x800u -#define ___GFP_WRITE 0x1000u -#define ___GFP_NOWARN 0x2000u -#define ___GFP_RETRY_MAYFAIL 0x4000u -#define ___GFP_NOFAIL 0x8000u -#define ___GFP_NORETRY 0x10000u -#define ___GFP_MEMALLOC 0x20000u -#define ___GFP_COMP 0x40000u -#define ___GFP_NOMEMALLOC 0x80000u -#define ___GFP_HARDWALL 0x100000u -#define ___GFP_THISNODE 0x200000u -#define ___GFP_ACCOUNT 0x400000u -#define ___GFP_ZEROTAGS 0x800000u -#ifdef CONFIG_KASAN_HW_TAGS -#define ___GFP_SKIP_ZERO 0x1000000u -#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u -#define ___GFP_SKIP_KASAN_POISON 0x4000000u -#else -#define ___GFP_SKIP_ZERO 0 -#define ___GFP_SKIP_KASAN_UNPOISON 0 -#define ___GFP_SKIP_KASAN_POISON 0 -#endif -#ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x8000000u -#else -#define ___GFP_NOLOCKDEP 0 -#endif -/* If the above are modified, __GFP_BITS_SHIFT may need updating */ - -/* - * Physical address zone modifiers (see linux/mmzone.h - low four bits) - * - * Do not put any conditional on these. If necessary modify the definitions - * without the underscores and use them consistently. The definitions here may - * be used in bit comparisons. - */ -#define __GFP_DMA ((__force gfp_t)___GFP_DMA) -#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) -#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) -#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ -#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) - -/** - * DOC: Page mobility and placement hints - * - * Page mobility and placement hints - * --------------------------------- - * - * These flags provide hints about how mobile the page is. Pages with similar - * mobility are placed within the same pageblocks to minimise problems due - * to external fragmentation. - * - * %__GFP_MOVABLE (also a zone modifier) indicates that the page can be - * moved by page migration during memory compaction or can be reclaimed. - * - * %__GFP_RECLAIMABLE is used for slab allocations that specify - * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. - * - * %__GFP_WRITE indicates the caller intends to dirty the page. Where possible, - * these pages will be spread between local zones to avoid all the dirty - * pages being in one zone (fair zone allocation policy). - * - * %__GFP_HARDWALL enforces the cpuset memory allocation policy. - * - * %__GFP_THISNODE forces the allocation to be satisfied from the requested - * node with no fallbacks or placement policy enforcements. - * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - */ -#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) -#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) -#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) -#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) -#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) - -/** - * DOC: Watermark modifiers - * - * Watermark modifiers -- controls access to emergency reserves - * ------------------------------------------------------------ - * - * %__GFP_HIGH indicates that the caller is high-priority and that granting - * the request is necessary before the system can make forward progress. - * For example, creating an IO context to clean pages. - * - * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is - * high priority. Users are typically interrupt handlers. This may be - * used in conjunction with %__GFP_HIGH - * - * %__GFP_MEMALLOC allows access to all memory. This should only be used when - * the caller guarantees the allocation will allow more memory to be freed - * very shortly e.g. process exiting or swapping. Users either should - * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). - * Users of this flag have to be extremely careful to not deplete the reserve - * completely and implement a throttling mechanism which controls the - * consumption of the reserve based on the amount of freed memory. - * Usage of a pre-allocated pool (e.g. mempool) should be always considered - * before using this flag. - * - * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. - * This takes precedence over the %__GFP_MEMALLOC flag if both are set. - */ -#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) -#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) -#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) -#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) - -/** - * DOC: Reclaim modifiers - * - * Reclaim modifiers - * ----------------- - * Please note that all the following flags are only applicable to sleepable - * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them). - * - * %__GFP_IO can start physical IO. - * - * %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the - * allocator recursing into the filesystem which might already be holding - * locks. - * - * %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. - * This flag can be cleared to avoid unnecessary delays when a fallback - * option is available. - * - * %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when - * the low watermark is reached and have it reclaim pages until the high - * watermark is reached. A caller may wish to clear this flag when fallback - * options are available and the reclaim is likely to disrupt the system. The - * canonical example is THP allocation where a fallback is cheap but - * reclaim/compaction may cause indirect stalls. - * - * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. - * - * The default allocator behavior depends on the request size. We have a concept - * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER). - * !costly allocations are too essential to fail so they are implicitly - * non-failing by default (with some exceptions like OOM victims might fail so - * the caller still has to check for failures) while costly requests try to be - * not disruptive and back off even without invoking the OOM killer. - * The following three modifiers might be used to override some of these - * implicit rules - * - * %__GFP_NORETRY: The VM implementation will try only very lightweight - * memory direct reclaim to get some memory under memory pressure (thus - * it can sleep). It will avoid disruptive actions like OOM killer. The - * caller must handle the failure which is quite likely to happen under - * heavy memory pressure. The flag is suitable when failure can easily be - * handled at small cost, such as reduced throughput - * - * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim - * procedures that have previously failed if there is some indication - * that progress has been made else where. It can wait for other - * tasks to attempt high level approaches to freeing memory such as - * compaction (which removes fragmentation) and page-out. - * There is still a definite limit to the number of retries, but it is - * a larger limit than with %__GFP_NORETRY. - * Allocations with this flag may fail, but only when there is - * genuinely little unused memory. While these allocations do not - * directly trigger the OOM killer, their failure indicates that - * the system is likely to need to use the OOM killer soon. The - * caller must handle failure, but can reasonably do so by failing - * a higher-level request, or completing it only in a much less - * efficient manner. - * If the allocation does fail, and the caller is in a position to - * free some non-essential memory, doing so could benefit the system - * as a whole. - * - * %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller - * cannot handle allocation failures. The allocation could block - * indefinitely but will never return with failure. Testing for - * failure is pointless. - * New users should be evaluated carefully (and the flag should be - * used only when there is no reasonable failure policy) but it is - * definitely preferable to use the flag rather than opencode endless - * loop around allocator. - * Using this flag for costly allocations is _highly_ discouraged. - */ -#define __GFP_IO ((__force gfp_t)___GFP_IO) -#define __GFP_FS ((__force gfp_t)___GFP_FS) -#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */ -#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */ -#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM)) -#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL) -#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) -#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) - -/** - * DOC: Action modifiers - * - * Action modifiers - * ---------------- - * - * %__GFP_NOWARN suppresses allocation failure reports. - * - * %__GFP_COMP address compound page metadata. - * - * %__GFP_ZERO returns a zeroed page on success. - * - * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself - * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that - * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting - * memory tags at the same time as zeroing memory has minimal additional - * performace impact. - * - * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. - * Only effective in HW_TAGS mode. - * - * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation. - * Typically, used for userspace pages. Only effective in HW_TAGS mode. - */ -#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) -#define __GFP_COMP ((__force gfp_t)___GFP_COMP) -#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) -#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) -#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) -#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) -#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) - -/* Disable lockdep for GFP context tracking */ -#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) - -/* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) -#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) - -/** - * DOC: Useful GFP flag combinations - * - * Useful GFP flag combinations - * ---------------------------- - * - * Useful GFP flag combinations that are commonly used. It is recommended - * that subsystems start with one of these combinations and then set/clear - * %__GFP_FOO flags as necessary. - * - * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower - * watermark is applied to allow access to "atomic reserves". - * The current implementation doesn't support NMI and few other strict - * non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. - * - * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires - * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. - * - * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. - * - * %GFP_NOWAIT is for kernel allocations that should not stall for direct - * reclaim, start physical IO or use any filesystem callback. - * - * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages - * that do not require the starting of any physical IO. - * Please try to avoid using this flag directly and instead use - * memalloc_noio_{save,restore} to mark the whole scope which cannot - * perform any IO with a short explanation why. All allocation requests - * will inherit GFP_NOIO implicitly. - * - * %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. - * Please try to avoid using this flag directly and instead use - * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't - * recurse into the FS layer with a short explanation why. All allocation - * requests will inherit GFP_NOFS implicitly. - * - * %GFP_USER is for userspace allocations that also need to be directly - * accessibly by the kernel or hardware. It is typically used by hardware - * for buffers that are mapped to userspace (e.g. graphics) that hardware - * still must DMA to. cpuset limits are enforced for these allocations. - * - * %GFP_DMA exists for historical reasons and should be avoided where possible. - * The flags indicates that the caller requires that the lowest zone be - * used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but - * it would require careful auditing as some users really require it and - * others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the - * lowest zone as a type of emergency reserve. - * - * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit - * address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory - * because the DMA32 kmalloc cache array is not implemented. - * (Reason: there is no such user in kernel). - * - * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, - * do not need to be directly accessible by the kernel but that cannot - * move once in use. An example may be a hardware allocation that maps - * data directly into userspace but has no addressing limitations. - * - * %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not - * need direct access to but can use kmap() when access is required. They - * are expected to be movable via page reclaim or page migration. Typically, - * pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE. - * - * %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They - * are compound allocations that will generally fail quickly if memory is not - * available and will not wake kswapd/kcompactd on failure. The _LIGHT - * version does not attempt reclaim/compaction at all and is by default used - * in page fault path, while the non-light is used by khugepaged. - */ -#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) -#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) -#define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT) -#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) -#define GFP_NOIO (__GFP_RECLAIM) -#define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) -#define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) -#define GFP_DMA __GFP_DMA -#define GFP_DMA32 __GFP_DMA32 -#define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) -#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ - __GFP_SKIP_KASAN_POISON) -#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ - __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) -#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) - /* Convert GFP flags to their corresponding migrate type */ #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) #define GFP_MOVABLE_SHIFT 3 diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h new file mode 100644 index 0000000000000..06fc85cee23fe --- /dev/null +++ b/include/linux/gfp_types.h @@ -0,0 +1,348 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_GFP_TYPES_H +#define __LINUX_GFP_TYPES_H + +/* The typedef is in types.h but we want the documentation here */ +#if 0 +/** + * typedef gfp_t - Memory allocation flags. + * + * GFP flags are commonly used throughout Linux to indicate how memory + * should be allocated. The GFP acronym stands for get_free_pages(), + * the underlying memory allocation function. Not every GFP flag is + * supported by every function which may allocate memory. Most users + * will want to use a plain ``GFP_KERNEL``. + */ +typedef unsigned int __bitwise gfp_t; +#endif + +/* + * In case of changes, please don't forget to update + * include/trace/events/mmflags.h and tools/perf/builtin-kmem.c + */ + +/* Plain integer GFP bitmasks. Do not use this directly. */ +#define ___GFP_DMA 0x01u +#define ___GFP_HIGHMEM 0x02u +#define ___GFP_DMA32 0x04u +#define ___GFP_MOVABLE 0x08u +#define ___GFP_RECLAIMABLE 0x10u +#define ___GFP_HIGH 0x20u +#define ___GFP_IO 0x40u +#define ___GFP_FS 0x80u +#define ___GFP_ZERO 0x100u +#define ___GFP_ATOMIC 0x200u +#define ___GFP_DIRECT_RECLAIM 0x400u +#define ___GFP_KSWAPD_RECLAIM 0x800u +#define ___GFP_WRITE 0x1000u +#define ___GFP_NOWARN 0x2000u +#define ___GFP_RETRY_MAYFAIL 0x4000u +#define ___GFP_NOFAIL 0x8000u +#define ___GFP_NORETRY 0x10000u +#define ___GFP_MEMALLOC 0x20000u +#define ___GFP_COMP 0x40000u +#define ___GFP_NOMEMALLOC 0x80000u +#define ___GFP_HARDWALL 0x100000u +#define ___GFP_THISNODE 0x200000u +#define ___GFP_ACCOUNT 0x400000u +#define ___GFP_ZEROTAGS 0x800000u +#ifdef CONFIG_KASAN_HW_TAGS +#define ___GFP_SKIP_ZERO 0x1000000u +#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u +#define ___GFP_SKIP_KASAN_POISON 0x4000000u +#else +#define ___GFP_SKIP_ZERO 0 +#define ___GFP_SKIP_KASAN_UNPOISON 0 +#define ___GFP_SKIP_KASAN_POISON 0 +#endif +#ifdef CONFIG_LOCKDEP +#define ___GFP_NOLOCKDEP 0x8000000u +#else +#define ___GFP_NOLOCKDEP 0 +#endif +/* If the above are modified, __GFP_BITS_SHIFT may need updating */ + +/* + * Physical address zone modifiers (see linux/mmzone.h - low four bits) + * + * Do not put any conditional on these. If necessary modify the definitions + * without the underscores and use them consistently. The definitions here may + * be used in bit comparisons. + */ +#define __GFP_DMA ((__force gfp_t)___GFP_DMA) +#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) +#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) +#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) + +/** + * DOC: Page mobility and placement hints + * + * Page mobility and placement hints + * --------------------------------- + * + * These flags provide hints about how mobile the page is. Pages with similar + * mobility are placed within the same pageblocks to minimise problems due + * to external fragmentation. + * + * %__GFP_MOVABLE (also a zone modifier) indicates that the page can be + * moved by page migration during memory compaction or can be reclaimed. + * + * %__GFP_RECLAIMABLE is used for slab allocations that specify + * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. + * + * %__GFP_WRITE indicates the caller intends to dirty the page. Where possible, + * these pages will be spread between local zones to avoid all the dirty + * pages being in one zone (fair zone allocation policy). + * + * %__GFP_HARDWALL enforces the cpuset memory allocation policy. + * + * %__GFP_THISNODE forces the allocation to be satisfied from the requested + * node with no fallbacks or placement policy enforcements. + * + * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. + */ +#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) +#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) +#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) +#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) +#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) + +/** + * DOC: Watermark modifiers + * + * Watermark modifiers -- controls access to emergency reserves + * ------------------------------------------------------------ + * + * %__GFP_HIGH indicates that the caller is high-priority and that granting + * the request is necessary before the system can make forward progress. + * For example, creating an IO context to clean pages. + * + * %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is + * high priority. Users are typically interrupt handlers. This may be + * used in conjunction with %__GFP_HIGH + * + * %__GFP_MEMALLOC allows access to all memory. This should only be used when + * the caller guarantees the allocation will allow more memory to be freed + * very shortly e.g. process exiting or swapping. Users either should + * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). + * Users of this flag have to be extremely careful to not deplete the reserve + * completely and implement a throttling mechanism which controls the + * consumption of the reserve based on the amount of freed memory. + * Usage of a pre-allocated pool (e.g. mempool) should be always considered + * before using this flag. + * + * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. + * This takes precedence over the %__GFP_MEMALLOC flag if both are set. + */ +#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) +#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) +#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) +#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) + +/** + * DOC: Reclaim modifiers + * + * Reclaim modifiers + * ----------------- + * Please note that all the following flags are only applicable to sleepable + * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them). + * + * %__GFP_IO can start physical IO. + * + * %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the + * allocator recursing into the filesystem which might already be holding + * locks. + * + * %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. + * This flag can be cleared to avoid unnecessary delays when a fallback + * option is available. + * + * %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when + * the low watermark is reached and have it reclaim pages until the high + * watermark is reached. A caller may wish to clear this flag when fallback + * options are available and the reclaim is likely to disrupt the system. The + * canonical example is THP allocation where a fallback is cheap but + * reclaim/compaction may cause indirect stalls. + * + * %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. + * + * The default allocator behavior depends on the request size. We have a concept + * of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER). + * !costly allocations are too essential to fail so they are implicitly + * non-failing by default (with some exceptions like OOM victims might fail so + * the caller still has to check for failures) while costly requests try to be + * not disruptive and back off even without invoking the OOM killer. + * The following three modifiers might be used to override some of these + * implicit rules + * + * %__GFP_NORETRY: The VM implementation will try only very lightweight + * memory direct reclaim to get some memory under memory pressure (thus + * it can sleep). It will avoid disruptive actions like OOM killer. The + * caller must handle the failure which is quite likely to happen under + * heavy memory pressure. The flag is suitable when failure can easily be + * handled at small cost, such as reduced throughput + * + * %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim + * procedures that have previously failed if there is some indication + * that progress has been made else where. It can wait for other + * tasks to attempt high level approaches to freeing memory such as + * compaction (which removes fragmentation) and page-out. + * There is still a definite limit to the number of retries, but it is + * a larger limit than with %__GFP_NORETRY. + * Allocations with this flag may fail, but only when there is + * genuinely little unused memory. While these allocations do not + * directly trigger the OOM killer, their failure indicates that + * the system is likely to need to use the OOM killer soon. The + * caller must handle failure, but can reasonably do so by failing + * a higher-level request, or completing it only in a much less + * efficient manner. + * If the allocation does fail, and the caller is in a position to + * free some non-essential memory, doing so could benefit the system + * as a whole. + * + * %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller + * cannot handle allocation failures. The allocation could block + * indefinitely but will never return with failure. Testing for + * failure is pointless. + * New users should be evaluated carefully (and the flag should be + * used only when there is no reasonable failure policy) but it is + * definitely preferable to use the flag rather than opencode endless + * loop around allocator. + * Using this flag for costly allocations is _highly_ discouraged. + */ +#define __GFP_IO ((__force gfp_t)___GFP_IO) +#define __GFP_FS ((__force gfp_t)___GFP_FS) +#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */ +#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */ +#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM)) +#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL) +#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) +#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) + +/** + * DOC: Action modifiers + * + * Action modifiers + * ---------------- + * + * %__GFP_NOWARN suppresses allocation failure reports. + * + * %__GFP_COMP address compound page metadata. + * + * %__GFP_ZERO returns a zeroed page on success. + * + * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself + * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that + * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting + * memory tags at the same time as zeroing memory has minimal additional + * performace impact. + * + * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. + * Only effective in HW_TAGS mode. + * + * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation. + * Typically, used for userspace pages. Only effective in HW_TAGS mode. + */ +#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) +#define __GFP_COMP ((__force gfp_t)___GFP_COMP) +#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) +#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) +#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) +#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) +#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) + +/* Disable lockdep for GFP context tracking */ +#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) + +/* Room for N __GFP_FOO bits */ +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) + +/** + * DOC: Useful GFP flag combinations + * + * Useful GFP flag combinations + * ---------------------------- + * + * Useful GFP flag combinations that are commonly used. It is recommended + * that subsystems start with one of these combinations and then set/clear + * %__GFP_FOO flags as necessary. + * + * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower + * watermark is applied to allow access to "atomic reserves". + * The current implementation doesn't support NMI and few other strict + * non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT. + * + * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires + * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. + * + * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is + * accounted to kmemcg. + * + * %GFP_NOWAIT is for kernel allocations that should not stall for direct + * reclaim, start physical IO or use any filesystem callback. + * + * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages + * that do not require the starting of any physical IO. + * Please try to avoid using this flag directly and instead use + * memalloc_noio_{save,restore} to mark the whole scope which cannot + * perform any IO with a short explanation why. All allocation requests + * will inherit GFP_NOIO implicitly. + * + * %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. + * Please try to avoid using this flag directly and instead use + * memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't + * recurse into the FS layer with a short explanation why. All allocation + * requests will inherit GFP_NOFS implicitly. + * + * %GFP_USER is for userspace allocations that also need to be directly + * accessibly by the kernel or hardware. It is typically used by hardware + * for buffers that are mapped to userspace (e.g. graphics) that hardware + * still must DMA to. cpuset limits are enforced for these allocations. + * + * %GFP_DMA exists for historical reasons and should be avoided where possible. + * The flags indicates that the caller requires that the lowest zone be + * used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but + * it would require careful auditing as some users really require it and + * others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the + * lowest zone as a type of emergency reserve. + * + * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit + * address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory + * because the DMA32 kmalloc cache array is not implemented. + * (Reason: there is no such user in kernel). + * + * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, + * do not need to be directly accessible by the kernel but that cannot + * move once in use. An example may be a hardware allocation that maps + * data directly into userspace but has no addressing limitations. + * + * %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not + * need direct access to but can use kmap() when access is required. They + * are expected to be movable via page reclaim or page migration. Typically, + * pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE. + * + * %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They + * are compound allocations that will generally fail quickly if memory is not + * available and will not wake kswapd/kcompactd on failure. The _LIGHT + * version does not attempt reclaim/compaction at all and is by default used + * in page fault path, while the non-light is used by khugepaged. + */ +#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) +#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) +#define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT) +#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) +#define GFP_NOIO (__GFP_RECLAIM) +#define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) +#define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) +#define GFP_DMA __GFP_DMA +#define GFP_DMA32 __GFP_DMA32 +#define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) +#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ + __GFP_SKIP_KASAN_POISON) +#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ + __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) +#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) + +#endif /* __LINUX_GFP_TYPES_H */ diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h index e3e8c8662b490..9209856b66444 100644 --- a/include/linux/ipc_namespace.h +++ b/include/linux/ipc_namespace.h @@ -36,8 +36,6 @@ struct ipc_namespace { unsigned int msg_ctlmax; unsigned int msg_ctlmnb; unsigned int msg_ctlmni; - atomic_t msg_bytes; - atomic_t msg_hdrs; size_t shm_ctlmax; size_t shm_ctlall; @@ -77,6 +75,9 @@ struct ipc_namespace { struct llist_node mnt_llist; struct ns_common ns; + int padding[16]; + atomic_t msg_bytes; + atomic_t msg_hdrs; } __randomize_layout; extern struct ipc_namespace init_ipc_ns; diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index e79d6e0b14e8e..9f6e197434018 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -45,7 +45,7 @@ /* * The default maximum commit age, in seconds. */ -#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 +#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 #ifdef CONFIG_JBD2_DEBUG /* diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 0b4f17418f64c..e24cf9144b24f 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -19,6 +19,10 @@ struct stable_node; struct mem_cgroup; #ifdef CONFIG_KSM +int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long *vm_flags); +int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long *vm_flags); int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags); int __ksm_enter(struct mm_struct *mm); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c29ab4c0cd5c6..e6321ec7621d5 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -87,6 +87,7 @@ struct page { */ union { struct list_head lru; + /* Or, for the Unevictable "LRU list" slot */ struct { /* Always even, to negate PageTail */ @@ -94,6 +95,10 @@ struct page { /* Count page's or folio's mlocks */ unsigned int mlock_count; }; + + /* Or, free page */ + struct list_head buddy_list; + struct list_head pcp_list; }; /* See page-flags.h for PAGE_MAPPING_FLAGS */ struct address_space *mapping; diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index aab70355d64f3..299259cfe4627 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -355,15 +355,18 @@ enum zone_watermarks { }; /* - * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER plus one additional - * for pageblock size for THP if configured. + * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. One additional list + * for THP which will usually be GFP_MOVABLE. Even if it is another type, + * it should not contribute to serious fragmentation causing THP allocation + * failures. */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define NR_PCP_THP 1 #else #define NR_PCP_THP 0 #endif -#define NR_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1 + NR_PCP_THP)) +#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1)) +#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP) /* * Shift to encode migratetype and order in the same integer, with order @@ -379,6 +382,7 @@ enum zone_watermarks { /* Fields and list protected by pagesets local_lock in page_alloc.c */ struct per_cpu_pages { + spinlock_t lock; /* Protects lists field */ int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ int batch; /* chunk size for buddy add/remove */ @@ -389,7 +393,7 @@ struct per_cpu_pages { /* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[NR_PCP_LISTS]; -}; +} ____cacheline_aligned_in_smp; struct per_cpu_zonestat { #ifdef CONFIG_SMP diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 679591301994d..685193470d64c 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -8,6 +8,7 @@ struct page_counter { atomic_long_t usage; + unsigned long padding[7]; unsigned long min; unsigned long low; unsigned long high; diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ce96866fbec40..232073d5e8c42 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1202,7 +1202,7 @@ struct readahead_control { ._index = i, \ } -#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) +#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) void page_cache_ra_unbounded(struct readahead_control *, unsigned long nr_to_read, unsigned long lookahead_count); diff --git a/include/linux/sched.h b/include/linux/sched.h index c46f3a63b758f..6258b32dffd78 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1050,6 +1050,7 @@ struct task_struct { /* Cached requested key. */ struct key *cached_requested_key; #endif + int fsync_count; /* * executable name, excluding path. diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index a34b0f9a9972e..82afad91de9fb 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -917,6 +917,7 @@ asmlinkage long sys_madvise(unsigned long start, size_t len, int behavior); asmlinkage long sys_process_madvise(int pidfd, const struct iovec __user *vec, size_t vlen, int behavior, unsigned int flags); asmlinkage long sys_process_mrelease(int pidfd, unsigned int flags); +asmlinkage long sys_pmadv_ksm(int pidfd, int behavior, unsigned int flags); asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, unsigned long prot, unsigned long pgoff, unsigned long flags); diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index 33a4240e6a6f1..82213f9c4c17f 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, #ifdef CONFIG_USER_NS +extern int unprivileged_userns_clone; + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { if (ns) @@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); struct ns_common *ns_get_owner(struct ns_common *ns); #else +#define unprivileged_userns_clone 0 + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { return &init_user_ns; diff --git a/include/linux/wait.h b/include/linux/wait.h index 851e07da2583f..85653fc332748 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); +extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); @@ -1163,6 +1164,7 @@ do { \ */ void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 45fa180cc56ad..40f7e6d04af0c 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -886,8 +886,11 @@ __SYSCALL(__NR_futex_waitv, sys_futex_waitv) #define __NR_set_mempolicy_home_node 450 __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) +#define __NR_pmadv_ksm 451 +__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) + #undef __NR_syscalls -#define __NR_syscalls 451 +#define __NR_syscalls 452 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h index d174914a837db..bf8e2af101a3c 100644 --- a/include/uapi/linux/if_bonding.h +++ b/include/uapi/linux/if_bonding.h @@ -82,7 +82,7 @@ #define BOND_STATE_ACTIVE 0 /* link is active */ #define BOND_STATE_BACKUP 1 /* link is backup */ -#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ +#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ diff --git a/init/Kconfig b/init/Kconfig index c7900e8975f18..11ec35b657793 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -116,6 +116,10 @@ config THREAD_INFO_IN_TASK menu "General setup" +config CACHY + bool "Some kernel tweaks by CachyOS" + default y + config BROKEN bool @@ -1235,6 +1239,22 @@ config USER_NS If unsure, say N. +config USER_NS_UNPRIVILEGED + bool "Allow unprivileged users to create namespaces" + default y + depends on USER_NS + help + When disabled, unprivileged users will not be able to create + new namespaces. Allowing users to create their own namespaces + has been part of several recent local privilege escalation + exploits, so if you need user namespaces but are + paranoid^Wsecurity-conscious you want to disable this. + + This setting can be overridden at runtime via the + kernel.unprivileged_userns_clone sysctl. + + If unsure, say Y. + config PID_NS bool "PID Namespaces" default y @@ -1403,11 +1423,13 @@ config CC_OPTIMIZE_FOR_PERFORMANCE config CC_OPTIMIZE_FOR_PERFORMANCE_O3 bool "Optimize more for performance (-O3)" - depends on ARC help Choosing this option will pass "-O3" to your compiler to optimize the kernel yet more for performance. + This option is EXPERIMENTAL; you may encounter compiler bugs and/or + kernel bugs with this option enabled. + config CC_OPTIMIZE_FOR_SIZE bool "Optimize for size (-Os)" help diff --git a/init/do_mounts.c b/init/do_mounts.c index 7058e14ad5f70..ce4e841ea7053 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) if (strcmp(name, "/dev/ram") == 0) return Root_RAM0; #ifdef CONFIG_BLOCK - if (strncmp(name, "PARTUUID=", 9) == 0) - return devt_from_partuuid(name + 9); + if (strncmp(name, "PARTUUID=", 9) == 0) { + dev_t res; + int needtowait = 40<<1; + res = devt_from_partuuid(name + 9); + while (!res && needtowait) { + /* waiting 0.5 sec */ + msleep(500); + res = devt_from_partuuid(name + 9); + needtowait--; + } + return res; + } if (strncmp(name, "PARTLABEL=", 10) == 0) return devt_from_partlabel(name + 10); if (strncmp(name, "/dev/", 5) == 0) @@ -612,7 +622,9 @@ void __init prepare_namespace(void) * For example, it is not atypical to wait 5 seconds here * for the touchpad of a laptop to initialize. */ + async_synchronize_full(); wait_for_device_probe(); + async_synchronize_full(); md_run_setup(); diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index 38ef6d06888ef..0f78364efd4f2 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz @@ -40,6 +40,27 @@ choice on SMP and NUMA systems and exactly dividing by both PAL and NTSC frame rates for video and multimedia work. + config HZ_500 + bool "500 HZ" + help + 500 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_600 + bool "600 HZ" + help + 600 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_750 + bool "750 HZ" + help + 750 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + config HZ_1000 bool "1000 HZ" help @@ -53,6 +74,9 @@ config HZ default 100 if HZ_100 default 250 if HZ_250 default 300 if HZ_300 + default 500 if HZ_500 + default 600 if HZ_600 + default 750 if HZ_750 default 1000 if HZ_1000 config SCHED_HRTICK diff --git a/kernel/fork.c b/kernel/fork.c index 9d44f2d46c696..e5ba00a7e3b27 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -99,6 +99,10 @@ #include #include +#ifdef CONFIG_USER_NS +#include +#endif + #include #include #include @@ -1995,6 +1999,10 @@ static __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); + if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. @@ -3143,6 +3151,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; + if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + err = -EPERM; + if (!capable(CAP_SYS_ADMIN)) + goto bad_unshare_out; + } + err = check_unshare_flags(unshare_flags); if (err) goto bad_unshare_out; diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 65f0262f635e1..8a8f77930818d 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) struct task_struct *new, *owner; unsigned long flags, new_flags; enum owner_state state; + int i = 0; lockdep_assert_preemption_disabled(); @@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) break; } - cpu_relax(); + if (i++ > 1000) + cpu_relax(); } return state; diff --git a/kernel/module/internal.h b/kernel/module/internal.h index ec104c2950c39..851ea72471a0e 100644 --- a/kernel/module/internal.h +++ b/kernel/module/internal.h @@ -53,6 +53,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; extern const s32 __start___kcrctab[]; extern const s32 __start___kcrctab_gpl[]; +extern struct boot_params boot_params; + struct load_info { const char *name; /* pointer to module in temporary copy, freed at end of load_module() */ diff --git a/kernel/module/main.c b/kernel/module/main.c index 0548151dd9339..f9212da570f62 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include "internal.h" diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c index 9a8f4f0f63297..1b6af54d88c15 100644 --- a/kernel/module/procfs.c +++ b/kernel/module/procfs.c @@ -141,6 +141,19 @@ static const struct proc_ops modules_proc_ops = { static int __init proc_modules_init(void) { proc_create("modules", 0, NULL, &modules_proc_ops); + +#ifdef CONFIG_MODULE_SIG_FORCE + switch (boot_params.secure_boot) { + case efi_secureboot_mode_unset: + case efi_secureboot_mode_unknown: + case efi_secureboot_mode_disabled: + /* + * sig_unenforce is only applied if SecureBoot is not + * enabled. + */ + sig_enforce = !sig_unenforce; + } +#endif return 0; } module_init(proc_modules_init); diff --git a/kernel/module/signing.c b/kernel/module/signing.c index a2ff4242e623d..876e93758e914 100644 --- a/kernel/module/signing.c +++ b/kernel/module/signing.c @@ -21,6 +21,10 @@ static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); module_param(sig_enforce, bool_enable_only, 0644); +/* Allow disabling module signature requirement by adding boot param */ +static bool sig_unenforce = false; +module_param(sig_unenforce, bool_enable_only, 0644); + /* * Export sig_enforce kernel cmdline parameter to allow other subsystems rely diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index bb3d63bdf4ae8..e0e356eb038d8 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -959,6 +959,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, PN(se.exec_start); PN(se.vruntime); PN(se.sum_exec_runtime); + P(fsync_count); nr_switches = p->nvcsw + p->nivcsw; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 77b2048a93262..78559747322f4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -68,9 +68,13 @@ * * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_latency = 3000000ULL; +static unsigned int normalized_sysctl_sched_latency = 3000000ULL; +#else unsigned int sysctl_sched_latency = 6000000ULL; static unsigned int normalized_sysctl_sched_latency = 6000000ULL; - +#endif /* * The initial- and re-scaling of tunables is configurable * @@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; * * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_min_granularity = 400000ULL; +static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; +#else unsigned int sysctl_sched_min_granularity = 750000ULL; static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; +#endif /* * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. @@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; * * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_wakeup_granularity = 500000UL; +static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; +#else unsigned int sysctl_sched_wakeup_granularity = 1000000UL; static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; +#endif const_debug unsigned int sysctl_sched_migration_cost = 500000UL; @@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ +#ifdef CONFIG_CACHY +static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; +#else static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +#endif #ifdef CONFIG_SYSCTL static struct ctl_table sched_fair_sysctls[] = { diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 9860bb9a847cf..c7f045873a5fb 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ } EXPORT_SYMBOL_GPL(add_wait_queue_priority); +void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + __add_wait_queue(wq_head, wq_entry); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); + void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) { unsigned long flags; @@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent } EXPORT_SYMBOL(prepare_to_wait_exclusive); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + if (list_empty(&wq_entry->entry)) + __add_wait_queue(wq_head, wq_entry); + set_current_state(state); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); + void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) { wq_entry->flags = flags; diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index a492f159624fa..dc765f3eff16f 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -291,6 +291,7 @@ COND_SYSCALL(mincore); COND_SYSCALL(madvise); COND_SYSCALL(process_madvise); COND_SYSCALL(process_mrelease); +COND_SYSCALL(pmadv_ksm); COND_SYSCALL(remap_file_pages); COND_SYSCALL(mbind); COND_SYSCALL(get_mempolicy); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 35d0342195132..5e35535807e41 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -89,6 +89,9 @@ #ifdef CONFIG_PERF_EVENTS static const int six_hundred_forty_kb = 640 * 1024; #endif +#ifdef CONFIG_USER_NS +#include +#endif static const int ngroups_max = NGROUPS_MAX; @@ -1608,6 +1611,15 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, +#ifdef CONFIG_USER_NS + { + .procname = "unprivileged_userns_clone", + .data = &unprivileged_userns_clone, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #ifdef CONFIG_PROC_SYSCTL { .procname = "tainted", diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 5481ba44a8d68..423ab2563ad75 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -21,6 +21,13 @@ #include #include +/* sysctl */ +#ifdef CONFIG_USER_NS_UNPRIVILEGED +int unprivileged_userns_clone = 1; +#else +int unprivileged_userns_clone; +#endif + static struct kmem_cache *user_ns_cachep __read_mostly; static DEFINE_MUTEX(userns_state_mutex); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index ecb0e8346e653..7415713167da8 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; int __read_mostly watchdog_user_enabled = 1; int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; int __read_mostly soft_watchdog_user_enabled = 1; -int __read_mostly watchdog_thresh = 10; +int __read_mostly watchdog_thresh = 40; static int __read_mostly nmi_watchdog_available; struct cpumask watchdog_cpumask __read_mostly; diff --git a/lib/bitmap.c b/lib/bitmap.c index b18e31ea6e664..2b67cd6576924 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -237,7 +237,7 @@ void bitmap_cut(unsigned long *dst, const unsigned long *src, } EXPORT_SYMBOL(bitmap_cut); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -275,7 +275,7 @@ void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_xor); -int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -333,10 +333,9 @@ bool __bitmap_subset(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_subset); -int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - unsigned int k, lim = bits/BITS_PER_LONG; - int w = 0; + unsigned long k, w = 0, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; k++) w += hweight_long(bitmap[k]); @@ -1564,7 +1563,7 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits) /* Clear tail bits in the last element of array beyond nbits. */ if (nbits % 64) - buf[-1] &= GENMASK_ULL(nbits % 64, 0); + buf[-1] &= GENMASK_ULL((nbits - 1) % 64, 0); } EXPORT_SYMBOL(bitmap_to_arr64); #endif diff --git a/lib/cpumask.c b/lib/cpumask.c index a971a82d2f436..f0ae119be8c41 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -7,61 +7,6 @@ #include #include -/** - * cpumask_next - get the next cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) - * @srcp: the cpumask pointer - * - * Returns >= nr_cpu_ids if no further cpus set. - */ -unsigned int cpumask_next(int n, const struct cpumask *srcp) -{ - /* -1 is a legal arg here. */ - if (n != -1) - cpumask_check(n); - return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); -} -EXPORT_SYMBOL(cpumask_next); - -/** - * cpumask_next_and - get the next cpu in *src1p & *src2p - * @n: the cpu prior to the place to search (ie. return will be > @n) - * @src1p: the first cpumask pointer - * @src2p: the second cpumask pointer - * - * Returns >= nr_cpu_ids if no further cpus set in both. - */ -int cpumask_next_and(int n, const struct cpumask *src1p, - const struct cpumask *src2p) -{ - /* -1 is a legal arg here. */ - if (n != -1) - cpumask_check(n); - return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), - nr_cpumask_bits, n + 1); -} -EXPORT_SYMBOL(cpumask_next_and); - -/** - * cpumask_any_but - return a "random" in a cpumask, but not this one. - * @mask: the cpumask to search - * @cpu: the cpu to ignore. - * - * Often used to find any cpu but smp_processor_id() in a mask. - * Returns >= nr_cpu_ids if no cpus set. - */ -int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) -{ - unsigned int i; - - cpumask_check(cpu); - for_each_cpu(i, mask) - if (i != cpu) - break; - return i; -} -EXPORT_SYMBOL(cpumask_any_but); - /** * cpumask_next_wrap - helper to implement for_each_cpu_wrap * @n: the cpu prior to the place to search @@ -74,9 +19,9 @@ EXPORT_SYMBOL(cpumask_any_but); * Note: the @wrap argument is required for the start condition when * we cannot assume @start is set in @mask. */ -int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) +unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) { - int next; + unsigned int next; again: next = cpumask_next(n, mask); @@ -125,34 +70,6 @@ bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) } EXPORT_SYMBOL(alloc_cpumask_var_node); -bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) -{ - return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); -} -EXPORT_SYMBOL(zalloc_cpumask_var_node); - -/** - * alloc_cpumask_var - allocate a struct cpumask - * @mask: pointer to cpumask_var_t where the cpumask is returned - * @flags: GFP_ flags - * - * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is - * a nop returning a constant 1 (in ). - * - * See alloc_cpumask_var_node. - */ -bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) -{ - return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); -} -EXPORT_SYMBOL(alloc_cpumask_var); - -bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) -{ - return alloc_cpumask_var(mask, flags | __GFP_ZERO); -} -EXPORT_SYMBOL(zalloc_cpumask_var); - /** * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena. * @mask: pointer to cpumask_var_t where the cpumask is returned @@ -205,7 +122,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) */ unsigned int cpumask_local_spread(unsigned int i, int node) { - int cpu; + unsigned int cpu; /* Wrap: we always want a cpu. */ i %= num_online_cpus(); @@ -243,10 +160,10 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); * * Returns >= nr_cpu_ids if the intersection is empty. */ -int cpumask_any_and_distribute(const struct cpumask *src1p, +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) { - int next, prev; + unsigned int next, prev; /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); @@ -262,9 +179,9 @@ int cpumask_any_and_distribute(const struct cpumask *src1p, } EXPORT_SYMBOL(cpumask_any_and_distribute); -int cpumask_any_distribute(const struct cpumask *srcp) +unsigned int cpumask_any_distribute(const struct cpumask *srcp) { - int next, prev; + unsigned int next, prev; /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); diff --git a/lib/find_bit.c b/lib/find_bit.c index 1b8e4b2a9cba8..03cbf1d3dbb94 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -19,72 +19,144 @@ #include #include -#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ - !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ - !defined(find_next_and_bit) /* - * This is a common helper function for find_next_bit, find_next_zero_bit, and - * find_next_and_bit. The differences are: + * This is a common helper functions for find_{first,next}_bit{,_le}. + * Internal parameters are: * - The "invert" argument, which is XORed with each fetched word before - * searching it for one bits. - * - The optional "addr2", which is anded with "addr1" if present. + * searching it for set bits; to implement find_*_zero_bit(). + * - The optional "addr2", which is ANDed with "addr1" if present; to + * implement find_*_and_bit(). + * - The "need_swab" that converts words to BE format; to implement + * find_*_le() on big-endian machines. */ -unsigned long _find_next_bit(const unsigned long *addr1, +static inline +unsigned long __find_first_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long invert, bool need_swab) +{ + unsigned long idx, val; + + for (idx = 0; idx * BITS_PER_LONG < size; idx++) { + val = addr1[idx]; + if (addr2) + val &= addr2[idx]; + + val ^= invert; + + if (val) { + if (need_swab) + val = swab(val); + return min(idx * BITS_PER_LONG + __ffs(val), size); + } + } + + return size; +} + +static inline unsigned long __find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le) + unsigned long start, unsigned long invert, bool need_swab) { unsigned long tmp, mask; if (unlikely(start >= nbits)) return nbits; + if (IS_ALIGNED(start, BITS_PER_LONG)) + goto aligned; + + /* Handle 1st word. */ tmp = addr1[start / BITS_PER_LONG]; if (addr2) tmp &= addr2[start / BITS_PER_LONG]; tmp ^= invert; - /* Handle 1st word. */ mask = BITMAP_FIRST_WORD_MASK(start); - if (le) + if (need_swab) mask = swab(mask); tmp &= mask; start = round_down(start, BITS_PER_LONG); - - while (!tmp) { - start += BITS_PER_LONG; - if (start >= nbits) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; + if (tmp) { + if (need_swab) + tmp = swab(tmp); + return min(start + __ffs(tmp), nbits); } - if (le) - tmp = swab(tmp); + start += BITS_PER_LONG; + if (start >= nbits) + return nbits; + +aligned: + return start + __find_first_bit(addr1 + start/BITS_PER_LONG, + addr2 ? addr2 + start/BITS_PER_LONG : NULL, + nbits - start, invert, need_swab); +} - return min(start + __ffs(tmp), nbits); +#ifndef find_next_bit +unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) +{ + return __find_next_bit(addr, NULL, nbits, start, 0UL, 0); } EXPORT_SYMBOL(_find_next_bit); #endif +#ifndef find_next_and_bit +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start) +{ + return __find_next_bit(addr1, addr2, nbits, start, 0UL, 0); +} +EXPORT_SYMBOL(_find_next_and_bit); +#endif + +#ifndef find_next_zero_bit +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start) +{ + return __find_next_bit(addr, NULL, nbits, start, ~0UL, 0); +} +EXPORT_SYMBOL(_find_next_zero_bit); +#endif + +#ifdef __BIG_ENDIAN +#ifndef find_first_zero_bit_le +/* + * Find the first cleared bit in an LE memory region. + */ +unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size) +{ + return __find_first_bit(addr, NULL, size, ~0UL, true); +} +EXPORT_SYMBOL(_find_first_zero_bit_le); +#endif + +#ifndef find_next_zero_bit_le +unsigned long _find_next_zero_bit_le(const void *addr, unsigned + long size, unsigned long offset) +{ + return __find_next_bit(addr, NULL, size, offset, ~0UL, 1); +} +EXPORT_SYMBOL(_find_next_zero_bit_le); +#endif + +#ifndef find_next_bit_le +unsigned long _find_next_bit_le(const void *addr, unsigned + long size, unsigned long offset) +{ + return __find_next_bit(addr, NULL, size, offset, 0UL, 1); +} +EXPORT_SYMBOL(_find_next_bit_le); +#endif +#endif /* __BIG_ENDIAN */ + #ifndef find_first_bit /* * Find the first set bit in a memory region. */ unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx]) - return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); - } - - return size; + return __find_first_bit(addr, NULL, size, 0UL, false); } EXPORT_SYMBOL(_find_first_bit); #endif @@ -97,15 +169,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size) { - unsigned long idx, val; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - val = addr1[idx] & addr2[idx]; - if (val) - return min(idx * BITS_PER_LONG + __ffs(val), size); - } - - return size; + return __find_first_bit(addr1, addr2, size, 0UL, false); } EXPORT_SYMBOL(_find_first_and_bit); #endif @@ -116,14 +180,7 @@ EXPORT_SYMBOL(_find_first_and_bit); */ unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx] != ~0UL) - return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); - } - - return size; + return __find_first_bit(addr, NULL, size, ~0UL, false); } EXPORT_SYMBOL(_find_first_zero_bit); #endif diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 39b74221f4a7c..ec3eab8cd6b1e 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) for (best = NULL, algo = raid6_recov_algos; *algo; algo++) if (!best || (*algo)->priority > best->priority) - if (!(*algo)->valid || (*algo)->valid()) + if (!(*algo)->valid || (*algo)->valid()) { best = *algo; + break; + } if (best) { raid6_2data_recov = best->data2; diff --git a/lib/string.c b/lib/string.c index 6f334420f6871..2e60e9e40535d 100644 --- a/lib/string.c +++ b/lib/string.c @@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) EXPORT_SYMBOL(strnstr); #endif +#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 + +#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) + +#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) + +#define MEMCHR_MASK_GEN(mask) \ + do { \ + mask *= 0x01010101; \ + mask |= mask << 32; \ + } while (0) + +#else + +#define MEMCHR_MASK_GEN(mask) \ + do { \ + mask |= mask << 8; \ + mask |= mask << 16; \ + mask |= mask << 32; \ + } while (0) + +#endif + #ifndef __HAVE_ARCH_MEMCHR /** * memchr - Find a character in an area of memory. - * @s: The memory area + * @p: The memory area * @c: The byte to search for - * @n: The size of the area. + * @length: The size of the area. * * returns the address of the first occurrence of @c, or %NULL * if @c is not found */ -void *memchr(const void *s, int c, size_t n) +void *memchr(const void *p, int c, unsigned long length) { - const unsigned char *p = s; - while (n-- != 0) { - if ((unsigned char)c == *p++) { - return (void *)(p - 1); + u64 mask, val; + const void *end = p + length; + + c &= 0xff; + if (p <= end - 8) { + mask = c; + MEMCHR_MASK_GEN(mask); + + for (; p <= end - 8; p += 8) { + val = *(u64 *)p ^ mask; + if ((val + 0xfefefefefefefeffu) & + (~val & 0x8080808080808080u)) + break; } } + + for (; p < end; p++) + if (*(unsigned char *)p == c) + return (void *)p; + return NULL; } EXPORT_SYMBOL(memchr); @@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) return check_bytes8(start, value, bytes); value64 = value; -#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 - value64 *= 0x0101010101010101ULL; -#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) - value64 *= 0x01010101; - value64 |= value64 << 32; -#else - value64 |= value64 << 8; - value64 |= value64 << 16; - value64 |= value64 << 32; -#endif + MEMCHR_MASK_GEN(value64); prefix = (unsigned long)start % 8; if (prefix) { diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index d5923a640457d..98754ff9fe686 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -604,6 +604,12 @@ static void __init test_bitmap_arr64(void) pr_err("bitmap_copy_arr64(nbits == %d:" " tail is not safely cleared: %d\n", nbits, next_bit); + if ((nbits % 64) && + (arr[(nbits - 1) / 64] & ~GENMASK_ULL((nbits - 1) % 64, 0))) + pr_err("bitmap_to_arr64(nbits == %d): tail is not safely cleared: 0x%016llx (must be 0x%016llx)\n", + nbits, arr[(nbits - 1) / 64], + GENMASK_ULL((nbits - 1) % 64, 0)); + if (nbits < EXP1_IN_BITS - 64) expect_eq_uint(arr[DIV_ROUND_UP(nbits, 64)], 0xa5a5a5a5); } @@ -869,6 +875,67 @@ static void __init test_bitmap_print_buf(void) } } +static void __init test_bitmap_const_eval(void) +{ + DECLARE_BITMAP(bitmap, BITS_PER_LONG); + unsigned long initvar = BIT(2); + unsigned long bitopvar = 0; + unsigned long var = 0; + int res; + + /* + * Compilers must be able to optimize all of those to compile-time + * constants on any supported optimization level (-O2, -Os) and any + * architecture. Otherwise, trigger a build bug. + * The whole function gets optimized out then, there's nothing to do + * in runtime. + */ + + /* + * Equals to `unsigned long bitmap[1] = { GENMASK(6, 5), }`. + * Clang on s390 optimizes bitops at compile-time as intended, but at + * the same time stops treating @bitmap and @bitopvar as compile-time + * constants after regular test_bit() is executed, thus triggering the + * build bugs below. So, call const_test_bit() there directly until + * the compiler is fixed. + */ + bitmap_clear(bitmap, 0, BITS_PER_LONG); +#if defined(__s390__) && defined(__clang__) + if (!const_test_bit(7, bitmap)) +#else + if (!test_bit(7, bitmap)) +#endif + bitmap_set(bitmap, 5, 2); + + /* Equals to `unsigned long bitopvar = BIT(20)` */ + __change_bit(31, &bitopvar); + bitmap_shift_right(&bitopvar, &bitopvar, 11, BITS_PER_LONG); + + /* Equals to `unsigned long var = BIT(25)` */ + var |= BIT(25); + if (var & BIT(0)) + var ^= GENMASK(9, 6); + + /* __const_hweight<32|64>(GENMASK(6, 5)) == 2 */ + res = bitmap_weight(bitmap, 20); + BUILD_BUG_ON(!__builtin_constant_p(res)); + BUILD_BUG_ON(res != 2); + + /* !(BIT(31) & BIT(18)) == 1 */ + res = !test_bit(18, &bitopvar); + BUILD_BUG_ON(!__builtin_constant_p(res)); + BUILD_BUG_ON(!res); + + /* BIT(2) & GENMASK(14, 8) == 0 */ + res = initvar & GENMASK(14, 8); + BUILD_BUG_ON(!__builtin_constant_p(res)); + BUILD_BUG_ON(res); + + /* ~BIT(25) */ + BUILD_BUG_ON(!__builtin_constant_p(~var)); + BUILD_BUG_ON(~var != ~BIT(25)); +} + static void __init selftest(void) { test_zero_clear(); @@ -884,6 +951,7 @@ static void __init selftest(void) test_for_each_set_clump8(); test_bitmap_cut(); test_bitmap_print_buf(); + test_bitmap_const_eval(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c index b0424d23ac57f..fb941a5b70f54 100644 --- a/lib/zstd/compress/zstd_double_fast.c +++ b/lib/zstd/compress/zstd_double_fast.c @@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( return (size_t)(iend - anchor); } +#define ZSTD_GEN_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ + } + +ZSTD_GEN_FN(noDict, 4) +ZSTD_GEN_FN(noDict, 5) +ZSTD_GEN_FN(noDict, 6) +ZSTD_GEN_FN(noDict, 7) + +ZSTD_GEN_FN(dictMatchState, 4) +ZSTD_GEN_FN(dictMatchState, 5) +ZSTD_GEN_FN(dictMatchState, 6) +ZSTD_GEN_FN(dictMatchState, 7) + +#undef ZSTD_GEN_FN + size_t ZSTD_compressBlock_doubleFast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); } } @@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); } } @@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ @@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } +#define ZSTD_GEN_FN(mls) \ + static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_FN(4) +ZSTD_GEN_FN(5) +ZSTD_GEN_FN(6) +ZSTD_GEN_FN(7) + +#undef ZSTD_GEN_FN + size_t ZSTD_compressBlock_doubleFast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); } } diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c index 96b7d48e2868e..e0652e31d421a 100644 --- a/lib/zstd/compress/zstd_fast.c +++ b/lib/zstd/compress/zstd_fast.c @@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( return (size_t)(iend - anchor); } +#define ZSTD_GEN_FN(mls) \ + static size_t ZSTD_compressBlock_fast_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_FN(4) +ZSTD_GEN_FN(5) +ZSTD_GEN_FN(6) +ZSTD_GEN_FN(7) + +#undef ZSTD_GEN_FN size_t ZSTD_compressBlock_fast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); } } @@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( return (size_t)(iend - anchor); } +#define ZSTD_GEN_FN(mls) \ + static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_FN(4) +ZSTD_GEN_FN(5) +ZSTD_GEN_FN(6) +ZSTD_GEN_FN(7) + +#undef ZSTD_GEN_FN + size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); } } @@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ @@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( return (size_t)(iend - anchor); } +#define ZSTD_GEN_FN(mls) \ + static size_t ZSTD_compressBlock_fast_extDict_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_FN(4) +ZSTD_GEN_FN(5) +ZSTD_GEN_FN(6) +ZSTD_GEN_FN(7) + +#undef ZSTD_GEN_FN size_t ZSTD_compressBlock_fast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); } } diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c index fb54d4e28a2bc..1db22db5b1e91 100644 --- a/lib/zstd/compress/zstd_lazy.c +++ b/lib/zstd/compress/zstd_lazy.c @@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, } -static size_t -ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); - } -} - - -static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); - } -} - - -static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); - } -} - - - /* ********************************* * Hash Chain ***********************************/ @@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE_TEMPLATE -size_t ZSTD_HcFindBestMatch_generic ( +size_t ZSTD_HcFindBestMatch( ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, @@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( return ml; } +typedef size_t (*searchMax_f)( + ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); - } -} +/* + * This struct contains the functions necessary for lazy to search. + * Currently, that is only searchMax. However, it is still valuable to have the + * VTable because this makes it easier to add more functions to the VTable later. + */ +typedef struct { + searchMax_f searchMax; +} ZSTD_LazyVTable; + +#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ + static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ + ZSTD_BtFindBestMatch_##dictMode##_##mls \ + }; +#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ + static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ + ZSTD_HcFindBestMatch_##dictMode##_##mls \ + }; -static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) +/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) + +#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ + { \ + &ZSTD_BtVTable_##dictMode##_4, \ + &ZSTD_BtVTable_##dictMode##_5, \ + &ZSTD_BtVTable_##dictMode##_6 \ } -} - -static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); +#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ + { \ + &ZSTD_HcVTable_##dictMode##_4, \ + &ZSTD_HcVTable_##dictMode##_5, \ + &ZSTD_HcVTable_##dictMode##_6 \ } -} - -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); +#define GEN_ZSTD_VTABLE_ARRAY(X) \ + { \ + X(noDict), \ + X(extDict), \ + X(dictMatchState), \ + X(dedicatedDictSearch) \ } -} - /* ******************************* * Common parser - lazy strategy *********************************/ typedef enum { search_hashChain, search_binaryTree } searchMethod_e; +static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) +{ + /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ + ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); + ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); + /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ + + U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); + switch (searchMethod) { + case search_hashChain: + return hcVTables[dictMode][mls - 4]; + case search_binaryTree: + return btVTables[dictMode][mls - 4]; + default: + return NULL; + } +} + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_lazy_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, @@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( const U32 prefixLowestIndex = ms->window.dictLimit; const BYTE* const prefixLowest = base + prefixLowestIndex; - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - /* * This table is indexed first by the four ZSTD_dictMode_e values, and then * by the two searchMethod_e values. NULLs are placed for configurations * that should never occur (extDict modes go to the other implementation * below and there is no DDSS for binary tree search yet). */ - const searchMax_f searchFuncs[4][2] = { - { - ZSTD_HcFindBestMatch_selectMLS, - ZSTD_BtFindBestMatch_selectMLS - }, - { - NULL, - NULL - }, - { - ZSTD_HcFindBestMatch_dictMatchState_selectMLS, - ZSTD_BtFindBestMatch_dictMatchState_selectMLS - }, - { - ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, - NULL - } - }; - - searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; + searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; const int isDMS = dictMode == ZSTD_dictMatchState; @@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const dictStart = dictBase + ms->window.lowLimit; const U32 windowLog = ms->cParams.windowLog; - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; U32 offset_1 = rep[0], offset_2 = rep[1]; diff --git a/mm/compaction.c b/mm/compaction.c index 1f89b969c12bf..2462ff734a10f 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1726,7 +1726,7 @@ typedef enum { * Allow userspace to control policy on scanning the unevictable LRU for * compactable pages. */ -#ifdef CONFIG_PREEMPT_RT +#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) int sysctl_compact_unevictable_allowed __read_mostly = 0; #else int sysctl_compact_unevictable_allowed __read_mostly = 1; @@ -2718,7 +2718,11 @@ static void compact_nodes(void) * aggressively the kernel should compact memory in the * background. It takes values in the range [0, 100]. */ +#ifdef CONFIG_CACHY +unsigned int __read_mostly sysctl_compaction_proactiveness; +#else unsigned int __read_mostly sysctl_compaction_proactiveness = 20; +#endif int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) diff --git a/mm/ksm.c b/mm/ksm.c index 54f78c9eecaee..e670da229e2c9 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -131,6 +131,8 @@ struct mm_slot { * @address: the next address inside that to be scanned * @rmap_list: link to the next rmap to be scanned in the rmap_list * @seqnr: count of completed full scans (needed when removing unstable node) + * @new_ksmpages_of_this_scanning: count of the new merged KSM pages in the + * current scanning of mm_lists (cleared after ksm_do_scan() ends) * * There is only the one ksm_scan instance of this cursor structure. */ @@ -139,6 +141,7 @@ struct ksm_scan { unsigned long address; struct rmap_item **rmap_list; unsigned long seqnr; + unsigned long new_ksmpages_of_this_scanning; }; /** @@ -277,6 +280,31 @@ static unsigned int zero_checksum __read_mostly; /* Whether to merge empty (zeroed) pages with actual zero pages */ static bool ksm_use_zero_pages __read_mostly; +/* + * Work in auto-mode. + * Maximum number of multiplicative factor of pages_to_scan. + */ +static unsigned int ksm_max_scanning_factor = 32; + +/* + * Work in auto-mode. + * The multiplicative factor of pages_to_scan. + * Real pages to scan equals to the product of scanning_factor + * and pages_to_scan + */ +static unsigned int scanning_factor = 1; + +/* + * Work in auto-mode. + * Default ksm_auto_threshold_percent: 20, means 20%. When free memory + * is lower than total memory * ksm_auto_threshold_percent/100, ksmd will + * be triggered to compare and merge pages. + */ +unsigned int ksm_auto_threshold_percent = 20; + +/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */ +static bool auto_triggered; + #ifdef CONFIG_NUMA /* Zeroed when merging across nodes is not allowed */ static unsigned int ksm_merge_across_nodes = 1; @@ -290,6 +318,7 @@ static int ksm_nr_node_ids = 1; #define KSM_RUN_MERGE 1 #define KSM_RUN_UNMERGE 2 #define KSM_RUN_OFFLINE 4 +#define KSM_RUN_AUTO 8 static unsigned long ksm_run = KSM_RUN_STOP; static void wait_while_offlining(void); @@ -2020,6 +2049,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, rmap_item->address |= STABLE_FLAG; hlist_add_head(&rmap_item->hlist, &stable_node->hlist); + ksm_scan.new_ksmpages_of_this_scanning++; + if (rmap_item->hlist.next) ksm_pages_sharing++; else @@ -2385,6 +2416,23 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) return NULL; } +/* + * enhance_scanning_factor(): + * double the values of scanning_factor, but not more than + * ksm_max_scanning_factor. + */ +static inline void enhance_scanning_factor(void) +{ + scanning_factor = scanning_factor << 1; + if (scanning_factor > ksm_max_scanning_factor) + scanning_factor = ksm_max_scanning_factor; +} + +static inline void reset_scanning_factor(void) +{ + scanning_factor = 1; +} + /** * ksm_do_scan - the ksm scanner main worker function. * @scan_npages: number of pages we want to scan before we return. @@ -2399,14 +2447,51 @@ static void ksm_do_scan(unsigned int scan_npages) rmap_item = scan_get_next_rmap_item(&page); if (!rmap_item) return; - cmp_and_merge_page(page, rmap_item); + if (ksm_run & KSM_RUN_AUTO && !auto_triggered) { + /* + * This should happens only when ksm_run is KSM_RUN_AUTO + * and free memory threshold still not reached. + * The reason to calculate it's checksum is to reduce the + * waiting time the rmap_item is added to unstable tree. + */ + rmap_item->oldchecksum = calc_checksum(page); + } else + cmp_and_merge_page(page, rmap_item); + put_page(page); } } +/* Work in auto mode, should reset auto_triggered ? */ +static bool should_untrigger_ksmd_to_merge(void) +{ + unsigned long total_ram_pages, free_pages; + + total_ram_pages = totalram_pages(); + free_pages = global_zone_page_state(NR_FREE_PAGES); + + return free_pages > (total_ram_pages * + ksm_auto_threshold_percent / 100) + + (total_ram_pages >> 4); +} + +/* Work in auto mode, should ksmd start to merge ? */ +static bool should_trigger_ksmd_to_merge(void) +{ + unsigned long total_ram_pages, free_pages; + + total_ram_pages = totalram_pages(); + free_pages = global_zone_page_state(NR_FREE_PAGES); + + return free_pages < (total_ram_pages * + ksm_auto_threshold_percent / 100); +} + static int ksmd_should_run(void) { - return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); + if (!list_empty(&ksm_mm_head.mm_list)) + return ksm_run & KSM_RUN_AUTO || ksm_run & KSM_RUN_MERGE; + return 0; } static int ksm_scan_thread(void *nothing) @@ -2419,17 +2504,58 @@ static int ksm_scan_thread(void *nothing) while (!kthread_should_stop()) { mutex_lock(&ksm_thread_mutex); wait_while_offlining(); - if (ksmd_should_run()) - ksm_do_scan(ksm_thread_pages_to_scan); + if (ksmd_should_run()) { + if (ksm_run & KSM_RUN_AUTO) { + /* + * If free memory is not lower than threshold, we only scan + * 5 pages and just calculate pages' checksum and not compare + * and merge them. Otherwise, do real scanning and merging as + * scanning-enhanced algorithm. + */ + if (!auto_triggered) + ksm_do_scan(5); + else + ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor); + } + else + ksm_do_scan(ksm_thread_pages_to_scan); + } mutex_unlock(&ksm_thread_mutex); + /* + * If there are new ksm pages after scanning, then we + * can enhance scanning_factor to improve ksm_thread's + * pages_to_scan to speed up scanning. Otherwaise, we + * reset scanning_factor to be one, so that to recover + * the normal state because there is greater probability + * of getting no new KsmPages in the next scanning. + */ + if (ksm_run & KSM_RUN_AUTO) { + if (ksm_scan.new_ksmpages_of_this_scanning > 0) + enhance_scanning_factor(); + else + reset_scanning_factor(); + + ksm_scan.new_ksmpages_of_this_scanning = 0; + + if (should_trigger_ksmd_to_merge()) + auto_triggered = true; + else if (should_untrigger_ksmd_to_merge()) + auto_triggered = false; + } + try_to_freeze(); if (ksmd_should_run()) { sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); - wait_event_interruptible_timeout(ksm_iter_wait, - sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), - msecs_to_jiffies(sleep_ms)); + if (sleep_ms >= 1000) + wait_event_interruptible_timeout(ksm_iter_wait, + sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), + msecs_to_jiffies(round_jiffies_relative(sleep_ms))); + else + wait_event_interruptible_timeout(ksm_iter_wait, + sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), + msecs_to_jiffies(sleep_ms)); } else { wait_event_freezable(ksm_thread_wait, ksmd_should_run() || kthread_should_stop()); @@ -2438,54 +2564,78 @@ static int ksm_scan_thread(void *nothing) return 0; } -int ksm_madvise(struct vm_area_struct *vma, unsigned long start, - unsigned long end, int advice, unsigned long *vm_flags) +int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long *vm_flags) { - struct mm_struct *mm = vma->vm_mm; int err; - switch (advice) { - case MADV_MERGEABLE: - /* - * Be somewhat over-protective for now! - */ - if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | - VM_PFNMAP | VM_IO | VM_DONTEXPAND | - VM_HUGETLB | VM_MIXEDMAP)) - return 0; /* just ignore the advice */ + /* + * Be somewhat over-protective for now! + */ + if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO | VM_DONTEXPAND | + VM_HUGETLB | VM_MIXEDMAP)) + return 0; /* just ignore the advice */ - if (vma_is_dax(vma)) - return 0; + if (vma_is_dax(vma)) + return 0; #ifdef VM_SAO if (*vm_flags & VM_SAO) return 0; #endif #ifdef VM_SPARC_ADI - if (*vm_flags & VM_SPARC_ADI) - return 0; + if (*vm_flags & VM_SPARC_ADI) + return 0; #endif - if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { - err = __ksm_enter(mm); - if (err) - return err; - } + if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { + err = __ksm_enter(mm); + if (err) + return err; + } - *vm_flags |= VM_MERGEABLE; - break; + *vm_flags |= VM_MERGEABLE; - case MADV_UNMERGEABLE: - if (!(*vm_flags & VM_MERGEABLE)) - return 0; /* just ignore the advice */ + return 0; +} - if (vma->anon_vma) { - err = unmerge_ksm_pages(vma, start, end); - if (err) - return err; - } +int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long *vm_flags) +{ + int err; + + if (!(*vm_flags & VM_MERGEABLE)) + return 0; /* just ignore the advice */ + + if (vma->anon_vma) { + err = unmerge_ksm_pages(vma, start, end); + if (err) + return err; + } + + *vm_flags &= ~VM_MERGEABLE; + + return 0; +} + +int ksm_madvise(struct vm_area_struct *vma, unsigned long start, + unsigned long end, int advice, unsigned long *vm_flags) +{ + struct mm_struct *mm = vma->vm_mm; + int err; - *vm_flags &= ~VM_MERGEABLE; + switch (advice) { + case MADV_MERGEABLE: + err = ksm_madvise_merge(mm, vma, vm_flags); + if (err) + return err; + break; + + case MADV_UNMERGEABLE: + err = ksm_madvise_unmerge(vma, start, end, vm_flags); + if (err) + return err; break; } @@ -2906,7 +3056,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, err = kstrtouint(buf, 10, &flags); if (err) return -EINVAL; - if (flags > KSM_RUN_UNMERGE) + if (flags > KSM_RUN_UNMERGE && flags != KSM_RUN_AUTO) return -EINVAL; /* @@ -2932,13 +3082,88 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, } mutex_unlock(&ksm_thread_mutex); - if (flags & KSM_RUN_MERGE) + if (flags & KSM_RUN_MERGE || flags & KSM_RUN_AUTO) wake_up_interruptible(&ksm_thread_wait); return count; } KSM_ATTR(run); +static ssize_t ksmd_status_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int len = 0; + + if (ksm_run & KSM_RUN_AUTO) { + len += sysfs_emit_at(buf, len, "mode: auto\n"); + len += sysfs_emit_at(buf, len, "auto_triggered %d\n", + auto_triggered); + len += sysfs_emit_at(buf, len, "scanning_factor: %u\n", + scanning_factor); + } else if (ksm_run & KSM_RUN_MERGE) + len += sysfs_emit_at(buf, len, "mode: on\n"); + else if (ksm_run & KSM_RUN_UNMERGE) + len += sysfs_emit_at(buf, len, "mode: unmerge\n"); + else + len += sysfs_emit_at(buf, len, "mode: off\n"); + + + return len; +} +KSM_ATTR_RO(ksmd_status); + +static ssize_t max_scanning_factor_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", ksm_max_scanning_factor); +} + +static ssize_t max_scanning_factor_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + unsigned int value; + int err; + + err = kstrtouint(buf, 10, &value); + if (err) + return -EINVAL; + + if (value < 1) + return -EINVAL; + + ksm_max_scanning_factor = value; + + return count; +} +KSM_ATTR(max_scanning_factor); + +static ssize_t auto_threshold_percent_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", ksm_auto_threshold_percent); +} + +static ssize_t auto_threshold_percent_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + unsigned int value; + int err; + + err = kstrtouint(buf, 10, &value); + if (err) + return -EINVAL; + + if (value > 100) + return -EINVAL; + + ksm_auto_threshold_percent = value; + + return count; +} +KSM_ATTR(auto_threshold_percent); + #ifdef CONFIG_NUMA static ssize_t merge_across_nodes_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -3149,6 +3374,9 @@ static struct attribute *ksm_attrs[] = { &sleep_millisecs_attr.attr, &pages_to_scan_attr.attr, &run_attr.attr, + &ksmd_status_attr.attr, + &max_scanning_factor_attr.attr, + &auto_threshold_percent_attr.attr, &pages_shared_attr.attr, &pages_sharing_attr.attr, &pages_unshared_attr.attr, @@ -3180,6 +3408,7 @@ static int __init ksm_init(void) zero_checksum = calc_checksum(ZERO_PAGE(0)); /* Default to false for backwards compatibility */ ksm_use_zero_pages = false; + auto_triggered = false; err = ksm_slab_init(); if (err) diff --git a/mm/madvise.c b/mm/madvise.c index 0316bbc6441b2..b71963da64641 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1173,6 +1173,10 @@ process_madvise_behavior_valid(int behavior) case MADV_COLD: case MADV_PAGEOUT: case MADV_WILLNEED: +#ifdef CONFIG_KSM + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: +#endif return true; default: return false; @@ -1493,3 +1497,114 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, out: return ret; } + +SYSCALL_DEFINE3(pmadv_ksm, int, pidfd, int, behaviour, unsigned int, flags) +{ +#ifdef CONFIG_KSM + ssize_t ret; + struct pid *pid; + struct task_struct *task; + struct mm_struct *mm; + unsigned int f_flags; + struct vm_area_struct *vma; + + if (flags != 0) { + ret = -EINVAL; + goto out; + } + + switch (behaviour) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + break; + default: + ret = -EINVAL; + goto out; + break; + } + + pid = pidfd_get_pid(pidfd, &f_flags); + if (IS_ERR(pid)) { + ret = PTR_ERR(pid); + goto out; + } + + task = get_pid_task(pid, PIDTYPE_PID); + if (!task) { + ret = -ESRCH; + goto put_pid; + } + + /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR_OR_NULL(mm)) { + ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH; + goto release_task; + } + + /* Require CAP_SYS_NICE for influencing process performance. */ + if (!capable(CAP_SYS_NICE)) { + ret = -EPERM; + goto release_mm; + } + + if (mmap_write_lock_killable(mm)) { + ret = -EINTR; + goto release_mm; + } + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + switch (behaviour) { + case MADV_MERGEABLE: + ret = ksm_madvise_merge(vma->vm_mm, vma, &vma->vm_flags); + break; + case MADV_UNMERGEABLE: + ret = ksm_madvise_unmerge(vma, vma->vm_start, vma->vm_end, &vma->vm_flags); + break; + default: + /* look, ma, no brain */ + break; + } + if (ret) + break; + } + + mmap_write_unlock(mm); + +release_mm: + mmput(mm); +release_task: + put_task_struct(task); +put_pid: + put_pid(pid); +out: + return ret; +#else /* CONFIG_KSM */ + return -ENOSYS; +#endif /* CONFIG_KSM */ +} + +#ifdef CONFIG_KSM +static ssize_t ksm_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", __NR_pmadv_ksm); +} +static struct kobj_attribute pmadv_ksm_attr = __ATTR_RO(ksm); + +static struct attribute *pmadv_sysfs_attrs[] = { + &pmadv_ksm_attr.attr, + NULL, +}; + +static const struct attribute_group pmadv_sysfs_attr_group = { + .attrs = pmadv_sysfs_attrs, + .name = "pmadv", +}; + +static int __init pmadv_sysfs_init(void) +{ + return sysfs_create_group(kernel_kobj, &pmadv_sysfs_attr_group); +} +subsys_initcall(pmadv_sysfs_init); +#endif /* CONFIG_KSM */ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 618c366a2f074..b6888b0486b67 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); x = __this_cpu_add_return(stats_updates, abs(val)); - if (x > MEMCG_CHARGE_BATCH) { + if (x > MEMCG_CHARGE_BATCH * 128) { atomic_add(x / MEMCG_CHARGE_BATCH, &stats_flush_threshold); __this_cpu_write(stats_updates, 0); } diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 55c2776ae6999..ddda9f55a8595 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -70,7 +70,11 @@ static long ratelimit_pages = 32; /* * Start background writeback (via writeback threads) at this percentage */ +#ifdef CONFIG_CACHY +static int dirty_background_ratio = 5; +#else static int dirty_background_ratio = 10; +#endif /* * dirty_background_bytes starts at 0 (disabled) so that it is a function of @@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; /* * The interval between `kupdate'-style writebacks */ +#ifdef CONFIG_CACHY +unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ +#else unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ +#endif EXPORT_SYMBOL_GPL(dirty_writeback_interval); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b5b14b78c4fd4..c9cc9c6c6dfbd 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -126,13 +126,97 @@ typedef int __bitwise fpi_t; static DEFINE_MUTEX(pcp_batch_high_lock); #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) -struct pagesets { - local_lock_t lock; -}; -static DEFINE_PER_CPU(struct pagesets, pagesets) = { - .lock = INIT_LOCAL_LOCK(lock), -}; +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) +/* + * On SMP, spin_trylock is sufficient protection. + * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. + */ +#define pcp_trylock_prepare(flags) do { } while (0) +#define pcp_trylock_finish(flag) do { } while (0) +#else + +/* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ +#define pcp_trylock_prepare(flags) local_irq_save(flags) +#define pcp_trylock_finish(flags) local_irq_restore(flags) +#endif + +/* + * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid + * a migration causing the wrong PCP to be locked and remote memory being + * potentially allocated, pin the task to the CPU for the lookup+lock. + * preempt_disable is used on !RT because it is faster than migrate_disable. + * migrate_disable is used on RT because otherwise RT spinlock usage is + * interfered with and a high priority task cannot preempt the allocator. + */ +#ifndef CONFIG_PREEMPT_RT +#define pcpu_task_pin() preempt_disable() +#define pcpu_task_unpin() preempt_enable() +#else +#define pcpu_task_pin() migrate_disable() +#define pcpu_task_unpin() migrate_enable() +#endif +/* + * Generic helper to lookup and a per-cpu variable with an embedded spinlock. + * Return value should be used with equivalent unlock helper. + */ +#define pcpu_spin_lock(type, member, ptr) \ +({ \ + type *_ret; \ + pcpu_task_pin(); \ + _ret = this_cpu_ptr(ptr); \ + spin_lock(&_ret->member); \ + _ret; \ +}) + +#define pcpu_spin_lock_irqsave(type, member, ptr, flags) \ +({ \ + type *_ret; \ + pcpu_task_pin(); \ + _ret = this_cpu_ptr(ptr); \ + spin_lock_irqsave(&_ret->member, flags); \ + _ret; \ +}) + +#define pcpu_spin_trylock_irqsave(type, member, ptr, flags) \ +({ \ + type *_ret; \ + pcpu_task_pin(); \ + _ret = this_cpu_ptr(ptr); \ + if (!spin_trylock_irqsave(&_ret->member, flags)) { \ + pcpu_task_unpin(); \ + _ret = NULL; \ + } \ + _ret; \ +}) + +#define pcpu_spin_unlock(member, ptr) \ +({ \ + spin_unlock(&ptr->member); \ + pcpu_task_unpin(); \ +}) + +#define pcpu_spin_unlock_irqrestore(member, ptr, flags) \ +({ \ + spin_unlock_irqrestore(&ptr->member, flags); \ + pcpu_task_unpin(); \ +}) + +/* struct per_cpu_pages specific helpers. */ +#define pcp_spin_lock(ptr) \ + pcpu_spin_lock(struct per_cpu_pages, lock, ptr) + +#define pcp_spin_lock_irqsave(ptr, flags) \ + pcpu_spin_lock_irqsave(struct per_cpu_pages, lock, ptr, flags) + +#define pcp_spin_trylock_irqsave(ptr, flags) \ + pcpu_spin_trylock_irqsave(struct per_cpu_pages, lock, ptr, flags) + +#define pcp_spin_unlock(ptr) \ + pcpu_spin_unlock(lock, ptr) + +#define pcp_spin_unlock_irqrestore(ptr, flags) \ + pcpu_spin_unlock_irqrestore(lock, ptr, flags) #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); @@ -151,13 +235,7 @@ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif -/* work_structs for global per-cpu drains */ -struct pcpu_drain { - struct zone *zone; - struct work_struct work; -}; static DEFINE_MUTEX(pcpu_drain_mutex); -static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY volatile unsigned long latent_entropy __latent_entropy; @@ -653,7 +731,7 @@ static inline unsigned int order_to_pindex(int migratetype, int order) #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (order > PAGE_ALLOC_COSTLY_ORDER) { VM_BUG_ON(order != pageblock_order); - base = PAGE_ALLOC_COSTLY_ORDER + 1; + return NR_LOWORDER_PCP_LISTS; } #else VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); @@ -667,7 +745,7 @@ static inline int pindex_to_order(unsigned int pindex) int order = pindex / MIGRATE_PCPTYPES; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (order > PAGE_ALLOC_COSTLY_ORDER) + if (pindex == NR_LOWORDER_PCP_LISTS) order = pageblock_order; #else VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); @@ -785,7 +863,7 @@ static inline bool set_page_guard(struct zone *zone, struct page *page, return false; __SetPageGuard(page); - INIT_LIST_HEAD(&page->lru); + INIT_LIST_HEAD(&page->buddy_list); set_page_private(page, order); /* Guard pages are not available for any usage */ __mod_zone_freepage_state(zone, -(1 << order), migratetype); @@ -928,7 +1006,7 @@ static inline void add_to_free_list(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_add(&page->lru, &area->free_list[migratetype]); + list_add(&page->buddy_list, &area->free_list[migratetype]); area->nr_free++; } @@ -938,7 +1016,7 @@ static inline void add_to_free_list_tail(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_add_tail(&page->lru, &area->free_list[migratetype]); + list_add_tail(&page->buddy_list, &area->free_list[migratetype]); area->nr_free++; } @@ -952,7 +1030,7 @@ static inline void move_to_free_list(struct page *page, struct zone *zone, { struct free_area *area = &zone->free_area[order]; - list_move_tail(&page->lru, &area->free_list[migratetype]); + list_move_tail(&page->buddy_list, &area->free_list[migratetype]); } static inline void del_page_from_free_list(struct page *page, struct zone *zone, @@ -962,7 +1040,7 @@ static inline void del_page_from_free_list(struct page *page, struct zone *zone, if (page_reported(page)) __ClearPageReported(page); - list_del(&page->lru); + list_del(&page->buddy_list); __ClearPageBuddy(page); set_page_private(page, 0); zone->free_area[order].nr_free--; @@ -1473,10 +1551,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, /* Ensure requested pindex is drained first. */ pindex = pindex - 1; - /* - * local_lock_irq held so equivalent to spin_lock_irqsave for - * both PREEMPT_RT and non-PREEMPT_RT configurations. - */ + /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ spin_lock(&zone->lock); isolated_pageblocks = has_isolate_pageblock(zone); @@ -1504,11 +1579,11 @@ static void free_pcppages_bulk(struct zone *zone, int count, do { int mt; - page = list_last_entry(list, struct page, lru); + page = list_last_entry(list, struct page, pcp_list); mt = get_pcppage_migratetype(page); /* must delete to avoid corrupting pcp list */ - list_del(&page->lru); + list_del(&page->pcp_list); count -= nr_pages; pcp->count -= nr_pages; @@ -3044,10 +3119,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, { int i, allocated = 0; - /* - * local_lock_irq held so equivalent to spin_lock_irqsave for - * both PREEMPT_RT and non-PREEMPT_RT configurations. - */ + /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ spin_lock(&zone->lock); for (i = 0; i < count; ++i) { struct page *page = __rmqueue(zone, order, migratetype, @@ -3068,7 +3140,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * for IO devices that can merge IO requests if the physical * pages are ordered properly. */ - list_add_tail(&page->lru, list); + list_add_tail(&page->pcp_list, list); allocated++; if (is_migrate_cma(get_pcppage_migratetype(page))) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, @@ -3091,51 +3163,48 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * Called from the vmstat counter updater to drain pagesets of this * currently executing processor on remote nodes after they have * expired. - * - * Note that this function must be called with the thread pinned to - * a single processor. */ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) { - unsigned long flags; int to_drain, batch; - local_lock_irqsave(&pagesets.lock, flags); batch = READ_ONCE(pcp->batch); to_drain = min(pcp->count, batch); - if (to_drain > 0) + if (to_drain > 0) { + unsigned long flags; + + /* + * free_pcppages_bulk expects IRQs disabled for zone->lock + * so even though pcp->lock is not intended to be IRQ-safe, + * it's needed in this context. + */ + spin_lock_irqsave(&pcp->lock, flags); free_pcppages_bulk(zone, to_drain, pcp, 0); - local_unlock_irqrestore(&pagesets.lock, flags); + spin_unlock_irqrestore(&pcp->lock, flags); + } } #endif /* * Drain pcplists of the indicated processor and zone. - * - * The processor must either be the current processor and the - * thread pinned to the current processor or a processor that - * is not online. */ static void drain_pages_zone(unsigned int cpu, struct zone *zone) { - unsigned long flags; struct per_cpu_pages *pcp; - local_lock_irqsave(&pagesets.lock, flags); - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - if (pcp->count) - free_pcppages_bulk(zone, pcp->count, pcp, 0); + if (pcp->count) { + unsigned long flags; - local_unlock_irqrestore(&pagesets.lock, flags); + /* See drain_zone_pages on why this is disabling IRQs */ + spin_lock_irqsave(&pcp->lock, flags); + free_pcppages_bulk(zone, pcp->count, pcp, 0); + spin_unlock_irqrestore(&pcp->lock, flags); + } } /* * Drain pcplists of all zones on the indicated processor. - * - * The processor must either be the current processor and the - * thread pinned to the current processor or a processor that - * is not online. */ static void drain_pages(unsigned int cpu) { @@ -3148,9 +3217,6 @@ static void drain_pages(unsigned int cpu) /* * Spill all of this CPU's per-cpu pages back into the buddy allocator. - * - * The CPU has to be pinned. When zone parameter is non-NULL, spill just - * the single zone's pages. */ void drain_local_pages(struct zone *zone) { @@ -3162,24 +3228,6 @@ void drain_local_pages(struct zone *zone) drain_pages(cpu); } -static void drain_local_pages_wq(struct work_struct *work) -{ - struct pcpu_drain *drain; - - drain = container_of(work, struct pcpu_drain, work); - - /* - * drain_all_pages doesn't use proper cpu hotplug protection so - * we can race with cpu offline when the WQ can move this from - * a cpu pinned worker to an unbound one. We can operate on a different - * cpu which is alright but we also have to make sure to not move to - * a different one. - */ - migrate_disable(); - drain_local_pages(drain->zone); - migrate_enable(); -} - /* * The implementation of drain_all_pages(), exposing an extra parameter to * drain on all cpus. @@ -3200,13 +3248,6 @@ static void __drain_all_pages(struct zone *zone, bool force_all_cpus) */ static cpumask_t cpus_with_pcps; - /* - * Make sure nobody triggers this path before mm_percpu_wq is fully - * initialized. - */ - if (WARN_ON_ONCE(!mm_percpu_wq)) - return; - /* * Do not drain if one is already in progress unless it's specific to * a zone. Such callers are primarily CMA and memory hotplug and need @@ -3256,14 +3297,11 @@ static void __drain_all_pages(struct zone *zone, bool force_all_cpus) } for_each_cpu(cpu, &cpus_with_pcps) { - struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); - - drain->zone = zone; - INIT_WORK(&drain->work, drain_local_pages_wq); - queue_work_on(cpu, mm_percpu_wq, &drain->work); + if (zone) + drain_pages_zone(cpu, zone); + else + drain_pages(cpu); } - for_each_cpu(cpu, &cpus_with_pcps) - flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); mutex_unlock(&pcpu_drain_mutex); } @@ -3272,8 +3310,6 @@ static void __drain_all_pages(struct zone *zone, bool force_all_cpus) * Spill all the per-cpu pages from all CPUs back into the buddy allocator. * * When zone parameter is non-NULL, spill just the single zone's pages. - * - * Note that this can be extremely slow as the draining happens in a workqueue. */ void drain_all_pages(struct zone *zone) { @@ -3318,7 +3354,7 @@ void mark_free_pages(struct zone *zone) for_each_migratetype_order(order, t) { list_for_each_entry(page, - &zone->free_area[order].free_list[t], lru) { + &zone->free_area[order].free_list[t], buddy_list) { unsigned long i; pfn = page_to_pfn(page); @@ -3395,19 +3431,17 @@ static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, return min(READ_ONCE(pcp->batch) << 2, high); } -static void free_unref_page_commit(struct page *page, int migratetype, +static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, + struct page *page, int migratetype, unsigned int order) { - struct zone *zone = page_zone(page); - struct per_cpu_pages *pcp; int high; int pindex; bool free_high; __count_vm_event(PGFREE); - pcp = this_cpu_ptr(zone->per_cpu_pageset); pindex = order_to_pindex(migratetype, order); - list_add(&page->lru, &pcp->lists[pindex]); + list_add(&page->pcp_list, &pcp->lists[pindex]); pcp->count += 1 << order; /* @@ -3432,6 +3466,9 @@ static void free_unref_page_commit(struct page *page, int migratetype, void free_unref_page(struct page *page, unsigned int order) { unsigned long flags; + unsigned long __maybe_unused UP_flags; + struct per_cpu_pages *pcp; + struct zone *zone; unsigned long pfn = page_to_pfn(page); int migratetype; @@ -3454,9 +3491,16 @@ void free_unref_page(struct page *page, unsigned int order) migratetype = MIGRATE_MOVABLE; } - local_lock_irqsave(&pagesets.lock, flags); - free_unref_page_commit(page, migratetype, order); - local_unlock_irqrestore(&pagesets.lock, flags); + zone = page_zone(page); + pcp_trylock_prepare(UP_flags); + pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); + if (pcp) { + free_unref_page_commit(zone, pcp, page, migratetype, order); + pcp_spin_unlock_irqrestore(pcp, flags); + } else { + free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); + } + pcp_trylock_finish(UP_flags); } /* @@ -3465,6 +3509,8 @@ void free_unref_page(struct page *page, unsigned int order) void free_unref_page_list(struct list_head *list) { struct page *page, *next; + struct per_cpu_pages *pcp = NULL; + struct zone *locked_zone = NULL; unsigned long flags; int batch_count = 0; int migratetype; @@ -3489,8 +3535,18 @@ void free_unref_page_list(struct list_head *list) } } - local_lock_irqsave(&pagesets.lock, flags); list_for_each_entry_safe(page, next, list, lru) { + struct zone *zone = page_zone(page); + + /* Different zone, different pcp lock. */ + if (zone != locked_zone) { + if (pcp) + pcp_spin_unlock_irqrestore(pcp, flags); + + locked_zone = zone; + pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); + } + /* * Non-isolated types over MIGRATE_PCPTYPES get added * to the MIGRATE_MOVABLE pcp list. @@ -3500,19 +3556,21 @@ void free_unref_page_list(struct list_head *list) migratetype = MIGRATE_MOVABLE; trace_mm_page_free_batched(page); - free_unref_page_commit(page, migratetype, 0); + free_unref_page_commit(zone, pcp, page, migratetype, 0); /* * Guard against excessive IRQ disabled times when we get * a large list of pages to free. */ if (++batch_count == SWAP_CLUSTER_MAX) { - local_unlock_irqrestore(&pagesets.lock, flags); + pcp_spin_unlock_irqrestore(pcp, flags); batch_count = 0; - local_lock_irqsave(&pagesets.lock, flags); + pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); } } - local_unlock_irqrestore(&pagesets.lock, flags); + + if (pcp) + pcp_spin_unlock_irqrestore(pcp, flags); } /* @@ -3637,6 +3695,43 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, #endif } +static __always_inline +struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, + unsigned int order, unsigned int alloc_flags, + int migratetype) +{ + struct page *page; + unsigned long flags; + + do { + page = NULL; + spin_lock_irqsave(&zone->lock, flags); + /* + * order-0 request can reach here when the pcplist is skipped + * due to non-CMA allocation context. HIGHATOMIC area is + * reserved for high-order atomic allocation, so order-0 + * request should skip it. + */ + if (order > 0 && alloc_flags & ALLOC_HARDER) + page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); + if (!page) { + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (!page) { + spin_unlock_irqrestore(&zone->lock, flags); + return NULL; + } + } + __mod_zone_freepage_state(zone, -(1 << order), + get_pcppage_migratetype(page)); + spin_unlock_irqrestore(&zone->lock, flags); + } while (check_new_pages(page, order)); + + __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); + zone_statistics(preferred_zone, zone, 1); + + return page; +} + /* Remove page from the per-cpu list, caller must protect the list */ static inline struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, @@ -3670,8 +3765,8 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, return NULL; } - page = list_first_entry(list, struct page, lru); - list_del(&page->lru); + page = list_first_entry(list, struct page, pcp_list); + list_del(&page->pcp_list); pcp->count -= 1 << order; } while (check_new_pcp(page, order)); @@ -3688,19 +3783,29 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, struct list_head *list; struct page *page; unsigned long flags; + unsigned long __maybe_unused UP_flags; - local_lock_irqsave(&pagesets.lock, flags); + /* + * spin_trylock may fail due to a parallel drain. In the future, the + * trylock will also protect against IRQ reentrancy. + */ + pcp_trylock_prepare(UP_flags); + pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); + if (!pcp) { + pcp_trylock_finish(UP_flags); + return NULL; + } /* * On allocation, reduce the number of pages that are batch freed. * See nr_pcp_free() where free_factor is increased for subsequent * frees. */ - pcp = this_cpu_ptr(zone->per_cpu_pageset); pcp->free_factor >>= 1; list = &pcp->lists[order_to_pindex(migratetype, order)]; page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); - local_unlock_irqrestore(&pagesets.lock, flags); + pcp_spin_unlock_irqrestore(pcp, flags); + pcp_trylock_finish(UP_flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); zone_statistics(preferred_zone, zone, 1); @@ -3717,9 +3822,14 @@ struct page *rmqueue(struct zone *preferred_zone, gfp_t gfp_flags, unsigned int alloc_flags, int migratetype) { - unsigned long flags; struct page *page; + /* + * We most definitely don't want callers attempting to + * allocate greater than order-1 page units with __GFP_NOFAIL. + */ + WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); + if (likely(pcp_allowed_order(order))) { /* * MIGRATE_MOVABLE pcplist could have the pages on CMA area and @@ -3729,53 +3839,23 @@ struct page *rmqueue(struct zone *preferred_zone, migratetype != MIGRATE_MOVABLE) { page = rmqueue_pcplist(preferred_zone, zone, order, gfp_flags, migratetype, alloc_flags); - goto out; + if (likely(page)) + goto out; } } - /* - * We most definitely don't want callers attempting to - * allocate greater than order-1 page units with __GFP_NOFAIL. - */ - WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); - - do { - page = NULL; - spin_lock_irqsave(&zone->lock, flags); - /* - * order-0 request can reach here when the pcplist is skipped - * due to non-CMA allocation context. HIGHATOMIC area is - * reserved for high-order atomic allocation, so order-0 - * request should skip it. - */ - if (order > 0 && alloc_flags & ALLOC_HARDER) - page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); - if (!page) { - page = __rmqueue(zone, order, migratetype, alloc_flags); - if (!page) - goto failed; - } - __mod_zone_freepage_state(zone, -(1 << order), - get_pcppage_migratetype(page)); - spin_unlock_irqrestore(&zone->lock, flags); - } while (check_new_pages(page, order)); - - __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); - zone_statistics(preferred_zone, zone, 1); + page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, + migratetype); out: /* Separate test+clear to avoid unnecessary atomics */ - if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { + if (unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) { clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); wakeup_kswapd(zone, 0, 0, zone_idx(zone)); } VM_BUG_ON_PAGE(page && bad_range(zone, page), page); return page; - -failed: - spin_unlock_irqrestore(&zone->lock, flags); - return NULL; } #ifdef CONFIG_FAIL_PAGE_ALLOC @@ -5252,6 +5332,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, { struct page *page; unsigned long flags; + unsigned long __maybe_unused UP_flags; struct zone *zone; struct zoneref *z; struct per_cpu_pages *pcp; @@ -5332,11 +5413,14 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, if (unlikely(!zone)) goto failed; + /* Is a parallel drain in progress? */ + pcp_trylock_prepare(UP_flags); + pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); + if (!pcp) + goto failed_irq; + /* Attempt the batch allocation */ - local_lock_irqsave(&pagesets.lock, flags); - pcp = this_cpu_ptr(zone->per_cpu_pageset); pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; - while (nr_populated < nr_pages) { /* Skip existing pages */ @@ -5349,8 +5433,10 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, pcp, pcp_list); if (unlikely(!page)) { /* Try and allocate at least one page */ - if (!nr_account) + if (!nr_account) { + pcp_spin_unlock_irqrestore(pcp, flags); goto failed_irq; + } break; } nr_account++; @@ -5363,7 +5449,8 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, nr_populated++; } - local_unlock_irqrestore(&pagesets.lock, flags); + pcp_spin_unlock_irqrestore(pcp, flags); + pcp_trylock_finish(UP_flags); __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); @@ -5372,7 +5459,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, return nr_populated; failed_irq: - local_unlock_irqrestore(&pagesets.lock, flags); + pcp_trylock_finish(UP_flags); failed: page = __alloc_pages(gfp, 0, preferred_nid, nodemask); @@ -6893,11 +6980,11 @@ static int zone_batchsize(struct zone *zone) /* * The number of pages to batch allocate is either ~0.1% - * of the zone or 1MB, whichever is smaller. The batch + * of the zone or 4MB, whichever is smaller. The batch * size is striking a balance between allocation latency * and zone lock contention. */ - batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); + batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); batch /= 4; /* We effectively *= 4 below */ if (batch < 1) batch = 1; @@ -6975,6 +7062,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) * historical relationship between high and batch. */ high = max(high, batch << 2); + high = max(high, 1024); return high; #else @@ -7012,6 +7100,7 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta memset(pcp, 0, sizeof(*pcp)); memset(pzstats, 0, sizeof(*pzstats)); + spin_lock_init(&pcp->lock); for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) INIT_LIST_HEAD(&pcp->lists[pindex]); diff --git a/mm/vmpressure.c b/mm/vmpressure.c index b52644771cc43..11a4b0e3b583c 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; * essence, they are percents: the higher the value, the more number * unsuccessful reclaims there were. */ +#ifdef CONFIG_CACHY +static const unsigned int vmpressure_level_med = 65; +#else static const unsigned int vmpressure_level_med = 60; +#endif static const unsigned int vmpressure_level_critical = 95; /* diff --git a/mm/vmscan.c b/mm/vmscan.c index f7d9a683e3a7d..82b19744d7b0d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -176,7 +176,11 @@ struct scan_control { /* * From 0 .. 200. Higher means more swappy. */ +#ifdef CONFIG_CACHY +int vm_swappiness = 20; +#else int vm_swappiness = 60; +#endif static void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs) diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index eb31c7158b39c..8cefa0b62fb90 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) * having to remove and re-insert us on the wait queue. */ for (;;) { - prepare_to_wait_exclusive(sk_sleep(sk), &wait, + prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); release_sock(sk); if (reqsk_queue_empty(&icsk->icsk_accept_queue)) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index dd41c2c6459db..bc94259326ee1 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4662,8 +4662,8 @@ void __init tcp_init(void) tcp_init_mem(); /* Set per-socket limits to no more than 1/128 the pressure threshold */ limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); - max_wshare = min(4UL*1024*1024, limit); - max_rshare = min(6UL*1024*1024, limit); + max_wshare = min(16UL*1024*1024, limit); + max_rshare = min(16UL*1024*1024, limit); init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM; init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; diff --git a/tools/include/asm-generic/bitops/non-atomic.h b/tools/include/asm-generic/bitops/non-atomic.h index 7e10c4b50c5db..0c472a8334083 100644 --- a/tools/include/asm-generic/bitops/non-atomic.h +++ b/tools/include/asm-generic/bitops/non-atomic.h @@ -2,10 +2,10 @@ #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ -#include +#include /** - * __set_bit - Set a bit in memory + * ___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -13,7 +13,8 @@ * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -21,7 +22,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr) *p |= mask; } -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -30,7 +32,7 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) } /** - * __change_bit - Toggle a bit in memory + * ___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -38,7 +40,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -47,7 +50,7 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_set_bit - Set a bit and return its old value + * ___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -55,7 +58,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -66,7 +70,7 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * ___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -74,7 +78,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -85,8 +90,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) +static __always_inline bool +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -97,11 +102,12 @@ static inline int __test_and_change_bit(int nr, } /** - * test_bit - Determine whether a bit is set + * _test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ -static inline int test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index afdf93bebaaf1..ae1852e391423 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -11,10 +11,10 @@ #define DECLARE_BITMAP(name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] -int __bitmap_weight(const unsigned long *bitmap, int bits); +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); bool __bitmap_equal(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); @@ -45,7 +45,7 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits); } -static inline int bitmap_empty(const unsigned long *src, unsigned nbits) +static inline bool bitmap_empty(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -53,7 +53,7 @@ static inline int bitmap_empty(const unsigned long *src, unsigned nbits) return find_first_bit(src, nbits) == nbits; } -static inline int bitmap_full(const unsigned long *src, unsigned int nbits) +static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); @@ -61,7 +61,7 @@ static inline int bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static inline unsigned long bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -146,7 +146,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, * @src2: operand 2 * @nbits: size of bitmap */ -static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) diff --git a/tools/include/linux/bitops.h b/tools/include/linux/bitops.h index 5fca38fe1ba83..f18683b95ea67 100644 --- a/tools/include/linux/bitops.h +++ b/tools/include/linux/bitops.h @@ -25,6 +25,22 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +/* + * Defined here because those may be needed by architecture-specific static + * inlines. + */ + +#define bitop(op, nr, addr) \ + op(nr, addr) + +#define __set_bit(nr, addr) bitop(___set_bit, nr, addr) +#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) +#define __change_bit(nr, addr) bitop(___change_bit, nr, addr) +#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr) +#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) +#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) +#define test_bit(nr, addr) bitop(_test_bit, nr, addr) + /* * Include this here because some architectures need generic_ffs/fls in * scope diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c index 354f8cdc08803..e1fafc131a49c 100644 --- a/tools/lib/bitmap.c +++ b/tools/lib/bitmap.c @@ -5,9 +5,9 @@ */ #include -int __bitmap_weight(const unsigned long *bitmap, int bits) +unsigned long __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - int k, w = 0, lim = bits/BITS_PER_LONG; + unsigned long k, w = 0, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; k++) w += hweight_long(bitmap[k]); @@ -57,7 +57,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, return ret; } -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; -- 2.37.1 From c1a27a51d0d5f21eacf676b29821dcf3c0ce3106 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:31:38 +0200 Subject: [PATCH 03/16] sched Signed-off-by: Peter Jung --- drivers/powercap/dtpm_cpu.c | 33 +- drivers/thermal/cpufreq_cooling.c | 6 +- include/linux/cgroup-defs.h | 4 + include/linux/kernel_stat.h | 7 + include/linux/sched.h | 3 +- include/linux/sched/rt.h | 8 - include/linux/sched/topology.h | 1 + kernel/cgroup/rstat.c | 44 +- kernel/rseq.c | 23 +- kernel/sched/core.c | 215 +++--- kernel/sched/core_sched.c | 15 +- kernel/sched/cpudeadline.c | 2 +- kernel/sched/cpufreq_schedutil.c | 5 +- kernel/sched/cputime.c | 15 + kernel/sched/deadline.c | 34 +- kernel/sched/fair.c | 875 ++++++++++++++-------- kernel/sched/features.h | 5 +- kernel/sched/pelt.h | 40 +- kernel/sched/rt.c | 19 +- kernel/sched/sched.h | 98 ++- kernel/sched/topology.c | 23 +- tools/testing/selftests/rseq/rseq-riscv.h | 50 +- tools/testing/selftests/rseq/rseq.c | 3 +- 23 files changed, 970 insertions(+), 558 deletions(-) diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index f5eced0842b36..6a88eb7e9f757 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -71,34 +71,19 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) static u64 scale_pd_power_uw(struct cpumask *pd_mask, u64 power) { - unsigned long max = 0, sum_util = 0; + unsigned long max, sum_util = 0; int cpu; - for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { - - /* - * The capacity is the same for all CPUs belonging to - * the same perf domain, so a single call to - * arch_scale_cpu_capacity() is enough. However, we - * need the CPU parameter to be initialized by the - * loop, so the call ends up in this block. - * - * We can initialize 'max' with a cpumask_first() call - * before the loop but the bits computation is not - * worth given the arch_scale_cpu_capacity() just - * returns a value where the resulting assembly code - * will be optimized by the compiler. - */ - max = arch_scale_cpu_capacity(cpu); - sum_util += sched_cpu_util(cpu, max); - } - /* - * In the improbable case where all the CPUs of the perf - * domain are offline, 'max' will be zero and will lead to an - * illegal operation with a zero division. + * The capacity is the same for all CPUs belonging to + * the same perf domain. */ - return max ? (power * ((sum_util << 10) / max)) >> 10 : 0; + max = arch_scale_cpu_capacity(cpumask_first(pd_mask)); + + for_each_cpu_and(cpu, pd_mask, cpu_online_mask) + sum_util += sched_cpu_util(cpu); + + return (power * ((sum_util << 10) / max)) >> 10; } static u64 get_pd_power_uw(struct dtpm *dtpm) diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index b8151d95a8068..b263b0fde03ca 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -137,11 +137,9 @@ static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu, int cpu_idx) { - unsigned long max = arch_scale_cpu_capacity(cpu); - unsigned long util; + unsigned long util = sched_cpu_util(cpu); - util = sched_cpu_util(cpu, max); - return (util * 100) / max; + return (util * 100) / arch_scale_cpu_capacity(cpu); } #else /* !CONFIG_SMP */ static u32 get_load(struct cpufreq_cooling_device *cpufreq_cdev, int cpu, diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index d4427d0a0e187..187b54a9567ec 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -288,6 +288,10 @@ struct css_set { struct cgroup_base_stat { struct task_cputime cputime; + +#ifdef CONFIG_SCHED_CORE + u64 forceidle_sum; +#endif }; /* diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 69ae6b2784645..ddb5a358fd829 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -28,6 +28,9 @@ enum cpu_usage_stat { CPUTIME_STEAL, CPUTIME_GUEST, CPUTIME_GUEST_NICE, +#ifdef CONFIG_SCHED_CORE + CPUTIME_FORCEIDLE, +#endif NR_STATS, }; @@ -115,4 +118,8 @@ extern void account_process_tick(struct task_struct *, int user); extern void account_idle_ticks(unsigned long ticks); +#ifdef CONFIG_SCHED_CORE +extern void __account_forceidle_time(struct task_struct *tsk, u64 delta); +#endif + #endif /* _LINUX_KERNEL_STAT_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 6258b32dffd78..0b3052d3a0511 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -539,6 +539,7 @@ struct sched_entity { struct rb_node run_node; struct list_head group_node; unsigned int on_rq; + unsigned int migrated; u64 exec_start; u64 sum_exec_runtime; @@ -2258,7 +2259,7 @@ static inline bool owner_on_cpu(struct task_struct *owner) } /* Returns effective CPU energy utilization, as seen by the scheduler */ -unsigned long sched_cpu_util(int cpu, unsigned long max); +unsigned long sched_cpu_util(int cpu); #endif /* CONFIG_SMP */ #ifdef CONFIG_RSEQ diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index e5af028c08b49..994c25640e156 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -39,20 +39,12 @@ static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *p) } extern void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task); extern void rt_mutex_adjust_pi(struct task_struct *p); -static inline bool tsk_is_pi_blocked(struct task_struct *tsk) -{ - return tsk->pi_blocked_on != NULL; -} #else static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) { return NULL; } # define rt_mutex_adjust_pi(p) do { } while (0) -static inline bool tsk_is_pi_blocked(struct task_struct *tsk) -{ - return false; -} #endif extern void normalize_rt_tasks(void); diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 56cffe42abbc4..816df6cc444e1 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -81,6 +81,7 @@ struct sched_domain_shared { atomic_t ref; atomic_t nr_busy_cpus; int has_idle_cores; + int nr_idle_scan; }; struct sched_domain { diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index 24b5c2ab55983..feb59380c8962 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -310,6 +310,9 @@ static void cgroup_base_stat_add(struct cgroup_base_stat *dst_bstat, dst_bstat->cputime.utime += src_bstat->cputime.utime; dst_bstat->cputime.stime += src_bstat->cputime.stime; dst_bstat->cputime.sum_exec_runtime += src_bstat->cputime.sum_exec_runtime; +#ifdef CONFIG_SCHED_CORE + dst_bstat->forceidle_sum += src_bstat->forceidle_sum; +#endif } static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat, @@ -318,6 +321,9 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat, dst_bstat->cputime.utime -= src_bstat->cputime.utime; dst_bstat->cputime.stime -= src_bstat->cputime.stime; dst_bstat->cputime.sum_exec_runtime -= src_bstat->cputime.sum_exec_runtime; +#ifdef CONFIG_SCHED_CORE + dst_bstat->forceidle_sum -= src_bstat->forceidle_sum; +#endif } static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu) @@ -398,6 +404,11 @@ void __cgroup_account_cputime_field(struct cgroup *cgrp, case CPUTIME_SOFTIRQ: rstatc->bstat.cputime.stime += delta_exec; break; +#ifdef CONFIG_SCHED_CORE + case CPUTIME_FORCEIDLE: + rstatc->bstat.forceidle_sum += delta_exec; + break; +#endif default: break; } @@ -411,8 +422,9 @@ void __cgroup_account_cputime_field(struct cgroup *cgrp, * with how it is done by __cgroup_account_cputime_field for each bit of * cpu time attributed to a cgroup. */ -static void root_cgroup_cputime(struct task_cputime *cputime) +static void root_cgroup_cputime(struct cgroup_base_stat *bstat) { + struct task_cputime *cputime = &bstat->cputime; int i; cputime->stime = 0; @@ -438,6 +450,10 @@ static void root_cgroup_cputime(struct task_cputime *cputime) cputime->sum_exec_runtime += user; cputime->sum_exec_runtime += sys; cputime->sum_exec_runtime += cpustat[CPUTIME_STEAL]; + +#ifdef CONFIG_SCHED_CORE + bstat->forceidle_sum += cpustat[CPUTIME_FORCEIDLE]; +#endif } } @@ -445,27 +461,43 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq) { struct cgroup *cgrp = seq_css(seq)->cgroup; u64 usage, utime, stime; - struct task_cputime cputime; + struct cgroup_base_stat bstat; +#ifdef CONFIG_SCHED_CORE + u64 forceidle_time; +#endif if (cgroup_parent(cgrp)) { cgroup_rstat_flush_hold(cgrp); usage = cgrp->bstat.cputime.sum_exec_runtime; cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime, &utime, &stime); +#ifdef CONFIG_SCHED_CORE + forceidle_time = cgrp->bstat.forceidle_sum; +#endif cgroup_rstat_flush_release(); } else { - root_cgroup_cputime(&cputime); - usage = cputime.sum_exec_runtime; - utime = cputime.utime; - stime = cputime.stime; + root_cgroup_cputime(&bstat); + usage = bstat.cputime.sum_exec_runtime; + utime = bstat.cputime.utime; + stime = bstat.cputime.stime; +#ifdef CONFIG_SCHED_CORE + forceidle_time = bstat.forceidle_sum; +#endif } do_div(usage, NSEC_PER_USEC); do_div(utime, NSEC_PER_USEC); do_div(stime, NSEC_PER_USEC); +#ifdef CONFIG_SCHED_CORE + do_div(forceidle_time, NSEC_PER_USEC); +#endif seq_printf(seq, "usage_usec %llu\n" "user_usec %llu\n" "system_usec %llu\n", usage, utime, stime); + +#ifdef CONFIG_SCHED_CORE + seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time); +#endif } diff --git a/kernel/rseq.c b/kernel/rseq.c index 97ac20b4f7387..bda8175f8f993 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -18,8 +18,9 @@ #define CREATE_TRACE_POINTS #include -#define RSEQ_CS_PREEMPT_MIGRATE_FLAGS (RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE | \ - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT) +#define RSEQ_CS_NO_RESTART_FLAGS (RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT | \ + RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL | \ + RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE) /* * @@ -175,23 +176,15 @@ static int rseq_need_restart(struct task_struct *t, u32 cs_flags) u32 flags, event_mask; int ret; + if (WARN_ON_ONCE(cs_flags & RSEQ_CS_NO_RESTART_FLAGS) || cs_flags) + return -EINVAL; + /* Get thread flags. */ ret = get_user(flags, &t->rseq->flags); if (ret) return ret; - /* Take critical section flags into account. */ - flags |= cs_flags; - - /* - * Restart on signal can only be inhibited when restart on - * preempt and restart on migrate are inhibited too. Otherwise, - * a preempted signal handler could fail to restart the prior - * execution context on sigreturn. - */ - if (unlikely((flags & RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL) && - (flags & RSEQ_CS_PREEMPT_MIGRATE_FLAGS) != - RSEQ_CS_PREEMPT_MIGRATE_FLAGS)) + if (WARN_ON_ONCE(flags & RSEQ_CS_NO_RESTART_FLAGS) || flags) return -EINVAL; /* @@ -203,7 +196,7 @@ static int rseq_need_restart(struct task_struct *t, u32 cs_flags) t->rseq_event_mask = 0; preempt_enable(); - return !!(event_mask & ~flags); + return !!event_mask; } static int clear_rseq_cs(struct task_struct *t) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecdc..5555e49c4e120 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -873,15 +873,11 @@ static inline void hrtick_rq_init(struct rq *rq) ({ \ typeof(ptr) _ptr = (ptr); \ typeof(mask) _mask = (mask); \ - typeof(*_ptr) _old, _val = *_ptr; \ + typeof(*_ptr) _val = *_ptr; \ \ - for (;;) { \ - _old = cmpxchg(_ptr, _val, _val | _mask); \ - if (_old == _val) \ - break; \ - _val = _old; \ - } \ - _old; \ + do { \ + } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ + _val; \ }) #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) @@ -890,7 +886,7 @@ static inline void hrtick_rq_init(struct rq *rq) * this avoids any races wrt polling state changes and thereby avoids * spurious IPIs. */ -static bool set_nr_and_not_polling(struct task_struct *p) +static inline bool set_nr_and_not_polling(struct task_struct *p) { struct thread_info *ti = task_thread_info(p); return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); @@ -905,30 +901,28 @@ static bool set_nr_and_not_polling(struct task_struct *p) static bool set_nr_if_polling(struct task_struct *p) { struct thread_info *ti = task_thread_info(p); - typeof(ti->flags) old, val = READ_ONCE(ti->flags); + typeof(ti->flags) val = READ_ONCE(ti->flags); for (;;) { if (!(val & _TIF_POLLING_NRFLAG)) return false; if (val & _TIF_NEED_RESCHED) return true; - old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); - if (old == val) + if (try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)) break; - val = old; } return true; } #else -static bool set_nr_and_not_polling(struct task_struct *p) +static inline bool set_nr_and_not_polling(struct task_struct *p) { set_tsk_need_resched(p); return true; } #ifdef CONFIG_SMP -static bool set_nr_if_polling(struct task_struct *p) +static inline bool set_nr_if_polling(struct task_struct *p) { return false; } @@ -3808,7 +3802,7 @@ bool cpus_share_cache(int this_cpu, int that_cpu) return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } -static inline bool ttwu_queue_cond(int cpu, int wake_flags) +static inline bool ttwu_queue_cond(int cpu) { /* * Do not complicate things with the async wake_list while the CPU is @@ -3824,13 +3818,21 @@ static inline bool ttwu_queue_cond(int cpu, int wake_flags) if (!cpus_share_cache(smp_processor_id(), cpu)) return true; + if (cpu == smp_processor_id()) + return false; + /* - * If the task is descheduling and the only running task on the - * CPU then use the wakelist to offload the task activation to - * the soon-to-be-idle CPU as the current CPU is likely busy. - * nr_running is checked to avoid unnecessary task stacking. + * If the wakee cpu is idle, or the task is descheduling and the + * only running task on the CPU, then use the wakelist to offload + * the task activation to the idle (or soon-to-be-idle) CPU as + * the current CPU is likely busy. nr_running is checked to + * avoid unnecessary task stacking. + * + * Note that we can only get here with (wakee) p->on_rq=0, + * p->on_cpu can be whatever, we've done the dequeue, so + * the wakee has been accounted out of ->nr_running. */ - if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1) + if (!cpu_rq(cpu)->nr_running) return true; return false; @@ -3838,10 +3840,7 @@ static inline bool ttwu_queue_cond(int cpu, int wake_flags) static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) { - if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) { - if (WARN_ON_ONCE(cpu == smp_processor_id())) - return false; - + if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu)) { sched_clock_cpu(cpu); /* Sync clocks across CPUs */ __ttwu_queue_wakelist(p, cpu, wake_flags); return true; @@ -4163,7 +4162,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) * scheduling. */ if (smp_load_acquire(&p->on_cpu) && - ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU)) + ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) goto unlock; /* @@ -4753,7 +4752,8 @@ static inline void prepare_task(struct task_struct *next) * Claim the task as running, we do this before switching to it * such that any running task will have this set. * - * See the ttwu() WF_ON_CPU case and its ordering comment. + * See the smp_load_acquire(&p->on_cpu) case in ttwu() and + * its ordering comment. */ WRITE_ONCE(next->on_cpu, 1); #endif @@ -6500,8 +6500,12 @@ static inline void sched_submit_work(struct task_struct *tsk) io_wq_worker_sleeping(tsk); } - if (tsk_is_pi_blocked(tsk)) - return; + /* + * spinlock and rwlock must not flush block requests. This will + * deadlock if the callback attempts to acquire a lock which is + * already acquired. + */ + SCHED_WARN_ON(current->__state & TASK_RTLOCK_WAIT); /* * If we are going to sleep and we have plugged IO queued, @@ -6998,17 +7002,29 @@ void set_user_nice(struct task_struct *p, long nice) EXPORT_SYMBOL(set_user_nice); /* - * can_nice - check if a task can reduce its nice value + * is_nice_reduction - check if nice value is an actual reduction + * + * Similar to can_nice() but does not perform a capability check. + * * @p: task * @nice: nice value */ -int can_nice(const struct task_struct *p, const int nice) +static bool is_nice_reduction(const struct task_struct *p, const int nice) { /* Convert nice value [19,-20] to rlimit style value [1,40]: */ int nice_rlim = nice_to_rlimit(nice); - return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || - capable(CAP_SYS_NICE)); + return (nice_rlim <= task_rlimit(p, RLIMIT_NICE)); +} + +/* + * can_nice - check if a task can reduce its nice value + * @p: task + * @nice: nice value + */ +int can_nice(const struct task_struct *p, const int nice) +{ + return is_nice_reduction(p, nice) || capable(CAP_SYS_NICE); } #ifdef __ARCH_WANT_SYS_NICE @@ -7137,12 +7153,14 @@ struct task_struct *idle_task(int cpu) * required to meet deadlines. */ unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, - unsigned long max, enum cpu_util_type type, + enum cpu_util_type type, struct task_struct *p) { - unsigned long dl_util, util, irq; + unsigned long dl_util, util, irq, max; struct rq *rq = cpu_rq(cpu); + max = arch_scale_cpu_capacity(cpu); + if (!uclamp_is_used() && type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { return max; @@ -7222,10 +7240,9 @@ unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, return min(max, util); } -unsigned long sched_cpu_util(int cpu, unsigned long max) +unsigned long sched_cpu_util(int cpu) { - return effective_cpu_util(cpu, cpu_util_cfs(cpu), max, - ENERGY_UTIL, NULL); + return effective_cpu_util(cpu, cpu_util_cfs(cpu), ENERGY_UTIL, NULL); } #endif /* CONFIG_SMP */ @@ -7287,6 +7304,69 @@ static bool check_same_owner(struct task_struct *p) return match; } +/* + * Allow unprivileged RT tasks to decrease priority. + * Only issue a capable test if needed and only once to avoid an audit + * event on permitted non-privileged operations: + */ +static int user_check_sched_setscheduler(struct task_struct *p, + const struct sched_attr *attr, + int policy, int reset_on_fork) +{ + if (fair_policy(policy)) { + if (attr->sched_nice < task_nice(p) && + !is_nice_reduction(p, attr->sched_nice)) + goto req_priv; + } + + if (rt_policy(policy)) { + unsigned long rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO); + + /* Can't set/change the rt policy: */ + if (policy != p->policy && !rlim_rtprio) + goto req_priv; + + /* Can't increase priority: */ + if (attr->sched_priority > p->rt_priority && + attr->sched_priority > rlim_rtprio) + goto req_priv; + } + + /* + * Can't set/change SCHED_DEADLINE policy at all for now + * (safest behavior); in the future we would like to allow + * unprivileged DL tasks to increase their relative deadline + * or reduce their runtime (both ways reducing utilization) + */ + if (dl_policy(policy)) + goto req_priv; + + /* + * Treat SCHED_IDLE as nice 20. Only allow a switch to + * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. + */ + if (task_has_idle_policy(p) && !idle_policy(policy)) { + if (!is_nice_reduction(p, task_nice(p))) + goto req_priv; + } + + /* Can't change other user's priorities: */ + if (!check_same_owner(p)) + goto req_priv; + + /* Normal users shall not reset the sched_reset_on_fork flag: */ + if (p->sched_reset_on_fork && !reset_on_fork) + goto req_priv; + + return 0; + +req_priv: + if (!capable(CAP_SYS_NICE)) + return -EPERM; + + return 0; +} + static int __sched_setscheduler(struct task_struct *p, const struct sched_attr *attr, bool user, bool pi) @@ -7328,58 +7408,11 @@ static int __sched_setscheduler(struct task_struct *p, (rt_policy(policy) != (attr->sched_priority != 0))) return -EINVAL; - /* - * Allow unprivileged RT tasks to decrease priority: - */ - if (user && !capable(CAP_SYS_NICE)) { - if (fair_policy(policy)) { - if (attr->sched_nice < task_nice(p) && - !can_nice(p, attr->sched_nice)) - return -EPERM; - } - - if (rt_policy(policy)) { - unsigned long rlim_rtprio = - task_rlimit(p, RLIMIT_RTPRIO); - - /* Can't set/change the rt policy: */ - if (policy != p->policy && !rlim_rtprio) - return -EPERM; - - /* Can't increase priority: */ - if (attr->sched_priority > p->rt_priority && - attr->sched_priority > rlim_rtprio) - return -EPERM; - } - - /* - * Can't set/change SCHED_DEADLINE policy at all for now - * (safest behavior); in the future we would like to allow - * unprivileged DL tasks to increase their relative deadline - * or reduce their runtime (both ways reducing utilization) - */ - if (dl_policy(policy)) - return -EPERM; - - /* - * Treat SCHED_IDLE as nice 20. Only allow a switch to - * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. - */ - if (task_has_idle_policy(p) && !idle_policy(policy)) { - if (!can_nice(p, task_nice(p))) - return -EPERM; - } - - /* Can't change other user's priorities: */ - if (!check_same_owner(p)) - return -EPERM; - - /* Normal users shall not reset the sched_reset_on_fork flag: */ - if (p->sched_reset_on_fork && !reset_on_fork) - return -EPERM; - } - if (user) { + retval = user_check_sched_setscheduler(p, attr, policy, reset_on_fork); + if (retval) + return retval; + if (attr->sched_flags & SCHED_FLAG_SUGOV) return -EINVAL; @@ -9531,7 +9564,7 @@ static struct kmem_cache *task_group_cache __read_mostly; #endif DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); -DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); +DECLARE_PER_CPU(cpumask_var_t, select_rq_mask); void __init sched_init(void) { @@ -9580,7 +9613,7 @@ void __init sched_init(void) for_each_possible_cpu(i) { per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( cpumask_size(), GFP_KERNEL, cpu_to_node(i)); - per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node( + per_cpu(select_rq_mask, i) = (cpumask_var_t)kzalloc_node( cpumask_size(), GFP_KERNEL, cpu_to_node(i)); } #endif /* CONFIG_CPUMASK_OFFSTACK */ diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index 38a2cec21014d..93878cb2a46dc 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -56,7 +56,6 @@ static unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long old_cookie; struct rq_flags rf; struct rq *rq; - bool enqueued; rq = task_rq_lock(p, &rf); @@ -68,14 +67,16 @@ static unsigned long sched_core_update_cookie(struct task_struct *p, */ SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq)); - enqueued = sched_core_enqueued(p); - if (enqueued) + if (sched_core_enqueued(p)) sched_core_dequeue(rq, p, DEQUEUE_SAVE); old_cookie = p->core_cookie; p->core_cookie = cookie; - if (enqueued) + /* + * Consider the cases: !prev_cookie and !cookie. + */ + if (cookie && task_on_rq_queued(p)) sched_core_enqueue(rq, p); /* @@ -277,7 +278,11 @@ void __sched_core_account_forceidle(struct rq *rq) if (p == rq_i->idle) continue; - __schedstat_add(p->stats.core_forceidle_sum, delta); + /* + * Note: this will account forceidle to the current cpu, even + * if it comes from our SMT sibling. + */ + __account_forceidle_time(p, delta); } } diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c index 02d970a879edd..57c92d751bcd7 100644 --- a/kernel/sched/cpudeadline.c +++ b/kernel/sched/cpudeadline.c @@ -123,7 +123,7 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, unsigned long cap, max_cap = 0; int cpu, max_cpu = -1; - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return 1; /* Ensure the capacity of the CPUs fits the task. */ diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 3dbf351d12d52..1207c78f85c11 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -157,11 +157,10 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy, static void sugov_get_util(struct sugov_cpu *sg_cpu) { struct rq *rq = cpu_rq(sg_cpu->cpu); - unsigned long max = arch_scale_cpu_capacity(sg_cpu->cpu); - sg_cpu->max = max; + sg_cpu->max = arch_scale_cpu_capacity(sg_cpu->cpu); sg_cpu->bw_dl = cpu_bw_dl(rq); - sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu), max, + sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu), FREQUENCY_UTIL, NULL); } diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 78a233d43757f..95fc778537434 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -226,6 +226,21 @@ void account_idle_time(u64 cputime) cpustat[CPUTIME_IDLE] += cputime; } + +#ifdef CONFIG_SCHED_CORE +/* + * Account for forceidle time due to core scheduling. + * + * REQUIRES: schedstat is enabled. + */ +void __account_forceidle_time(struct task_struct *p, u64 delta) +{ + __schedstat_add(p->stats.core_forceidle_sum, delta); + + task_group_account_field(p, CPUTIME_FORCEIDLE, delta); +} +#endif + /* * When a guest is interrupted for a longer amount of time, missed clock * ticks are not redelivered later. Due to that, this function may on diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 7bf561262cb86..1d9c90958baa1 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -30,14 +30,16 @@ static struct ctl_table sched_dl_sysctls[] = { .data = &sysctl_sched_dl_period_max, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_douintvec_minmax, + .extra1 = (void *)&sysctl_sched_dl_period_min, }, { .procname = "sched_deadline_period_min_us", .data = &sysctl_sched_dl_period_min, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_douintvec_minmax, + .extra2 = (void *)&sysctl_sched_dl_period_max, }, {} }; @@ -122,15 +124,12 @@ static inline int dl_bw_cpus(int i) return cpus; } -static inline unsigned long __dl_bw_capacity(int i) +static inline unsigned long __dl_bw_capacity(const struct cpumask *mask) { - struct root_domain *rd = cpu_rq(i)->rd; unsigned long cap = 0; + int i; - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), - "sched RCU must be held"); - - for_each_cpu_and(i, rd->span, cpu_active_mask) + for_each_cpu_and(i, mask, cpu_active_mask) cap += capacity_orig_of(i); return cap; @@ -142,11 +141,14 @@ static inline unsigned long __dl_bw_capacity(int i) */ static inline unsigned long dl_bw_capacity(int i) { - if (!static_branch_unlikely(&sched_asym_cpucapacity) && + if (!sched_asym_cpucap_active() && capacity_orig_of(i) == SCHED_CAPACITY_SCALE) { return dl_bw_cpus(i) << SCHED_CAPACITY_SHIFT; } else { - return __dl_bw_capacity(i); + RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), + "sched RCU must be held"); + + return __dl_bw_capacity(cpu_rq(i)->rd->span); } } @@ -1847,7 +1849,7 @@ select_task_rq_dl(struct task_struct *p, int cpu, int flags) * Take the capacity of the CPU into account to * ensure it fits the requirement of the task. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) + if (sched_asym_cpucap_active()) select_rq |= !dl_task_fits_capacity(p, cpu); if (select_rq) { @@ -3005,17 +3007,15 @@ bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr) int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial) { - int ret = 1, trial_cpus; + unsigned long flags, cap; struct dl_bw *cur_dl_b; - unsigned long flags; + int ret = 1; rcu_read_lock_sched(); cur_dl_b = dl_bw_of(cpumask_any(cur)); - trial_cpus = cpumask_weight(trial); - + cap = __dl_bw_capacity(trial); raw_spin_lock_irqsave(&cur_dl_b->lock, flags); - if (cur_dl_b->bw != -1 && - cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw) + if (__dl_overflow(cur_dl_b, cap, 0, 0)) ret = 0; raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); rcu_read_unlock_sched(); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 78559747322f4..95b6183e070b6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -630,11 +630,8 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) } /* ensure we never gain time by being placed backwards. */ - cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); -#ifndef CONFIG_64BIT - smp_wmb(); - cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; -#endif + u64_u32_store(cfs_rq->min_vruntime, + max_vruntime(cfs_rq->min_vruntime, vruntime)); } static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) @@ -1066,6 +1063,7 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) /* * We are starting a new run period: */ + se->migrated = 0; se->exec_start = rq_clock_task(rq_of(cfs_rq)); } @@ -1073,6 +1071,33 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) * Scheduling class queueing methods: */ +#ifdef CONFIG_NUMA +#define NUMA_IMBALANCE_MIN 2 + +static inline long +adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) +{ + /* + * Allow a NUMA imbalance if busy CPUs is less than the maximum + * threshold. Above this threshold, individual tasks may be contending + * for both memory bandwidth and any shared HT resources. This is an + * approximation as the number of running tasks may not be related to + * the number of busy CPUs due to sched_setaffinity. + */ + if (dst_running > imb_numa_nr) + return imbalance; + + /* + * Allow a small imbalance based on a simple pair of communicating + * tasks that remain local when the destination is lightly loaded. + */ + if (imbalance <= NUMA_IMBALANCE_MIN) + return 0; + + return imbalance; +} +#endif /* CONFIG_NUMA */ + #ifdef CONFIG_NUMA_BALANCING /* * Approximate time to scan a full NUMA task in ms. The task scan period is @@ -1566,8 +1591,6 @@ struct task_numa_env { static unsigned long cpu_load(struct rq *rq); static unsigned long cpu_runnable(struct rq *rq); -static inline long adjust_numa_imbalance(int imbalance, - int dst_running, int imb_numa_nr); static inline enum numa_type numa_classify(unsigned int imbalance_pct, @@ -1808,6 +1831,15 @@ static bool task_numa_compare(struct task_numa_env *env, */ cur_ng = rcu_dereference(cur->numa_group); if (cur_ng == p_ng) { + /* + * Do not swap within a group or between tasks that have + * no group if there is spare capacity. Swapping does + * not address the load imbalance and helps one task at + * the cost of punishing another. + */ + if (env->dst_stats.node_type == node_has_spare) + goto unlock; + imp = taskimp + task_weight(cur, env->src_nid, dist) - task_weight(cur, env->dst_nid, dist); /* @@ -2903,6 +2935,7 @@ void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) p->node_stamp = 0; p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; p->numa_scan_period = sysctl_numa_balancing_scan_delay; + p->numa_migrate_retry = 0; /* Protect against double add, see task_tick_numa and task_numa_work */ p->numa_work.next = &p->numa_work; p->numa_faults = NULL; @@ -3162,6 +3195,8 @@ void reweight_task(struct task_struct *p, int prio) load->inv_weight = sched_prio_to_wmult[prio]; } +static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); + #ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP /* @@ -3272,8 +3307,6 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) } #endif /* CONFIG_SMP */ -static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); - /* * Recomputes the group entity based on the current state of its group * runqueue. @@ -3331,6 +3364,34 @@ static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) } #ifdef CONFIG_SMP +static inline bool load_avg_is_decayed(struct sched_avg *sa) +{ + if (sa->load_sum) + return false; + + if (sa->util_sum) + return false; + + if (sa->runnable_sum) + return false; + + /* + * _avg must be null when _sum are null because _avg = _sum / divider + * Make sure that rounding and/or propagation of PELT values never + * break this. + */ + SCHED_WARN_ON(sa->load_avg || + sa->util_avg || + sa->runnable_avg); + + return true; +} + +static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) +{ + return u64_u32_load_copy(cfs_rq->avg.last_update_time, + cfs_rq->last_update_time_copy); +} #ifdef CONFIG_FAIR_GROUP_SCHED /* * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list @@ -3363,27 +3424,12 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) if (cfs_rq->load.weight) return false; - if (cfs_rq->avg.load_sum) - return false; - - if (cfs_rq->avg.util_sum) - return false; - - if (cfs_rq->avg.runnable_sum) + if (!load_avg_is_decayed(&cfs_rq->avg)) return false; if (child_cfs_rq_on_list(cfs_rq)) return false; - /* - * _avg must be null when _sum are null because _avg = _sum / divider - * Make sure that rounding and/or propagation of PELT values never - * break this. - */ - SCHED_WARN_ON(cfs_rq->avg.load_avg || - cfs_rq->avg.util_avg || - cfs_rq->avg.runnable_avg); - return true; } @@ -3441,27 +3487,9 @@ void set_task_rq_fair(struct sched_entity *se, if (!(se->avg.last_update_time && prev)) return; -#ifndef CONFIG_64BIT - { - u64 p_last_update_time_copy; - u64 n_last_update_time_copy; - - do { - p_last_update_time_copy = prev->load_last_update_time_copy; - n_last_update_time_copy = next->load_last_update_time_copy; - - smp_rmb(); + p_last_update_time = cfs_rq_last_update_time(prev); + n_last_update_time = cfs_rq_last_update_time(next); - p_last_update_time = prev->avg.last_update_time; - n_last_update_time = next->avg.last_update_time; - - } while (p_last_update_time != p_last_update_time_copy || - n_last_update_time != n_last_update_time_copy); - } -#else - p_last_update_time = prev->avg.last_update_time; - n_last_update_time = next->avg.last_update_time; -#endif __update_load_avg_blocked_se(p_last_update_time, se); se->avg.last_update_time = n_last_update_time; } @@ -3740,6 +3768,89 @@ static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum #endif /* CONFIG_FAIR_GROUP_SCHED */ +#ifdef CONFIG_NO_HZ_COMMON +static inline void migrate_se_pelt_lag(struct sched_entity *se) +{ + u64 throttled = 0, now, lut; + struct cfs_rq *cfs_rq; + struct rq *rq; + bool is_idle; + + if (load_avg_is_decayed(&se->avg)) + return; + + cfs_rq = cfs_rq_of(se); + rq = rq_of(cfs_rq); + + rcu_read_lock(); + is_idle = is_idle_task(rcu_dereference(rq->curr)); + rcu_read_unlock(); + + /* + * The lag estimation comes with a cost we don't want to pay all the + * time. Hence, limiting to the case where the source CPU is idle and + * we know we are at the greatest risk to have an outdated clock. + */ + if (!is_idle) + return; + + /* + * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: + * + * last_update_time (the cfs_rq's last_update_time) + * = cfs_rq_clock_pelt()@cfs_rq_idle + * = rq_clock_pelt()@cfs_rq_idle + * - cfs->throttled_clock_pelt_time@cfs_rq_idle + * + * cfs_idle_lag (delta between rq's update and cfs_rq's update) + * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle + * + * rq_idle_lag (delta between now and rq's update) + * = sched_clock_cpu() - rq_clock()@rq_idle + * + * We can then write: + * + * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + + * sched_clock_cpu() - rq_clock()@rq_idle + * Where: + * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle + * rq_clock()@rq_idle is rq->clock_idle + * cfs->throttled_clock_pelt_time@cfs_rq_idle + * is cfs_rq->throttled_pelt_idle + */ + +#ifdef CONFIG_CFS_BANDWIDTH + throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); + /* The clock has been stopped for throttling */ + if (throttled == U64_MAX) + return; +#endif + now = u64_u32_load(rq->clock_pelt_idle); + /* + * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case + * is observed the old clock_pelt_idle value and the new clock_idle, + * which lead to an underestimation. The opposite would lead to an + * overestimation. + */ + smp_rmb(); + lut = cfs_rq_last_update_time(cfs_rq); + + now -= throttled; + if (now < lut) + /* + * cfs_rq->avg.last_update_time is more recent than our + * estimation, let's use it. + */ + now = lut; + else + now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); + + __update_load_avg_blocked_se(now, se); +} +#else +static void migrate_se_pelt_lag(struct sched_entity *se) {} +#endif + /** * update_cfs_rq_load_avg - update the cfs_rq's load/util averages * @now: current time, as per cfs_rq_clock_pelt() @@ -3814,12 +3925,9 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) } decayed |= __update_load_avg_cfs_rq(now, cfs_rq); - -#ifndef CONFIG_64BIT - smp_wmb(); - cfs_rq->load_last_update_time_copy = sa->last_update_time; -#endif - + u64_u32_store_copy(sa->last_update_time, + cfs_rq->last_update_time_copy, + sa->last_update_time); return decayed; } @@ -3951,27 +4059,6 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s } } -#ifndef CONFIG_64BIT -static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) -{ - u64 last_update_time_copy; - u64 last_update_time; - - do { - last_update_time_copy = cfs_rq->load_last_update_time_copy; - smp_rmb(); - last_update_time = cfs_rq->avg.last_update_time; - } while (last_update_time != last_update_time_copy); - - return last_update_time; -} -#else -static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) -{ - return cfs_rq->avg.last_update_time; -} -#endif - /* * Synchronize entity load avg of dequeued entity without locking * the previous rq. @@ -4194,7 +4281,7 @@ static inline int task_fits_capacity(struct task_struct *p, static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return; if (!p || p->nr_cpus_allowed == 1) { @@ -4273,17 +4360,18 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) { u64 vruntime = cfs_rq->min_vruntime; - /* - * The 'current' period is already promised to the current tasks, - * however the extra weight of the new task will slow them down a - * little, place the new task so that it fits in the slot that - * stays open at the end. - */ - if (initial && sched_feat(START_DEBIT)) - vruntime += sched_vslice(cfs_rq, se); + if (unlikely(initial)) { /* task creation is once, wakeup is often */ + /* + * The 'current' period is already promised to the current + * tasks, however the extra weight of the new task will slow + * them down a little, place the new task so that it fits in + * the slot that stays open at the end. + */ + if (sched_feat(START_DEBIT)) + vruntime += sched_vslice(cfs_rq, se); - /* sleeps up to a single latency don't count. */ - if (!initial) { + } else { + /* sleeps up to a single latency don't count. */ unsigned long thresh; if (se_is_idle(se)) @@ -4304,6 +4392,27 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); } + +static void place_entity_migrate(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + if (!sched_feat(PLACE_MIGRATE)) + return; + + if (cfs_rq->nr_running < se->migrated) { + /* + * Migrated to a shorter runqueue, go first because + * we were under-served on the old runqueue. + */ + se->vruntime = cfs_rq->min_vruntime; + return; + } + + /* + * Migrated to a longer runqueue, go last because + * we got over-served on the old runqueue. + */ + se->vruntime = cfs_rq->min_vruntime + sched_vslice(cfs_rq, se); +} static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -4378,6 +4487,8 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) if (flags & ENQUEUE_WAKEUP) place_entity(cfs_rq, se, 0); + else if (se->migrated) + place_entity_migrate(cfs_rq, se); check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); @@ -4386,16 +4497,11 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) __enqueue_entity(cfs_rq, se); se->on_rq = 1; - /* - * When bandwidth control is enabled, cfs might have been removed - * because of a parent been throttled but cfs->nr_running > 1. Try to - * add it unconditionally. - */ - if (cfs_rq->nr_running == 1 || cfs_bandwidth_used()) - list_add_leaf_cfs_rq(cfs_rq); - - if (cfs_rq->nr_running == 1) + if (cfs_rq->nr_running == 1) { check_enqueue_throttle(cfs_rq); + if (!throttled_hierarchy(cfs_rq)) + list_add_leaf_cfs_rq(cfs_rq); + } } static void __clear_buddies_last(struct sched_entity *se) @@ -4495,6 +4601,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) update_min_vruntime(cfs_rq); + + if (cfs_rq->nr_running == 0) + update_idle_cfs_rq_clock_pelt(cfs_rq); } /* @@ -5010,11 +5119,18 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) /* update hierarchical throttle state */ walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); - /* Nothing to run but something to decay (on_list)? Complete the branch */ if (!cfs_rq->load.weight) { - if (cfs_rq->on_list) - goto unthrottle_throttle; - return; + if (!cfs_rq->on_list) + return; + /* + * Nothing to run but something to decay (on_list)? + * Complete the branch. + */ + for_each_sched_entity(se) { + if (list_add_leaf_cfs_rq(cfs_rq_of(se))) + break; + } + goto unthrottle_throttle; } task_delta = cfs_rq->h_nr_running; @@ -5052,31 +5168,12 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) /* end evaluation on encountering a throttled cfs_rq */ if (cfs_rq_throttled(qcfs_rq)) goto unthrottle_throttle; - - /* - * One parent has been throttled and cfs_rq removed from the - * list. Add it back to not break the leaf list. - */ - if (throttled_hierarchy(qcfs_rq)) - list_add_leaf_cfs_rq(qcfs_rq); } /* At this point se is NULL and we are at root level*/ add_nr_running(rq, task_delta); unthrottle_throttle: - /* - * The cfs_rq_throttled() breaks in the above iteration can result in - * incomplete leaf list maintenance, resulting in triggering the - * assertion below. - */ - for_each_sched_entity(se) { - struct cfs_rq *qcfs_rq = cfs_rq_of(se); - - if (list_add_leaf_cfs_rq(qcfs_rq)) - break; - } - assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ @@ -5731,13 +5828,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) /* end evaluation on encountering a throttled cfs_rq */ if (cfs_rq_throttled(cfs_rq)) goto enqueue_throttle; - - /* - * One parent has been throttled and cfs_rq removed from the - * list. Add it back to not break the leaf list. - */ - if (throttled_hierarchy(cfs_rq)) - list_add_leaf_cfs_rq(cfs_rq); } /* At this point se is NULL and we are at root level*/ @@ -5761,21 +5851,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_overutilized_status(rq); enqueue_throttle: - if (cfs_bandwidth_used()) { - /* - * When bandwidth control is enabled; the cfs_rq_throttled() - * breaks in the above iteration can result in incomplete - * leaf list maintenance, resulting in triggering the assertion - * below. - */ - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - - if (list_add_leaf_cfs_rq(cfs_rq)) - break; - } - } - assert_list_leaf_cfs_rq(rq); hrtick_update(rq); @@ -5862,7 +5937,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) /* Working cpumask for: load_balance, load_balance_newidle. */ DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); -DEFINE_PER_CPU(cpumask_var_t, select_idle_mask); +DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); #ifdef CONFIG_NO_HZ_COMMON @@ -6352,8 +6427,9 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd */ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) { - struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); + struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); int i, cpu, idle_cpu = -1, nr = INT_MAX; + struct sched_domain_shared *sd_share; struct rq *this_rq = this_rq(); int this = smp_processor_id(); struct sched_domain *this_sd; @@ -6393,6 +6469,17 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool time = cpu_clock(this); } + if (sched_feat(SIS_UTIL)) { + sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); + if (sd_share) { + /* because !--nr is the condition to stop scan */ + nr = READ_ONCE(sd_share->nr_idle_scan) + 1; + /* overloaded LLC is unlikely to have idle cpu/core */ + if (nr == 1) + return -1; + } + } + for_each_cpu_wrap(cpu, cpus, target + 1) { if (has_idle_core) { i = select_idle_core(p, cpu, cpus, &idle_cpu); @@ -6438,7 +6525,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) int cpu, best_cpu = -1; struct cpumask *cpus; - cpus = this_cpu_cpumask_var_ptr(select_idle_mask); + cpus = this_cpu_cpumask_var_ptr(select_rq_mask); cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); task_util = uclamp_task_util(p); @@ -6462,7 +6549,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) static inline bool asym_fits_capacity(unsigned long task_util, int cpu) { - if (static_branch_unlikely(&sched_asym_cpucapacity)) + if (sched_asym_cpucap_active()) return fits_capacity(task_util, capacity_of(cpu)); return true; @@ -6482,13 +6569,13 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * On asymmetric system, update task utilization because we will check * that the task fits with cpu's capacity. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { sync_entity_load_avg(&p->se); task_util = uclamp_task_util(p); } /* - * per-cpu select_idle_mask usage + * per-cpu select_rq_mask usage */ lockdep_assert_irqs_disabled(); @@ -6536,7 +6623,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * For asymmetric CPU capacity systems, our domain of interest is * sd_asym_cpucapacity rather than sd_llc. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); /* * On an asymmetric CPU capacity system where an exclusive @@ -6658,62 +6745,96 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) } /* - * compute_energy(): Estimates the energy that @pd would consume if @p was - * migrated to @dst_cpu. compute_energy() predicts what will be the utilization - * landscape of @pd's CPUs after the task migration, and uses the Energy Model - * to compute what would be the energy if we decided to actually migrate that - * task. + * energy_env - Utilization landscape for energy estimation. + * @task_busy_time: Utilization contribution by the task for which we test the + * placement. Given by eenv_task_busy_time(). + * @pd_busy_time: Utilization of the whole perf domain without the task + * contribution. Given by eenv_pd_busy_time(). + * @cpu_cap: Maximum CPU capacity for the perf domain. + * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). */ -static long -compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +struct energy_env { + unsigned long task_busy_time; + unsigned long pd_busy_time; + unsigned long cpu_cap; + unsigned long pd_cap; +}; + +/* + * Compute the task busy time for compute_energy(). This time cannot be + * injected directly into effective_cpu_util() because of the IRQ scaling. + * The latter only makes sense with the most recent CPUs where the task has + * run. + */ +static inline void eenv_task_busy_time(struct energy_env *eenv, + struct task_struct *p, int prev_cpu) { - struct cpumask *pd_mask = perf_domain_span(pd); - unsigned long cpu_cap = arch_scale_cpu_capacity(cpumask_first(pd_mask)); - unsigned long max_util = 0, sum_util = 0; - unsigned long _cpu_cap = cpu_cap; + unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); + unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); + + if (unlikely(irq >= max_cap)) + busy_time = max_cap; + else + busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); + + eenv->task_busy_time = busy_time; +} + +/* + * Compute the perf_domain (PD) busy time for compute_energy(). Based on the + * utilization for each @pd_cpus, it however doesn't take into account + * clamping since the ratio (utilization / cpu_capacity) is already enough to + * scale the EM reported power consumption at the (eventually clamped) + * cpu_capacity. + * + * The contribution of the task @p for which we want to estimate the + * energy cost is removed (by cpu_util_next()) and must be calculated + * separately (see eenv_task_busy_time). This ensures: + * + * - A stable PD utilization, no matter which CPU of that PD we want to place + * the task on. + * + * - A fair comparison between CPUs as the task contribution (task_util()) + * will always be the same no matter which CPU utilization we rely on + * (util_avg or util_est). + * + * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't + * exceed @eenv->pd_cap. + */ +static inline void eenv_pd_busy_time(struct energy_env *eenv, + struct cpumask *pd_cpus, + struct task_struct *p) +{ + unsigned long busy_time = 0; int cpu; - _cpu_cap -= arch_scale_thermal_pressure(cpumask_first(pd_mask)); + for_each_cpu(cpu, pd_cpus) { + unsigned long util = cpu_util_next(cpu, p, -1); - /* - * The capacity state of CPUs of the current rd can be driven by CPUs - * of another rd if they belong to the same pd. So, account for the - * utilization of these CPUs too by masking pd with cpu_online_mask - * instead of the rd span. - * - * If an entire pd is outside of the current rd, it will not appear in - * its pd list and will not be accounted by compute_energy(). - */ - for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { - unsigned long util_freq = cpu_util_next(cpu, p, dst_cpu); - unsigned long cpu_util, util_running = util_freq; - struct task_struct *tsk = NULL; + busy_time += effective_cpu_util(cpu, util, ENERGY_UTIL, NULL); + } - /* - * When @p is placed on @cpu: - * - * util_running = max(cpu_util, cpu_util_est) + - * max(task_util, _task_util_est) - * - * while cpu_util_next is: max(cpu_util + task_util, - * cpu_util_est + _task_util_est) - */ - if (cpu == dst_cpu) { - tsk = p; - util_running = - cpu_util_next(cpu, p, -1) + task_util_est(p); - } + eenv->pd_busy_time = min(eenv->pd_cap, busy_time); +} - /* - * Busy time computation: utilization clamping is not - * required since the ratio (sum_util / cpu_capacity) - * is already enough to scale the EM reported power - * consumption at the (eventually clamped) cpu_capacity. - */ - cpu_util = effective_cpu_util(cpu, util_running, cpu_cap, - ENERGY_UTIL, NULL); +/* + * Compute the maximum utilization for compute_energy() when the task @p + * is placed on the cpu @dst_cpu. + * + * Returns the maximum utilization among @eenv->cpus. This utilization can't + * exceed @eenv->cpu_cap. + */ +static inline unsigned long +eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, + struct task_struct *p, int dst_cpu) +{ + unsigned long max_util = 0; + int cpu; - sum_util += min(cpu_util, _cpu_cap); + for_each_cpu(cpu, pd_cpus) { + struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; + unsigned long util = cpu_util_next(cpu, p, dst_cpu); + unsigned long cpu_util; /* * Performance domain frequency: utilization clamping @@ -6722,12 +6843,29 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * NOTE: in case RT tasks are running, by default the * FREQUENCY_UTIL's utilization can be max OPP. */ - cpu_util = effective_cpu_util(cpu, util_freq, cpu_cap, - FREQUENCY_UTIL, tsk); - max_util = max(max_util, min(cpu_util, _cpu_cap)); + cpu_util = effective_cpu_util(cpu, util, FREQUENCY_UTIL, tsk); + max_util = max(max_util, cpu_util); } - return em_cpu_energy(pd->em_pd, max_util, sum_util, _cpu_cap); + return min(max_util, eenv->cpu_cap); +} + +/* + * compute_energy(): Use the Energy Model to estimate the energy that @pd would + * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task + * contribution is ignored. + */ +static inline unsigned long +compute_energy(struct energy_env *eenv, struct perf_domain *pd, + struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) +{ + unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); + unsigned long busy_time = eenv->pd_busy_time; + + if (dst_cpu >= 0) + busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); + + return em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); } /* @@ -6771,12 +6909,13 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) */ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) { + struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; - struct root_domain *rd = cpu_rq(smp_processor_id())->rd; - int cpu, best_energy_cpu = prev_cpu, target = -1; - unsigned long cpu_cap, util, base_energy = 0; + struct root_domain *rd = this_rq()->rd; + int cpu, best_energy_cpu, target = -1; struct sched_domain *sd; struct perf_domain *pd; + struct energy_env eenv; rcu_read_lock(); pd = rcu_dereference(rd->pd); @@ -6799,20 +6938,39 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) if (!task_util_est(p)) goto unlock; + eenv_task_busy_time(&eenv, p, prev_cpu); + for (; pd; pd = pd->next) { - unsigned long cur_delta, spare_cap, max_spare_cap = 0; + unsigned long cpu_cap, cpu_thermal_cap, util; + unsigned long cur_delta, max_spare_cap = 0; bool compute_prev_delta = false; - unsigned long base_energy_pd; int max_spare_cap_cpu = -1; + unsigned long base_energy; + + cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); + + if (cpumask_empty(cpus)) + continue; + + /* Account thermal pressure for the energy estimation */ + cpu = cpumask_first(cpus); + cpu_thermal_cap = arch_scale_cpu_capacity(cpu); + cpu_thermal_cap -= arch_scale_thermal_pressure(cpu); + + eenv.cpu_cap = cpu_thermal_cap; + eenv.pd_cap = 0; + + for_each_cpu(cpu, cpus) { + eenv.pd_cap += cpu_thermal_cap; + + if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) + continue; - for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { if (!cpumask_test_cpu(cpu, p->cpus_ptr)) continue; util = cpu_util_next(cpu, p, cpu); cpu_cap = capacity_of(cpu); - spare_cap = cpu_cap; - lsub_positive(&spare_cap, util); /* * Skip CPUs that cannot satisfy the capacity request. @@ -6825,15 +6983,17 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) if (!fits_capacity(util, cpu_cap)) continue; + lsub_positive(&cpu_cap, util); + if (cpu == prev_cpu) { /* Always use prev_cpu as a candidate. */ compute_prev_delta = true; - } else if (spare_cap > max_spare_cap) { + } else if (cpu_cap > max_spare_cap) { /* * Find the CPU with the maximum spare capacity * in the performance domain. */ - max_spare_cap = spare_cap; + max_spare_cap = cpu_cap; max_spare_cap_cpu = cpu; } } @@ -6841,25 +7001,29 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) if (max_spare_cap_cpu < 0 && !compute_prev_delta) continue; + eenv_pd_busy_time(&eenv, cpus, p); /* Compute the 'base' energy of the pd, without @p */ - base_energy_pd = compute_energy(p, -1, pd); - base_energy += base_energy_pd; + base_energy = compute_energy(&eenv, pd, cpus, p, -1); /* Evaluate the energy impact of using prev_cpu. */ if (compute_prev_delta) { - prev_delta = compute_energy(p, prev_cpu, pd); - if (prev_delta < base_energy_pd) + prev_delta = compute_energy(&eenv, pd, cpus, p, + prev_cpu); + /* CPU utilization has changed */ + if (prev_delta < base_energy) goto unlock; - prev_delta -= base_energy_pd; + prev_delta -= base_energy; best_delta = min(best_delta, prev_delta); } /* Evaluate the energy impact of using max_spare_cap_cpu. */ if (max_spare_cap_cpu >= 0) { - cur_delta = compute_energy(p, max_spare_cap_cpu, pd); - if (cur_delta < base_energy_pd) + cur_delta = compute_energy(&eenv, pd, cpus, p, + max_spare_cap_cpu); + /* CPU utilization has changed */ + if (cur_delta < base_energy) goto unlock; - cur_delta -= base_energy_pd; + cur_delta -= base_energy; if (cur_delta < best_delta) { best_delta = cur_delta; best_energy_cpu = max_spare_cap_cpu; @@ -6868,12 +7032,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) } rcu_read_unlock(); - /* - * Pick the best CPU if prev_cpu cannot be used, or if it saves at - * least 6% of the energy used by prev_cpu. - */ - if ((prev_delta == ULONG_MAX) || - (prev_delta - best_delta) > ((prev_delta + base_energy) >> 4)) + if (best_delta < prev_delta) target = best_energy_cpu; return target; @@ -6969,6 +7128,8 @@ static void detach_entity_cfs_rq(struct sched_entity *se); */ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) { + struct sched_entity *se = &p->se; + /* * As blocked tasks retain absolute vruntime the migration needs to * deal with this by subtracting the old and adding the new @@ -6976,23 +7137,9 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) * the task on the new runqueue. */ if (READ_ONCE(p->__state) == TASK_WAKING) { - struct sched_entity *se = &p->se; struct cfs_rq *cfs_rq = cfs_rq_of(se); - u64 min_vruntime; - -#ifndef CONFIG_64BIT - u64 min_vruntime_copy; - - do { - min_vruntime_copy = cfs_rq->min_vruntime_copy; - smp_rmb(); - min_vruntime = cfs_rq->min_vruntime; - } while (min_vruntime != min_vruntime_copy); -#else - min_vruntime = cfs_rq->min_vruntime; -#endif - se->vruntime -= min_vruntime; + se->vruntime -= u64_u32_load(cfs_rq->min_vruntime); } if (p->on_rq == TASK_ON_RQ_MIGRATING) { @@ -7001,25 +7148,30 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) * rq->lock and can modify state directly. */ lockdep_assert_rq_held(task_rq(p)); - detach_entity_cfs_rq(&p->se); + detach_entity_cfs_rq(se); } else { + remove_entity_load_avg(se); + /* - * We are supposed to update the task to "current" time, then - * its up to date and ready to go to new CPU/cfs_rq. But we - * have difficulty in getting what current time is, so simply - * throw away the out-of-date time. This will result in the - * wakee task is less decayed, but giving the wakee more load - * sounds not bad. + * Here, the task's PELT values have been updated according to + * the current rq's clock. But if that clock hasn't been + * updated in a while, a substantial idle time will be missed, + * leading to an inflation after wake-up on the new rq. + * + * Estimate the missing time from the cfs_rq last_update_time + * and update sched_avg to improve the PELT continuity after + * migration. */ - remove_entity_load_avg(&p->se); + migrate_se_pelt_lag(se); } /* Tell new CPU we are migrated */ - p->se.avg.last_update_time = 0; + se->avg.last_update_time = 0; /* We have migrated, no longer consider this task hot */ - p->se.exec_start = 0; + for_each_sched_entity(se) + se->migrated = READ_ONCE(cfs_rq_of(se)->nr_running) + !se->on_rq; update_scan_period(p, new_cpu); } @@ -7603,8 +7755,8 @@ enum group_type { */ group_fully_busy, /* - * SD_ASYM_CPUCAPACITY only: One task doesn't fit with CPU's capacity - * and must be migrated to a more powerful CPU. + * One task doesn't fit with CPU's capacity and must be migrated to a + * more powerful CPU. */ group_misfit_task, /* @@ -7705,6 +7857,9 @@ static int task_hot(struct task_struct *p, struct lb_env *env) if (sysctl_sched_migration_cost == 0) return 0; + if (p->se.migrated) + return 0; + delta = rq_clock_task(env->src_rq) - p->se.exec_start; return delta < (s64)sysctl_sched_migration_cost; @@ -8185,6 +8340,9 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { update_tg_load_avg(cfs_rq); + if (cfs_rq->nr_running == 0) + update_idle_cfs_rq_clock_pelt(cfs_rq); + if (cfs_rq == &rq->cfs) decayed = true; } @@ -8518,7 +8676,7 @@ static inline int sg_imbalanced(struct sched_group *group) /* * group_has_capacity returns true if the group has spare capacity that could * be used by some tasks. - * We consider that a group has spare capacity if the * number of task is + * We consider that a group has spare capacity if the number of task is * smaller than the number of CPUs or if the utilization is lower than the * available capacity for CFS tasks. * For the latter, we use a threshold to stabilize the state, to take into @@ -8687,6 +8845,19 @@ sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu); } +static inline bool +sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) +{ + /* + * When there is more than 1 task, the group_overloaded case already + * takes care of cpu with reduced capacity + */ + if (rq->cfs.h_nr_running != 1) + return false; + + return check_cpu_capacity(rq, sd); +} + /** * update_sg_lb_stats - Update sched_group's statistics for load balancing. * @env: The load balancing environment. @@ -8709,8 +8880,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, for_each_cpu_and(i, sched_group_span(group), env->cpus) { struct rq *rq = cpu_rq(i); + unsigned long load = cpu_load(rq); - sgs->group_load += cpu_load(rq); + sgs->group_load += load; sgs->group_util += cpu_util_cfs(i); sgs->group_runnable += cpu_runnable(rq); sgs->sum_h_nr_running += rq->cfs.h_nr_running; @@ -8740,11 +8912,17 @@ static inline void update_sg_lb_stats(struct lb_env *env, if (local_group) continue; - /* Check for a misfit task on the cpu */ - if (env->sd->flags & SD_ASYM_CPUCAPACITY && - sgs->group_misfit_task_load < rq->misfit_task_load) { - sgs->group_misfit_task_load = rq->misfit_task_load; - *sg_status |= SG_OVERLOAD; + if (env->sd->flags & SD_ASYM_CPUCAPACITY) { + /* Check for a misfit task on the cpu */ + if (sgs->group_misfit_task_load < rq->misfit_task_load) { + sgs->group_misfit_task_load = rq->misfit_task_load; + *sg_status |= SG_OVERLOAD; + } + } else if ((env->idle != CPU_NOT_IDLE) && + sched_reduced_capacity(rq, env->sd)) { + /* Check for a task running on a CPU with reduced capacity */ + if (sgs->group_misfit_task_load < load) + sgs->group_misfit_task_load = load; } } @@ -8797,7 +8975,8 @@ static bool update_sd_pick_busiest(struct lb_env *env, * CPUs in the group should either be possible to resolve * internally or be covered by avg_load imbalance (eventually). */ - if (sgs->group_type == group_misfit_task && + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + (sgs->group_type == group_misfit_task) && (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || sds->local_stat.group_type != group_has_spare)) return false; @@ -9075,16 +9254,6 @@ static bool update_pick_idlest(struct sched_group *idlest, return true; } -/* - * Allow a NUMA imbalance if busy CPUs is less than 25% of the domain. - * This is an approximation as the number of running tasks may not be - * related to the number of busy CPUs due to sched_setaffinity. - */ -static inline bool allow_numa_imbalance(int running, int imb_numa_nr) -{ - return running <= imb_numa_nr; -} - /* * find_idlest_group() finds and returns the least busy CPU group within the * domain. @@ -9201,7 +9370,9 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) break; case group_has_spare: +#ifdef CONFIG_NUMA if (sd->flags & SD_NUMA) { + int imb_numa_nr = sd->imb_numa_nr; #ifdef CONFIG_NUMA_BALANCING int idlest_cpu; /* @@ -9214,17 +9385,31 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) idlest_cpu = cpumask_first(sched_group_span(idlest)); if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) return idlest; -#endif +#endif /* CONFIG_NUMA_BALANCING */ /* * Otherwise, keep the task close to the wakeup source * and improve locality if the number of running tasks * would remain below threshold where an imbalance is - * allowed. If there is a real need of migration, - * periodic load balance will take care of it. + * allowed while accounting for the possibility the + * task is pinned to a subset of CPUs. If there is a + * real need of migration, periodic load balance will + * take care of it. */ - if (allow_numa_imbalance(local_sgs.sum_nr_running + 1, sd->imb_numa_nr)) + if (p->nr_cpus_allowed != NR_CPUS) { + struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); + + cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); + imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); + } + + imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); + if (!adjust_numa_imbalance(imbalance, + local_sgs.sum_nr_running + 1, + imb_numa_nr)) { return NULL; + } } +#endif /* CONFIG_NUMA */ /* * Select group with highest number of idle CPUs. We could also @@ -9240,6 +9425,77 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) return idlest; } +static void update_idle_cpu_scan(struct lb_env *env, + unsigned long sum_util) +{ + struct sched_domain_shared *sd_share; + int llc_weight, pct; + u64 x, y, tmp; + /* + * Update the number of CPUs to scan in LLC domain, which could + * be used as a hint in select_idle_cpu(). The update of sd_share + * could be expensive because it is within a shared cache line. + * So the write of this hint only occurs during periodic load + * balancing, rather than CPU_NEWLY_IDLE, because the latter + * can fire way more frequently than the former. + */ + if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) + return; + + llc_weight = per_cpu(sd_llc_size, env->dst_cpu); + if (env->sd->span_weight != llc_weight) + return; + + sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); + if (!sd_share) + return; + + /* + * The number of CPUs to search drops as sum_util increases, when + * sum_util hits 85% or above, the scan stops. + * The reason to choose 85% as the threshold is because this is the + * imbalance_pct(117) when a LLC sched group is overloaded. + * + * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] + * and y'= y / SCHED_CAPACITY_SCALE + * + * x is the ratio of sum_util compared to the CPU capacity: + * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) + * y' is the ratio of CPUs to be scanned in the LLC domain, + * and the number of CPUs to scan is calculated by: + * + * nr_scan = llc_weight * y' [2] + * + * When x hits the threshold of overloaded, AKA, when + * x = 100 / pct, y drops to 0. According to [1], + * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 + * + * Scale x by SCHED_CAPACITY_SCALE: + * x' = sum_util / llc_weight; [3] + * + * and finally [1] becomes: + * y = SCHED_CAPACITY_SCALE - + * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] + * + */ + /* equation [3] */ + x = sum_util; + do_div(x, llc_weight); + + /* equation [4] */ + pct = env->sd->imbalance_pct; + tmp = x * x * pct * pct; + do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); + tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); + y = SCHED_CAPACITY_SCALE - tmp; + + /* equation [2] */ + y *= llc_weight; + do_div(y, SCHED_CAPACITY_SCALE); + if ((int)y != sd_share->nr_idle_scan) + WRITE_ONCE(sd_share->nr_idle_scan, (int)y); +} + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. @@ -9252,6 +9508,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd struct sched_group *sg = env->sd->groups; struct sg_lb_stats *local = &sds->local_stat; struct sg_lb_stats tmp_sgs; + unsigned long sum_util = 0; int sg_status = 0; do { @@ -9284,6 +9541,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd sds->total_load += sgs->group_load; sds->total_capacity += sgs->group_capacity; + sum_util += sgs->group_util; sg = sg->next; } while (sg != env->sd->groups); @@ -9309,24 +9567,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED); trace_sched_overutilized_tp(rd, SG_OVERUTILIZED); } -} - -#define NUMA_IMBALANCE_MIN 2 - -static inline long adjust_numa_imbalance(int imbalance, - int dst_running, int imb_numa_nr) -{ - if (!allow_numa_imbalance(dst_running, imb_numa_nr)) - return imbalance; - - /* - * Allow a small imbalance based on a simple pair of communicating - * tasks that remain local when the destination is lightly loaded. - */ - if (imbalance <= NUMA_IMBALANCE_MIN) - return 0; - return imbalance; + update_idle_cpu_scan(env, sum_util); } /** @@ -9343,9 +9585,18 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s busiest = &sds->busiest_stat; if (busiest->group_type == group_misfit_task) { - /* Set imbalance to allow misfit tasks to be balanced. */ - env->migration_type = migrate_misfit; - env->imbalance = 1; + if (env->sd->flags & SD_ASYM_CPUCAPACITY) { + /* Set imbalance to allow misfit tasks to be balanced. */ + env->migration_type = migrate_misfit; + env->imbalance = 1; + } else { + /* + * Set load imbalance to allow moving task from cpu + * with reduced capacity. + */ + env->migration_type = migrate_load; + env->imbalance = busiest->group_misfit_task_load; + } return; } @@ -9413,7 +9664,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s */ env->migration_type = migrate_task; lsub_positive(&nr_diff, local->sum_nr_running); - env->imbalance = nr_diff >> 1; + env->imbalance = nr_diff; } else { /* @@ -9421,15 +9672,21 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * idle cpus. */ env->migration_type = migrate_task; - env->imbalance = max_t(long, 0, (local->idle_cpus - - busiest->idle_cpus) >> 1); + env->imbalance = max_t(long, 0, + (local->idle_cpus - busiest->idle_cpus)); } +#ifdef CONFIG_NUMA /* Consider allowing a small imbalance between NUMA groups */ if (env->sd->flags & SD_NUMA) { env->imbalance = adjust_numa_imbalance(env->imbalance, - local->sum_nr_running + 1, env->sd->imb_numa_nr); + local->sum_nr_running + 1, + env->sd->imb_numa_nr); } +#endif + + /* Number of tasks to move to restore balance */ + env->imbalance >>= 1; return; } @@ -9852,9 +10109,15 @@ static int should_we_balance(struct lb_env *env) /* * In the newly idle case, we will allow all the CPUs * to do the newly idle load balance. + * + * However, we bail out if we already have tasks or a wakeup pending, + * to optimize wakeup latency. */ - if (env->idle == CPU_NEWLY_IDLE) + if (env->idle == CPU_NEWLY_IDLE) { + if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) + return 0; return 1; + } /* Try to find first idle CPU */ for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { @@ -11305,9 +11568,13 @@ static inline bool vruntime_normalized(struct task_struct *p) */ static void propagate_entity_cfs_rq(struct sched_entity *se) { - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + if (cfs_rq_throttled(cfs_rq)) + return; - list_add_leaf_cfs_rq(cfs_rq_of(se)); + if (!throttled_hierarchy(cfs_rq)) + list_add_leaf_cfs_rq(cfs_rq); /* Start to propagate at parent */ se = se->parent; @@ -11315,14 +11582,13 @@ static void propagate_entity_cfs_rq(struct sched_entity *se) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); - if (!cfs_rq_throttled(cfs_rq)){ - update_load_avg(cfs_rq, se, UPDATE_TG); - list_add_leaf_cfs_rq(cfs_rq); - continue; - } + update_load_avg(cfs_rq, se, UPDATE_TG); - if (list_add_leaf_cfs_rq(cfs_rq)) + if (cfs_rq_throttled(cfs_rq)) break; + + if (!throttled_hierarchy(cfs_rq)) + list_add_leaf_cfs_rq(cfs_rq); } } #else @@ -11440,10 +11706,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) void init_cfs_rq(struct cfs_rq *cfs_rq) { cfs_rq->tasks_timeline = RB_ROOT_CACHED; - cfs_rq->min_vruntime = (u64)(-(1LL << 20)); -#ifndef CONFIG_64BIT - cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; -#endif + u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20))); #ifdef CONFIG_SMP raw_spin_lock_init(&cfs_rq->removed.lock); #endif diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 1cf435bbcd9ca..d58baad2cee2b 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -60,7 +60,8 @@ SCHED_FEAT(TTWU_QUEUE, true) /* * When doing wakeups, attempt to limit superfluous scans of the LLC domain. */ -SCHED_FEAT(SIS_PROP, true) +SCHED_FEAT(SIS_PROP, false) +SCHED_FEAT(SIS_UTIL, true) /* * Issue a WARN when we do multiple update_rq_clock() calls @@ -100,3 +101,5 @@ SCHED_FEAT(LATENCY_WARN, false) SCHED_FEAT(ALT_PERIOD, true) SCHED_FEAT(BASE_SLICE, true) + +SCHED_FEAT(PLACE_MIGRATE, true) \ No newline at end of file diff --git a/kernel/sched/pelt.h b/kernel/sched/pelt.h index 4ff2ed4f8fa15..3a0e0dc287219 100644 --- a/kernel/sched/pelt.h +++ b/kernel/sched/pelt.h @@ -61,6 +61,25 @@ static inline void cfs_se_util_change(struct sched_avg *avg) WRITE_ONCE(avg->util_est.enqueued, enqueued); } +static inline u64 rq_clock_pelt(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + assert_clock_updated(rq); + + return rq->clock_pelt - rq->lost_idle_time; +} + +/* The rq is idle, we can sync to clock_task */ +static inline void _update_idle_rq_clock_pelt(struct rq *rq) +{ + rq->clock_pelt = rq_clock_task(rq); + + u64_u32_store(rq->clock_idle, rq_clock(rq)); + /* Paired with smp_rmb in migrate_se_pelt_lag() */ + smp_wmb(); + u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq)); +} + /* * The clock_pelt scales the time to reflect the effective amount of * computation done during the running delta time but then sync back to @@ -76,8 +95,7 @@ static inline void cfs_se_util_change(struct sched_avg *avg) static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) { if (unlikely(is_idle_task(rq->curr))) { - /* The rq is idle, we can sync to clock_task */ - rq->clock_pelt = rq_clock_task(rq); + _update_idle_rq_clock_pelt(rq); return; } @@ -130,17 +148,23 @@ static inline void update_idle_rq_clock_pelt(struct rq *rq) */ if (util_sum >= divider) rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; + + _update_idle_rq_clock_pelt(rq); } -static inline u64 rq_clock_pelt(struct rq *rq) +#ifdef CONFIG_CFS_BANDWIDTH +static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { - lockdep_assert_rq_held(rq); - assert_clock_updated(rq); + u64 throttled; - return rq->clock_pelt - rq->lost_idle_time; + if (unlikely(cfs_rq->throttle_count)) + throttled = U64_MAX; + else + throttled = cfs_rq->throttled_clock_pelt_time; + + u64_u32_store(cfs_rq->throttled_pelt_idle, throttled); } -#ifdef CONFIG_CFS_BANDWIDTH /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { @@ -150,6 +174,7 @@ static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time; } #else +static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { return rq_clock_pelt(rq_of(cfs_rq)); @@ -204,6 +229,7 @@ update_rq_clock_pelt(struct rq *rq, s64 delta) { } static inline void update_idle_rq_clock_pelt(struct rq *rq) { } +static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } #endif diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 8c9ed96648409..054b6711e9619 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -480,7 +480,7 @@ static inline void rt_queue_push_tasks(struct rq *rq) #endif /* CONFIG_SMP */ static void enqueue_top_rt_rq(struct rt_rq *rt_rq); -static void dequeue_top_rt_rq(struct rt_rq *rt_rq); +static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count); static inline int on_rt_rq(struct sched_rt_entity *rt_se) { @@ -509,7 +509,7 @@ static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) unsigned int cpu_cap; /* Only heterogeneous systems can benefit from this check */ - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return true; min_cap = uclamp_eff_value(p, UCLAMP_MIN); @@ -601,7 +601,7 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) rt_se = rt_rq->tg->rt_se[cpu]; if (!rt_se) { - dequeue_top_rt_rq(rt_rq); + dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ cpufreq_update_util(rq_of_rt_rq(rt_rq), 0); } @@ -687,7 +687,7 @@ static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) { - dequeue_top_rt_rq(rt_rq); + dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); } static inline int rt_rq_throttled(struct rt_rq *rt_rq) @@ -1089,7 +1089,7 @@ static void update_curr_rt(struct rq *rq) } static void -dequeue_top_rt_rq(struct rt_rq *rt_rq) +dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count) { struct rq *rq = rq_of_rt_rq(rt_rq); @@ -1100,7 +1100,7 @@ dequeue_top_rt_rq(struct rt_rq *rt_rq) BUG_ON(!rq->nr_running); - sub_nr_running(rq, rt_rq->rt_nr_running); + sub_nr_running(rq, count); rt_rq->rt_queued = 0; } @@ -1486,18 +1486,21 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) { struct sched_rt_entity *back = NULL; + unsigned int rt_nr_running; for_each_sched_rt_entity(rt_se) { rt_se->back = back; back = rt_se; } - dequeue_top_rt_rq(rt_rq_of_se(back)); + rt_nr_running = rt_rq_of_se(back)->rt_nr_running; for (rt_se = back; rt_se; rt_se = rt_se->back) { if (on_rt_rq(rt_se)) __dequeue_rt_entity(rt_se, flags); } + + dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running); } static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) @@ -1894,7 +1897,7 @@ static int find_lowest_rq(struct task_struct *task) * If we're on asym system ensure we consider the different capacities * of the CPUs when searching for the lowest_mask. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, lowest_mask, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 47b89a0fc6e55..3ccd35c22f0fd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -320,21 +320,6 @@ struct dl_bw { u64 total_bw; }; -/* - * Verify the fitness of task @p to run on @cpu taking into account the - * CPU original capacity and the runtime/deadline ratio of the task. - * - * The function will return true if the CPU original capacity of the - * @cpu scaled by SCHED_CAPACITY_SCALE >= runtime/deadline ratio of the - * task and false otherwise. - */ -static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) -{ - unsigned long cap = arch_scale_cpu_capacity(cpu); - - return cap_scale(p->dl.dl_deadline, cap) >= p->dl.dl_runtime; -} - extern void init_dl_bw(struct dl_bw *dl_b); extern int sched_dl_global_validate(void); extern void sched_dl_do_global(void); @@ -520,6 +505,45 @@ struct cfs_bandwidth { }; #endif /* CONFIG_CGROUP_SCHED */ +/* + * u64_u32_load/u64_u32_store + * + * Use a copy of a u64 value to protect against data race. This is only + * applicable for 32-bits architectures. + */ +#ifdef CONFIG_64BIT +# define u64_u32_load_copy(var, copy) var +# define u64_u32_store_copy(var, copy, val) (var = val) +#else +# define u64_u32_load_copy(var, copy) \ +({ \ + u64 __val, __val_copy; \ + do { \ + __val_copy = copy; \ + /* \ + * paired with u64_u32_store_copy(), ordering access \ + * to var and copy. \ + */ \ + smp_rmb(); \ + __val = var; \ + } while (__val != __val_copy); \ + __val; \ +}) +# define u64_u32_store_copy(var, copy, val) \ +do { \ + typeof(val) __val = (val); \ + var = __val; \ + /* \ + * paired with u64_u32_load_copy(), ordering access to var and \ + * copy. \ + */ \ + smp_wmb(); \ + copy = __val; \ +} while (0) +#endif +# define u64_u32_load(var) u64_u32_load_copy(var, var##_copy) +# define u64_u32_store(var, val) u64_u32_store_copy(var, var##_copy, val) + /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -560,7 +584,7 @@ struct cfs_rq { */ struct sched_avg avg; #ifndef CONFIG_64BIT - u64 load_last_update_time_copy; + u64 last_update_time_copy; #endif struct { raw_spinlock_t lock ____cacheline_aligned; @@ -609,6 +633,10 @@ struct cfs_rq { int runtime_enabled; s64 runtime_remaining; + u64 throttled_pelt_idle; +#ifndef CONFIG_64BIT + u64 throttled_pelt_idle_copy; +#endif u64 throttled_clock; u64 throttled_clock_pelt; u64 throttled_clock_pelt_time; @@ -981,6 +1009,12 @@ struct rq { u64 clock_task ____cacheline_aligned; u64 clock_pelt; unsigned long lost_idle_time; + u64 clock_pelt_idle; + u64 clock_idle; +#ifndef CONFIG_64BIT + u64 clock_pelt_idle_copy; + u64 clock_idle_copy; +#endif atomic_t nr_iowait; @@ -1764,6 +1798,11 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); extern struct static_key_false sched_asym_cpucapacity; +static __always_inline bool sched_asym_cpucap_active(void) +{ + return static_branch_unlikely(&sched_asym_cpucapacity); +} + struct sched_group_capacity { atomic_t ref; /* @@ -1815,15 +1854,6 @@ static inline struct cpumask *group_balance_mask(struct sched_group *sg) return to_cpumask(sg->sgc->cpumask); } -/** - * group_first_cpu - Returns the first CPU in the cpumask of a sched_group. - * @group: The group whose first CPU is to be returned. - */ -static inline unsigned int group_first_cpu(struct sched_group *group) -{ - return cpumask_first(sched_group_span(group)); -} - extern int group_balance_cpu(struct sched_group *sg); #ifdef CONFIG_SCHED_DEBUG @@ -2044,7 +2074,6 @@ static inline int task_on_rq_migrating(struct task_struct *p) #define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ -#define WF_ON_CPU 0x40 /* Wakee is on_cpu */ #ifdef CONFIG_SMP static_assert(WF_EXEC == SD_BALANCE_EXEC); @@ -2852,9 +2881,24 @@ enum cpu_util_type { }; unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, - unsigned long max, enum cpu_util_type type, + enum cpu_util_type type, struct task_struct *p); +/* + * Verify the fitness of task @p to run on @cpu taking into account the + * CPU original capacity and the runtime/deadline ratio of the task. + * + * The function will return true if the original capacity of @cpu is + * greater than or equal to task's deadline density right shifted by + * (BW_SHIFT - SCHED_CAPACITY_SHIFT) and false otherwise. + */ +static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned long cap = arch_scale_cpu_capacity(cpu); + + return cap >= p->dl.dl_density >> (BW_SHIFT - SCHED_CAPACITY_SHIFT); +} + static inline unsigned long cpu_bw_dl(struct rq *rq) { return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 05b6c2ad90b9c..8739c2a5a54ea 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -2316,23 +2316,30 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att /* * For a single LLC per node, allow an - * imbalance up to 25% of the node. This is an - * arbitrary cutoff based on SMT-2 to balance - * between memory bandwidth and avoiding - * premature sharing of HT resources and SMT-4 - * or SMT-8 *may* benefit from a different - * cutoff. + * imbalance up to 12.5% of the node. This is + * arbitrary cutoff based two factors -- SMT and + * memory channels. For SMT-2, the intent is to + * avoid premature sharing of HT resources but + * SMT-4 or SMT-8 *may* benefit from a different + * cutoff. For memory channels, this is a very + * rough estimate of how many channels may be + * active and is based on recent CPUs with + * many cores. * * For multiple LLCs, allow an imbalance * until multiple tasks would share an LLC * on one node while LLCs on another node - * remain idle. + * remain idle. This assumes that there are + * enough logical CPUs per LLC to avoid SMT + * factors and that there is a correlation + * between LLCs and memory channels. */ nr_llcs = sd->span_weight / child->span_weight; if (nr_llcs == 1) - imb = sd->span_weight >> 2; + imb = sd->span_weight >> 3; else imb = nr_llcs; + imb = max(1U, imb); sd->imb_numa_nr = imb; /* Set span based on the first NUMA domain. */ diff --git a/tools/testing/selftests/rseq/rseq-riscv.h b/tools/testing/selftests/rseq/rseq-riscv.h index b86642f90d7fd..3a391c9bf4689 100644 --- a/tools/testing/selftests/rseq/rseq-riscv.h +++ b/tools/testing/selftests/rseq/rseq-riscv.h @@ -86,7 +86,7 @@ do { \ #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ - "la "RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ + "la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \ __rseq_str(label) ":\n" @@ -103,17 +103,17 @@ do { \ #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ - "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ - "lw "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ - "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ + "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ - "beq "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + "beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ @@ -127,12 +127,12 @@ do { \ REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ - "add "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ + "add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ RSEQ_ASM_TMP_REG_1 "\n" \ REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n" #define RSEQ_ASM_OP_R_ADD(count) \ - "add "RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ + "add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ ", %[" __rseq_str(count) "]\n" #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ @@ -194,8 +194,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -251,8 +251,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expectnot] "r" (expectnot), [load] "m" (*load), @@ -301,8 +301,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [count] "r" (count) RSEQ_INJECT_INPUT @@ -352,8 +352,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "m" (*v), [newv] "r" (newv), @@ -411,8 +411,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "m" (*v), [newv] "r" (newv), @@ -472,8 +472,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [v2] "m" (*v2), @@ -532,8 +532,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "m" (*v), [newv] "r" (newv), @@ -593,8 +593,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "m" (*v), [newv] "r" (newv), @@ -651,8 +651,8 @@ int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [ptr] "r" (ptr), [off] "er" (off), [inc] "er" (inc) diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index 986b9458efb26..4177f9507bbee 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -111,7 +111,8 @@ void rseq_init(void) libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); - if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p) { + if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p && + *libc_rseq_size_p != 0) { /* rseq registration owned by glibc */ rseq_offset = *libc_rseq_offset_p; rseq_size = *libc_rseq_size_p; -- 2.37.1 From baf079b53cb5cf70540763e4aee6b48db2393430 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:32:17 +0200 Subject: [PATCH 04/16] Extend-DAMOS-for-Proactive-LRU-lists-Sorting Signed-off-by: Peter Jung --- Documentation/admin-guide/mm/damon/index.rst | 1 + .../admin-guide/mm/damon/lru_sort.rst | 294 ++++++++++ Documentation/admin-guide/mm/damon/usage.rst | 2 + include/linux/damon.h | 4 + mm/damon/Kconfig | 8 + mm/damon/Makefile | 1 + mm/damon/dbgfs.c | 64 +- mm/damon/lru_sort.c | 548 ++++++++++++++++++ mm/damon/ops-common.c | 42 ++ mm/damon/ops-common.h | 2 + mm/damon/paddr.c | 60 +- mm/damon/reclaim.c | 13 +- mm/damon/sysfs.c | 2 + 13 files changed, 1016 insertions(+), 25 deletions(-) create mode 100644 Documentation/admin-guide/mm/damon/lru_sort.rst create mode 100644 mm/damon/lru_sort.c diff --git a/Documentation/admin-guide/mm/damon/index.rst b/Documentation/admin-guide/mm/damon/index.rst index 61aff88347f3c..53762770e0e44 100644 --- a/Documentation/admin-guide/mm/damon/index.rst +++ b/Documentation/admin-guide/mm/damon/index.rst @@ -14,3 +14,4 @@ optimize those. start usage reclaim + lru_sort diff --git a/Documentation/admin-guide/mm/damon/lru_sort.rst b/Documentation/admin-guide/mm/damon/lru_sort.rst new file mode 100644 index 0000000000000..c09cace806516 --- /dev/null +++ b/Documentation/admin-guide/mm/damon/lru_sort.rst @@ -0,0 +1,294 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================= +DAMON-based LRU-lists Sorting +============================= + +DAMON-based LRU-lists Sorting (DAMON_LRU_SORT) is a static kernel module that +aimed to be used for proactive and lightweight data access pattern based +(de)prioritization of pages on their LRU-lists for making LRU-lists a more +trusworthy data access pattern source. + +Where Proactive LRU-lists Sorting is Required? +============================================== + +As page-granularity access checking overhead could be significant on huge +systems, LRU lists are normally not proactively sorted but partially and +reactively sorted for special events including specific user requests, system +calls and memory pressure. As a result, LRU lists are sometimes not so +perfectly prepared to be used as a trustworthy access pattern source for some +situations including reclamation target pages selection under sudden memory +pressure. + +Because DAMON can identify access patterns of best-effort accuracy while +inducing only user-specified range of overhead, proactively running +DAMON_LRU_SORT could be helpful for making LRU lists more trustworthy access +pattern source with low and controlled overhead. + +How It Works? +============= + +DAMON_LRU_SORT finds hot pages (pages of memory regions that showing access +rates that higher than a user-specified threshold) and cold pages (pages of +memory regions that showing no access for a time that longer than a +user-specified threshold) using DAMON, and prioritizes hot pages while +deprioritizing cold pages on their LRU-lists. To avoid it consuming too much +CPU for the prioritizations, a CPU time usage limit can be configured. Under +the limit, it prioritizes and deprioritizes more hot and cold pages first, +respectively. System administrators can also configure under what situation +this scheme should automatically activated and deactivated with three memory +pressure watermarks. + +Its default parameters for hotness/coldness thresholds and CPU quota limit are +conservatively chosen. That is, the module under its default parameters could +be widely used without harm for common situations while providing a level of +benefits for systems having clear hot/cold access patterns under memory +pressure while consuming only a limited small portion of CPU time. + +Interface: Module Parameters +============================ + +To use this feature, you should first ensure your system is running on a kernel +that is built with ``CONFIG_DAMON_LRU_SORT=y``. + +To let sysadmins enable or disable it and tune for the given system, +DAMON_LRU_SORT utilizes module parameters. That is, you can put +``damon_lru_sort.=`` on the kernel boot command line or write +proper values to ``/sys/modules/damon_lru_sort/parameters/`` files. + +Below are the description of each parameter. + +enabled +------- + +Enable or disable DAMON_LRU_SORT. + +You can enable DAMON_LRU_SORT by setting the value of this parameter as ``Y``. +Setting it as ``N`` disables DAMON_LRU_SORT. Note that DAMON_LRU_SORT could do +no real monitoring and LRU-lists sorting due to the watermarks-based activation +condition. Refer to below descriptions for the watermarks parameter for this. + +commit_inputs +------------- + +Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. + +Input parameters that updated while DAMON_LRU_SORT is running are not applied +by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT reads values +of parametrs except ``enabled`` again. Once the re-reading is done, this +parameter is set as ``N``. If invalid parameters are found while the +re-reading, DAMON_LRU_SORT will be disabled. + +hot_thres_access_freq +--------------------- + +Access frequency threshold for hot memory regions identification in permil. + +If a memory region is accessed in frequency of this or higher, DAMON_LRU_SORT +identifies the region as hot, and mark it as accessed on the LRU list, so that +it could not be reclaimed under memory pressure. 50% by default. + +cold_min_age +------------ + +Time threshold for cold memory regions identification in microseconds. + +If a memory region is not accessed for this or longer time, DAMON_LRU_SORT +identifies the region as cold, and mark it as unaccessed on the LRU list, so +that it could be reclaimed first under memory pressure. 120 seconds by +default. + +quota_ms +-------- + +Limit of time for trying the LRU lists sorting in milliseconds. + +DAMON_LRU_SORT tries to use only up to this time within a time window +(quota_reset_interval_ms) for trying LRU lists sorting. This can be used +for limiting CPU consumption of DAMON_LRU_SORT. If the value is zero, the +limit is disabled. + +10 ms by default. + +quota_reset_interval_ms +----------------------- + +The time quota charge reset interval in milliseconds. + +The charge reset interval for the quota of time (quota_ms). That is, +DAMON_LRU_SORT does not try LRU-lists sorting for more than quota_ms +milliseconds or quota_sz bytes within quota_reset_interval_ms milliseconds. + +1 second by default. + +wmarks_interval +--------------- + +The watermarks check time interval in microseconds. + +Minimal time to wait before checking the watermarks, when DAMON_LRU_SORT is +enabled but inactive due to its watermarks rule. 5 seconds by default. + +wmarks_high +----------- + +Free memory rate (per thousand) for the high watermark. + +If free memory of the system in bytes per thousand bytes is higher than this, +DAMON_LRU_SORT becomes inactive, so it does nothing but periodically checks the +watermarks. 200 (20%) by default. + +wmarks_mid +---------- + +Free memory rate (per thousand) for the middle watermark. + +If free memory of the system in bytes per thousand bytes is between this and +the low watermark, DAMON_LRU_SORT becomes active, so starts the monitoring and +the LRU-lists sorting. 150 (15%) by default. + +wmarks_low +---------- + +Free memory rate (per thousand) for the low watermark. + +If free memory of the system in bytes per thousand bytes is lower than this, +DAMON_LRU_SORT becomes inactive, so it does nothing but periodically checks the +watermarks. 50 (5%) by default. + +sample_interval +--------------- + +Sampling interval for the monitoring in microseconds. + +The sampling interval of DAMON for the cold memory monitoring. Please refer to +the DAMON documentation (:doc:`usage`) for more detail. 5ms by default. + +aggr_interval +------------- + +Aggregation interval for the monitoring in microseconds. + +The aggregation interval of DAMON for the cold memory monitoring. Please +refer to the DAMON documentation (:doc:`usage`) for more detail. 100ms by +default. + +min_nr_regions +-------------- + +Minimum number of monitoring regions. + +The minimal number of monitoring regions of DAMON for the cold memory +monitoring. This can be used to set lower-bound of the monitoring quality. +But, setting this too high could result in increased monitoring overhead. +Please refer to the DAMON documentation (:doc:`usage`) for more detail. 10 by +default. + +max_nr_regions +-------------- + +Maximum number of monitoring regions. + +The maximum number of monitoring regions of DAMON for the cold memory +monitoring. This can be used to set upper-bound of the monitoring overhead. +However, setting this too low could result in bad monitoring quality. Please +refer to the DAMON documentation (:doc:`usage`) for more detail. 1000 by +defaults. + +monitor_region_start +-------------------- + +Start of target memory region in physical address. + +The start physical address of memory region that DAMON_LRU_SORT will do work +against. By default, biggest System RAM is used as the region. + +monitor_region_end +------------------ + +End of target memory region in physical address. + +The end physical address of memory region that DAMON_LRU_SORT will do work +against. By default, biggest System RAM is used as the region. + +kdamond_pid +----------- + +PID of the DAMON thread. + +If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. Else, +-1. + +nr_lru_sort_tried_hot_regions +----------------------------- + +Number of hot memory regions that tried to be LRU-sorted. + +bytes_lru_sort_tried_hot_regions +-------------------------------- + +Total bytes of hot memory regions that tried to be LRU-sorted. + +nr_lru_sorted_hot_regions +------------------------- + +Number of hot memory regions that successfully be LRU-sorted. + +bytes_lru_sorted_hot_regions +---------------------------- + +Total bytes of hot memory regions that successfully be LRU-sorted. + +nr_hot_quota_exceeds +-------------------- + +Number of times that the time quota limit for hot regions have exceeded. + +nr_lru_sort_tried_cold_regions +------------------------------ + +Number of cold memory regions that tried to be LRU-sorted. + +bytes_lru_sort_tried_cold_regions +--------------------------------- + +Total bytes of cold memory regions that tried to be LRU-sorted. + +nr_lru_sorted_cold_regions +-------------------------- + +Number of cold memory regions that successfully be LRU-sorted. + +bytes_lru_sorted_cold_regions +----------------------------- + +Total bytes of cold memory regions that successfully be LRU-sorted. + +nr_cold_quota_exceeds +--------------------- + +Number of times that the time quota limit for cold regions have exceeded. + +Example +======= + +Below runtime example commands make DAMON_LRU_SORT to find memory regions +having >=50% access frequency and LRU-prioritize while LRU-deprioritizing +memory regions that not accessed for 120 seconds. The prioritization and +deprioritization is limited to be done using only up to 1% CPU time to avoid +DAMON_LRU_SORT consuming too much CPU time for the (de)prioritization. It also +asks DAMON_LRU_SORT to do nothing if the system's free memory rate is more than +50%, but start the real works if it becomes lower than 40%. If DAMON_RECLAIM +doesn't make progress and therefore the free memory rate becomes lower than +20%, it asks DAMON_LRU_SORT to do nothing again, so that we can fall back to +the LRU-list based page granularity reclamation. :: + + # cd /sys/modules/damon_lru_sort/parameters + # echo 500 > hot_thres_access_freq + # echo 120000000 > cold_min_age + # echo 10 > quota_ms + # echo 1000 > quota_reset_interval_ms + # echo 500 > wmarks_high + # echo 400 > wmarks_mid + # echo 200 > wmarks_low + # echo Y > enabled diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 1bb7b72414b24..d822bf6355ce7 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -264,6 +264,8 @@ that can be written to and read from the file and their meaning are as below. - ``pageout``: Call ``madvise()`` for the region with ``MADV_PAGEOUT`` - ``hugepage``: Call ``madvise()`` for the region with ``MADV_HUGEPAGE`` - ``nohugepage``: Call ``madvise()`` for the region with ``MADV_NOHUGEPAGE`` + - ``lru_prio``: Prioritize the region on its LRU lists. + - ``lru_deprio``: Deprioritize the region on its LRU lists. - ``stat``: Do nothing but count the statistics schemes//access_pattern/ diff --git a/include/linux/damon.h b/include/linux/damon.h index 7c62da31ce4b5..01748597f9acc 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -86,6 +86,8 @@ struct damon_target { * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. + * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. * @DAMOS_STAT: Do nothing but count the stat. * @NR_DAMOS_ACTIONS: Total number of DAMOS actions */ @@ -95,6 +97,8 @@ enum damos_action { DAMOS_PAGEOUT, DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, + DAMOS_LRU_PRIO, + DAMOS_LRU_DEPRIO, DAMOS_STAT, /* Do nothing but only record the stat */ NR_DAMOS_ACTIONS, }; diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 9b559c76d6dd1..66265e3a9c659 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -92,4 +92,12 @@ config DAMON_RECLAIM reclamation under light memory pressure, while the traditional page scanning-based reclamation is used for heavy pressure. +config DAMON_LRU_SORT + bool "Build DAMON-based LRU-lists sorting (DAMON_LRU_SORT)" + depends on DAMON_PADDR + help + This builds the DAMON-based LRU-lists sorting subsystem. It tries to + protect frequently accessed (hot) pages while rarely accessed (cold) + pages reclaimed first under memory pressure. + endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile index dbf7190b4144a..3e6b8ad73858a 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_DAMON_PADDR) += ops-common.o paddr.o obj-$(CONFIG_DAMON_SYSFS) += sysfs.o obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o obj-$(CONFIG_DAMON_RECLAIM) += reclaim.o +obj-$(CONFIG_DAMON_LRU_SORT) += lru_sort.o diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index a0dab8b5e45f2..e2f5f3927d5e0 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -97,6 +97,31 @@ static ssize_t dbgfs_attrs_write(struct file *file, return ret; } +/* + * Return corresponding dbgfs' scheme action value (int) for the given + * damos_action if the given damos_action value is valid and supported by + * dbgfs, negative error code otherwise. + */ +static int damos_action_to_dbgfs_scheme_action(enum damos_action action) +{ + switch (action) { + case DAMOS_WILLNEED: + return 0; + case DAMOS_COLD: + return 1; + case DAMOS_PAGEOUT: + return 2; + case DAMOS_HUGEPAGE: + return 3; + case DAMOS_NOHUGEPAGE: + return 4; + case DAMOS_STAT: + return 5; + default: + return -EINVAL; + } +} + static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) { struct damos *s; @@ -109,7 +134,7 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, - s->action, + damos_action_to_dbgfs_scheme_action(s->action), s->quota.ms, s->quota.sz, s->quota.reset_interval, s->quota.weight_sz, @@ -160,18 +185,27 @@ static void free_schemes_arr(struct damos **schemes, ssize_t nr_schemes) kfree(schemes); } -static bool damos_action_valid(int action) +/* + * Return corresponding damos_action for the given dbgfs input for a scheme + * action if the input is valid, negative error code otherwise. + */ +static enum damos_action dbgfs_scheme_action_to_damos_action(int dbgfs_action) { - switch (action) { - case DAMOS_WILLNEED: - case DAMOS_COLD: - case DAMOS_PAGEOUT: - case DAMOS_HUGEPAGE: - case DAMOS_NOHUGEPAGE: - case DAMOS_STAT: - return true; + switch (dbgfs_action) { + case 0: + return DAMOS_WILLNEED; + case 1: + return DAMOS_COLD; + case 2: + return DAMOS_PAGEOUT; + case 3: + return DAMOS_HUGEPAGE; + case 4: + return DAMOS_NOHUGEPAGE; + case 5: + return DAMOS_STAT; default: - return false; + return -EINVAL; } } @@ -189,7 +223,8 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, int pos = 0, parsed, ret; unsigned long min_sz, max_sz; unsigned int min_nr_a, max_nr_a, min_age, max_age; - unsigned int action; + unsigned int action_input; + enum damos_action action; schemes = kmalloc_array(max_nr_schemes, sizeof(scheme), GFP_KERNEL); @@ -204,7 +239,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n", &min_sz, &max_sz, &min_nr_a, &max_nr_a, - &min_age, &max_age, &action, "a.ms, + &min_age, &max_age, &action_input, "a.ms, "a.sz, "a.reset_interval, "a.weight_sz, "a.weight_nr_accesses, "a.weight_age, &wmarks.metric, @@ -212,7 +247,8 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, &wmarks.low, &parsed); if (ret != 18) break; - if (!damos_action_valid(action)) + action = dbgfs_scheme_action_to_damos_action(action_input); + if ((int)action < 0) goto fail; if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age) diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c new file mode 100644 index 0000000000000..9de6f00a71c5d --- /dev/null +++ b/mm/damon/lru_sort.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON-based LRU-lists Sorting + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-lru-sort: " fmt + +#include +#include +#include +#include +#include + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "damon_lru_sort." + +/* + * Enable or disable DAMON_LRU_SORT. + * + * You can enable DAMON_LRU_SORT by setting the value of this parameter as + * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that + * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the + * watermarks-based activation condition. Refer to below descriptions for the + * watermarks parameter for this. + */ +static bool enabled __read_mostly; + +/* + * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. + * + * Input parameters that updated while DAMON_LRU_SORT is running are not + * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT + * reads values of parametrs except ``enabled`` again. Once the re-reading is + * done, this parameter is set as ``N``. If invalid parameters are found while + * the re-reading, DAMON_LRU_SORT will be disabled. + */ +static bool commit_inputs __read_mostly; +module_param(commit_inputs, bool, 0600); + +/* + * Access frequency threshold for hot memory regions identification in permil. + * + * If a memory region is accessed in frequency of this or higher, + * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the + * LRU list, so that it could not be reclaimed under memory pressure. 50% by + * default. + */ +static unsigned long hot_thres_access_freq = 500; +module_param(hot_thres_access_freq, ulong, 0600); + +/* + * Time threshold for cold memory regions identification in microseconds. + * + * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT + * identifies the region as cold, and mark it as unaccessed on the LRU list, so + * that it could be reclaimed first under memory pressure. 120 seconds by + * default. + */ +static unsigned long cold_min_age __read_mostly = 120000000; +module_param(cold_min_age, ulong, 0600); + +/* + * Limit of time for trying the LRU lists sorting in milliseconds. + * + * DAMON_LRU_SORT tries to use only up to this time within a time window + * (quota_reset_interval_ms) for trying LRU lists sorting. This can be used + * for limiting CPU consumption of DAMON_LRU_SORT. If the value is zero, the + * limit is disabled. + * + * 10 ms by default. + */ +static unsigned long quota_ms __read_mostly = 10; +module_param(quota_ms, ulong, 0600); + +/* + * The time quota charge reset interval in milliseconds. + * + * The charge reset interval for the quota of time (quota_ms). That is, + * DAMON_LRU_SORT does not try LRU-lists sorting for more than quota_ms + * milliseconds or quota_sz bytes within quota_reset_interval_ms milliseconds. + * + * 1 second by default. + */ +static unsigned long quota_reset_interval_ms __read_mostly = 1000; +module_param(quota_reset_interval_ms, ulong, 0600); + +/* + * The watermarks check time interval in microseconds. + * + * Minimal time to wait before checking the watermarks, when DAMON_LRU_SORT is + * enabled but inactive due to its watermarks rule. 5 seconds by default. + */ +static unsigned long wmarks_interval __read_mostly = 5000000; +module_param(wmarks_interval, ulong, 0600); + +/* + * Free memory rate (per thousand) for the high watermark. + * + * If free memory of the system in bytes per thousand bytes is higher than + * this, DAMON_LRU_SORT becomes inactive, so it does nothing but periodically + * checks the watermarks. 200 (20%) by default. + */ +static unsigned long wmarks_high __read_mostly = 200; +module_param(wmarks_high, ulong, 0600); + +/* + * Free memory rate (per thousand) for the middle watermark. + * + * If free memory of the system in bytes per thousand bytes is between this and + * the low watermark, DAMON_LRU_SORT becomes active, so starts the monitoring + * and the LRU-lists sorting. 150 (15%) by default. + */ +static unsigned long wmarks_mid __read_mostly = 150; +module_param(wmarks_mid, ulong, 0600); + +/* + * Free memory rate (per thousand) for the low watermark. + * + * If free memory of the system in bytes per thousand bytes is lower than this, + * DAMON_LRU_SORT becomes inactive, so it does nothing but periodically checks + * the watermarks. 50 (5%) by default. + */ +static unsigned long wmarks_low __read_mostly = 50; +module_param(wmarks_low, ulong, 0600); + +/* + * Sampling interval for the monitoring in microseconds. + * + * The sampling interval of DAMON for the hot/cold memory monitoring. Please + * refer to the DAMON documentation for more detail. 5 ms by default. + */ +static unsigned long sample_interval __read_mostly = 5000; +module_param(sample_interval, ulong, 0600); + +/* + * Aggregation interval for the monitoring in microseconds. + * + * The aggregation interval of DAMON for the hot/cold memory monitoring. + * Please refer to the DAMON documentation for more detail. 100 ms by default. + */ +static unsigned long aggr_interval __read_mostly = 100000; +module_param(aggr_interval, ulong, 0600); + +/* + * Minimum number of monitoring regions. + * + * The minimal number of monitoring regions of DAMON for the hot/cold memory + * monitoring. This can be used to set lower-bound of the monitoring quality. + * But, setting this too high could result in increased monitoring overhead. + * Please refer to the DAMON documentation for more detail. 10 by default. + */ +static unsigned long min_nr_regions __read_mostly = 10; +module_param(min_nr_regions, ulong, 0600); + +/* + * Maximum number of monitoring regions. + * + * The maximum number of monitoring regions of DAMON for the hot/cold memory + * monitoring. This can be used to set upper-bound of the monitoring overhead. + * However, setting this too low could result in bad monitoring quality. + * Please refer to the DAMON documentation for more detail. 1000 by default. + */ +static unsigned long max_nr_regions __read_mostly = 1000; +module_param(max_nr_regions, ulong, 0600); + +/* + * Start of the target memory region in physical address. + * + * The start physical address of memory region that DAMON_LRU_SORT will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_start __read_mostly; +module_param(monitor_region_start, ulong, 0600); + +/* + * End of the target memory region in physical address. + * + * The end physical address of memory region that DAMON_LRU_SORT will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_end __read_mostly; +module_param(monitor_region_end, ulong, 0600); + +/* + * PID of the DAMON thread + * + * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. + * Else, -1. + */ +static int kdamond_pid __read_mostly = -1; +module_param(kdamond_pid, int, 0400); + +/* + * Number of hot memory regions that tried to be LRU-sorted. + */ +static unsigned long nr_lru_sort_tried_hot_regions __read_mostly; +module_param(nr_lru_sort_tried_hot_regions, ulong, 0400); + +/* + * Total bytes of hot memory regions that tried to be LRU-sorted. + */ +static unsigned long bytes_lru_sort_tried_hot_regions __read_mostly; +module_param(bytes_lru_sort_tried_hot_regions, ulong, 0400); + +/* + * Number of hot memory regions that successfully be LRU-sorted. + */ +static unsigned long nr_lru_sorted_hot_regions __read_mostly; +module_param(nr_lru_sorted_hot_regions, ulong, 0400); + +/* + * Total bytes of hot memory regions that successfully be LRU-sorted. + */ +static unsigned long bytes_lru_sorted_hot_regions __read_mostly; +module_param(bytes_lru_sorted_hot_regions, ulong, 0400); + +/* + * Number of times that the time quota limit for hot regions have exceeded + */ +static unsigned long nr_hot_quota_exceeds __read_mostly; +module_param(nr_hot_quota_exceeds, ulong, 0400); + +/* + * Number of cold memory regions that tried to be LRU-sorted. + */ +static unsigned long nr_lru_sort_tried_cold_regions __read_mostly; +module_param(nr_lru_sort_tried_cold_regions, ulong, 0400); + +/* + * Total bytes of cold memory regions that tried to be LRU-sorted. + */ +static unsigned long bytes_lru_sort_tried_cold_regions __read_mostly; +module_param(bytes_lru_sort_tried_cold_regions, ulong, 0400); + +/* + * Number of cold memory regions that successfully be LRU-sorted. + */ +static unsigned long nr_lru_sorted_cold_regions __read_mostly; +module_param(nr_lru_sorted_cold_regions, ulong, 0400); + +/* + * Total bytes of cold memory regions that successfully be LRU-sorted. + */ +static unsigned long bytes_lru_sorted_cold_regions __read_mostly; +module_param(bytes_lru_sorted_cold_regions, ulong, 0400); + +/* + * Number of times that the time quota limit for cold regions have exceeded + */ +static unsigned long nr_cold_quota_exceeds __read_mostly; +module_param(nr_cold_quota_exceeds, ulong, 0400); + +static struct damon_ctx *ctx; +static struct damon_target *target; + +struct damon_lru_sort_ram_walk_arg { + unsigned long start; + unsigned long end; +}; + +static int walk_system_ram(struct resource *res, void *arg) +{ + struct damon_lru_sort_ram_walk_arg *a = arg; + + if (a->end - a->start < resource_size(res)) { + a->start = res->start; + a->end = res->end; + } + return 0; +} + +/* + * Find biggest 'System RAM' resource and store its start and end address in + * @start and @end, respectively. If no System RAM is found, returns false. + */ +static bool get_monitoring_region(unsigned long *start, unsigned long *end) +{ + struct damon_lru_sort_ram_walk_arg arg = {}; + + walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); + if (arg.end <= arg.start) + return false; + + *start = arg.start; + *end = arg.end; + return true; +} + +/* Create a DAMON-based operation scheme for hot memory regions */ +static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) +{ + struct damos_watermarks wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = wmarks_interval, + .high = wmarks_high, + .mid = wmarks_mid, + .low = wmarks_low, + }; + struct damos_quota quota = { + /* + * Do not try LRU-lists sorting of hot pages for more than half + * of quota_ms milliseconds within quota_reset_interval_ms. + */ + .ms = quota_ms / 2, + .sz = 0, + .reset_interval = quota_reset_interval_ms, + /* Within the quota, mark hotter regions accessed first. */ + .weight_sz = 0, + .weight_nr_accesses = 1, + .weight_age = 0, + }; + struct damos *scheme = damon_new_scheme( + /* Find regions having PAGE_SIZE or larger size */ + PAGE_SIZE, ULONG_MAX, + /* and accessed for more than the threshold */ + hot_thres, UINT_MAX, + /* no matter its age */ + 0, UINT_MAX, + /* prioritize those on LRU lists, as soon as found */ + DAMOS_LRU_PRIO, + /* under the quota. */ + "a, + /* (De)activate this according to the watermarks. */ + &wmarks); + + return scheme; +} + +/* Create a DAMON-based operation scheme for cold memory regions */ +static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) +{ + struct damos_watermarks wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = wmarks_interval, + .high = wmarks_high, + .mid = wmarks_mid, + .low = wmarks_low, + }; + struct damos_quota quota = { + /* + * Do not try LRU-lists sorting of cold pages for more than + * half of quota_ms milliseconds within + * quota_reset_interval_ms. + */ + .ms = quota_ms / 2, + .sz = 0, + .reset_interval = quota_reset_interval_ms, + /* Within the quota, mark colder regions not accessed first. */ + .weight_sz = 0, + .weight_nr_accesses = 0, + .weight_age = 1, + }; + struct damos *scheme = damon_new_scheme( + /* Find regions having PAGE_SIZE or larger size */ + PAGE_SIZE, ULONG_MAX, + /* and not accessed at all */ + 0, 0, + /* for cold_thres or more micro-seconds, and */ + cold_thres, UINT_MAX, + /* mark those as not accessed, as soon as found */ + DAMOS_LRU_DEPRIO, + /* under the quota. */ + "a, + /* (De)activate this according to the watermarks. */ + &wmarks); + + return scheme; +} + +static int damon_lru_sort_apply_parameters(void) +{ + struct damos *scheme, *next_scheme; + struct damon_addr_range addr_range; + unsigned int hot_thres, cold_thres; + int err = 0; + + err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, + min_nr_regions, max_nr_regions); + if (err) + return err; + + /* free previously set schemes */ + damon_for_each_scheme_safe(scheme, next_scheme, ctx) + damon_destroy_scheme(scheme); + + /* aggr_interval / sample_interval is the maximum nr_accesses */ + hot_thres = aggr_interval / sample_interval * hot_thres_access_freq / + 1000; + scheme = damon_lru_sort_new_hot_scheme(hot_thres); + if (!scheme) + return -ENOMEM; + damon_add_scheme(ctx, scheme); + + cold_thres = cold_min_age / aggr_interval; + scheme = damon_lru_sort_new_cold_scheme(cold_thres); + if (!scheme) + return -ENOMEM; + damon_add_scheme(ctx, scheme); + + if (monitor_region_start > monitor_region_end) + return -EINVAL; + if (!monitor_region_start && !monitor_region_end && + !get_monitoring_region(&monitor_region_start, + &monitor_region_end)) + return -EINVAL; + addr_range.start = monitor_region_start; + addr_range.end = monitor_region_end; + return damon_set_regions(target, &addr_range, 1); +} + +static int damon_lru_sort_turn(bool on) +{ + int err; + + if (!on) { + err = damon_stop(&ctx, 1); + if (!err) + kdamond_pid = -1; + return err; + } + + err = damon_lru_sort_apply_parameters(); + if (err) + return err; + + err = damon_start(&ctx, 1, true); + if (err) + return err; + kdamond_pid = ctx->kdamond->pid; + return 0; +} + +static struct delayed_work damon_lru_sort_timer; +static void damon_lru_sort_timer_fn(struct work_struct *work) +{ + static bool last_enabled; + bool now_enabled; + + now_enabled = enabled; + if (last_enabled != now_enabled) { + if (!damon_lru_sort_turn(now_enabled)) + last_enabled = now_enabled; + else + enabled = last_enabled; + } +} +static DECLARE_DELAYED_WORK(damon_lru_sort_timer, damon_lru_sort_timer_fn); + +static bool damon_lru_sort_initialized; + +static int damon_lru_sort_enabled_store(const char *val, + const struct kernel_param *kp) +{ + int rc = param_set_bool(val, kp); + + if (rc < 0) + return rc; + + if (!damon_lru_sort_initialized) + return rc; + + schedule_delayed_work(&damon_lru_sort_timer, 0); + + return 0; +} + +static const struct kernel_param_ops enabled_param_ops = { + .set = damon_lru_sort_enabled_store, + .get = param_get_bool, +}; + +module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); +MODULE_PARM_DESC(enabled, + "Enable or disable DAMON_LRU_SORT (default: disabled)"); + +static int damon_lru_sort_handle_commit_inputs(void) +{ + int err; + + if (!commit_inputs) + return 0; + + err = damon_lru_sort_apply_parameters(); + commit_inputs = false; + return err; +} + +static int damon_lru_sort_after_aggregation(struct damon_ctx *c) +{ + struct damos *s; + + /* update the stats parameter */ + damon_for_each_scheme(s, c) { + if (s->action == DAMOS_LRU_PRIO) { + nr_lru_sort_tried_hot_regions = s->stat.nr_tried; + bytes_lru_sort_tried_hot_regions = s->stat.sz_tried; + nr_lru_sorted_hot_regions = s->stat.nr_applied; + bytes_lru_sorted_hot_regions = s->stat.sz_applied; + nr_hot_quota_exceeds = s->stat.qt_exceeds; + } else if (s->action == DAMOS_LRU_DEPRIO) { + nr_lru_sort_tried_cold_regions = s->stat.nr_tried; + bytes_lru_sort_tried_cold_regions = s->stat.sz_tried; + nr_lru_sorted_cold_regions = s->stat.nr_applied; + bytes_lru_sorted_cold_regions = s->stat.sz_applied; + nr_cold_quota_exceeds = s->stat.qt_exceeds; + } + } + + return damon_lru_sort_handle_commit_inputs(); +} + +static int damon_lru_sort_after_wmarks_check(struct damon_ctx *c) +{ + return damon_lru_sort_handle_commit_inputs(); +} + +static int __init damon_lru_sort_init(void) +{ + ctx = damon_new_ctx(); + if (!ctx) + return -ENOMEM; + + if (damon_select_ops(ctx, DAMON_OPS_PADDR)) { + damon_destroy_ctx(ctx); + return -EINVAL; + } + + ctx->callback.after_wmarks_check = damon_lru_sort_after_wmarks_check; + ctx->callback.after_aggregation = damon_lru_sort_after_aggregation; + + target = damon_new_target(); + if (!target) { + damon_destroy_ctx(ctx); + return -ENOMEM; + } + damon_add_target(ctx, target); + + schedule_delayed_work(&damon_lru_sort_timer, 0); + + damon_lru_sort_initialized = true; + return 0; +} + +module_init(damon_lru_sort_init); diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 10ef20b2003f5..b1335de200e77 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -130,3 +130,45 @@ int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, /* Return coldness of the region */ return DAMOS_MAX_SCORE - hotness; } + +int damon_hot_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s) +{ + unsigned int max_nr_accesses; + int freq_subscore; + unsigned int age_in_sec; + int age_in_log, age_subscore; + unsigned int freq_weight = s->quota.weight_nr_accesses; + unsigned int age_weight = s->quota.weight_age; + int hotness; + + max_nr_accesses = c->aggr_interval / c->sample_interval; + freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; + + age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; + for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; + age_in_log++, age_in_sec >>= 1) + ; + + /* If frequency is 0, higher age means it's colder */ + if (freq_subscore == 0) + age_in_log *= -1; + + /* + * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. + * Scale it to be in [0, 100] and set it as age subscore. + */ + age_in_log += DAMON_MAX_AGE_IN_LOG; + age_subscore = age_in_log * DAMON_MAX_SUBSCORE / + DAMON_MAX_AGE_IN_LOG / 2; + + hotness = (freq_weight * freq_subscore + age_weight * age_subscore); + if (freq_weight + age_weight) + hotness /= freq_weight + age_weight; + /* + * Transform it to fit in [0, DAMOS_MAX_SCORE] + */ + hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + + return hotness; +} diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index e790cb5f8fe05..52329ff361cd0 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -14,3 +14,5 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, struct damos *s); +int damon_hot_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index b40ff5811bb22..dc131c6a54038 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -204,16 +204,11 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) return max_nr_accesses; } -static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, - struct damon_target *t, struct damon_region *r, - struct damos *scheme) +static unsigned long damon_pa_pageout(struct damon_region *r) { unsigned long addr, applied; LIST_HEAD(page_list); - if (scheme->action != DAMOS_PAGEOUT) - return 0; - for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { struct page *page = damon_get_page(PHYS_PFN(addr)); @@ -238,6 +233,55 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, return applied * PAGE_SIZE; } +static unsigned long damon_pa_mark_accessed(struct damon_region *r) +{ + unsigned long addr, applied = 0; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + mark_page_accessed(page); + put_page(page); + applied++; + } + return applied * PAGE_SIZE; +} + +static unsigned long damon_pa_deactivate_pages(struct damon_region *r) +{ + unsigned long addr, applied = 0; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + deactivate_page(page); + put_page(page); + applied++; + } + return applied * PAGE_SIZE; +} + +static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) +{ + switch (scheme->action) { + case DAMOS_PAGEOUT: + return damon_pa_pageout(r); + case DAMOS_LRU_PRIO: + return damon_pa_mark_accessed(r); + case DAMOS_LRU_DEPRIO: + return damon_pa_deactivate_pages(r); + default: + break; + } + return 0; +} + static int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme) @@ -245,6 +289,10 @@ static int damon_pa_scheme_score(struct damon_ctx *context, switch (scheme->action) { case DAMOS_PAGEOUT: return damon_pageout_score(context, r, scheme); + case DAMOS_LRU_PRIO: + return damon_hot_score(context, r, scheme); + case DAMOS_LRU_DEPRIO: + return damon_pageout_score(context, r, scheme); default: break; } diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 4b07c29effe97..d2d74200cd969 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -379,13 +379,14 @@ static bool damon_reclaim_initialized; static int enabled_store(const char *val, const struct kernel_param *kp) { - int rc = param_set_bool(val, kp); - - if (rc < 0) - return rc; + int rc; /* system_wq might not initialized yet */ if (!damon_reclaim_initialized) + return -EINVAL; + + rc = param_set_bool(val, kp); + if (rc < 0) return rc; if (enabled) @@ -441,8 +442,10 @@ static int __init damon_reclaim_init(void) if (!ctx) return -ENOMEM; - if (damon_select_ops(ctx, DAMON_OPS_PADDR)) + if (damon_select_ops(ctx, DAMON_OPS_PADDR)) { + damon_destroy_ctx(ctx); return -EINVAL; + } ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; ctx->callback.after_aggregation = damon_reclaim_after_aggregation; diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 09f9e8ca3d1fa..0f65db55fa28d 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -762,6 +762,8 @@ static const char * const damon_sysfs_damos_action_strs[] = { "pageout", "hugepage", "nohugepage", + "lru_prio", + "lru_deprio", "stat", }; -- 2.37.1 From 3d2004539ac2d8dd54c2ee9d530a6f021fd378e8 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:32:39 +0200 Subject: [PATCH 05/16] fixes Signed-off-by: Peter Jung --- .../admin-guide/kernel-parameters.txt | 9 + Makefile | 5 - arch/x86/kvm/emulate.c | 9 +- block/mq-deadline.c | 9 +- drivers/cpufreq/Kconfig | 2 - drivers/hid/hid-quirks.c | 2 + drivers/infiniband/core/addr.c | 1 + .../net/ethernet/aquantia/atlantic/aq_nic.c | 17 +- drivers/pci/quirks.c | 101 +++++++ drivers/tty/Kconfig | 13 + drivers/watchdog/sp5100_tco.c | 1 + fs/proc/base.c | 3 + fs/proc/internal.h | 1 + fs/proc/task_mmu.c | 273 +++++++++++++++--- include/uapi/linux/vt.h | 15 +- kernel/cpu.c | 1 + kernel/rcu/rcutorture.c | 2 +- kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 4 +- net/mac80211/airtime.c | 12 +- net/mac80211/rate.c | 12 +- net/mac80211/rc80211_minstrel_ht.c | 7 +- net/sched/Kconfig | 4 + 23 files changed, 432 insertions(+), 73 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index cc3ea8febc623..877a057d71c36 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4084,6 +4084,15 @@ nomsi [MSI] If the PCI_MSI kernel config parameter is enabled, this kernel boot option can be used to disable the use of MSI interrupts system-wide. + pcie_acs_override = + [PCIE] Override missing PCIe ACS support for: + downstream + All downstream ports - full ACS capabilities + multifunction + All multifunction devices - multifunction ACS subset + id:nnnn:nnnn + Specific device - full ACS capabilities + Specified as vid:did (vendor/device ID) in hex noioapicquirk [APIC] Disable all boot interrupt quirks. Safety option to keep boot IRQs enabled. This should never be necessary. diff --git a/Makefile b/Makefile index df92892325ae0..1a4593dd3d3be 100644 --- a/Makefile +++ b/Makefile @@ -992,11 +992,6 @@ KBUILD_CFLAGS += -fno-strict-overflow # Make sure -fstack-check isn't enabled (like gentoo apparently did) KBUILD_CFLAGS += -fno-stack-check -# conserve stack if available -ifdef CONFIG_CC_IS_GCC -KBUILD_CFLAGS += -fconserve-stack -endif - # Prohibit date/time macros, which would make the build non-deterministic KBUILD_CFLAGS += -Werror=date-time diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index f8382abe22ff8..1f096691b1745 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -244,6 +244,9 @@ enum x86_transfer_type { static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr) { + if (WARN_ON_ONCE(nr >= 16)) + nr &= 16 - 1; + if (!(ctxt->regs_valid & (1 << nr))) { ctxt->regs_valid |= 1 << nr; ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr); @@ -253,6 +256,9 @@ static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr) static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr) { + if (WARN_ON_ONCE(nr >= 16)) + nr &= 16 - 1; + ctxt->regs_valid |= 1 << nr; ctxt->regs_dirty |= 1 << nr; return &ctxt->_regs[nr]; @@ -266,9 +272,10 @@ static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr) static void writeback_registers(struct x86_emulate_ctxt *ctxt) { + unsigned long dirty = ctxt->regs_dirty; unsigned reg; - for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16) + for_each_set_bit(reg, &dirty, 16) ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); } diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 1a9e835e816cd..0ae521ee112bf 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -4,6 +4,9 @@ * for the blk-mq scheduling framework * * Copyright (C) 2016 Jens Axboe + * + * Tunes for responsiveness by Alexandre Frade + * (C) 2022 Alexandre Frade */ #include #include @@ -30,13 +33,13 @@ * See Documentation/block/deadline-iosched.rst */ static const int read_expire = HZ / 2; /* max time before a read is submitted. */ -static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */ +static const int write_expire = HZ; /* ditto for writes, these limits are SOFT! */ /* * Time after which to dispatch lower priority requests even if higher * priority requests are pending. */ static const int prio_aging_expire = 10 * HZ; -static const int writes_starved = 2; /* max times reads can starve a write */ +static const int writes_starved = 1; /* max times reads can starve a write */ static const int fifo_batch = 16; /* # of sequential requests treated as one by the above parameters. For throughput. */ @@ -635,7 +638,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) dd->fifo_expire[DD_READ] = read_expire; dd->fifo_expire[DD_WRITE] = write_expire; dd->writes_starved = writes_starved; - dd->front_merges = 1; + dd->front_merges = 0; dd->last_dir = DD_WRITE; dd->fifo_batch = fifo_batch; dd->prio_aging_expire = prio_aging_expire; diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index c3038cdc68650..ac0db6b8106da 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -72,7 +72,6 @@ config CPU_FREQ_DEFAULT_GOV_USERSPACE config CPU_FREQ_DEFAULT_GOV_ONDEMAND bool "ondemand" - depends on !(X86_INTEL_PSTATE && SMP) select CPU_FREQ_GOV_ONDEMAND select CPU_FREQ_GOV_PERFORMANCE help @@ -85,7 +84,6 @@ config CPU_FREQ_DEFAULT_GOV_ONDEMAND config CPU_FREQ_DEFAULT_GOV_CONSERVATIVE bool "conservative" - depends on !(X86_INTEL_PSTATE && SMP) select CPU_FREQ_GOV_CONSERVATIVE select CPU_FREQ_GOV_PERFORMANCE help diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index dc67717d2dabc..bee7076f9c82b 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -493,6 +493,8 @@ static const struct hid_device_id hid_have_special_driver[] = { #if IS_ENABLED(CONFIG_HID_MAGICMOUSE) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICMOUSE) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD) }, + { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2) }, #endif #if IS_ENABLED(CONFIG_HID_MAYFLASH) { HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3) }, diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index f253295795f0a..e74d36cfe66b8 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -820,6 +820,7 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid, union { struct sockaddr_in _sockaddr_in; struct sockaddr_in6 _sockaddr_in6; + struct sockaddr_ib _sockaddr_ib; } sgid_addr, dgid_addr; int ret; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c index e11cc29d3264c..8143a2e5c247a 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c @@ -268,9 +268,10 @@ static void aq_nic_polling_timer_cb(struct timer_list *t) struct aq_vec_s *aq_vec = NULL; unsigned int i = 0U; - for (i = 0U, aq_vec = self->aq_vec[0]; - self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) + for (i = 0U; self->aq_vecs > i; ++i) { + aq_vec = self->aq_vec[i]; aq_vec_isr(i, (void *)aq_vec); + } mod_timer(&self->polling_timer, jiffies + AQ_CFG_POLLING_TIMER_INTERVAL); @@ -1064,9 +1065,10 @@ u64 *aq_nic_get_stats(struct aq_nic_s *self, u64 *data) data += i; for (tc = 0U; tc < self->aq_nic_cfg.tcs; tc++) { - for (i = 0U, aq_vec = self->aq_vec[0]; - aq_vec && self->aq_vecs > i; - ++i, aq_vec = self->aq_vec[i]) { + for (i = 0U; self->aq_vecs > i; ++i) { + aq_vec = self->aq_vec[i]; + if (!aq_vec) + break; data += count; count = aq_vec_get_sw_stats(aq_vec, tc, data); } @@ -1400,9 +1402,10 @@ int aq_nic_stop(struct aq_nic_s *self) aq_ptp_irq_free(self); - for (i = 0U, aq_vec = self->aq_vec[0]; - self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) + for (i = 0U; self->aq_vecs > i; ++i) { + aq_vec = self->aq_vec[i]; aq_vec_stop(aq_vec); + } aq_ptp_ring_stop(self); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 41aeaa2351322..7651de329141b 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3364,6 +3364,106 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65f8, quirk_intel_mc_errata); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65f9, quirk_intel_mc_errata); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x65fa, quirk_intel_mc_errata); +static bool acs_on_downstream; +static bool acs_on_multifunction; + +#define NUM_ACS_IDS 16 +struct acs_on_id { + unsigned short vendor; + unsigned short device; +}; +static struct acs_on_id acs_on_ids[NUM_ACS_IDS]; +static u8 max_acs_id; + +static __init int pcie_acs_override_setup(char *p) +{ + if (!p) + return -EINVAL; + + while (*p) { + if (!strncmp(p, "downstream", 10)) + acs_on_downstream = true; + if (!strncmp(p, "multifunction", 13)) + acs_on_multifunction = true; + if (!strncmp(p, "id:", 3)) { + char opt[5]; + int ret; + long val; + + if (max_acs_id >= NUM_ACS_IDS - 1) { + pr_warn("Out of PCIe ACS override slots (%d)\n", + NUM_ACS_IDS); + goto next; + } + + p += 3; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].vendor = val; + + p += strcspn(p, ":"); + if (*p != ':') { + pr_warn("PCIe ACS invalid ID\n"); + goto next; + } + + p++; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].device = val; + max_acs_id++; + } +next: + p += strcspn(p, ","); + if (*p == ',') + p++; + } + + if (acs_on_downstream || acs_on_multifunction || max_acs_id) + pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n"); + + return 0; +} +early_param("pcie_acs_override", pcie_acs_override_setup); + +static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags) +{ + int i; + + /* Never override ACS for legacy devices or devices with ACS caps */ + if (!pci_is_pcie(dev) || + pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS)) + return -ENOTTY; + + for (i = 0; i < max_acs_id; i++) + if (acs_on_ids[i].vendor == dev->vendor && + acs_on_ids[i].device == dev->device) + return 1; + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_DOWNSTREAM: + case PCI_EXP_TYPE_ROOT_PORT: + if (acs_on_downstream) + return 1; + break; + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_UPSTREAM: + case PCI_EXP_TYPE_LEG_END: + case PCI_EXP_TYPE_RC_END: + if (acs_on_multifunction && dev->multifunction) + return 1; + } + + return -ENOTTY; +} /* * Ivytown NTB BAR sizes are misreported by the hardware due to an erratum. * To work around this, query the size it should be configured to by the @@ -4966,6 +5066,7 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_NXP, 0x8d9b, pci_quirk_nxp_rp_acs }, /* Zhaoxin Root/Downstream Ports */ { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, + { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides }, { 0 } }; diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig index cc30ff93e2e47..812116d48064e 100644 --- a/drivers/tty/Kconfig +++ b/drivers/tty/Kconfig @@ -75,6 +75,19 @@ config VT_CONSOLE_SLEEP def_bool y depends on VT_CONSOLE && PM_SLEEP +config NR_TTY_DEVICES + int "Maximum tty device number" + depends on VT + range 12 63 + default 63 + help + This option is used to change the number of tty devices in /dev. + The default value is 63. The lowest number you can set is 12, + 63 is also the upper limit so we don't overrun the serial + consoles. + + If unsure, say 63. + config HW_CONSOLE bool depends on VT diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index 86ffb58fbc854..ae54dd33e2336 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -402,6 +402,7 @@ static int sp5100_tco_setupdevice_mmio(struct device *dev, iounmap(addr); release_resource(res); + kfree(res); return ret; } diff --git a/fs/proc/base.c b/fs/proc/base.c index 8dfa36a99c742..87eb5d21944cb 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3237,6 +3237,9 @@ static const struct pid_entry tgid_base_stuff[] = { REG("smaps", S_IRUGO, proc_pid_smaps_operations), REG("smaps_rollup", S_IRUGO, proc_pid_smaps_rollup_operations), REG("pagemap", S_IRUSR, proc_pagemap_operations), +#ifdef CONFIG_MEM_SOFT_DIRTY + REG("pagemap_reset", S_IRUSR, proc_pagemap_reset_operations), +#endif #endif #ifdef CONFIG_SECURITY DIR("attr", S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations), diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 06a80f78433d8..4794d54d5fb72 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -300,6 +300,7 @@ extern const struct file_operations proc_pid_smaps_operations; extern const struct file_operations proc_pid_smaps_rollup_operations; extern const struct file_operations proc_clear_refs_operations; extern const struct file_operations proc_pagemap_operations; +extern const struct file_operations proc_pagemap_reset_operations; extern unsigned long task_vsize(struct mm_struct *); extern unsigned long task_statm(struct mm_struct *, diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 2d04e3470d4cd..4c241f988fbe0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1063,6 +1063,8 @@ enum clear_refs_types { struct clear_refs_private { enum clear_refs_types type; + unsigned long start, end; + bool clear_range; }; #ifdef CONFIG_MEM_SOFT_DIRTY @@ -1083,8 +1085,8 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, return page_maybe_dma_pinned(page); } -static inline void clear_soft_dirty(struct vm_area_struct *vma, - unsigned long addr, pte_t *pte) +static inline bool clear_soft_dirty(struct vm_area_struct *vma, + unsigned long addr, pte_t *pte) { /* * The soft-dirty tracker uses #PF-s to catch writes @@ -1093,37 +1095,46 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma, * of how soft-dirty works. */ pte_t ptent = *pte; + bool ret = false; if (pte_present(ptent)) { pte_t old_pte; if (pte_is_pinned(vma, addr, ptent)) - return; + return ret; old_pte = ptep_modify_prot_start(vma, addr, pte); + ret = pte_soft_dirty(old_pte); ptent = pte_wrprotect(old_pte); ptent = pte_clear_soft_dirty(ptent); ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); } else if (is_swap_pte(ptent)) { + ret = pte_swp_soft_dirty(ptent); ptent = pte_swp_clear_soft_dirty(ptent); set_pte_at(vma->vm_mm, addr, pte, ptent); } + return ret; } #else -static inline void clear_soft_dirty(struct vm_area_struct *vma, +static inline bool clear_soft_dirty(struct vm_area_struct *vma, unsigned long addr, pte_t *pte) { + return false; } #endif #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static inline bool clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { pmd_t old, pmd = *pmdp; + bool ret = false; if (pmd_present(pmd)) { /* See comment in change_huge_pmd() */ old = pmdp_invalidate(vma, addr, pmdp); + + ret = pmd_soft_dirty(old); + if (pmd_dirty(old)) pmd = pmd_mkdirty(pmd); if (pmd_young(old)) @@ -1134,14 +1145,17 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } else if (is_migration_entry(pmd_to_swp_entry(pmd))) { + ret = pmd_swp_soft_dirty(pmd); pmd = pmd_swp_clear_soft_dirty(pmd); set_pmd_at(vma->vm_mm, addr, pmdp, pmd); } + return ret; } #else -static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, +static inline bool clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { + return false; } #endif @@ -1154,6 +1168,8 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, spinlock_t *ptl; struct page *page; + BUG_ON(addr < cp->start || end > cp->end); + ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1210,9 +1226,11 @@ static int clear_refs_test_walk(unsigned long start, unsigned long end, struct clear_refs_private *cp = walk->private; struct vm_area_struct *vma = walk->vma; - if (vma->vm_flags & VM_PFNMAP) + if (!cp->clear_range && (vma->vm_flags & VM_PFNMAP)) return 1; + BUG_ON(start < cp->start || end > cp->end); + /* * Writing 1 to /proc/pid/clear_refs affects all pages. * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. @@ -1235,10 +1253,12 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct task_struct *task; - char buffer[PROC_NUMBUF]; + char buffer[18]; struct mm_struct *mm; struct vm_area_struct *vma; enum clear_refs_types type; + unsigned long start, end; + bool clear_range; int itype; int rv; @@ -1247,12 +1267,34 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, count = sizeof(buffer) - 1; if (copy_from_user(buffer, buf, count)) return -EFAULT; - rv = kstrtoint(strstrip(buffer), 10, &itype); - if (rv < 0) - return rv; - type = (enum clear_refs_types)itype; - if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) - return -EINVAL; + + if (buffer[0] == '6') + { + static int once; + + if (!once++) + printk(KERN_DEBUG "task_mmu: Using POC clear refs range implementation.\n"); + + if (count != 17) + return -EINVAL; + + type = CLEAR_REFS_SOFT_DIRTY; + start = *(unsigned long *)(buffer + 1); + end = *(unsigned long *)(buffer + 1 + 8); + } + else + { + rv = kstrtoint(strstrip(buffer), 10, &itype); + if (rv < 0) + return rv; + type = (enum clear_refs_types)itype; + + if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) + return -EINVAL; + + start = 0; + end = -1UL; + } task = get_proc_task(file_inode(file)); if (!task) @@ -1264,41 +1306,87 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, .type = type, }; - if (mmap_write_lock_killable(mm)) { - count = -EINTR; - goto out_mm; + if (start || end != -1UL) + { + start = min(start, mm->highest_vm_end) & PAGE_MASK; + end = min(end, mm->highest_vm_end) & PAGE_MASK; + + if (start >= end) + { + count = -EINVAL; + goto out_mm; + } + clear_range = true; + } + else + { + clear_range = false; } + + cp.start = start; + cp.end = end; + cp.clear_range = clear_range; + if (type == CLEAR_REFS_MM_HIWATER_RSS) { + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + /* * Writing 5 to /proc/pid/clear_refs resets the peak * resident set size to this mm's current rss value. */ reset_mm_hiwater_rss(mm); - goto out_unlock; + mmap_write_unlock(mm); + goto out_mm; } if (type == CLEAR_REFS_SOFT_DIRTY) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (!(vma->vm_flags & VM_SOFTDIRTY)) - continue; - vma->vm_flags &= ~VM_SOFTDIRTY; - vma_set_page_prot(vma); + if (mmap_read_lock_killable(mm)) { + count = -EINTR; + goto out_mm; } - + if (!clear_range) + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (!(vma->vm_flags & VM_SOFTDIRTY)) + continue; + mmap_read_unlock(mm); + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + for (vma = mm->mmap; vma; vma = vma->vm_next) { + vma->vm_flags &= ~VM_SOFTDIRTY; + vma_set_page_prot(vma); + } + mmap_write_downgrade(mm); + break; + } inc_tlb_flush_pending(mm); mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, - 0, NULL, mm, 0, -1UL); + 0, NULL, mm, start, end); mmu_notifier_invalidate_range_start(&range); } - walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, + else + { + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + } + walk_page_range(mm, start, end == -1UL ? mm->highest_vm_end : end, &clear_refs_walk_ops, &cp); if (type == CLEAR_REFS_SOFT_DIRTY) { mmu_notifier_invalidate_range_end(&range); flush_tlb_mm(mm); dec_tlb_flush_pending(mm); + mmap_read_unlock(mm); + } + else + { + mmap_write_unlock(mm); } -out_unlock: - mmap_write_unlock(mm); out_mm: mmput(mm); } @@ -1320,6 +1408,7 @@ struct pagemapread { int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ pagemap_entry_t *buffer; bool show_pfn; + bool reset; }; #define PAGEMAP_WALK_SIZE (PMD_SIZE) @@ -1330,6 +1419,7 @@ struct pagemapread { #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) #define PM_SOFT_DIRTY BIT_ULL(55) #define PM_MMAP_EXCLUSIVE BIT_ULL(56) +#define PM_SOFT_DIRTY_PAGE BIT_ULL(57) #define PM_UFFD_WP BIT_ULL(57) #define PM_FILE BIT_ULL(61) #define PM_SWAP BIT_ULL(62) @@ -1351,6 +1441,14 @@ static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme, return 0; } +static int add_addr_to_pagemap(unsigned long addr, struct pagemapread *pm) +{ + ((unsigned long *)pm->buffer)[pm->pos++] = addr; + if (pm->pos >= pm->len) + return PM_END_OF_BUFFER; + return 0; +} + static int pagemap_pte_hole(unsigned long start, unsigned long end, __always_unused int depth, struct mm_walk *walk) { @@ -1358,6 +1456,9 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, unsigned long addr = start; int err = 0; + if (pm->reset) + goto out; + while (addr < end) { struct vm_area_struct *vma = find_vma(walk->mm, addr); pagemap_entry_t pme = make_pme(0, 0); @@ -1392,8 +1493,9 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, } static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, - struct vm_area_struct *vma, unsigned long addr, pte_t pte) + struct vm_area_struct *vma, unsigned long addr, pte_t *pte_addr) { + pte_t pte = *pte_addr; u64 frame = 0, flags = 0; struct page *page = NULL; bool migration = false; @@ -1404,13 +1506,13 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; if (pte_uffd_wp(pte)) flags |= PM_UFFD_WP; } else if (is_swap_pte(pte)) { swp_entry_t entry; if (pte_swp_soft_dirty(pte)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; if (pte_swp_uffd_wp(pte)) flags |= PM_UFFD_WP; entry = pte_to_swp_entry(pte); @@ -1452,6 +1554,20 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, pmd_t pmd = *pmdp; struct page *page = NULL; + if (pm->reset) + { + if (clear_soft_dirty_pmd(vma, addr, pmdp)) + { + for (; addr != end; addr += PAGE_SIZE) + { + err = add_addr_to_pagemap(addr, pm); + if (err) + break; + } + } + goto trans_huge_done; + } + if (vma->vm_flags & VM_SOFTDIRTY) flags |= PM_SOFT_DIRTY; @@ -1460,7 +1576,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, flags |= PM_PRESENT; if (pmd_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; if (pmd_uffd_wp(pmd)) flags |= PM_UFFD_WP; if (pm->show_pfn) @@ -1480,7 +1596,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, } flags |= PM_SWAP; if (pmd_swp_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; + flags |= PM_SOFT_DIRTY | PM_SOFT_DIRTY_PAGE; if (pmd_swp_uffd_wp(pmd)) flags |= PM_UFFD_WP; VM_BUG_ON(!is_pmd_migration_entry(pmd)); @@ -1505,6 +1621,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, frame += (1 << MAX_SWAPFILES_SHIFT); } } +trans_huge_done: spin_unlock(ptl); return err; } @@ -1519,10 +1636,18 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, */ orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); for (; addr < end; pte++, addr += PAGE_SIZE) { - pagemap_entry_t pme; + if (pm->reset) + { + if (clear_soft_dirty(vma, addr, pte)) + err = add_addr_to_pagemap(addr, pm); + } + else + { + pagemap_entry_t pme; - pme = pte_to_pagemap_entry(pm, vma, addr, *pte); - err = add_to_pagemap(addr, &pme, pm); + pme = pte_to_pagemap_entry(pm, vma, addr, pte); + err = add_to_pagemap(addr, &pme, pm); + } if (err) break; } @@ -1620,8 +1745,8 @@ static const struct mm_walk_ops pagemap_ops = { * determine which areas of memory are actually mapped and llseek to * skip over unmapped regions. */ -static ssize_t pagemap_read(struct file *file, char __user *buf, - size_t count, loff_t *ppos) +static ssize_t do_pagemap_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos, bool reset) { struct mm_struct *mm = file->private_data; struct pagemapread pm; @@ -1630,6 +1755,8 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long start_vaddr; unsigned long end_vaddr; int ret = 0, copied = 0; + struct mmu_notifier_range range; + size_t buffer_len; if (!mm || !mmget_not_zero(mm)) goto out; @@ -1645,19 +1772,38 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); + pm.reset = reset; + + buffer_len = min(PAGEMAP_WALK_SIZE >> PAGE_SHIFT, count / PM_ENTRY_BYTES); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); - pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); + pm.buffer = kmalloc_array(buffer_len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) goto out_mm; src = *ppos; svpfn = src / PM_ENTRY_BYTES; - end_vaddr = mm->task_size; + + start_vaddr = svpfn << PAGE_SHIFT; + + if (reset) + { + if (count < sizeof(end_vaddr)) + { + ret = -EINVAL; + goto out_mm; + } + if (copy_from_user(&end_vaddr, buf, sizeof(end_vaddr))) + return -EFAULT; + end_vaddr = min(end_vaddr, mm->task_size); + } + else + { + end_vaddr = mm->task_size; + start_vaddr = end_vaddr; + } /* watch out for wraparound */ - start_vaddr = end_vaddr; if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) start_vaddr = untagged_addr(svpfn << PAGE_SHIFT); @@ -1677,18 +1823,35 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + pm.len = min(buffer_len, count / PM_ENTRY_BYTES); + + end = reset ? end_vaddr : (start_vaddr + (pm.len << PAGE_SHIFT)); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; + ret = mmap_read_lock_killable(mm); if (ret) goto out_free; + + if (reset) + { + inc_tlb_flush_pending(mm); + mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, + 0, NULL, mm, start_vaddr, end); + mmu_notifier_invalidate_range_start(&range); + } ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); + if (reset) + { + mmu_notifier_invalidate_range_end(&range); + flush_tlb_mm(mm); + dec_tlb_flush_pending(mm); + } mmap_read_unlock(mm); - start_vaddr = end; len = min(count, PM_ENTRY_BYTES * pm.pos); + BUG_ON(ret && ret != PM_END_OF_BUFFER); if (copy_to_user(buf, pm.buffer, len)) { ret = -EFAULT; goto out_free; @@ -1696,6 +1859,8 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, copied += len; buf += len; count -= len; + + start_vaddr = reset && pm.pos == pm.len ? ((unsigned long *)pm.buffer)[pm.pos - 1] + PAGE_SIZE : end; } *ppos += copied; if (!ret || ret == PM_END_OF_BUFFER) @@ -1709,6 +1874,18 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, return ret; } +static ssize_t pagemap_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return do_pagemap_read(file, buf, count, ppos, false); +} + +static ssize_t pagemap_reset_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return do_pagemap_read(file, buf, count, ppos, true); +} + static int pagemap_open(struct inode *inode, struct file *file) { struct mm_struct *mm; @@ -1735,6 +1912,14 @@ const struct file_operations proc_pagemap_operations = { .open = pagemap_open, .release = pagemap_release, }; + +const struct file_operations proc_pagemap_reset_operations = { + .llseek = mem_lseek, /* borrow this */ + .read = pagemap_reset_read, + .open = pagemap_open, + .release = pagemap_release, +}; + #endif /* CONFIG_PROC_PAGE_MONITOR */ #ifdef CONFIG_NUMA diff --git a/include/uapi/linux/vt.h b/include/uapi/linux/vt.h index e9d39c48520af..3bceead8da40b 100644 --- a/include/uapi/linux/vt.h +++ b/include/uapi/linux/vt.h @@ -3,12 +3,25 @@ #define _UAPI_LINUX_VT_H +/* + * We will make this definition solely for the purpose of making packages + * such as splashutils build, because they can not understand that + * NR_TTY_DEVICES is defined in the kernel configuration. + */ +#ifndef CONFIG_NR_TTY_DEVICES +#define CONFIG_NR_TTY_DEVICES 63 +#endif + /* * These constants are also useful for user-level apps (e.g., VC * resizing). */ #define MIN_NR_CONSOLES 1 /* must be at least 1 */ -#define MAX_NR_CONSOLES 63 /* serial lines start at 64 */ +/* + * NR_TTY_DEVICES: + * Value MUST be at least 12 and must never be higher then 63 + */ +#define MAX_NR_CONSOLES CONFIG_NR_TTY_DEVICES /* serial lines start above this */ /* Note: the ioctl VT_GETSTATE does not work for consoles 16 and higher (since it returns a short) */ diff --git a/kernel/cpu.c b/kernel/cpu.c index bbad5e375d3ba..64ee10502fc08 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2638,6 +2638,7 @@ void init_cpu_possible(const struct cpumask *src) void init_cpu_online(const struct cpumask *src) { cpumask_copy(&__cpu_online_mask, src); + atomic_set(&__num_online_cpus, cpumask_weight(cpu_online_mask)); } void set_cpu_online(unsigned int cpu, bool online) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7120165a93426..ec492e5417fad 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3341,7 +3341,7 @@ rcu_torture_init(void) t = per_cpu(ksoftirqd, cpu); WARN_ON_ONCE(!t); sp.sched_priority = 2; - sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + sched_setscheduler_nocheck(t, SCHED_RR, &sp); } } } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c25ba442044a6..fc993202dc35b 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4588,7 +4588,7 @@ static int __init rcu_spawn_gp_kthread(void) return 0; if (kthread_prio) { sp.sched_priority = kthread_prio; - sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + sched_setscheduler_nocheck(t, SCHED_RR, &sp); } rnp = rcu_get_root(); raw_spin_lock_irqsave_rcu_node(rnp, flags); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c8ba0fe17267c..271a23bd51409 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1006,7 +1006,7 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) struct sched_param sp; sp.sched_priority = kthread_prio; - sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); + sched_setscheduler_nocheck(current, SCHED_RR, &sp); #endif /* #ifdef CONFIG_RCU_BOOST */ WRITE_ONCE(rdp->rcuc_activity, jiffies); @@ -1194,7 +1194,7 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) rnp->boost_kthread_task = t; raw_spin_unlock_irqrestore_rcu_node(rnp, flags); sp.sched_priority = kthread_prio; - sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + sched_setscheduler_nocheck(t, SCHED_RR, &sp); wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ out: diff --git a/net/mac80211/airtime.c b/net/mac80211/airtime.c index 4bab1683652d7..d01f66168e05c 100644 --- a/net/mac80211/airtime.c +++ b/net/mac80211/airtime.c @@ -448,9 +448,17 @@ static u32 ieee80211_get_rate_duration(struct ieee80211_hw *hw, return 0; } - if (WARN_ON_ONCE((status->encoding != RX_ENC_HE && streams > 4) || - (status->encoding == RX_ENC_HE && streams > 8))) + if (unlikely(status->encoding != RX_ENC_HE && streams > 4)) { + pr_warn_once("%s: status->encoding != RX_ENC_HE (%u != %d) && streams > 4 (%d > 4)\n", + __func__, status->encoding, RX_ENC_HE, streams); return 0; + } + + if (unlikely(status->encoding == RX_ENC_HE && streams > 8)) { + pr_warn_once("%s: status->encoding == RX_ENC_HE (%u == %d) && streams > 8 (%d > 8)\n", + __func__, status->encoding, RX_ENC_HE, streams); + return 0; + } duration = airtime_mcs_groups[group].duration[idx]; duration <<= airtime_mcs_groups[group].shift; diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c index ae9700e0a1a5b..608a062c56d7e 100644 --- a/net/mac80211/rate.c +++ b/net/mac80211/rate.c @@ -664,7 +664,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif, * needs to be fixed. */ if (rates[i].flags & IEEE80211_TX_RC_MCS) { - WARN_ON(rates[i].idx > 76); + if (unlikely(rates[i].idx > 76)) + pr_warn("%s: rates[%d].idx == %d > 76\n", + __func__, i, rates[i].idx); if (!(rates[i].flags & IEEE80211_TX_RC_USE_RTS_CTS) && info->control.use_cts_prot) @@ -674,7 +676,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif, } if (rates[i].flags & IEEE80211_TX_RC_VHT_MCS) { - WARN_ON(ieee80211_rate_get_vht_mcs(&rates[i]) > 9); + if (unlikely(ieee80211_rate_get_vht_mcs(&rates[i]) > 9)) + pr_warn("%s: ieee80211_rate_get_vht_mcs(&rates[%d]) == %d > 9\n", + __func__, i, ieee80211_rate_get_vht_mcs(&rates[i])); continue; } @@ -685,7 +689,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif, } /* RC is busted */ - if (WARN_ON_ONCE(rates[i].idx >= sband->n_bitrates)) { + if (unlikely(rates[i].idx >= sband->n_bitrates)) { + pr_warn_once("%s: rates[%d].idx == %d >= sband->n_bitrates == %d\n", + __func__, i, rates[i].idx, sband->n_bitrates); rates[i].idx = -1; continue; } diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c index 5f27e6746762a..b6dc3a9523861 100644 --- a/net/mac80211/rc80211_minstrel_ht.c +++ b/net/mac80211/rc80211_minstrel_ht.c @@ -508,8 +508,13 @@ minstrel_ht_get_tp_avg(struct minstrel_ht_sta *mi, int group, int rate, if (minstrel_ht_is_legacy_group(group)) overhead = mi->overhead_legacy; - else + else { ampdu_len = minstrel_ht_avg_ampdu_len(mi); + if (unlikely(!ampdu_len)) { + pr_err_once("%s: ampdu_len == 0!\n", __func__); + ampdu_len = 1; + } + } nsecs = 1000 * overhead / ampdu_len; nsecs += minstrel_mcs_groups[group].duration[rate] << diff --git a/net/sched/Kconfig b/net/sched/Kconfig index 1e8ab4749c6c3..75122fd6506b1 100644 --- a/net/sched/Kconfig +++ b/net/sched/Kconfig @@ -474,6 +474,9 @@ choice config DEFAULT_SFQ bool "Stochastic Fair Queue" if NET_SCH_SFQ + config DEFAULT_CAKE + bool "Common Applications Kept Enhanced" if NET_SCH_CAKE + config DEFAULT_PFIFO_FAST bool "Priority FIFO Fast" endchoice @@ -485,6 +488,7 @@ config DEFAULT_NET_SCH default "fq_codel" if DEFAULT_FQ_CODEL default "fq_pie" if DEFAULT_FQ_PIE default "sfq" if DEFAULT_SFQ + default "cake" if DEFAULT_CAKE default "pfifo_fast" endif -- 2.37.1 From 5f1e552cfd1f0b751241643208c9438191135eb7 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:32:54 +0200 Subject: [PATCH 06/16] fs-patches Signed-off-by: Peter Jung --- block/blk-cgroup.c | 1 + block/blk-ioc.c | 2 + block/blk-ioprio.c | 57 +------ block/blk-ioprio.h | 9 + block/blk-mq.c | 11 ++ block/blk-throttle.c | 175 +++++++++++++------ block/blk-throttle.h | 20 ++- block/ioprio.c | 58 ++++++- fs/btrfs/delayed-inode.c | 358 ++++++++++++++++++++++++++------------- fs/btrfs/delayed-inode.h | 7 + fs/btrfs/file.c | 2 +- fs/btrfs/inode.c | 31 +++- fs/btrfs/reflink.c | 8 +- fs/btrfs/tree-log.c | 18 +- fs/btrfs/tree-log.h | 3 + fs/btrfs/volumes.h | 7 + include/linux/ioprio.h | 24 ++- 17 files changed, 530 insertions(+), 261 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 764e740b0c0f0..6906981563f8e 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1299,6 +1299,7 @@ int blkcg_init_queue(struct request_queue *q) ret = blk_iolatency_init(q); if (ret) { blk_throtl_exit(q); + blk_ioprio_exit(q); goto err_destroy_all; } diff --git a/block/blk-ioc.c b/block/blk-ioc.c index df9cfe4ca5328..63fc020424082 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -247,6 +247,8 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); #endif + ioc->ioprio = IOPRIO_DEFAULT; + return ioc; } diff --git a/block/blk-ioprio.c b/block/blk-ioprio.c index 79e797f5d1949..c00060a02c6ef 100644 --- a/block/blk-ioprio.c +++ b/block/blk-ioprio.c @@ -62,7 +62,6 @@ struct ioprio_blkg { struct ioprio_blkcg { struct blkcg_policy_data cpd; enum prio_policy prio_policy; - bool prio_set; }; static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) @@ -113,7 +112,6 @@ static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, if (ret < 0) return ret; blkcg->prio_policy = ret; - blkcg->prio_set = true; return nbytes; } @@ -183,26 +181,20 @@ static struct blkcg_policy ioprio_policy = { .pd_free_fn = ioprio_free_pd, }; -struct blk_ioprio { - struct rq_qos rqos; -}; - -static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq, - struct bio *bio) +void blkcg_set_ioprio(struct bio *bio) { struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); u16 prio; - if (!blkcg->prio_set) + if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE) return; /* * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers * correspond to a lower priority. Hence, the max_t() below selects * the lower priority of bi_ioprio and the cgroup I/O priority class. - * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the - * bio I/O priority is not modified. If the bio I/O priority equals - * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio. + * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O + * priority is assigned to the bio. */ prio = max_t(u16, bio->bi_ioprio, IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); @@ -210,49 +202,14 @@ static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq, bio->bi_ioprio = prio; } -static void blkcg_ioprio_exit(struct rq_qos *rqos) +void blk_ioprio_exit(struct request_queue *q) { - struct blk_ioprio *blkioprio_blkg = - container_of(rqos, typeof(*blkioprio_blkg), rqos); - - blkcg_deactivate_policy(rqos->q, &ioprio_policy); - kfree(blkioprio_blkg); + blkcg_deactivate_policy(q, &ioprio_policy); } -static struct rq_qos_ops blkcg_ioprio_ops = { - .track = blkcg_ioprio_track, - .exit = blkcg_ioprio_exit, -}; - int blk_ioprio_init(struct request_queue *q) { - struct blk_ioprio *blkioprio_blkg; - struct rq_qos *rqos; - int ret; - - blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL); - if (!blkioprio_blkg) - return -ENOMEM; - - ret = blkcg_activate_policy(q, &ioprio_policy); - if (ret) { - kfree(blkioprio_blkg); - return ret; - } - - rqos = &blkioprio_blkg->rqos; - rqos->id = RQ_QOS_IOPRIO; - rqos->ops = &blkcg_ioprio_ops; - rqos->q = q; - - /* - * Registering the rq-qos policy after activating the blk-cgroup - * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the - * rq-qos callbacks. - */ - rq_qos_add(q, rqos); - - return 0; + return blkcg_activate_policy(q, &ioprio_policy); } static int __init ioprio_init(void) diff --git a/block/blk-ioprio.h b/block/blk-ioprio.h index a7785c2f1aea7..5a1eb550e178c 100644 --- a/block/blk-ioprio.h +++ b/block/blk-ioprio.h @@ -6,14 +6,23 @@ #include struct request_queue; +struct bio; #ifdef CONFIG_BLK_CGROUP_IOPRIO int blk_ioprio_init(struct request_queue *q); +void blk_ioprio_exit(struct request_queue *q); +void blkcg_set_ioprio(struct bio *bio); #else static inline int blk_ioprio_init(struct request_queue *q) { return 0; } +static inline void blk_ioprio_exit(struct request_queue *q) +{ +} +static inline void blkcg_set_ioprio(struct bio *bio) +{ +} #endif #endif /* _BLK_IOPRIO_H_ */ diff --git a/block/blk-mq.c b/block/blk-mq.c index 93d9d60980fb5..c81f579ace592 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -42,6 +42,7 @@ #include "blk-stat.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" +#include "blk-ioprio.h" static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); @@ -2783,6 +2784,14 @@ static inline struct request *blk_mq_get_cached_request(struct request_queue *q, return rq; } +static void bio_set_ioprio(struct bio *bio) +{ + /* Nobody set ioprio so far? Initialize it based on task's nice value */ + if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE) + bio->bi_ioprio = get_current_ioprio(); + blkcg_set_ioprio(bio); +} + /** * blk_mq_submit_bio - Create and send a request to block device. * @bio: Bio pointer. @@ -2812,6 +2821,8 @@ void blk_mq_submit_bio(struct bio *bio) if (!bio_integrity_prep(bio)) return; + bio_set_ioprio(bio); + rq = blk_mq_get_cached_request(q, plug, &bio, nr_segs); if (!rq) { if (!bio) diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 139b2d7a99e2f..7818c68e0471b 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -329,8 +329,8 @@ static struct bio *throtl_pop_queued(struct list_head *queued, /* init a service_queue, assumes the caller zeroed it */ static void throtl_service_queue_init(struct throtl_service_queue *sq) { - INIT_LIST_HEAD(&sq->queued[0]); - INIT_LIST_HEAD(&sq->queued[1]); + INIT_LIST_HEAD(&sq->queued[READ]); + INIT_LIST_HEAD(&sq->queued[WRITE]); sq->pending_tree = RB_ROOT_CACHED; timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0); } @@ -520,7 +520,6 @@ static void throtl_rb_erase(struct rb_node *n, { rb_erase_cached(n, &parent_sq->pending_tree); RB_CLEAR_NODE(n); - --parent_sq->nr_pending; } static void update_min_dispatch_time(struct throtl_service_queue *parent_sq) @@ -562,19 +561,16 @@ static void tg_service_queue_add(struct throtl_grp *tg) static void throtl_enqueue_tg(struct throtl_grp *tg) { - if (!(tg->flags & THROTL_TG_PENDING)) { - tg_service_queue_add(tg); - tg->flags |= THROTL_TG_PENDING; - tg->service_queue.parent_sq->nr_pending++; - } + tg_service_queue_add(tg); + tg->service_queue.parent_sq->nr_pending++; } static void throtl_dequeue_tg(struct throtl_grp *tg) { - if (tg->flags & THROTL_TG_PENDING) { - throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); - tg->flags &= ~THROTL_TG_PENDING; - } + struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; + + throtl_rb_erase(&tg->rb_node, parent_sq); + --parent_sq->nr_pending; } /* Call with queue lock held */ @@ -639,6 +635,8 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, { tg->bytes_disp[rw] = 0; tg->io_disp[rw] = 0; + tg->bytes_skipped[rw] = 0; + tg->io_skipped[rw] = 0; /* * Previous slice has expired. We must have trimmed it after last @@ -656,12 +654,17 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, tg->slice_end[rw], jiffies); } -static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw) +static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, + bool clear_skipped) { tg->bytes_disp[rw] = 0; tg->io_disp[rw] = 0; tg->slice_start[rw] = jiffies; tg->slice_end[rw] = jiffies + tg->td->throtl_slice; + if (clear_skipped) { + tg->bytes_skipped[rw] = 0; + tg->io_skipped[rw] = 0; + } throtl_log(&tg->service_queue, "[%c] new slice start=%lu end=%lu jiffies=%lu", @@ -754,33 +757,20 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) tg->slice_start[rw], tg->slice_end[rw], jiffies); } -static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, - u32 iops_limit, unsigned long *wait) +static unsigned int calculate_io_allowed(u32 iops_limit, + unsigned long jiffy_elapsed) { - bool rw = bio_data_dir(bio); unsigned int io_allowed; - unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; u64 tmp; - if (iops_limit == UINT_MAX) { - if (wait) - *wait = 0; - return true; - } - - jiffy_elapsed = jiffies - tg->slice_start[rw]; - - /* Round up to the next throttle slice, wait time must be nonzero */ - jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); - /* - * jiffy_elapsed_rnd should not be a big value as minimum iops can be + * jiffy_elapsed should not be a big value as minimum iops can be * 1 then at max jiffy elapsed should be equivalent of 1 second as we * will allow dispatch after 1 second and after that slice should * have been trimmed. */ - tmp = (u64)iops_limit * jiffy_elapsed_rnd; + tmp = (u64)iops_limit * jiffy_elapsed; do_div(tmp, HZ); if (tmp > UINT_MAX) @@ -788,6 +778,73 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, else io_allowed = tmp; + return io_allowed; +} + +static u64 calculate_bytes_allowed(u64 bps_limit, unsigned long jiffy_elapsed) +{ + return mul_u64_u64_div_u64(bps_limit, (u64)jiffy_elapsed, (u64)HZ); +} + +static void __tg_update_skipped(struct throtl_grp *tg, bool rw) +{ + unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; + u64 bps_limit = tg_bps_limit(tg, rw); + u32 iops_limit = tg_iops_limit(tg, rw); + + /* + * If config is updated while bios are still throttled, calculate and + * accumulate how many bytes/io are waited across changes. And + * bytes/io_skipped will be used to calculate new wait time under new + * configuration. + * + * Following calculation won't overflow as long as bios that are + * dispatched later won't preempt already throttled bios. Even if such + * overflow do happen, there should be no problem because unsigned is + * used here, and bytes_skipped/io_skipped will be updated correctly. + */ + if (bps_limit != U64_MAX) + tg->bytes_skipped[rw] += + calculate_bytes_allowed(bps_limit, jiffy_elapsed) - + tg->bytes_disp[rw]; + if (iops_limit != UINT_MAX) + tg->io_skipped[rw] += + calculate_io_allowed(iops_limit, jiffy_elapsed) - + tg->io_disp[rw]; +} + +static void tg_update_skipped(struct throtl_grp *tg) +{ + if (tg->service_queue.nr_queued[READ]) + __tg_update_skipped(tg, READ); + if (tg->service_queue.nr_queued[WRITE]) + __tg_update_skipped(tg, WRITE); + + /* see comments in struct throtl_grp for meaning of these fields. */ + throtl_log(&tg->service_queue, "%s: %llu %llu %u %u\n", __func__, + tg->bytes_skipped[READ], tg->bytes_skipped[WRITE], + tg->io_skipped[READ], tg->io_skipped[WRITE]); +} + +static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, + u32 iops_limit, unsigned long *wait) +{ + bool rw = bio_data_dir(bio); + unsigned int io_allowed; + unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; + + if (iops_limit == UINT_MAX) { + if (wait) + *wait = 0; + return true; + } + + jiffy_elapsed = jiffies - tg->slice_start[rw]; + + /* Round up to the next throttle slice, wait time must be nonzero */ + jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); + io_allowed = calculate_io_allowed(iops_limit, jiffy_elapsed_rnd) + + tg->io_skipped[rw]; if (tg->io_disp[rw] + 1 <= io_allowed) { if (wait) *wait = 0; @@ -806,12 +863,12 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, u64 bps_limit, unsigned long *wait) { bool rw = bio_data_dir(bio); - u64 bytes_allowed, extra_bytes, tmp; + u64 bytes_allowed, extra_bytes; unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; unsigned int bio_size = throtl_bio_data_size(bio); /* no need to throttle if this bio's bytes have been accounted */ - if (bps_limit == U64_MAX || bio_flagged(bio, BIO_THROTTLED)) { + if (bps_limit == U64_MAX) { if (wait) *wait = 0; return true; @@ -824,11 +881,8 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, jiffy_elapsed_rnd = tg->td->throtl_slice; jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); - - tmp = bps_limit * jiffy_elapsed_rnd; - do_div(tmp, HZ); - bytes_allowed = tmp; - + bytes_allowed = calculate_bytes_allowed(bps_limit, jiffy_elapsed_rnd) + + tg->bytes_skipped[rw]; if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) { if (wait) *wait = 0; @@ -889,7 +943,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, * slice and it should be extended instead. */ if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) - throtl_start_new_slice(tg, rw); + throtl_start_new_slice(tg, rw, true); else { if (time_before(tg->slice_end[rw], jiffies + tg->td->throtl_slice)) @@ -921,11 +975,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) unsigned int bio_size = throtl_bio_data_size(bio); /* Charge the bio to the group */ - if (!bio_flagged(bio, BIO_THROTTLED)) { - tg->bytes_disp[rw] += bio_size; - tg->last_bytes_disp[rw] += bio_size; - } - + tg->bytes_disp[rw] += bio_size; + tg->last_bytes_disp[rw] += bio_size; tg->io_disp[rw]++; tg->last_io_disp[rw]++; @@ -968,8 +1019,9 @@ static void throtl_add_bio_tg(struct bio *bio, struct throtl_qnode *qn, throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); + if (!sq->nr_queued[READ] && !sq->nr_queued[WRITE]) + throtl_enqueue_tg(tg); sq->nr_queued[rw]++; - throtl_enqueue_tg(tg); } static void tg_update_disptime(struct throtl_grp *tg) @@ -990,9 +1042,9 @@ static void tg_update_disptime(struct throtl_grp *tg) disptime = jiffies + min_wait; /* Update dispatch time */ - throtl_dequeue_tg(tg); + throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); tg->disptime = disptime; - throtl_enqueue_tg(tg); + tg_service_queue_add(tg); /* see throtl_add_bio_tg() */ tg->flags &= ~THROTL_TG_WAS_EMPTY; @@ -1101,13 +1153,13 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq) if (time_before(jiffies, tg->disptime)) break; - throtl_dequeue_tg(tg); - nr_disp += throtl_dispatch_tg(tg); sq = &tg->service_queue; - if (sq->nr_queued[0] || sq->nr_queued[1]) + if (sq->nr_queued[READ] || sq->nr_queued[WRITE]) tg_update_disptime(tg); + else + throtl_dequeue_tg(tg); if (nr_disp >= THROTL_QUANTUM) break; @@ -1321,10 +1373,10 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) * that a group's limit are dropped suddenly and we don't want to * account recently dispatched IO with new low rate. */ - throtl_start_new_slice(tg, READ); - throtl_start_new_slice(tg, WRITE); + throtl_start_new_slice(tg, READ, false); + throtl_start_new_slice(tg, WRITE, false); - if (tg->flags & THROTL_TG_PENDING) { + if (sq->nr_queued[READ] || sq->nr_queued[WRITE]) { tg_update_disptime(tg); throtl_schedule_next_dispatch(sq->parent_sq, true); } @@ -1350,6 +1402,7 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, v = U64_MAX; tg = blkg_to_tg(ctx.blkg); + tg_update_skipped(tg); if (is_u64) *(u64 *)((void *)tg + of_cft(of)->private) = v; @@ -1536,6 +1589,7 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, return ret; tg = blkg_to_tg(ctx.blkg); + tg_update_skipped(tg); v[0] = tg->bps_conf[READ][index]; v[1] = tg->bps_conf[WRITE][index]; @@ -2121,6 +2175,23 @@ bool __blk_throtl_bio(struct bio *bio) tg->last_low_overflow_time[rw] = jiffies; throtl_downgrade_check(tg); throtl_upgrade_check(tg); + + /* + * Splited bios can be re-entered because iops limit should be + * counted again, however, bps limit should not. Since bps limit + * will be counted again while dispatching it, compensate the + * over-accounting here. Since decrement of 'bytes_disp' can't + * handle the case that new slice is started, increase + * 'bytes_skipped' instead. + */ + if (bio_flagged(bio, BIO_THROTTLED)) { + unsigned int bio_size = throtl_bio_data_size(bio); + + tg->bytes_skipped[rw] += bio_size; + if (tg->last_bytes_disp[rw] >= bio_size) + tg->last_bytes_disp[rw] -= bio_size; + } + /* throtl is FIFO - if bios are already queued, should queue */ if (sq->nr_queued[rw]) break; diff --git a/block/blk-throttle.h b/block/blk-throttle.h index c1b6029961272..2ae5ac8fe76ec 100644 --- a/block/blk-throttle.h +++ b/block/blk-throttle.h @@ -53,10 +53,9 @@ struct throtl_service_queue { }; enum tg_state_flags { - THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */ - THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */ - THROTL_TG_HAS_IOPS_LIMIT = 1 << 2, /* tg has iops limit */ - THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */ + THROTL_TG_WAS_EMPTY = 1 << 0, /* bio_lists[] became non-empty */ + THROTL_TG_HAS_IOPS_LIMIT = 1 << 1, /* tg has iops limit */ + THROTL_TG_CANCELING = 1 << 2, /* starts to cancel bio */ }; enum { @@ -115,6 +114,19 @@ struct throtl_grp { uint64_t bytes_disp[2]; /* Number of bio's dispatched in current slice */ unsigned int io_disp[2]; + /* + * The following two fields are updated when new configuration is + * submitted while some bios are still throttled, they record how many + * bytes/io are waited already in previous configuration, and they will + * be used to calculate wait time under new configuration. + * + * Number of bytes will be skipped in current slice. In addition, this + * field will help to handle re-entered bio for bps limit, see details + * in __blk_throtl_bio(). + */ + uint64_t bytes_skipped[2]; + /* Number of bio will be skipped in current slice */ + unsigned int io_skipped[2]; unsigned long last_low_overflow_time[2]; diff --git a/block/ioprio.c b/block/ioprio.c index 2fe068fcaad58..09af41ac6157c 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -138,6 +138,32 @@ SYSCALL_DEFINE3(ioprio_set, int, which, int, who, int, ioprio) return ret; } +/* + * If the task has set an I/O priority, use that. Otherwise, return + * the default I/O priority. + * + * Expected to be called for current task or with task_lock() held to keep + * io_context stable. + */ +int __get_task_ioprio(struct task_struct *p) +{ + struct io_context *ioc = p->io_context; + int prio; + + if (p != current) + lockdep_assert_held(&p->alloc_lock); + if (ioc) + prio = ioc->ioprio; + else + prio = IOPRIO_DEFAULT; + + if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE) + prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(p), + task_nice_ioprio(p)); + return prio; +} +EXPORT_SYMBOL(__get_task_ioprio); + static int get_task_ioprio(struct task_struct *p) { int ret; @@ -145,22 +171,38 @@ static int get_task_ioprio(struct task_struct *p) ret = security_task_getioprio(p); if (ret) goto out; - ret = IOPRIO_DEFAULT; + task_lock(p); + ret = __get_task_ioprio(p); + task_unlock(p); +out: + return ret; +} + +/* + * Return raw IO priority value as set by userspace. We use this for + * ioprio_get(pid, IOPRIO_WHO_PROCESS) so that we keep historical behavior and + * also so that userspace can distinguish unset IO priority (which just gets + * overriden based on task's nice value) from IO priority set to some value. + */ +static int get_task_raw_ioprio(struct task_struct *p) +{ + int ret; + + ret = security_task_getioprio(p); + if (ret) + goto out; task_lock(p); if (p->io_context) ret = p->io_context->ioprio; + else + ret = IOPRIO_DEFAULT; task_unlock(p); out: return ret; } -int ioprio_best(unsigned short aprio, unsigned short bprio) +static int ioprio_best(unsigned short aprio, unsigned short bprio) { - if (!ioprio_valid(aprio)) - aprio = IOPRIO_DEFAULT; - if (!ioprio_valid(bprio)) - bprio = IOPRIO_DEFAULT; - return min(aprio, bprio); } @@ -181,7 +223,7 @@ SYSCALL_DEFINE2(ioprio_get, int, which, int, who) else p = find_task_by_vpid(who); if (p) - ret = get_task_ioprio(p); + ret = get_task_raw_ioprio(p); break; case IOPRIO_WHO_PGRP: if (!who) diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 748bf6b0d8600..e131a697c0cc4 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -52,18 +52,6 @@ static inline void btrfs_init_delayed_node( INIT_LIST_HEAD(&delayed_node->p_list); } -static inline int btrfs_is_continuous_delayed_item( - struct btrfs_delayed_item *item1, - struct btrfs_delayed_item *item2) -{ - if (item1->key.type == BTRFS_DIR_INDEX_KEY && - item1->key.objectid == item2->key.objectid && - item1->key.type == item2->key.type && - item1->key.offset + 1 == item2->key.offset) - return 1; - return 0; -} - static struct btrfs_delayed_node *btrfs_get_delayed_node( struct btrfs_inode *btrfs_inode) { @@ -398,8 +386,7 @@ static struct btrfs_delayed_item *__btrfs_lookup_delayed_insertion_item( } static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, - struct btrfs_delayed_item *ins, - int action) + struct btrfs_delayed_item *ins) { struct rb_node **p, *node; struct rb_node *parent_node = NULL; @@ -408,9 +395,9 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, int cmp; bool leftmost = true; - if (action == BTRFS_DELAYED_INSERTION_ITEM) + if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM) root = &delayed_node->ins_root; - else if (action == BTRFS_DELAYED_DELETION_ITEM) + else if (ins->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) root = &delayed_node->del_root; else BUG(); @@ -436,32 +423,19 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, rb_link_node(node, parent_node, p); rb_insert_color_cached(node, root, leftmost); ins->delayed_node = delayed_node; - ins->ins_or_del = action; - if (ins->key.type == BTRFS_DIR_INDEX_KEY && - action == BTRFS_DELAYED_INSERTION_ITEM && + /* Delayed items are always for dir index items. */ + ASSERT(ins->key.type == BTRFS_DIR_INDEX_KEY); + + if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM && ins->key.offset >= delayed_node->index_cnt) - delayed_node->index_cnt = ins->key.offset + 1; + delayed_node->index_cnt = ins->key.offset + 1; delayed_node->count++; atomic_inc(&delayed_node->root->fs_info->delayed_root->items); return 0; } -static int __btrfs_add_delayed_insertion_item(struct btrfs_delayed_node *node, - struct btrfs_delayed_item *item) -{ - return __btrfs_add_delayed_item(node, item, - BTRFS_DELAYED_INSERTION_ITEM); -} - -static int __btrfs_add_delayed_deletion_item(struct btrfs_delayed_node *node, - struct btrfs_delayed_item *item) -{ - return __btrfs_add_delayed_item(node, item, - BTRFS_DELAYED_DELETION_ITEM); -} - static void finish_one_item(struct btrfs_delayed_root *delayed_root) { int seq = atomic_inc_return(&delayed_root->items_seq); @@ -573,7 +547,13 @@ static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, trace_btrfs_space_reservation(fs_info, "delayed_item", item->key.objectid, num_bytes, 1); - item->bytes_reserved = num_bytes; + /* + * For insertions we track reserved metadata space by accounting + * for the number of leaves that will be used, based on the delayed + * node's index_items_size field. + */ + if (item->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) + item->bytes_reserved = num_bytes; } return ret; @@ -672,23 +652,41 @@ static void btrfs_delayed_inode_release_metadata(struct btrfs_fs_info *fs_info, } /* - * Insert a single delayed item or a batch of delayed items that have consecutive - * keys if they exist. + * Insert a single delayed item or a batch of delayed items, as many as possible + * that fit in a leaf. The delayed items (dir index keys) are sorted by their key + * in the rbtree, and if there's a gap between two consecutive dir index items, + * then it means at some point we had delayed dir indexes to add but they got + * removed (by btrfs_delete_delayed_dir_index()) before we attempted to flush them + * into the subvolume tree. Dir index keys also have their offsets coming from a + * monotonically increasing counter, so we can't get new keys with an offset that + * fits within a gap between delayed dir index items. */ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, struct btrfs_delayed_item *first_item) { + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_delayed_node *node = first_item->delayed_node; LIST_HEAD(item_list); struct btrfs_delayed_item *curr; struct btrfs_delayed_item *next; - const int max_size = BTRFS_LEAF_DATA_SIZE(root->fs_info); + const int max_size = BTRFS_LEAF_DATA_SIZE(fs_info); struct btrfs_item_batch batch; int total_size; char *ins_data = NULL; int ret; + lockdep_assert_held(&node->mutex); + + /* + * For delayed items to insert, we track reserved metadata bytes based + * on the number of leaves that we will use. + * See btrfs_insert_delayed_dir_index() and + * btrfs_delayed_item_reserve_metadata()). + */ + ASSERT(first_item->bytes_reserved == 0); + list_add_tail(&first_item->tree_list, &item_list); batch.total_data_size = first_item->data_len; batch.nr = 1; @@ -699,9 +697,11 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, int next_size; next = __btrfs_next_delayed_item(curr); - if (!next || !btrfs_is_continuous_delayed_item(curr, next)) + if (!next) break; + ASSERT(next->bytes_reserved == 0); + next_size = next->data_len + sizeof(struct btrfs_item); if (total_size + next_size > max_size) break; @@ -760,9 +760,20 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, list_for_each_entry_safe(curr, next, &item_list, tree_list) { list_del(&curr->tree_list); - btrfs_delayed_item_release_metadata(root, curr); btrfs_release_delayed_item(curr); } + + /* + * We inserted one batch of items into a leaf, now release one unit of + * metadata space from the delayed block reserve. + */ + if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + btrfs_calc_insert_metadata_size(fs_info, 1), + NULL); + + ASSERT(node->index_items_size >= total_size); + node->index_items_size -= total_size; out: kfree(ins_data); return ret; @@ -796,62 +807,75 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_delayed_item *item) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_delayed_item *curr, *next; - struct extent_buffer *leaf; - struct btrfs_key key; - struct list_head head; - int nitems, i, last_item; - int ret = 0; + struct extent_buffer *leaf = path->nodes[0]; + LIST_HEAD(batch_list); + int nitems, slot, last_slot; + int ret; + u64 total_reserved_size = item->bytes_reserved; - BUG_ON(!path->nodes[0]); + ASSERT(leaf != NULL); - leaf = path->nodes[0]; + slot = path->slots[0]; + last_slot = btrfs_header_nritems(leaf) - 1; + /* + * Our caller always gives us a path pointing to an existing item, so + * this can not happen. + */ + ASSERT(slot <= last_slot); + if (WARN_ON(slot > last_slot)) + return -ENOENT; - i = path->slots[0]; - last_item = btrfs_header_nritems(leaf) - 1; - if (i > last_item) - return -ENOENT; /* FIXME: Is errno suitable? */ + nitems = 1; + curr = item; + list_add_tail(&curr->tree_list, &batch_list); - next = item; - INIT_LIST_HEAD(&head); - btrfs_item_key_to_cpu(leaf, &key, i); - nitems = 0; /* - * count the number of the dir index items that we can delete in batch + * Keep checking if the next delayed item matches the next item in the + * leaf - if so, we can add it to the batch of items to delete from the + * leaf. */ - while (btrfs_comp_cpu_keys(&next->key, &key) == 0) { - list_add_tail(&next->tree_list, &head); - nitems++; + while (slot < last_slot) { + struct btrfs_key key; - curr = next; next = __btrfs_next_delayed_item(curr); if (!next) break; - if (!btrfs_is_continuous_delayed_item(curr, next)) - break; - - i++; - if (i > last_item) + slot++; + btrfs_item_key_to_cpu(leaf, &key, slot); + if (btrfs_comp_cpu_keys(&next->key, &key) != 0) break; - btrfs_item_key_to_cpu(leaf, &key, i); + nitems++; + curr = next; + list_add_tail(&curr->tree_list, &batch_list); + total_reserved_size += curr->bytes_reserved; } - if (!nitems) - return 0; - ret = btrfs_del_items(trans, root, path, path->slots[0], nitems); if (ret) - goto out; + return ret; + + /* In case of BTRFS_FS_LOG_RECOVERING items won't have reserved space */ + if (total_reserved_size > 0) { + /* + * Check btrfs_delayed_item_reserve_metadata() to see why we + * don't need to release/reserve qgroup space. + */ + trace_btrfs_space_reservation(fs_info, "delayed_item", + item->key.objectid, total_reserved_size, + 0); + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + total_reserved_size, NULL); + } - list_for_each_entry_safe(curr, next, &head, tree_list) { - btrfs_delayed_item_release_metadata(root, curr); + list_for_each_entry_safe(curr, next, &batch_list, tree_list) { list_del(&curr->tree_list); btrfs_release_delayed_item(curr); } -out: - return ret; + return 0; } static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, @@ -859,43 +883,52 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_delayed_node *node) { - struct btrfs_delayed_item *curr, *prev; int ret = 0; -do_again: - mutex_lock(&node->mutex); - curr = __btrfs_first_delayed_deletion_item(node); - if (!curr) - goto delete_fail; + while (ret == 0) { + struct btrfs_delayed_item *item; + + mutex_lock(&node->mutex); + item = __btrfs_first_delayed_deletion_item(node); + if (!item) { + mutex_unlock(&node->mutex); + break; + } + + ret = btrfs_search_slot(trans, root, &item->key, path, -1, 1); + if (ret > 0) { + /* + * There's no matching item in the leaf. This means we + * have already deleted this item in a past run of the + * delayed items. We ignore errors when running delayed + * items from an async context, through a work queue job + * running btrfs_async_run_delayed_root(), and don't + * release delayed items that failed to complete. This + * is because we will retry later, and at transaction + * commit time we always run delayed items and will + * then deal with errors if they fail to run again. + * + * So just release delayed items for which we can't find + * an item in the tree, and move to the next item. + */ + btrfs_release_path(path); + btrfs_release_delayed_item(item); + ret = 0; + } else if (ret == 0) { + ret = btrfs_batch_delete_items(trans, root, path, item); + btrfs_release_path(path); + } - ret = btrfs_search_slot(trans, root, &curr->key, path, -1, 1); - if (ret < 0) - goto delete_fail; - else if (ret > 0) { /* - * can't find the item which the node points to, so this node - * is invalid, just drop it. + * We unlock and relock on each iteration, this is to prevent + * blocking other tasks for too long while we are being run from + * the async context (work queue job). Those tasks are typically + * running system calls like creat/mkdir/rename/unlink/etc which + * need to add delayed items to this delayed node. */ - prev = curr; - curr = __btrfs_next_delayed_item(prev); - btrfs_release_delayed_item(prev); - ret = 0; - btrfs_release_path(path); - if (curr) { - mutex_unlock(&node->mutex); - goto do_again; - } else - goto delete_fail; + mutex_unlock(&node->mutex); } - btrfs_batch_delete_items(trans, root, path, curr); - btrfs_release_path(path); - mutex_unlock(&node->mutex); - goto do_again; - -delete_fail: - btrfs_release_path(path); - mutex_unlock(&node->mutex); return ret; } @@ -1347,6 +1380,23 @@ void btrfs_balance_delayed_items(struct btrfs_fs_info *fs_info) btrfs_wq_run_delayed_node(delayed_root, fs_info, BTRFS_DELAYED_BATCH); } +/* + * Return how many leaves of metadata we will use for inserting a given amount + * (in bytes) of dir index items (including sizeof struct btrfs_item). + */ +static unsigned int num_dir_index_leaves(const struct btrfs_fs_info *fs_info, + u32 index_items_size) +{ + const unsigned int leaf_data_size = BTRFS_LEAF_DATA_SIZE(fs_info); + unsigned int result; + + result = index_items_size / leaf_data_size; + if ((index_items_size % leaf_data_size) != 0) + result++; + + return result; +} + /* Will return 0 or -ENOMEM */ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, const char *name, int name_len, @@ -1354,9 +1404,13 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, struct btrfs_disk_key *disk_key, u8 type, u64 index) { + struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_delayed_node *delayed_node; struct btrfs_delayed_item *delayed_item; struct btrfs_dir_item *dir_item; + unsigned int leaves_before; + unsigned int leaves_after; + u32 data_len; int ret; delayed_node = btrfs_get_or_create_delayed_node(dir); @@ -1372,6 +1426,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, delayed_item->key.objectid = btrfs_ino(dir); delayed_item->key.type = BTRFS_DIR_INDEX_KEY; delayed_item->key.offset = index; + delayed_item->ins_or_del = BTRFS_DELAYED_INSERTION_ITEM; dir_item = (struct btrfs_dir_item *)delayed_item->data; dir_item->location = *disk_key; @@ -1381,15 +1436,47 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, btrfs_set_stack_dir_type(dir_item, type); memcpy((char *)(dir_item + 1), name, name_len); - ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, delayed_item); - /* - * we have reserved enough space when we start a new transaction, - * so reserving metadata failure is impossible - */ - BUG_ON(ret); + data_len = delayed_item->data_len + sizeof(struct btrfs_item); mutex_lock(&delayed_node->mutex); - ret = __btrfs_add_delayed_insertion_item(delayed_node, delayed_item); + leaves_before = num_dir_index_leaves(fs_info, + delayed_node->index_items_size); + leaves_after = num_dir_index_leaves(fs_info, + delayed_node->index_items_size + + data_len); + + ASSERT(leaves_after >= leaves_before); + ASSERT((leaves_after - leaves_before) <= 1); + + if (leaves_after > leaves_before) { + ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, + delayed_item); + /* + * Space was reserved for a dir index item insertion when we + * started the transaction, so getting a failure here should be + * impossible. + */ + if (WARN_ON(ret)) { + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_item(delayed_item); + goto release_node; + } + } else if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { + const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + + /* + * Adding the new dir index item does not require touching another + * leaf, so we can release 1 unit of metadata that was previously + * reserved when starting the transaction. This applies only to + * the case where we had a transaction start and excludes the + * transaction join case (when replaying log trees). + */ + btrfs_block_rsv_release(fs_info, trans->block_rsv, bytes, NULL); + ASSERT(trans->bytes_reserved >= bytes); + trans->bytes_reserved -= bytes; + } + + ret = __btrfs_add_delayed_item(delayed_node, delayed_item); if (unlikely(ret)) { btrfs_err(trans->fs_info, "err add delayed dir index item(name: %.*s) into the insertion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)", @@ -1397,6 +1484,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, delayed_node->inode_id, ret); BUG(); } + delayed_node->index_items_size += data_len; mutex_unlock(&delayed_node->mutex); release_node: @@ -1409,6 +1497,9 @@ static int btrfs_delete_delayed_insertion_item(struct btrfs_fs_info *fs_info, struct btrfs_key *key) { struct btrfs_delayed_item *item; + unsigned int leaves_before; + unsigned int leaves_after; + u32 data_len; mutex_lock(&node->mutex); item = __btrfs_lookup_delayed_insertion_item(node, key); @@ -1417,7 +1508,32 @@ static int btrfs_delete_delayed_insertion_item(struct btrfs_fs_info *fs_info, return 1; } - btrfs_delayed_item_release_metadata(node->root, item); + /* + * For delayed items to insert, we track reserved metadata bytes based + * on the number of leaves that we will use. + * See btrfs_insert_delayed_dir_index() and + * btrfs_delayed_item_reserve_metadata()). + */ + ASSERT(item->bytes_reserved == 0); + + data_len = item->data_len + sizeof(struct btrfs_item); + ASSERT(node->index_items_size >= data_len); + + leaves_before = num_dir_index_leaves(fs_info, node->index_items_size); + node->index_items_size -= data_len; + leaves_after = num_dir_index_leaves(fs_info, node->index_items_size); + + ASSERT(leaves_after <= leaves_before); + ASSERT((leaves_before - leaves_after) <= 1); + + if (leaves_after < leaves_before && + !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { + const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + bytes, NULL); + } + btrfs_release_delayed_item(item); mutex_unlock(&node->mutex); return 0; @@ -1451,6 +1567,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, } item->key = item_key; + item->ins_or_del = BTRFS_DELAYED_DELETION_ITEM; ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, item); /* @@ -1465,7 +1582,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, } mutex_lock(&node->mutex); - ret = __btrfs_add_delayed_deletion_item(node, item); + ret = __btrfs_add_delayed_item(node, item); if (unlikely(ret)) { btrfs_err(trans->fs_info, "err add delayed dir index item(index: %llu) into the deletion tree of the delayed node(root id: %llu, inode id: %llu, errno: %d)", @@ -1833,12 +1950,25 @@ static void __btrfs_kill_delayed_node(struct btrfs_delayed_node *delayed_node) mutex_lock(&delayed_node->mutex); curr_item = __btrfs_first_delayed_insertion_item(delayed_node); while (curr_item) { - btrfs_delayed_item_release_metadata(root, curr_item); prev_item = curr_item; curr_item = __btrfs_next_delayed_item(prev_item); btrfs_release_delayed_item(prev_item); } + if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && + delayed_node->index_items_size > 0) { + unsigned int num_leaves; + u64 bytes; + + num_leaves = num_dir_index_leaves(fs_info, + delayed_node->index_items_size); + bytes = btrfs_calc_insert_metadata_size(fs_info, num_leaves); + + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + bytes, NULL); + } + delayed_node->index_items_size = 0; + curr_item = __btrfs_first_delayed_deletion_item(delayed_node); while (curr_item) { btrfs_delayed_item_release_metadata(root, curr_item); diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h index b2412160c5bc9..2af0d82274791 100644 --- a/fs/btrfs/delayed-inode.h +++ b/fs/btrfs/delayed-inode.h @@ -55,6 +55,13 @@ struct btrfs_delayed_node { struct mutex mutex; struct btrfs_inode_item inode_item; refcount_t refs; + /* + * The sum of the sizes of all index items we will be inserting for + * this inode (if it represents a directory), including the size of + * struct btrfs_item. 32 bits is enough, as we flush delayed items + * way before reaching the 32 bits limit. Protected by @mutex. + */ + u32 index_items_size; u64 index_cnt; unsigned long flags; int count; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 9dfde1af8a64a..89c6d7ff19874 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2308,7 +2308,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) btrfs_release_log_ctx_extents(&ctx); if (ret < 0) { /* Fallthrough and commit/free transaction. */ - ret = 1; + ret = BTRFS_LOG_FORCE_COMMIT; } /* we've logged all the items and now have a consistent diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index d50448bf8eedd..d6e9b596c7bae 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -6366,7 +6366,13 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, } btrfs_mark_buffer_dirty(path->nodes[0]); - btrfs_release_path(path); + /* + * We don't need the path anymore, plus inheriting properties, adding + * ACLs, security xattrs, orphan item or adding the link, will result in + * allocating yet another path. So just free our path. + */ + btrfs_free_path(path); + path = NULL; if (args->subvol) { struct inode *parent; @@ -6423,8 +6429,7 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans, goto discard; } - ret = 0; - goto out; + return 0; discard: /* @@ -7588,8 +7593,12 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, const u64 data_alloc_len = length; bool unlock_extents = false; + /* + * Cap the size of reads to that usually seen in buffered I/O as we need + * to allocate a contiguous array for the checksums. + */ if (!write) - len = min_t(u64, len, fs_info->sectorsize); + len = min_t(u64, len, fs_info->sectorsize * BTRFS_MAX_BIO_SECTORS); lockstart = start; lockend = start + len - 1; @@ -9560,15 +9569,21 @@ static int btrfs_rename2(struct user_namespace *mnt_userns, struct inode *old_di struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { + int ret; + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) return -EINVAL; if (flags & RENAME_EXCHANGE) - return btrfs_rename_exchange(old_dir, old_dentry, new_dir, - new_dentry); + ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir, + new_dentry); + else + ret = btrfs_rename(mnt_userns, old_dir, old_dentry, new_dir, + new_dentry, flags); + + btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info); - return btrfs_rename(mnt_userns, old_dir, old_dentry, new_dir, - new_dentry, flags); + return ret; } struct btrfs_delalloc_work { diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index a3549d587464a..977e0d218d79d 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -5,6 +5,7 @@ #include "compression.h" #include "ctree.h" #include "delalloc-space.h" +#include "disk-io.h" #include "reflink.h" #include "transaction.h" #include "subpage.h" @@ -658,7 +659,8 @@ static void btrfs_double_mmap_unlock(struct inode *inode1, struct inode *inode2) static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len, struct inode *dst, u64 dst_loff) { - const u64 bs = BTRFS_I(src)->root->fs_info->sb->s_blocksize; + struct btrfs_fs_info *fs_info = BTRFS_I(src)->root->fs_info; + const u64 bs = fs_info->sb->s_blocksize; int ret; /* @@ -669,6 +671,8 @@ static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len, ret = btrfs_clone(src, dst, loff, len, ALIGN(len, bs), dst_loff, 1); btrfs_double_extent_unlock(src, loff, dst, dst_loff, len); + btrfs_btree_balance_dirty(fs_info); + return ret; } @@ -778,6 +782,8 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, round_down(destoff, PAGE_SIZE), round_up(destoff + len, PAGE_SIZE) - 1); + btrfs_btree_balance_dirty(fs_info); + return ret; } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 370388fadf960..c94713c811bbc 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -171,7 +171,7 @@ static int start_log_trans(struct btrfs_trans_handle *trans, int index = (root->log_transid + 1) % 2; if (btrfs_need_log_full_commit(trans)) { - ret = -EAGAIN; + ret = BTRFS_LOG_FORCE_COMMIT; goto out; } @@ -194,7 +194,7 @@ static int start_log_trans(struct btrfs_trans_handle *trans, * writing. */ if (zoned && !created) { - ret = -EAGAIN; + ret = BTRFS_LOG_FORCE_COMMIT; goto out; } @@ -3121,7 +3121,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, /* bail out if we need to do a full commit */ if (btrfs_need_log_full_commit(trans)) { - ret = -EAGAIN; + ret = BTRFS_LOG_FORCE_COMMIT; mutex_unlock(&root->log_mutex); goto out; } @@ -3222,7 +3222,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, } btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); - ret = -EAGAIN; + ret = BTRFS_LOG_FORCE_COMMIT; goto out; } @@ -3261,7 +3261,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, blk_finish_plug(&plug); btrfs_wait_tree_log_extents(log, mark); mutex_unlock(&log_root_tree->log_mutex); - ret = -EAGAIN; + ret = BTRFS_LOG_FORCE_COMMIT; goto out_wake_log_root; } @@ -5848,7 +5848,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, inode_only == LOG_INODE_ALL && inode->last_unlink_trans >= trans->transid) { btrfs_set_log_full_commit(trans); - ret = 1; + ret = BTRFS_LOG_FORCE_COMMIT; goto out_unlock; } @@ -6562,12 +6562,12 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, bool log_dentries = false; if (btrfs_test_opt(fs_info, NOTREELOG)) { - ret = 1; + ret = BTRFS_LOG_FORCE_COMMIT; goto end_no_trans; } if (btrfs_root_refs(&root->root_item) == 0) { - ret = 1; + ret = BTRFS_LOG_FORCE_COMMIT; goto end_no_trans; } @@ -6665,7 +6665,7 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, end_trans: if (ret < 0) { btrfs_set_log_full_commit(trans); - ret = 1; + ret = BTRFS_LOG_FORCE_COMMIT; } if (ret) diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index 1620f8170629e..57ab5f3b8dc77 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -12,6 +12,9 @@ /* return value for btrfs_log_dentry_safe that means we don't need to log it at all */ #define BTRFS_NO_LOG_SYNC 256 +/* We can't use the tree log for whatever reason, force a transaction commit */ +#define BTRFS_LOG_FORCE_COMMIT (1) + struct btrfs_log_ctx { int log_ret; int log_transid; diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 6721002000ee0..b495cb6f74cf3 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -354,6 +354,13 @@ struct btrfs_fs_devices { - 2 * sizeof(struct btrfs_chunk)) \ / sizeof(struct btrfs_stripe) + 1) +/* + * Maximum number of sectors for a single bio to limit the size of the + * checksum array. This matches the number of bio_vecs per bio and thus the + * I/O size for buffered I/O. + */ +#define BTRFS_MAX_BIO_SECTORS (256) + /* * Additional info to pass along bio. * diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 3f53bc27a19bf..7578d4f6a969a 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -11,7 +11,7 @@ /* * Default IO priority. */ -#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_BE_NORM) +#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0) /* * Check that a priority value has a valid class. @@ -46,23 +46,19 @@ static inline int task_nice_ioclass(struct task_struct *task) return IOPRIO_CLASS_BE; } -/* - * If the calling process has set an I/O priority, use that. Otherwise, return - * the default I/O priority. - */ -static inline int get_current_ioprio(void) +#ifdef CONFIG_BLOCK +int __get_task_ioprio(struct task_struct *p); +#else +static inline int __get_task_ioprio(struct task_struct *p) { - struct io_context *ioc = current->io_context; - - if (ioc) - return ioc->ioprio; return IOPRIO_DEFAULT; } +#endif /* CONFIG_BLOCK */ -/* - * For inheritance, return the highest of the two given priorities - */ -extern int ioprio_best(unsigned short aprio, unsigned short bprio); +static inline int get_current_ioprio(void) +{ + return __get_task_ioprio(current); +} extern int set_task_ioprio(struct task_struct *task, int ioprio); -- 2.37.1 From e8270141ba6ff597ad747b3f17b7f8217b9f64a4 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:33:12 +0200 Subject: [PATCH 07/16] futex-winesync Signed-off-by: Peter Jung --- Documentation/admin-guide/devices.txt | 3 +- Documentation/userspace-api/index.rst | 1 + .../userspace-api/ioctl/ioctl-number.rst | 2 + Documentation/userspace-api/winesync.rst | 444 +++++ MAINTAINERS | 9 + drivers/misc/Kconfig | 11 + drivers/misc/Makefile | 1 + drivers/misc/winesync.c | 1212 ++++++++++++++ include/linux/miscdevice.h | 1 + include/uapi/linux/futex.h | 13 + include/uapi/linux/winesync.h | 71 + kernel/futex/syscalls.c | 75 +- tools/testing/selftests/Makefile | 1 + .../selftests/drivers/winesync/Makefile | 8 + .../testing/selftests/drivers/winesync/config | 1 + .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ 16 files changed, 3330 insertions(+), 2 deletions(-) create mode 100644 Documentation/userspace-api/winesync.rst create mode 100644 drivers/misc/winesync.c create mode 100644 include/uapi/linux/winesync.h create mode 100644 tools/testing/selftests/drivers/winesync/Makefile create mode 100644 tools/testing/selftests/drivers/winesync/config create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt index 9764d6edb1892..a4696d3b4a5a0 100644 --- a/Documentation/admin-guide/devices.txt +++ b/Documentation/admin-guide/devices.txt @@ -376,8 +376,9 @@ 240 = /dev/userio Serio driver testing device 241 = /dev/vhost-vsock Host kernel driver for virtio vsock 242 = /dev/rfkill Turning off radio transmissions (rfkill) + 243 = /dev/winesync Wine synchronization primitive device - 243-254 Reserved for local use + 244-254 Reserved for local use 255 Reserved for MISC_DYNAMIC_MINOR 11 char Raw keyboard device (Linux/SPARC only) diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index a61eac0c73f82..0bf697ddcb09b 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -29,6 +29,7 @@ place where this information is gathered. sysfs-platform_profile vduse futex2 + winesync .. only:: subproject and html diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index fcab013e47c93..f617f2c56421b 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -375,6 +375,8 @@ Code Seq# Include File Comments 0xF6 all LTTng Linux Trace Toolkit Next Generation +0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives + 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver 0xFD all linux/dm-ioctl.h diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst new file mode 100644 index 0000000000000..f0110d2744c70 --- /dev/null +++ b/Documentation/userspace-api/winesync.rst @@ -0,0 +1,444 @@ +===================================== +Wine synchronization primitive driver +===================================== + +This page documents the user-space API for the winesync driver. + +winesync is a support driver for emulation of NT synchronization +primitives by the Wine project or other NT emulators. It exists +because implementation in user-space, using existing tools, cannot +simultaneously satisfy performance, correctness, and security +constraints. It is implemented entirely in software, and does not +drive any hardware device. + +This interface is meant as a compatibility tool only, and should not +be used for general synchronization. Instead use generic, versatile +interfaces such as futex(2) and poll(2). + +Synchronization primitives +========================== + +The winesync driver exposes three types of synchronization primitives: +semaphores, mutexes, and events. + +A semaphore holds a single volatile 32-bit counter, and a static +32-bit integer denoting the maximum value. It is considered signaled +when the counter is nonzero. The counter is decremented by one when a +wait is satisfied. Both the initial and maximum count are established +when the semaphore is created. + +A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit +identifier denoting its owner. A mutex is considered signaled when its +owner is zero (indicating that it is not owned). The recursion count +is incremented when a wait is satisfied, and ownership is set to the +given identifier. + +A mutex also holds an internal flag denoting whether its previous +owner has died; such a mutex is said to be inconsistent. Owner death +is not tracked automatically based on thread death, but rather must be +communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex +is inherently considered unowned. + +Except for the "unowned" semantics of zero, the actual value of the +owner identifier is not interpreted by the winesync driver at all. The +intended use is to store a thread identifier; however, the winesync +driver does not actually validate that a calling thread provides +consistent or unique identifiers. + +An event holds a volatile boolean state denoting whether it is +signaled or not. There are two types of events, auto-reset and +manual-reset. An auto-reset event is designaled when a wait is +satisfied; a manual-reset event is not. The event type is specified +when the event is created. + +Unless specified otherwise, all operations on an object are atomic and +totally ordered with respect to other operations on the same object. + +Objects are represented by unsigned 32-bit integers. + +Char device +=========== + +The winesync driver creates a single char device /dev/winesync. Each +file description opened on the device represents a unique namespace. +That is, objects created on one open file description are shared +across all its individual descriptors, but are not shared with other +open() calls on the same device. The same file description may be +shared across multiple processes. + +ioctl reference +=============== + +All operations on the device are done through ioctls. There are three +structures used in ioctl calls:: + + struct winesync_sem_args { + __u32 sem; + __u32 count; + __u32 max; + }; + + struct winesync_mutex_args { + __u32 mutex; + __u32 owner; + __u32 count; + }; + + struct winesync_event_args { + __u32 event; + __u32 signaled; + __u32 manual; + }; + + struct winesync_wait_args { + __u64 timeout; + __u64 objs; + __u32 count; + __u32 owner; + __u32 index; + __u32 pad; + }; + +Depending on the ioctl, members of the structure may be used as input, +output, or not at all. All ioctls return 0 on success. + +The ioctls are as follows: + +.. c:macro:: WINESYNC_IOC_CREATE_SEM + + Create a semaphore object. Takes a pointer to struct + :c:type:`winesync_sem_args`, which is used as follows: + + .. list-table:: + + * - ``sem`` + - On output, contains the identifier of the created semaphore. + * - ``count`` + - Initial count of the semaphore. + * - ``max`` + - Maximum count of the semaphore. + + Fails with ``EINVAL`` if ``count`` is greater than ``max``. + +.. c:macro:: WINESYNC_IOC_CREATE_MUTEX + + Create a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + .. list-table:: + + * - ``mutex`` + - On output, contains the identifier of the created mutex. + * - ``count`` + - Initial recursion count of the mutex. + * - ``owner`` + - Initial owner of the mutex. + + If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is + zero and ``count`` is nonzero, the function fails with ``EINVAL``. + +.. c:macro:: WINESYNC_IOC_CREATE_EVENT + + Create an event object. Takes a pointer to struct + :c:type:`winesync_event_args`, which is used as follows: + + .. list-table:: + + * - ``event`` + - On output, contains the identifier of the created event. + * - ``signaled`` + - If nonzero, the event is initially signaled, otherwise + nonsignaled. + * - ``manual`` + - If nonzero, the event is a manual-reset event, otherwise + auto-reset. + +.. c:macro:: WINESYNC_IOC_DELETE + + Delete an object of any type. Takes an input-only pointer to a + 32-bit integer denoting the object to delete. + + Wait ioctls currently in progress are not interrupted, and behave as + if the object remains valid. + +.. c:macro:: WINESYNC_IOC_PUT_SEM + + Post to a semaphore object. Takes a pointer to struct + :c:type:`winesync_sem_args`, which is used as follows: + + .. list-table:: + + * - ``sem`` + - Semaphore object to post to. + * - ``count`` + - Count to add to the semaphore. On output, contains the + previous count of the semaphore. + * - ``max`` + - Not used. + + If adding ``count`` to the semaphore's current count would raise the + latter past the semaphore's maximum count, the ioctl fails with + ``EOVERFLOW`` and the semaphore is not affected. If raising the + semaphore's count causes it to become signaled, eligible threads + waiting on this semaphore will be woken and the semaphore's count + decremented appropriately. + +.. c:macro:: WINESYNC_IOC_PUT_MUTEX + + Release a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + .. list-table:: + + * - ``mutex`` + - Mutex object to release. + * - ``owner`` + - Mutex owner identifier. + * - ``count`` + - On output, contains the previous recursion count. + + If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` + is not the current owner of the mutex, the ioctl fails with + ``EPERM``. + + The mutex's count will be decremented by one. If decrementing the + mutex's count causes it to become zero, the mutex is marked as + unowned and signaled, and eligible threads waiting on it will be + woken as appropriate. + +.. c:macro:: WINESYNC_IOC_SET_EVENT + + Signal an event object. Takes a pointer to struct + :c:type:`winesync_event_args`, which is used as follows: + + .. list-table:: + + * - ``event`` + - Event object to set. + * - ``signaled`` + - On output, contains the previous state of the event. + * - ``manual`` + - Unused. + + Eligible threads will be woken, and auto-reset events will be + designaled appropriately. + +.. c:macro:: WINESYNC_IOC_RESET_EVENT + + Designal an event object. Takes a pointer to struct + :c:type:`winesync_event_args`, which is used as follows: + + .. list-table:: + + * - ``event`` + - Event object to reset. + * - ``signaled`` + - On output, contains the previous state of the event. + * - ``manual`` + - Unused. + +.. c:macro:: WINESYNC_IOC_PULSE_EVENT + + Wake threads waiting on an event object without leaving it in a + signaled state. Takes a pointer to struct + :c:type:`winesync_event_args`, which is used as follows: + + .. list-table:: + + * - ``event`` + - Event object to pulse. + * - ``signaled`` + - On output, contains the previous state of the event. + * - ``manual`` + - Unused. + + A pulse operation can be thought of as a set followed by a reset, + performed as a single atomic operation. If two threads are waiting + on an auto-reset event which is pulsed, only one will be woken. If + two threads are waiting a manual-reset event which is pulsed, both + will be woken. However, in both cases, the event will be unsignaled + afterwards, and a simultaneous read operation will always report the + event as unsignaled. + +.. c:macro:: WINESYNC_IOC_READ_SEM + + Read the current state of a semaphore object. Takes a pointer to + struct :c:type:`winesync_sem_args`, which is used as follows: + + .. list-table:: + + * - ``sem`` + - Semaphore object to read. + * - ``count`` + - On output, contains the current count of the semaphore. + * - ``max`` + - On output, contains the maximum count of the semaphore. + +.. c:macro:: WINESYNC_IOC_READ_MUTEX + + Read the current state of a mutex object. Takes a pointer to struct + :c:type:`winesync_mutex_args`, which is used as follows: + + .. list-table:: + + * - ``mutex`` + - Mutex object to read. + * - ``owner`` + - On output, contains the current owner of the mutex, or zero + if the mutex is not currently owned. + * - ``count`` + - On output, contains the current recursion count of the mutex. + + If the mutex is marked as inconsistent, the function fails with + ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to + zero. + +.. c:macro:: WINESYNC_IOC_READ_EVENT + + Read the current state of an event object. Takes a pointer to struct + :c:type:`winesync_event_args`, which is used as follows: + + .. list-table:: + + * - ``event`` + - Event object. + * - ``signaled`` + - On output, contains the current state of the event. + * - ``manual`` + - On output, contains 1 if the event is a manual-reset event, + and 0 otherwise. + +.. c:macro:: WINESYNC_IOC_KILL_OWNER + + Mark any mutexes owned by the given owner as unowned and + inconsistent. Takes an input-only pointer to a 32-bit integer + denoting the owner. If the owner is zero, the ioctl fails with + ``EINVAL``. + + For each mutex currently owned by the given owner, eligible threads + waiting on said mutex will be woken as appropriate (and such waits + will fail with ``EOWNERDEAD``, as described below). + + The operation as a whole is not atomic; however, the modification of + each mutex is atomic and totally ordered with respect to other + operations on the same mutex. + +.. c:macro:: WINESYNC_IOC_WAIT_ANY + + Poll on any of a list of objects, atomically acquiring at most one. + Takes a pointer to struct :c:type:`winesync_wait_args`, which is + used as follows: + + .. list-table:: + + * - ``timeout`` + - Optional pointer to a 64-bit struct :c:type:`timespec` + (specified as an integer so that the structure has the same + size regardless of architecture). The timeout is specified in + absolute format, as measured against the MONOTONIC clock. If + the timeout is equal to or earlier than the current time, the + function returns immediately without sleeping. If ``timeout`` + is zero, i.e. NULL, the function will sleep until an object + is signaled, and will not fail with ``ETIMEDOUT``. + * - ``objs`` + - Pointer to an array of ``count`` 32-bit object identifiers + (specified as an integer so that the structure has the same + size regardless of architecture). If any identifier is + invalid, the function fails with ``EINVAL``. + * - ``count`` + - Number of object identifiers specified in the ``objs`` array. + * - ``owner`` + - Mutex owner identifier. If any object in ``objs`` is a mutex, + the ioctl will attempt to acquire that mutex on behalf of + ``owner``. If ``owner`` is zero, the ioctl fails with + ``EINVAL``. + * - ``index`` + - On success, contains the index (into ``objs``) of the object + which was signaled. If ``alert`` was signaled instead, + this contains ``count``. + * - ``alert`` + - Optional event object identifier. If nonzero, this specifies + an "alert" event object which, if signaled, will terminate + the wait. If nonzero, the identifier must point to a valid + event. + + This function attempts to acquire one of the given objects. If + unable to do so, it sleeps until an object becomes signaled, + subsequently acquiring it, or the timeout expires. In the latter + case the ioctl fails with ``ETIMEDOUT``. The function only acquires + one object, even if multiple objects are signaled. + + A semaphore is considered to be signaled if its count is nonzero, + and is acquired by decrementing its count by one. A mutex is + considered to be signaled if it is unowned or if its owner matches + the ``owner`` argument, and is acquired by incrementing its + recursion count by one and setting its owner to the ``owner`` + argument. An auto-reset event is acquired by designaling it; a + manual-reset event is not affected by acquisition. + + Acquisition is atomic and totally ordered with respect to other + operations on the same object. If two wait operations (with + different ``owner`` identifiers) are queued on the same mutex, only + one is signaled. If two wait operations are queued on the same + semaphore, and a value of one is posted to it, only one is signaled. + The order in which threads are signaled is not specified. + + If an inconsistent mutex is acquired, the ioctl fails with + ``EOWNERDEAD``. Although this is a failure return, the function may + otherwise be considered successful. The mutex is marked as owned by + the given owner (with a recursion count of 1) and as no longer + inconsistent, and ``index`` is still set to the index of the mutex. + + The ``alert`` argument is an "extra" event which can terminate the + wait, independently of all other objects. If members of ``objs`` and + ``alert`` are both simultaneously signaled, a member of ``objs`` + will always be given priority and acquired first. Aside from this, + for "any" waits, there is no difference between passing an event as + this parameter, and passing it as an additional object at the end of + the ``objs`` array. For "all" waits, there is an additional + difference, as described below. + + It is valid to pass the same object more than once, including by + passing the same event in the ``objs`` array and in ``alert``. If a + wakeup occurs due to that object being signaled, ``index`` is set to + the lowest index corresponding to that object. + + The function may fail with ``EINTR`` if a signal is received. + +.. c:macro:: WINESYNC_IOC_WAIT_ALL + + Poll on a list of objects, atomically acquiring all of them. Takes a + pointer to struct :c:type:`winesync_wait_args`, which is used + identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is + always filled with zero on success if not woken via alert. + + This function attempts to simultaneously acquire all of the given + objects. If unable to do so, it sleeps until all objects become + simultaneously signaled, subsequently acquiring them, or the timeout + expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and + no objects are modified. + + Objects may become signaled and subsequently designaled (through + acquisition by other threads) while this thread is sleeping. Only + once all objects are simultaneously signaled does the ioctl acquire + them and return. The entire acquisition is atomic and totally + ordered with respect to other operations on any of the given + objects. + + If an inconsistent mutex is acquired, the ioctl fails with + ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects + are nevertheless marked as acquired. Note that if multiple mutex + objects are specified, there is no way to know which were marked as + inconsistent. + + As with "any" waits, the ``alert`` argument is an "extra" event + which can terminate the wait. Critically, however, an "all" wait + will succeed if all members in ``objs`` are signaled, *or* if + ``alert`` is signaled. In the latter case ``index`` will be set to + ``count``. As with "any" waits, if both conditions are filled, the + former takes priority, and objects in ``objs`` will be acquired. + + Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same + object more than once, nor is it valid to pass the same object in + ``objs`` and in ``alert`` If this is attempted, the function fails + with ``EINVAL``. diff --git a/MAINTAINERS b/MAINTAINERS index 5da47941f0998..d09396ec20737 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -21604,6 +21604,15 @@ M: David Härdeman S: Maintained F: drivers/media/rc/winbond-cir.c +WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER +M: Zebediah Figura +L: wine-devel@winehq.org +S: Supported +F: Documentation/userspace-api/winesync.rst +F: drivers/misc/winesync.c +F: include/uapi/linux/winesync.h +F: tools/testing/selftests/drivers/winesync/ + WINSYSTEMS EBC-C384 WATCHDOG DRIVER M: William Breathitt Gray L: linux-watchdog@vger.kernel.org diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 41d2bb0ae23a3..3a1450cb86876 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -483,6 +483,17 @@ config OPEN_DICE If unsure, say N. +config WINESYNC + tristate "Synchronization primitives for Wine" + help + This module provides kernel support for synchronization primitives + used by Wine. It is not a hardware driver. + + To compile this driver as a module, choose M here: the + module will be called winesync. + + If unsure, say N. + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 70e800e9127f8..d9f269c101ff4 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -58,5 +58,6 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ obj-$(CONFIG_UACCE) += uacce/ obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o +obj-$(CONFIG_WINESYNC) += winesync.o obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o obj-$(CONFIG_OPEN_DICE) += open-dice.o diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c new file mode 100644 index 0000000000000..7a28f58dbbf20 --- /dev/null +++ b/drivers/misc/winesync.c @@ -0,0 +1,1212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * winesync.c - Kernel driver for Wine synchronization primitives + * + * Copyright (C) 2021 Zebediah Figura + */ + +#include +#include +#include +#include +#include +#include + +#define WINESYNC_NAME "winesync" + +enum winesync_type { + WINESYNC_TYPE_SEM, + WINESYNC_TYPE_MUTEX, + WINESYNC_TYPE_EVENT, +}; + +struct winesync_obj { + struct rcu_head rhead; + struct kref refcount; + spinlock_t lock; + + /* + * any_waiters is protected by the object lock, but all_waiters is + * protected by the device wait_all_lock. + */ + struct list_head any_waiters; + struct list_head all_waiters; + + /* + * Hint describing how many tasks are queued on this object in a + * wait-all operation. + * + * Any time we do a wake, we may need to wake "all" waiters as well as + * "any" waiters. In order to atomically wake "all" waiters, we must + * lock all of the objects, and that means grabbing the wait_all_lock + * below (and, due to lock ordering rules, before locking this object). + * However, wait-all is a rare operation, and grabbing the wait-all + * lock for every wake would create unnecessary contention. Therefore we + * first check whether all_hint is zero, and, if it is, we skip trying + * to wake "all" waiters. + * + * This hint isn't protected by any lock. It might change during the + * course of a wake, but there's no meaningful race there; it's only a + * hint. + * + * Since wait requests must originate from user-space threads, we're + * limited here by PID_MAX_LIMIT, so there's no risk of saturation. + */ + atomic_t all_hint; + + enum winesync_type type; + + /* The following fields are protected by the object lock. */ + union { + struct { + __u32 count; + __u32 max; + } sem; + struct { + __u32 count; + __u32 owner; + bool ownerdead; + } mutex; + struct { + bool manual; + bool signaled; + } event; + } u; +}; + +struct winesync_q_entry { + struct list_head node; + struct winesync_q *q; + struct winesync_obj *obj; + __u32 index; +}; + +struct winesync_q { + struct task_struct *task; + __u32 owner; + + /* + * Protected via atomic_cmpxchg(). Only the thread that wins the + * compare-and-swap may actually change object states and wake this + * task. + */ + atomic_t signaled; + + bool all; + bool ownerdead; + __u32 count; + struct winesync_q_entry entries[]; +}; + +struct winesync_device { + /* + * Wait-all operations must atomically grab all objects, and be totally + * ordered with respect to each other and wait-any operations. If one + * thread is trying to acquire several objects, another thread cannot + * touch the object at the same time. + * + * We achieve this by grabbing multiple object locks at the same time. + * However, this creates a lock ordering problem. To solve that problem, + * wait_all_lock is taken first whenever multiple objects must be locked + * at the same time. + */ + spinlock_t wait_all_lock; + + struct xarray objects; +}; + +static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) +{ + struct winesync_obj *obj; + + rcu_read_lock(); + obj = xa_load(&dev->objects, id); + if (obj && !kref_get_unless_zero(&obj->refcount)) + obj = NULL; + rcu_read_unlock(); + + return obj; +} + +static void destroy_obj(struct kref *ref) +{ + struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); + + kfree_rcu(obj, rhead); +} + +static void put_obj(struct winesync_obj *obj) +{ + kref_put(&obj->refcount, destroy_obj); +} + +static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, + enum winesync_type type) +{ + struct winesync_obj *obj = get_obj(dev, id); + + if (obj && obj->type != type) { + put_obj(obj); + return NULL; + } + return obj; +} + +static int winesync_char_open(struct inode *inode, struct file *file) +{ + struct winesync_device *dev; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + spin_lock_init(&dev->wait_all_lock); + + xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); + + file->private_data = dev; + return nonseekable_open(inode, file); +} + +static int winesync_char_release(struct inode *inode, struct file *file) +{ + struct winesync_device *dev = file->private_data; + struct winesync_obj *obj; + unsigned long id; + + xa_for_each(&dev->objects, id, obj) + put_obj(obj); + + xa_destroy(&dev->objects); + + kfree(dev); + + return 0; +} + +static void init_obj(struct winesync_obj *obj) +{ + kref_init(&obj->refcount); + atomic_set(&obj->all_hint, 0); + spin_lock_init(&obj->lock); + INIT_LIST_HEAD(&obj->any_waiters); + INIT_LIST_HEAD(&obj->all_waiters); +} + +static bool is_signaled(struct winesync_obj *obj, __u32 owner) +{ + lockdep_assert_held(&obj->lock); + + switch (obj->type) { + case WINESYNC_TYPE_SEM: + return !!obj->u.sem.count; + case WINESYNC_TYPE_MUTEX: + if (obj->u.mutex.owner && obj->u.mutex.owner != owner) + return false; + return obj->u.mutex.count < UINT_MAX; + case WINESYNC_TYPE_EVENT: + return obj->u.event.signaled; + } + + WARN(1, "bad object type %#x\n", obj->type); + return false; +} + +/* + * "locked_obj" is an optional pointer to an object which is already locked and + * should not be locked again. This is necessary so that changing an object's + * state and waking it can be a single atomic operation. + */ +static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, + struct winesync_obj *locked_obj) +{ + __u32 count = q->count; + bool can_wake = true; + __u32 i; + + lockdep_assert_held(&dev->wait_all_lock); + if (locked_obj) + lockdep_assert_held(&locked_obj->lock); + + for (i = 0; i < count; i++) { + if (q->entries[i].obj != locked_obj) + spin_lock(&q->entries[i].obj->lock); + } + + for (i = 0; i < count; i++) { + if (!is_signaled(q->entries[i].obj, q->owner)) { + can_wake = false; + break; + } + } + + if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { + for (i = 0; i < count; i++) { + struct winesync_obj *obj = q->entries[i].obj; + + switch (obj->type) { + case WINESYNC_TYPE_SEM: + obj->u.sem.count--; + break; + case WINESYNC_TYPE_MUTEX: + if (obj->u.mutex.ownerdead) + q->ownerdead = true; + obj->u.mutex.ownerdead = false; + obj->u.mutex.count++; + obj->u.mutex.owner = q->owner; + break; + case WINESYNC_TYPE_EVENT: + if (!obj->u.event.manual) + obj->u.event.signaled = false; + break; + } + } + wake_up_process(q->task); + } + + for (i = 0; i < count; i++) { + if (q->entries[i].obj != locked_obj) + spin_unlock(&q->entries[i].obj->lock); + } +} + +static void try_wake_all_obj(struct winesync_device *dev, + struct winesync_obj *obj) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&dev->wait_all_lock); + lockdep_assert_held(&obj->lock); + + list_for_each_entry(entry, &obj->all_waiters, node) + try_wake_all(dev, entry->q, obj); +} + +static void try_wake_any_sem(struct winesync_obj *sem) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&sem->lock); + + list_for_each_entry(entry, &sem->any_waiters, node) { + struct winesync_q *q = entry->q; + + if (!sem->u.sem.count) + break; + + if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + sem->u.sem.count--; + wake_up_process(q->task); + } + } +} + +static void try_wake_any_mutex(struct winesync_obj *mutex) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&mutex->lock); + + list_for_each_entry(entry, &mutex->any_waiters, node) { + struct winesync_q *q = entry->q; + + if (mutex->u.mutex.count == UINT_MAX) + break; + if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) + continue; + + if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + if (mutex->u.mutex.ownerdead) + q->ownerdead = true; + mutex->u.mutex.ownerdead = false; + mutex->u.mutex.count++; + mutex->u.mutex.owner = q->owner; + wake_up_process(q->task); + } + } +} + +static void try_wake_any_event(struct winesync_obj *event) +{ + struct winesync_q_entry *entry; + + lockdep_assert_held(&event->lock); + + list_for_each_entry(entry, &event->any_waiters, node) { + struct winesync_q *q = entry->q; + + if (!event->u.event.signaled) + break; + + if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { + if (!event->u.event.manual) + event->u.event.signaled = false; + wake_up_process(q->task); + } + } +} + +static int winesync_create_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + __u32 id; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (args.count > args.max) + return -EINVAL; + + sem = kzalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return -ENOMEM; + + init_obj(sem); + sem->type = WINESYNC_TYPE_SEM; + sem->u.sem.count = args.count; + sem->u.sem.max = args.max; + + ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); + if (ret < 0) { + kfree(sem); + return ret; + } + + return put_user(id, &user_args->sem); +} + +static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + __u32 id; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (!args.owner != !args.count) + return -EINVAL; + + mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); + if (!mutex) + return -ENOMEM; + + init_obj(mutex); + mutex->type = WINESYNC_TYPE_MUTEX; + mutex->u.mutex.count = args.count; + mutex->u.mutex.owner = args.owner; + + ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); + if (ret < 0) { + kfree(mutex); + return ret; + } + + return put_user(id, &user_args->mutex); +} + +static int winesync_create_event(struct winesync_device *dev, void __user *argp) +{ + struct winesync_event_args __user *user_args = argp; + struct winesync_event_args args; + struct winesync_obj *event; + __u32 id; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return -ENOMEM; + + init_obj(event); + event->type = WINESYNC_TYPE_EVENT; + event->u.event.manual = args.manual; + event->u.event.signaled = args.signaled; + + ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); + if (ret < 0) { + kfree(event); + return ret; + } + + return put_user(id, &user_args->event); +} + +static int winesync_delete(struct winesync_device *dev, void __user *argp) +{ + struct winesync_obj *obj; + __u32 id; + + if (get_user(id, (__u32 __user *)argp)) + return -EFAULT; + + obj = xa_erase(&dev->objects, id); + if (!obj) + return -EINVAL; + + put_obj(obj); + return 0; +} + +/* + * Actually change the semaphore state, returning -EOVERFLOW if it is made + * invalid. + */ +static int put_sem_state(struct winesync_obj *sem, __u32 count) +{ + lockdep_assert_held(&sem->lock); + + if (sem->u.sem.count + count < sem->u.sem.count || + sem->u.sem.count + count > sem->u.sem.max) + return -EOVERFLOW; + + sem->u.sem.count += count; + return 0; +} + +static int winesync_put_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + __u32 prev_count; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); + if (!sem) + return -EINVAL; + + if (atomic_read(&sem->all_hint) > 0) { + spin_lock(&dev->wait_all_lock); + spin_lock(&sem->lock); + + prev_count = sem->u.sem.count; + ret = put_sem_state(sem, args.count); + if (!ret) { + try_wake_all_obj(dev, sem); + try_wake_any_sem(sem); + } + + spin_unlock(&sem->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&sem->lock); + + prev_count = sem->u.sem.count; + ret = put_sem_state(sem, args.count); + if (!ret) + try_wake_any_sem(sem); + + spin_unlock(&sem->lock); + } + + put_obj(sem); + + if (!ret && put_user(prev_count, &user_args->count)) + ret = -EFAULT; + + return ret; +} + +/* + * Actually change the mutex state, returning -EPERM if not the owner. + */ +static int put_mutex_state(struct winesync_obj *mutex, + const struct winesync_mutex_args *args) +{ + lockdep_assert_held(&mutex->lock); + + if (mutex->u.mutex.owner != args->owner) + return -EPERM; + + if (!--mutex->u.mutex.count) + mutex->u.mutex.owner = 0; + return 0; +} + +static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + __u32 prev_count; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (!args.owner) + return -EINVAL; + + mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); + if (!mutex) + return -EINVAL; + + if (atomic_read(&mutex->all_hint) > 0) { + spin_lock(&dev->wait_all_lock); + spin_lock(&mutex->lock); + + prev_count = mutex->u.mutex.count; + ret = put_mutex_state(mutex, &args); + if (!ret) { + try_wake_all_obj(dev, mutex); + try_wake_any_mutex(mutex); + } + + spin_unlock(&mutex->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&mutex->lock); + + prev_count = mutex->u.mutex.count; + ret = put_mutex_state(mutex, &args); + if (!ret) + try_wake_any_mutex(mutex); + + spin_unlock(&mutex->lock); + } + + put_obj(mutex); + + if (!ret && put_user(prev_count, &user_args->count)) + ret = -EFAULT; + + return ret; +} + +static int winesync_read_sem(struct winesync_device *dev, void __user *argp) +{ + struct winesync_sem_args __user *user_args = argp; + struct winesync_sem_args args; + struct winesync_obj *sem; + __u32 id; + + if (get_user(id, &user_args->sem)) + return -EFAULT; + + sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); + if (!sem) + return -EINVAL; + + args.sem = id; + spin_lock(&sem->lock); + args.count = sem->u.sem.count; + args.max = sem->u.sem.max; + spin_unlock(&sem->lock); + + put_obj(sem); + + if (copy_to_user(user_args, &args, sizeof(args))) + return -EFAULT; + return 0; +} + +static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) +{ + struct winesync_mutex_args __user *user_args = argp; + struct winesync_mutex_args args; + struct winesync_obj *mutex; + __u32 id; + int ret; + + if (get_user(id, &user_args->mutex)) + return -EFAULT; + + mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); + if (!mutex) + return -EINVAL; + + args.mutex = id; + spin_lock(&mutex->lock); + args.count = mutex->u.mutex.count; + args.owner = mutex->u.mutex.owner; + ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; + spin_unlock(&mutex->lock); + + put_obj(mutex); + + if (copy_to_user(user_args, &args, sizeof(args))) + return -EFAULT; + return ret; +} + +static int winesync_read_event(struct winesync_device *dev, void __user *argp) +{ + struct winesync_event_args __user *user_args = argp; + struct winesync_event_args args; + struct winesync_obj *event; + __u32 id; + + if (get_user(id, &user_args->event)) + return -EFAULT; + + event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); + if (!event) + return -EINVAL; + + args.event = id; + spin_lock(&event->lock); + args.manual = event->u.event.manual; + args.signaled = event->u.event.signaled; + spin_unlock(&event->lock); + + put_obj(event); + + if (copy_to_user(user_args, &args, sizeof(args))) + return -EFAULT; + return 0; +} + +/* + * Actually change the mutex state to mark its owner as dead. + */ +static void put_mutex_ownerdead_state(struct winesync_obj *mutex) +{ + lockdep_assert_held(&mutex->lock); + + mutex->u.mutex.ownerdead = true; + mutex->u.mutex.owner = 0; + mutex->u.mutex.count = 0; +} + +static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) +{ + struct winesync_obj *obj; + unsigned long id; + __u32 owner; + + if (get_user(owner, (__u32 __user *)argp)) + return -EFAULT; + if (!owner) + return -EINVAL; + + rcu_read_lock(); + + xa_for_each(&dev->objects, id, obj) { + if (!kref_get_unless_zero(&obj->refcount)) + continue; + + if (obj->type != WINESYNC_TYPE_MUTEX) { + put_obj(obj); + continue; + } + + if (atomic_read(&obj->all_hint) > 0) { + spin_lock(&dev->wait_all_lock); + spin_lock(&obj->lock); + + if (obj->u.mutex.owner == owner) { + put_mutex_ownerdead_state(obj); + try_wake_all_obj(dev, obj); + try_wake_any_mutex(obj); + } + + spin_unlock(&obj->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&obj->lock); + + if (obj->u.mutex.owner == owner) { + put_mutex_ownerdead_state(obj); + try_wake_any_mutex(obj); + } + + spin_unlock(&obj->lock); + } + + put_obj(obj); + } + + rcu_read_unlock(); + + return 0; +} + +static int winesync_set_event(struct winesync_device *dev, void __user *argp, + bool pulse) +{ + struct winesync_event_args __user *user_args = argp; + struct winesync_event_args args; + struct winesync_obj *event; + bool prev_state; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); + if (!event) + return -EINVAL; + + if (atomic_read(&event->all_hint) > 0) { + spin_lock(&dev->wait_all_lock); + spin_lock(&event->lock); + + prev_state = event->u.event.signaled; + event->u.event.signaled = true; + try_wake_all_obj(dev, event); + try_wake_any_event(event); + if (pulse) + event->u.event.signaled = false; + + spin_unlock(&event->lock); + spin_unlock(&dev->wait_all_lock); + } else { + spin_lock(&event->lock); + + prev_state = event->u.event.signaled; + event->u.event.signaled = true; + try_wake_any_event(event); + if (pulse) + event->u.event.signaled = false; + + spin_unlock(&event->lock); + } + + put_obj(event); + + if (put_user(prev_state, &user_args->signaled)) + return -EFAULT; + + return 0; +} + +static int winesync_reset_event(struct winesync_device *dev, void __user *argp) +{ + struct winesync_event_args __user *user_args = argp; + struct winesync_event_args args; + struct winesync_obj *event; + bool prev_state; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); + if (!event) + return -EINVAL; + + spin_lock(&event->lock); + + prev_state = event->u.event.signaled; + event->u.event.signaled = false; + + spin_unlock(&event->lock); + + put_obj(event); + + if (put_user(prev_state, &user_args->signaled)) + return -EFAULT; + + return 0; +} + +static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) +{ + int ret = 0; + + do { + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + set_current_state(TASK_INTERRUPTIBLE); + if (atomic_read(&q->signaled) != -1) { + ret = 0; + break; + } + ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); + } while (ret < 0); + __set_current_state(TASK_RUNNING); + + return ret; +} + +/* + * Allocate and initialize the winesync_q structure, but do not queue us yet. + * Also, calculate the relative timeout. + */ +static int setup_wait(struct winesync_device *dev, + const struct winesync_wait_args *args, bool all, + ktime_t *ret_timeout, struct winesync_q **ret_q) +{ + const __u32 count = args->count; + struct winesync_q *q; + ktime_t timeout = 0; + __u32 total_count; + __u32 *ids; + __u32 i, j; + + if (!args->owner) + return -EINVAL; + + if (args->timeout) { + struct timespec64 to; + + if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) + return -EFAULT; + if (!timespec64_valid(&to)) + return -EINVAL; + + timeout = timespec64_to_ns(&to); + } + + total_count = count; + if (args->alert) + total_count++; + + ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; + if (copy_from_user(ids, u64_to_user_ptr(args->objs), + array_size(count, sizeof(*ids)))) { + kfree(ids); + return -EFAULT; + } + if (args->alert) + ids[count] = args->alert; + + q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); + if (!q) { + kfree(ids); + return -ENOMEM; + } + q->task = current; + q->owner = args->owner; + atomic_set(&q->signaled, -1); + q->all = all; + q->ownerdead = false; + q->count = count; + + for (i = 0; i < total_count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = get_obj(dev, ids[i]); + + if (!obj) + goto err; + + if (all) { + /* Check that the objects are all distinct. */ + for (j = 0; j < i; j++) { + if (obj == q->entries[j].obj) { + put_obj(obj); + goto err; + } + } + } + + entry->obj = obj; + entry->q = q; + entry->index = i; + } + + kfree(ids); + + *ret_q = q; + *ret_timeout = timeout; + return 0; + +err: + for (j = 0; j < i; j++) + put_obj(q->entries[j].obj); + kfree(ids); + kfree(q); + return -EINVAL; +} + +static void try_wake_any_obj(struct winesync_obj *obj) +{ + switch (obj->type) { + case WINESYNC_TYPE_SEM: + try_wake_any_sem(obj); + break; + case WINESYNC_TYPE_MUTEX: + try_wake_any_mutex(obj); + break; + case WINESYNC_TYPE_EVENT: + try_wake_any_event(obj); + break; + } +} + +static int winesync_wait_any(struct winesync_device *dev, void __user *argp) +{ + struct winesync_wait_args args; + struct winesync_q *q; + __u32 i, total_count; + ktime_t timeout; + int signaled; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + ret = setup_wait(dev, &args, false, &timeout, &q); + if (ret < 0) + return ret; + + total_count = args.count; + if (args.alert) + total_count++; + + /* queue ourselves */ + + for (i = 0; i < total_count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = entry->obj; + + spin_lock(&obj->lock); + list_add_tail(&entry->node, &obj->any_waiters); + spin_unlock(&obj->lock); + } + + /* + * Check if we are already signaled. + * + * Note that the API requires that normal objects are checked before + * the alert event. Hence we queue the alert event last, and check + * objects in order. + */ + + for (i = 0; i < total_count; i++) { + struct winesync_obj *obj = q->entries[i].obj; + + if (atomic_read(&q->signaled) != -1) + break; + + spin_lock(&obj->lock); + try_wake_any_obj(obj); + spin_unlock(&obj->lock); + } + + /* sleep */ + + ret = winesync_schedule(q, args.timeout ? &timeout : NULL); + + /* and finally, unqueue */ + + for (i = 0; i < total_count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = entry->obj; + + spin_lock(&obj->lock); + list_del(&entry->node); + spin_unlock(&obj->lock); + + put_obj(obj); + } + + signaled = atomic_read(&q->signaled); + if (signaled != -1) { + struct winesync_wait_args __user *user_args = argp; + + /* even if we caught a signal, we need to communicate success */ + ret = q->ownerdead ? -EOWNERDEAD : 0; + + if (put_user(signaled, &user_args->index)) + ret = -EFAULT; + } else if (!ret) { + ret = -ETIMEDOUT; + } + + kfree(q); + return ret; +} + +static int winesync_wait_all(struct winesync_device *dev, void __user *argp) +{ + struct winesync_wait_args args; + struct winesync_q *q; + ktime_t timeout; + int signaled; + __u32 i; + int ret; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + ret = setup_wait(dev, &args, true, &timeout, &q); + if (ret < 0) + return ret; + + /* queue ourselves */ + + spin_lock(&dev->wait_all_lock); + + for (i = 0; i < args.count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = entry->obj; + + atomic_inc(&obj->all_hint); + + /* + * obj->all_waiters is protected by dev->wait_all_lock rather + * than obj->lock, so there is no need to acquire it here. + */ + list_add_tail(&entry->node, &obj->all_waiters); + } + if (args.alert) { + struct winesync_q_entry *entry = &q->entries[args.count]; + struct winesync_obj *obj = entry->obj; + + spin_lock(&obj->lock); + list_add_tail(&entry->node, &obj->any_waiters); + spin_unlock(&obj->lock); + } + + /* check if we are already signaled */ + + try_wake_all(dev, q, NULL); + + spin_unlock(&dev->wait_all_lock); + + /* + * Check if the alert event is signaled, making sure to do so only + * after checking if the other objects are signaled. + */ + + if (args.alert) { + struct winesync_obj *obj = q->entries[args.count].obj; + + if (atomic_read(&q->signaled) == -1) { + spin_lock(&obj->lock); + try_wake_any_obj(obj); + spin_unlock(&obj->lock); + } + } + + /* sleep */ + + ret = winesync_schedule(q, args.timeout ? &timeout : NULL); + + /* and finally, unqueue */ + + spin_lock(&dev->wait_all_lock); + + for (i = 0; i < args.count; i++) { + struct winesync_q_entry *entry = &q->entries[i]; + struct winesync_obj *obj = entry->obj; + + /* + * obj->all_waiters is protected by dev->wait_all_lock rather + * than obj->lock, so there is no need to acquire it here. + */ + list_del(&entry->node); + + atomic_dec(&obj->all_hint); + + put_obj(obj); + } + if (args.alert) { + struct winesync_q_entry *entry = &q->entries[args.count]; + struct winesync_obj *obj = entry->obj; + + spin_lock(&obj->lock); + list_del(&entry->node); + spin_unlock(&obj->lock); + + put_obj(obj); + } + + spin_unlock(&dev->wait_all_lock); + + signaled = atomic_read(&q->signaled); + if (signaled != -1) { + struct winesync_wait_args __user *user_args = argp; + + /* even if we caught a signal, we need to communicate success */ + ret = q->ownerdead ? -EOWNERDEAD : 0; + + if (put_user(signaled, &user_args->index)) + ret = -EFAULT; + } else if (!ret) { + ret = -ETIMEDOUT; + } + + kfree(q); + return ret; +} + +static long winesync_char_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + struct winesync_device *dev = file->private_data; + void __user *argp = (void __user *)parm; + + switch (cmd) { + case WINESYNC_IOC_CREATE_EVENT: + return winesync_create_event(dev, argp); + case WINESYNC_IOC_CREATE_MUTEX: + return winesync_create_mutex(dev, argp); + case WINESYNC_IOC_CREATE_SEM: + return winesync_create_sem(dev, argp); + case WINESYNC_IOC_DELETE: + return winesync_delete(dev, argp); + case WINESYNC_IOC_KILL_OWNER: + return winesync_kill_owner(dev, argp); + case WINESYNC_IOC_PULSE_EVENT: + return winesync_set_event(dev, argp, true); + case WINESYNC_IOC_PUT_MUTEX: + return winesync_put_mutex(dev, argp); + case WINESYNC_IOC_PUT_SEM: + return winesync_put_sem(dev, argp); + case WINESYNC_IOC_READ_EVENT: + return winesync_read_event(dev, argp); + case WINESYNC_IOC_READ_MUTEX: + return winesync_read_mutex(dev, argp); + case WINESYNC_IOC_READ_SEM: + return winesync_read_sem(dev, argp); + case WINESYNC_IOC_RESET_EVENT: + return winesync_reset_event(dev, argp); + case WINESYNC_IOC_SET_EVENT: + return winesync_set_event(dev, argp, false); + case WINESYNC_IOC_WAIT_ALL: + return winesync_wait_all(dev, argp); + case WINESYNC_IOC_WAIT_ANY: + return winesync_wait_any(dev, argp); + default: + return -ENOSYS; + } +} + +static const struct file_operations winesync_fops = { + .owner = THIS_MODULE, + .open = winesync_char_open, + .release = winesync_char_release, + .unlocked_ioctl = winesync_char_ioctl, + .compat_ioctl = winesync_char_ioctl, + .llseek = no_llseek, +}; + +static struct miscdevice winesync_misc = { + .minor = WINESYNC_MINOR, + .name = WINESYNC_NAME, + .fops = &winesync_fops, +}; + +static int __init winesync_init(void) +{ + return misc_register(&winesync_misc); +} + +static void __exit winesync_exit(void) +{ + misc_deregister(&winesync_misc); +} + +module_init(winesync_init); +module_exit(winesync_exit); + +MODULE_AUTHOR("Zebediah Figura"); +MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("devname:" WINESYNC_NAME); +MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h index 0676f18093f92..350aecfcfb290 100644 --- a/include/linux/miscdevice.h +++ b/include/linux/miscdevice.h @@ -71,6 +71,7 @@ #define USERIO_MINOR 240 #define VHOST_VSOCK_MINOR 241 #define RFKILL_MINOR 242 +#define WINESYNC_MINOR 243 #define MISC_DYNAMIC_MINOR 255 struct device; diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 71a5df8d26898..d375ab21cbf83 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -22,6 +22,7 @@ #define FUTEX_WAIT_REQUEUE_PI 11 #define FUTEX_CMP_REQUEUE_PI 12 #define FUTEX_LOCK_PI2 13 +#define FUTEX_WAIT_MULTIPLE 31 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 @@ -68,6 +69,18 @@ struct futex_waitv { __u32 __reserved; }; +/** + * struct futex_wait_block - Block of futexes to be waited for + * @uaddr: User address of the futex + * @val: Futex value expected by userspace + * @bitset: Bitset for the optional bitmasked wakeup + */ +struct futex_wait_block { + __u32 __user *uaddr; + __u32 val; + __u32 bitset; +}; + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h new file mode 100644 index 0000000000000..5b4e369f74693 --- /dev/null +++ b/include/uapi/linux/winesync.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Kernel support for Wine synchronization primitives + * + * Copyright (C) 2021 Zebediah Figura + */ + +#ifndef __LINUX_WINESYNC_H +#define __LINUX_WINESYNC_H + +#include + +struct winesync_sem_args { + __u32 sem; + __u32 count; + __u32 max; +}; + +struct winesync_mutex_args { + __u32 mutex; + __u32 owner; + __u32 count; +}; + +struct winesync_event_args { + __u32 event; + __u32 manual; + __u32 signaled; +}; + +struct winesync_wait_args { + __u64 timeout; + __u64 objs; + __u32 count; + __u32 owner; + __u32 index; + __u32 alert; +}; + +#define WINESYNC_IOC_BASE 0xf7 + +#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ + struct winesync_sem_args) +#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) +#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ + struct winesync_sem_args) +#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ + struct winesync_wait_args) +#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ + struct winesync_wait_args) +#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ + struct winesync_mutex_args) +#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ + struct winesync_mutex_args) +#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) +#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ + struct winesync_sem_args) +#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ + struct winesync_mutex_args) +#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ + struct winesync_event_args) +#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ + struct winesync_event_args) +#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ + struct winesync_event_args) +#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ + struct winesync_event_args) +#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ + struct winesync_event_args) + +#endif diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 086a22d1adb78..c6f5f1e84e099 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -142,6 +142,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd) case FUTEX_LOCK_PI2: case FUTEX_WAIT_BITSET: case FUTEX_WAIT_REQUEUE_PI: + case FUTEX_WAIT_MULTIPLE: return true; } return false; @@ -154,13 +155,79 @@ futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t) return -EINVAL; *t = timespec64_to_ktime(*ts); - if (cmd == FUTEX_WAIT) + if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE) *t = ktime_add_safe(ktime_get(), *t); else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME)) *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t); return 0; } +/** + * futex_read_wait_block - Read an array of futex_wait_block from userspace + * @uaddr: Userspace address of the block + * @count: Number of blocks to be read + * + * This function creates and allocate an array of futex_q (we zero it to + * initialize the fields) and then, for each futex_wait_block element from + * userspace, fill a futex_q element with proper values. + */ +inline struct futex_vector *futex_read_wait_block(u32 __user *uaddr, u32 count) +{ + unsigned int i; + struct futex_vector *futexv; + struct futex_wait_block fwb; + struct futex_wait_block __user *entry = + (struct futex_wait_block __user *)uaddr; + + if (!count || count > FUTEX_WAITV_MAX) + return ERR_PTR(-EINVAL); + + futexv = kcalloc(count, sizeof(*futexv), GFP_KERNEL); + if (!futexv) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < count; i++) { + if (copy_from_user(&fwb, &entry[i], sizeof(fwb))) { + kfree(futexv); + return ERR_PTR(-EFAULT); + } + + futexv[i].w.flags = FUTEX_32; + futexv[i].w.val = fwb.val; + futexv[i].w.uaddr = (uintptr_t) (fwb.uaddr); + futexv[i].q = futex_q_init; + } + + return futexv; +} + +int futex_wait_multiple(struct futex_vector *vs, unsigned int count, + struct hrtimer_sleeper *to); + +int futex_opcode_31(ktime_t *abs_time, u32 __user *uaddr, int count) +{ + int ret; + struct futex_vector *vs; + struct hrtimer_sleeper *to = NULL, timeout; + + to = futex_setup_timer(abs_time, &timeout, 0, 0); + + vs = futex_read_wait_block(uaddr, count); + + if (IS_ERR(vs)) + return PTR_ERR(vs); + + ret = futex_wait_multiple(vs, count, abs_time ? to : NULL); + kfree(vs); + + if (to) { + hrtimer_cancel(&to->timer); + destroy_hrtimer_on_stack(&to->timer); + } + + return ret; +} + SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, const struct __kernel_timespec __user *, utime, u32 __user *, uaddr2, u32, val3) @@ -180,6 +247,9 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, tp = &t; } + if (cmd == FUTEX_WAIT_MULTIPLE) + return futex_opcode_31(tp, uaddr, val); + return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } @@ -370,6 +440,9 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, tp = &t; } + if (cmd == FUTEX_WAIT_MULTIPLE) + return futex_opcode_31(tp, uaddr, val); + return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } #endif /* CONFIG_COMPAT_32BIT_TIME */ diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index de11992dc5776..32abac9c2fa5a 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -12,6 +12,7 @@ TARGETS += cpu-hotplug TARGETS += damon TARGETS += drivers/dma-buf TARGETS += drivers/s390x/uvdevice +TARGETS += drivers/winesync TARGETS += efivarfs TARGETS += exec TARGETS += filesystems diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile new file mode 100644 index 0000000000000..43b39fdeea10e --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/Makefile @@ -0,0 +1,8 @@ +# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only +TEST_GEN_PROGS := winesync + +top_srcdir =../../../../.. +CFLAGS += -I$(top_srcdir)/usr/include +LDLIBS += -lpthread + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config new file mode 100644 index 0000000000000..60539c826d062 --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/config @@ -0,0 +1 @@ +CONFIG_WINESYNC=y diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c new file mode 100644 index 0000000000000..169e922484b00 --- /dev/null +++ b/tools/testing/selftests/drivers/winesync/winesync.c @@ -0,0 +1,1479 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Various unit tests for the "winesync" synchronization primitive driver. + * + * Copyright (C) 2021 Zebediah Figura + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../kselftest_harness.h" + +static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) +{ + struct winesync_sem_args args; + int ret; + + args.sem = sem; + args.count = 0xdeadbeef; + args.max = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); + *count = args.count; + *max = args.max; + return ret; +} + +#define check_sem_state(fd, sem, count, max) \ + ({ \ + __u32 __count, __max; \ + int ret = read_sem_state((fd), (sem), &__count, &__max); \ + EXPECT_EQ(0, ret); \ + EXPECT_EQ((count), __count); \ + EXPECT_EQ((max), __max); \ + }) + +static int put_sem(int fd, __u32 sem, __u32 *count) +{ + struct winesync_sem_args args; + int ret; + + args.sem = sem; + args.count = *count; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); + *count = args.count; + return ret; +} + +static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) +{ + struct winesync_mutex_args args; + int ret; + + args.mutex = mutex; + args.count = 0xdeadbeef; + args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); + *count = args.count; + *owner = args.owner; + return ret; +} + +#define check_mutex_state(fd, mutex, count, owner) \ + ({ \ + __u32 __count, __owner; \ + int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ + EXPECT_EQ(0, ret); \ + EXPECT_EQ((count), __count); \ + EXPECT_EQ((owner), __owner); \ + }) + +static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) +{ + struct winesync_mutex_args args; + int ret; + + args.mutex = mutex; + args.owner = owner; + args.count = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); + *count = args.count; + return ret; +} + +static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) +{ + struct winesync_event_args args; + int ret; + + args.event = event; + args.signaled = 0xdeadbeef; + args.manual = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); + *signaled = args.signaled; + *manual = args.manual; + return ret; +} + +#define check_event_state(fd, event, signaled, manual) \ + ({ \ + __u32 __signaled, __manual; \ + int ret = read_event_state((fd), (event), \ + &__signaled, &__manual); \ + EXPECT_EQ(0, ret); \ + EXPECT_EQ((signaled), __signaled); \ + EXPECT_EQ((manual), __manual); \ + }) + +static int wait_objs(int fd, unsigned long request, __u32 count, + const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) +{ + struct winesync_wait_args args = {0}; + struct timespec timeout; + int ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + args.timeout = (uintptr_t)&timeout; + args.count = count; + args.objs = (uintptr_t)objs; + args.owner = owner; + args.index = 0xdeadbeef; + args.alert = alert; + ret = ioctl(fd, request, &args); + *index = args.index; + return ret; +} + +static int wait_any(int fd, __u32 count, const __u32 *objs, + __u32 owner, __u32 *index) +{ + return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, + count, objs, owner, 0, index); +} + +static int wait_all(int fd, __u32 count, const __u32 *objs, + __u32 owner, __u32 *index) +{ + return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, + count, objs, owner, 0, index); +} + +static int wait_any_alert(int fd, __u32 count, const __u32 *objs, + __u32 owner, __u32 alert, __u32 *index) +{ + return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, + count, objs, owner, alert, index); +} + +static int wait_all_alert(int fd, __u32 count, const __u32 *objs, + __u32 owner, __u32 alert, __u32 *index) +{ + return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, + count, objs, owner, alert, index); +} + +TEST(semaphore_state) +{ + struct winesync_sem_args sem_args; + struct timespec timeout; + __u32 sem, count, index; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 3; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + sem_args.count = 2; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + check_sem_state(fd, sem, 2, 2); + + count = 0; + ret = put_sem(fd, sem, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, count); + check_sem_state(fd, sem, 2, 2); + + count = 1; + ret = put_sem(fd, sem, &count); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + check_sem_state(fd, sem, 2, 2); + + ret = wait_any(fd, 1, &sem, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem, 1, 2); + + ret = wait_any(fd, 1, &sem, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem, 0, 2); + + ret = wait_any(fd, 1, &sem, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + count = 3; + ret = put_sem(fd, sem, &count); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOVERFLOW, errno); + check_sem_state(fd, sem, 0, 2); + + count = 2; + ret = put_sem(fd, sem, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, count); + check_sem_state(fd, sem, 2, 2); + + ret = wait_any(fd, 1, &sem, 123, &index); + EXPECT_EQ(0, ret); + ret = wait_any(fd, 1, &sem, 123, &index); + EXPECT_EQ(0, ret); + + count = 1; + ret = put_sem(fd, sem, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, count); + check_sem_state(fd, sem, 1, 2); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(mutex_state) +{ + struct winesync_mutex_args mutex_args; + __u32 mutex, owner, count, index; + struct timespec timeout; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + mutex_args.owner = 123; + mutex_args.count = 0; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + mutex_args.owner = 0; + mutex_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + mutex_args.owner = 123; + mutex_args.count = 2; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + mutex = mutex_args.mutex; + check_mutex_state(fd, mutex, 2, 123); + + ret = put_mutex(fd, mutex, 0, &count); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = put_mutex(fd, mutex, 456, &count); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EPERM, errno); + check_mutex_state(fd, mutex, 2, 123); + + ret = put_mutex(fd, mutex, 123, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, count); + check_mutex_state(fd, mutex, 1, 123); + + ret = put_mutex(fd, mutex, 123, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, count); + check_mutex_state(fd, mutex, 0, 0); + + ret = put_mutex(fd, mutex, 123, &count); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EPERM, errno); + + ret = wait_any(fd, 1, &mutex, 456, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_mutex_state(fd, mutex, 1, 456); + + ret = wait_any(fd, 1, &mutex, 456, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_mutex_state(fd, mutex, 2, 456); + + ret = put_mutex(fd, mutex, 456, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, count); + check_mutex_state(fd, mutex, 1, 456); + + ret = wait_any(fd, 1, &mutex, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + owner = 0; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + check_mutex_state(fd, mutex, 1, 456); + + owner = 456; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + ret = wait_any(fd, 1, &mutex, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, index); + check_mutex_state(fd, mutex, 1, 123); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + mutex_args.count = 0xdeadbeef; + mutex_args.owner = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, mutex_args.count); + EXPECT_EQ(0, mutex_args.owner); + + ret = wait_any(fd, 1, &mutex, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(0, index); + check_mutex_state(fd, mutex, 1, 123); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); + EXPECT_EQ(0, ret); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + mutex = mutex_args.mutex; + check_mutex_state(fd, mutex, 0, 0); + + ret = wait_any(fd, 1, &mutex, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_mutex_state(fd, mutex, 1, 123); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(manual_event_state) +{ + struct winesync_event_args event_args; + __u32 index; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + event_args.manual = 1; + event_args.signaled = 0; + event_args.event = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, event_args.event); + check_event_state(fd, event_args.event, 0, 1); + + event_args.signaled = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 1, 1); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + check_event_state(fd, event_args.event, 1, 1); + + ret = wait_any(fd, 1, &event_args.event, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_event_state(fd, event_args.event, 1, 1); + + event_args.signaled = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + check_event_state(fd, event_args.event, 0, 1); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 1); + + ret = wait_any(fd, 1, &event_args.event, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + check_event_state(fd, event_args.event, 0, 1); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 1); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(auto_event_state) +{ + struct winesync_event_args event_args; + __u32 index; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + event_args.manual = 0; + event_args.signaled = 1; + event_args.event = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, event_args.event); + + check_event_state(fd, event_args.event, 1, 0); + + event_args.signaled = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + check_event_state(fd, event_args.event, 1, 0); + + ret = wait_any(fd, 1, &event_args.event, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_event_state(fd, event_args.event, 0, 0); + + event_args.signaled = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 0); + + ret = wait_any(fd, 1, &event_args.event, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + check_event_state(fd, event_args.event, 0, 0); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 0); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(test_wait_any) +{ + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + __u32 objs[2], owner, index; + struct timespec timeout; + int fd, ret; + + clock_gettime(CLOCK_MONOTONIC, &timeout); + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 2; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + ret = wait_any(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 1, 3); + check_mutex_state(fd, mutex_args.mutex, 0, 0); + + ret = wait_any(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 0, 0); + + ret = wait_any(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 123); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = wait_any(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 123); + + ret = wait_any(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 2, 123); + + ret = wait_any(fd, 2, objs, 456, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + ret = wait_any(fd, 2, objs, 456, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + EXPECT_EQ(1, index); + + ret = wait_any(fd, 2, objs, 456, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, index); + + /* test waiting on the same object twice */ + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + objs[0] = objs[1] = sem_args.sem; + ret = wait_any(fd, 2, objs, 456, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, wait_args.index); + check_sem_state(fd, sem_args.sem, 1, 3); + + ret = wait_any(fd, 0, NULL, 456, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(test_wait_all) +{ + struct winesync_event_args event_args = {0}; + struct winesync_mutex_args mutex_args = {0}; + struct winesync_sem_args sem_args = {0}; + __u32 objs[2], owner, index; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 2; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 0; + mutex_args.count = 0; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + event_args.manual = true; + event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 1, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 123); + + ret = wait_all(fd, 2, objs, 456, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + check_sem_state(fd, sem_args.sem, 1, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 123); + + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 2, 123); + + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + check_sem_state(fd, sem_args.sem, 0, 3); + check_mutex_state(fd, mutex_args.mutex, 2, 123); + + sem_args.count = 3; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 2, 3); + check_mutex_state(fd, mutex_args.mutex, 3, 123); + + owner = 123; + ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); + EXPECT_EQ(0, ret); + + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EOWNERDEAD, errno); + check_sem_state(fd, sem_args.sem, 1, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 123); + + objs[0] = sem_args.sem; + objs[1] = event_args.event; + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + check_sem_state(fd, sem_args.sem, 0, 3); + check_event_state(fd, event_args.event, 1, 1); + + /* test waiting on the same object twice */ + objs[0] = objs[1] = sem_args.sem; + ret = wait_all(fd, 2, objs, 123, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(invalid_objects) +{ + struct winesync_event_args event_args = {0}; + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + __u32 objs[2] = {0}; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + wait_args.objs = (uintptr_t)objs; + wait_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + sem_args.max = 1; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + + mutex_args.mutex = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + event_args.event = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + objs[0] = sem_args.sem; + objs[1] = sem_args.sem + 1; + wait_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + objs[0] = sem_args.sem + 1; + objs[1] = sem_args.sem; + ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + + sem_args.sem = mutex_args.mutex; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + close(fd); +} + +struct wake_args +{ + int fd; + __u32 obj; +}; + +struct wait_args +{ + int fd; + unsigned long request; + struct winesync_wait_args *args; + int ret; + int err; +}; + +static void *wait_thread(void *arg) +{ + struct wait_args *args = arg; + + args->ret = ioctl(args->fd, args->request, args->args); + args->err = errno; + return NULL; +} + +static void get_abs_timeout(struct timespec *timeout, clockid_t clock, + unsigned int ms) +{ + clock_gettime(clock, timeout); + timeout->tv_nsec += ms * 1000000; + timeout->tv_sec += (timeout->tv_nsec / 1000000000); + timeout->tv_nsec %= 1000000000; +} + +static int wait_for_thread(pthread_t thread, unsigned int ms) +{ + struct timespec timeout; + get_abs_timeout(&timeout, CLOCK_REALTIME, ms); + return pthread_timedjoin_np(thread, NULL, &timeout); +} + +TEST(wake_any) +{ + struct winesync_event_args event_args = {0}; + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct wait_args thread_args; + __u32 objs[2], count, index; + struct timespec timeout; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 0; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 123; + mutex_args.count = 1; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + + /* test waking the semaphore */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 456; + wait_args.index = 0xdeadbeef; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ANY; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + check_sem_state(fd, sem_args.sem, 0, 3); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(0, wait_args.index); + + /* test waking the mutex */ + + /* first grab it again for owner 123 */ + ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.owner = 456; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = put_mutex(fd, mutex_args.mutex, 123, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + ret = put_mutex(fd, mutex_args.mutex, 123, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, mutex_args.count); + check_mutex_state(fd, mutex_args.mutex, 1, 456); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + /* test waking events */ + + event_args.manual = false; + event_args.signaled = false; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + objs[1] = event_args.event; + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 0); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 0); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + event_args.manual = true; + event_args.signaled = false; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + objs[1] = event_args.event; + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 1, 1); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, event_args.signaled); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, event_args.signaled); + check_event_state(fd, event_args.event, 0, 1); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(1, wait_args.index); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + /* delete an object while it's being waited on */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); + wait_args.owner = 123; + objs[1] = mutex_args.mutex; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 200); + EXPECT_EQ(0, ret); + EXPECT_EQ(-1, thread_args.ret); + EXPECT_EQ(ETIMEDOUT, thread_args.err); + + close(fd); +} + +TEST(wake_all) +{ + struct winesync_event_args manual_event_args = {0}; + struct winesync_event_args auto_event_args = {0}; + struct winesync_mutex_args mutex_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct wait_args thread_args; + __u32 objs[4], count, index; + struct timespec timeout; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 0; + sem_args.max = 3; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + + mutex_args.owner = 123; + mutex_args.count = 1; + mutex_args.mutex = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, mutex_args.mutex); + + manual_event_args.manual = true; + manual_event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); + EXPECT_EQ(0, ret); + + auto_event_args.manual = false; + auto_event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); + EXPECT_EQ(0, ret); + + objs[0] = sem_args.sem; + objs[1] = mutex_args.mutex; + objs[2] = manual_event_args.event; + objs[3] = auto_event_args.event; + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 4; + wait_args.owner = 456; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ALL; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + check_sem_state(fd, sem_args.sem, 1, 3); + + ret = wait_any(fd, 1, &sem_args.sem, 123, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + ret = put_mutex(fd, mutex_args.mutex, 123, &count); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, count); + + ret = pthread_tryjoin_np(thread, NULL); + EXPECT_EQ(EBUSY, ret); + + check_mutex_state(fd, mutex_args.mutex, 0, 0); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, manual_event_args.signaled); + + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, sem_args.count); + check_sem_state(fd, sem_args.sem, 2, 3); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, auto_event_args.signaled); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, manual_event_args.signaled); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, auto_event_args.signaled); + + check_sem_state(fd, sem_args.sem, 1, 3); + check_mutex_state(fd, mutex_args.mutex, 1, 456); + check_event_state(fd, manual_event_args.event, 1, 1); + check_event_state(fd, auto_event_args.event, 0, 0); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + + /* delete an object while it's being waited on */ + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); + wait_args.owner = 123; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 200); + EXPECT_EQ(0, ret); + EXPECT_EQ(-1, thread_args.ret); + EXPECT_EQ(ETIMEDOUT, thread_args.err); + + close(fd); +} + +TEST(alert_any) +{ + struct winesync_event_args event_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct wait_args thread_args; + struct timespec timeout; + __u32 objs[2], index; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 0; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + objs[0] = sem_args.sem; + + sem_args.count = 1; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + objs[1] = sem_args.sem; + + event_args.manual = true; + event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(1, index); + + ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, index); + + /* test wakeup via alert */ + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + wait_args.alert = event_args.event; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ANY; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(2, wait_args.index); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + /* test with an auto-reset event */ + + event_args.manual = false; + event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + sem_args.sem = objs[0]; + sem_args.count = 1; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + + ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, index); + + ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST(alert_all) +{ + struct winesync_event_args event_args = {0}; + struct winesync_wait_args wait_args = {0}; + struct winesync_sem_args sem_args = {0}; + struct wait_args thread_args; + struct timespec timeout; + __u32 objs[2], index; + pthread_t thread; + int fd, ret; + + fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); + ASSERT_LE(0, fd); + + sem_args.count = 2; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + objs[0] = sem_args.sem; + + sem_args.count = 1; + sem_args.max = 2; + sem_args.sem = 0xdeadbeef; + ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); + EXPECT_EQ(0, ret); + EXPECT_NE(0xdeadbeef, sem_args.sem); + objs[1] = sem_args.sem; + + event_args.manual = true; + event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, index); + + /* test wakeup via alert */ + + ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); + wait_args.timeout = (uintptr_t)&timeout; + wait_args.objs = (uintptr_t)objs; + wait_args.count = 2; + wait_args.owner = 123; + wait_args.index = 0xdeadbeef; + wait_args.alert = event_args.event; + thread_args.fd = fd; + thread_args.args = &wait_args; + thread_args.request = WINESYNC_IOC_WAIT_ALL; + ret = pthread_create(&thread, NULL, wait_thread, &thread_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(ETIMEDOUT, ret); + + ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); + EXPECT_EQ(0, ret); + + ret = wait_for_thread(thread, 100); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, thread_args.ret); + EXPECT_EQ(2, wait_args.index); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + /* test with an auto-reset event */ + + event_args.manual = false; + event_args.signaled = true; + ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); + EXPECT_EQ(0, ret); + + sem_args.sem = objs[1]; + sem_args.count = 2; + ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); + EXPECT_EQ(0, ret); + + ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, index); + + ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(0, ret); + EXPECT_EQ(2, index); + + ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ETIMEDOUT, errno); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); + EXPECT_EQ(0, ret); + + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); + EXPECT_EQ(0, ret); + ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); + EXPECT_EQ(0, ret); + + close(fd); +} + +TEST_HARNESS_MAIN -- 2.37.1 From 2fdf0bbc9405571a3231f7772e21dd295fb41b87 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:33:31 +0200 Subject: [PATCH 08/16] hwmon Signed-off-by: Peter Jung --- Documentation/ABI/testing/sysfs-class-hwmon | 9 + .../bindings/hwmon/national,lm90.yaml | 131 + .../devicetree/bindings/trivial-devices.yaml | 2 + Documentation/hwmon/aquacomputer_d5next.rst | 17 +- Documentation/hwmon/asus_ec_sensors.rst | 4 + Documentation/hwmon/dell-smm-hwmon.rst | 3 + Documentation/hwmon/index.rst | 1 + Documentation/hwmon/lm90.rst | 233 +- Documentation/hwmon/lt7182s.rst | 92 + Documentation/hwmon/pmbus-core.rst | 9 + drivers/hwmon/Kconfig | 27 +- drivers/hwmon/aquacomputer_d5next.c | 316 +- drivers/hwmon/aspeed-pwm-tacho.c | 2 +- drivers/hwmon/asus-ec-sensors.c | 108 +- drivers/hwmon/asus_wmi_sensors.c | 12 +- drivers/hwmon/dell-smm-hwmon.c | 93 +- drivers/hwmon/drivetemp.c | 1 + drivers/hwmon/f71882fg.c | 2 + drivers/hwmon/gsc-hwmon.c | 3 + drivers/hwmon/lm75.h | 3 +- drivers/hwmon/lm90.c | 2556 +++++++++++------ drivers/hwmon/mcp3021.c | 99 +- drivers/hwmon/nct6775-platform.c | 13 +- drivers/hwmon/occ/common.c | 8 +- drivers/hwmon/occ/p9_sbe.c | 6 +- drivers/hwmon/pmbus/Kconfig | 9 + drivers/hwmon/pmbus/Makefile | 1 + drivers/hwmon/pmbus/lt7182s.c | 195 ++ drivers/hwmon/pmbus/ltc2978.c | 44 +- drivers/hwmon/pmbus/pmbus.h | 10 +- drivers/hwmon/pmbus/pmbus_core.c | 446 ++- drivers/hwmon/sch56xx-common.c | 44 +- drivers/hwmon/sht15.c | 17 +- drivers/hwmon/tps23861.c | 14 +- 34 files changed, 3309 insertions(+), 1221 deletions(-) create mode 100644 Documentation/hwmon/lt7182s.rst create mode 100644 drivers/hwmon/pmbus/lt7182s.c diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon index 653d4c75eddb3..7271781a23b28 100644 --- a/Documentation/ABI/testing/sysfs-class-hwmon +++ b/Documentation/ABI/testing/sysfs-class-hwmon @@ -938,3 +938,12 @@ Description: - 1: enable RW + +What: /sys/class/hwmon/hwmonX/device/pec +Description: + PEC support on I2C devices + + - 0, off, n: disable + - 1, on, y: enable + + RW diff --git a/Documentation/devicetree/bindings/hwmon/national,lm90.yaml b/Documentation/devicetree/bindings/hwmon/national,lm90.yaml index b046578498524..e1719839faf03 100644 --- a/Documentation/devicetree/bindings/hwmon/national,lm90.yaml +++ b/Documentation/devicetree/bindings/hwmon/national,lm90.yaml @@ -16,6 +16,7 @@ properties: - adi,adm1032 - adi,adt7461 - adi,adt7461a + - adi,adt7481 - dallas,max6646 - dallas,max6647 - dallas,max6649 @@ -50,6 +51,12 @@ properties: "#thermal-sensor-cells": const: 1 + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + vcc-supply: description: phandle to the regulator that provides the +VCC supply @@ -61,6 +68,29 @@ required: - compatible - reg +patternProperties: + "^channel@([0-2])$": + type: object + description: Represents channels of the device and their specific configuration. + + properties: + reg: + description: The channel number. 0 is local channel, 1-2 are remote channels. + items: + minimum: 0 + maximum: 2 + + label: + description: A descriptive name for this channel, like "ambient" or "psu". + + temperature-offset-millicelsius: + description: Temperature offset to be added to or subtracted from remote temperature measurements. + + required: + - reg + + additionalProperties: false + allOf: - if: not: @@ -70,12 +100,84 @@ allOf: enum: - adi,adt7461 - adi,adt7461a + - adi,adt7481 - ti,tmp451 - ti,tmp461 then: properties: ti,extended-range-enable: false + - if: + properties: + compatible: + contains: + enum: + - dallas,max6646 + - dallas,max6647 + - dallas,max6649 + - dallas,max6657 + - dallas,max6658 + - dallas,max6659 + - dallas,max6695 + - dallas,max6696 + then: + patternProperties: + "^channel@([0-2])$": + properties: + temperature-offset-millicelsius: false + + - if: + properties: + compatible: + contains: + enum: + - adi,adt7461 + - adi,adt7461a + - adi,adt7481 + - onnn,nct1008 + then: + patternProperties: + "^channel@([0-2])$": + properties: + temperature-offset-millicelsius: + maximum: 127750 + + - if: + properties: + compatible: + contains: + enum: + - adi,adm1032 + - dallas,max6680 + - dallas,max6681 + - gmt,g781 + - national,lm86 + - national,lm89 + - national,lm90 + - national,lm99 + - nxp,sa56004 + - winbond,w83l771 + then: + patternProperties: + "^channel@([0-2])$": + properties: + temperature-offset-millicelsius: + maximum: 127875 + + - if: + properties: + compatible: + contains: + enum: + - ti,tmp451 + - ti,tmp461 + then: + patternProperties: + "^channel@([0-2])$": + properties: + temperature-offset-millicelsius: + maximum: 127937 + additionalProperties: false examples: @@ -94,3 +196,32 @@ examples: #thermal-sensor-cells = <1>; }; }; + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + sensor@4c { + compatible = "adi,adt7481"; + reg = <0x4c>; + #address-cells = <1>; + #size-cells = <0>; + + channel@0 { + reg = <0x0>; + label = "local"; + }; + + channel@1 { + reg = <0x1>; + label = "front"; + temperature-offset-millicelsius = <4000>; + }; + + channel@2 { + reg = <0x2>; + label = "back"; + temperature-offset-millicelsius = <750>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 6aafa71806a3d..5d87b8426ff4c 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -41,6 +41,8 @@ properties: - adi,adp5585-02 # Analog Devices ADP5589 Keypad Decoder and I/O Expansion - adi,adp5589 + # Analog Devices LT7182S Dual Channel 6A, 20V PolyPhase Step-Down Silent Switcher + - adi,lt7182s # AMS iAQ-Core VOC Sensor - ams,iaq-core # i2c serial eeprom (24cxx) diff --git a/Documentation/hwmon/aquacomputer_d5next.rst b/Documentation/hwmon/aquacomputer_d5next.rst index 717e28226cde9..33649a1e3a05f 100644 --- a/Documentation/hwmon/aquacomputer_d5next.rst +++ b/Documentation/hwmon/aquacomputer_d5next.rst @@ -9,6 +9,7 @@ Supported devices: * Aquacomputer Farbwerk RGB controller * Aquacomputer Farbwerk 360 RGB controller * Aquacomputer Octo fan controller +* Aquacomputer Quadro fan controller Author: Aleksa Savic @@ -33,6 +34,9 @@ better suited for userspace tools. The Octo exposes four temperature sensors and eight PWM controllable fans, along with their speed (in RPM), power, voltage and current. +The Quadro exposes four temperature sensors, a flow sensor and four PWM controllable +fans, along with their speed (in RPM), power, voltage and current. + The Farbwerk and Farbwerk 360 expose four temperature sensors. Depending on the device, not all sysfs and debugfs entries will be available. @@ -45,13 +49,14 @@ the kernel and supports hotswapping. Sysfs entries ------------- -================ ============================================= +================ ============================================== temp[1-4]_input Temperature sensors (in millidegrees Celsius) -fan[1-2]_input Pump/fan speed (in RPM) -power[1-2]_input Pump/fan power (in micro Watts) -in[0-2]_input Pump/fan voltage (in milli Volts) -curr[1-2]_input Pump/fan current (in milli Amperes) -================ ============================================= +fan[1-8]_input Pump/fan speed (in RPM) / Flow speed (in dL/h) +power[1-8]_input Pump/fan power (in micro Watts) +in[0-7]_input Pump/fan voltage (in milli Volts) +curr[1-8]_input Pump/fan current (in milli Amperes) +pwm[1-8] Fan PWM (0 - 255) +================ ============================================== Debugfs entries --------------- diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst index 78ca69eda8778..02f4ad314a1eb 100644 --- a/Documentation/hwmon/asus_ec_sensors.rst +++ b/Documentation/hwmon/asus_ec_sensors.rst @@ -13,12 +13,16 @@ Supported boards: * ROG CROSSHAIR VIII FORMULA * ROG CROSSHAIR VIII HERO * ROG CROSSHAIR VIII IMPACT + * ROG MAXIMUS XI HERO + * ROG MAXIMUS XI HERO (WI-FI) * ROG STRIX B550-E GAMING * ROG STRIX B550-I GAMING * ROG STRIX X570-E GAMING * ROG STRIX X570-E GAMING WIFI II * ROG STRIX X570-F GAMING * ROG STRIX X570-I GAMING + * ROG STRIX Z690-A GAMING WIFI D4 + * ROG ZENITH II EXTREME Authors: - Eugene Shalygin diff --git a/Documentation/hwmon/dell-smm-hwmon.rst b/Documentation/hwmon/dell-smm-hwmon.rst index e5d85e40972c2..d8f1d6859b964 100644 --- a/Documentation/hwmon/dell-smm-hwmon.rst +++ b/Documentation/hwmon/dell-smm-hwmon.rst @@ -46,6 +46,9 @@ temp[1-10]_input RO Temperature reading in milli-degrees temp[1-10]_label RO Temperature sensor label. =============================== ======= ======================================= +Due to the nature of the SMM interface, each pwmX attribute controls +fan number X. + Disabling automatic BIOS fan control ------------------------------------ diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index a72c16872ec2d..f7113b0f8b2a6 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -109,6 +109,7 @@ Hardware Monitoring Kernel Drivers lm95234 lm95245 lochnagar + lt7182s ltc2992 ltc2945 ltc2947 diff --git a/Documentation/hwmon/lm90.rst b/Documentation/hwmon/lm90.rst index 05391fb4042d9..23af17a0ab442 100644 --- a/Documentation/hwmon/lm90.rst +++ b/Documentation/hwmon/lm90.rst @@ -3,6 +3,14 @@ Kernel driver lm90 Supported chips: + * National Semiconductor LM84 + + Prefix: 'lm84' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the National Semiconductor website + * National Semiconductor LM90 Prefix: 'lm90' @@ -43,6 +51,30 @@ Supported chips: http://www.national.com/mpf/LM/LM86.html + * Analog Devices ADM1020 + + Prefix: 'adm1020' + + Addresses scanned: I2C 0x4c - 0x4e + + Datasheet: Publicly available at the Analog Devices website + + * Analog Devices ADM1021 + + Prefix: 'adm1021' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the Analog Devices website + + * Analog Devices ADM1021A/ADM1023 + + Prefix: 'adm1023' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the Analog Devices website + * Analog Devices ADM1032 Prefix: 'adm1032' @@ -73,6 +105,36 @@ Supported chips: https://www.onsemi.com/PowerSolutions/product.do?id=ADT7461A + * Analog Devices ADT7481 + + Prefix: 'adt7481' + + Addresses scanned: I2C 0x4b and 0x4c + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=ADT7481 + + * Analog Devices ADT7482 + + Prefix: 'adt7482' + + Addresses scanned: I2C 0x4c + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=ADT7482 + + * Analog Devices ADT7483A + + Prefix: 'adt7483a' + + Addresses scanned: I2C 0x18, 0x19, 0x1a, 0x29, 0x2a, 0x2b, 0x4c, 0x4d, 0x4e + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=ADT7483A + * ON Semiconductor NCT1008 Prefix: 'nct1008' @@ -83,6 +145,72 @@ Supported chips: https://www.onsemi.com/PowerSolutions/product.do?id=NCT1008 + * ON Semiconductor NCT210 + + Prefix: 'adm1021' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=NCT210 + + * ON Semiconductor NCT214 + + Prefix: 'nct214' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=NCT214 + + * ON Semiconductor NCT218 + + Prefix: 'nct218' + + Addresses scanned: I2C 0x4c - 0x4d + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=NCT218 + + * ON Semiconductor NCT72 + + Prefix: 'nct72' + + Addresses scanned: I2C 0x4c - 0x4d + + Datasheet: Publicly available at the ON Semiconductor website + + https://www.onsemi.com/PowerSolutions/product.do?id=NCT72 + + * Maxim MAX1617 + + Prefix: 'max1617' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the Maxim website + + * Maxim MAX1617A + + Prefix: 'max1617a' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the Maxim website + + * Maxim MAX6642 + + Prefix: 'max6642' + + Addresses scanned: I2C 0x48-0x4f + + Datasheet: Publicly available at the Maxim website + + http://datasheets.maxim-ic.com/en/ds/MAX6642.pdf + * Maxim MAX6646 Prefix: 'max6646' @@ -105,7 +233,7 @@ Supported chips: * Maxim MAX6648 - Prefix: 'max6646' + Prefix: 'max6648' Addresses scanned: I2C 0x4c @@ -191,7 +319,7 @@ Supported chips: * Maxim MAX6692 - Prefix: 'max6646' + Prefix: 'max6648' Addresses scanned: I2C 0x4c @@ -275,6 +403,46 @@ Supported chips: https://www.ti.com/lit/gpn/tmp461 + * Philips NE1617, NE1617A + + Prefix: 'max1617' (probably detected as a max1617) + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheets: Publicly available at the Philips website + + * Philips NE1618 + + Prefix: 'ne1618' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheets: Publicly available at the Philips website + + * Genesys Logic GL523SM + + Prefix: 'gl523sm' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: + + * TI THMC10 + + Prefix: 'thmc10' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the TI website + + * Onsemi MC1066 + + Prefix: 'mc1066' + + Addresses scanned: I2C 0x18 - 0x1a, 0x29 - 0x2b, 0x4c - 0x4e + + Datasheet: Publicly available at the Onsemi website + Author: Jean Delvare @@ -285,6 +453,12 @@ The LM90 is a digital temperature sensor. It senses its own temperature as well as the temperature of up to one external diode. It is compatible with many other devices, many of which are supported by this driver. +The family of chips supported by this driver is derived from MAX1617. +This chip as well as various compatible chips support a local and a remote +temperature sensor with 8 bit accuracy. Later chips provide improved accuracy +and other additional features such as hysteresis and temperature offset +registers. + Note that there is no easy way to differentiate between the MAX6657, MAX6658 and MAX6659 variants. The extra features of the MAX6659 are only supported by this driver if the chip is located at address 0x4d or 0x4e, @@ -292,15 +466,31 @@ or if the chip type is explicitly selected as max6659. The MAX6680 and MAX6681 only differ in their pinout, therefore they obviously can't (and don't need to) be distinguished. -The specificity of this family of chipsets over the ADM1021/LM84 -family is that it features critical limits with hysteresis, and an -increased resolution of the remote temperature measurement. - The different chipsets of the family are not strictly identical, although very similar. For reference, here comes a non-exhaustive list of specific features: +LM84: + * 8 bit sensor resolution + +ADM1020, ADM1021, GL523SM, MAX1617, NE1617, NE1617A, THMC10: + * 8 bit sensor resolution + * Low temperature limits + +NCT210, NE1618: + * 11 bit sensor resolution for remote temperature sensor + * Low temperature limits + +ADM1021A, ADM1023: + * Temperature offset register for remote temperature sensor + * 11 bit resolution for remote temperature sensor + * Low temperature limits + LM90: + * 11 bit resolution for remote temperature sensor + * Temperature offset register for remote temperature sensor + * Low and critical temperature limits + * Configurable conversion rate * Filter and alert configuration register at 0xBF. * ALERT is triggered by temperatures over critical limits. @@ -322,8 +512,31 @@ ADM1032: ADT7461, ADT7461A, NCT1008: * Extended temperature range (breaks compatibility) * Lower resolution for remote temperature + * SMBus PEC support for Write Byte and Receive Byte transactions. + * 10 bit temperature resolution + +ADT7481, ADT7482, ADT7483: + * Temperature offset register + * SMBus PEC support + * 10 bit temperature resolution for external sensors + * Two remote sensors + * Selectable address (ADT7483) + +MAX6642: + * No critical limit register + * Conversion rate not configurable + * Better local resolution (10 bit) + * 10 bit external sensor resolution + +MAX6646, MAX6647, MAX6649: + * Better local resolution + * Extended range unsigned external temperature + +MAX6648, MAX6692: + * Better local resolution + * Unsigned temperature -MAX6654: +MAX6654, MAX6690: * Better local resolution * Selectable address * Remote sensor type selection @@ -423,6 +636,6 @@ two transactions will typically mean twice as much delay waiting for transaction completion, effectively doubling the register cache refresh time. I guess reliability comes at a price, but it's quite expensive this time. -So, as not everyone might enjoy the slowdown, PEC can be disabled through -sysfs. Just write 0 to the "pec" file and PEC will be disabled. Write 1 -to that file to enable PEC again. +So, as not everyone might enjoy the slowdown, PEC is disabled by default and +can be enabled through sysfs. Just write 1 to the "pec" file and PEC will be +enabled. Write 0 to that file to disable PEC again. diff --git a/Documentation/hwmon/lt7182s.rst b/Documentation/hwmon/lt7182s.rst new file mode 100644 index 0000000000000..f7268311b1914 --- /dev/null +++ b/Documentation/hwmon/lt7182s.rst @@ -0,0 +1,92 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver lt7182s +===================== + +Supported chips: + + * ADI LT7182S + + Prefix: 'lt7182s' + + Addresses scanned: - + + Datasheet: https://www.analog.com/en/products/lt7182s.html + +Author: Guenter Roeck + + +Description +----------- + +LT7182S is a Dual Channel 6A, 20V PolyPhase Step-Down Silent Switcher with +Digital Power System Management support. + + +Usage Notes +----------- + +This driver does not probe for PMBus devices. You will have to instantiate +devices explicitly. + +Example: the following commands will load the driver for a LT7182S +at address 0x4f on I2C bus #4:: + + # modprobe lt7182s + # echo lt7182s 0x4f > /sys/bus/i2c/devices/i2c-4/new_device + +It can also be instantiated by declaring an entry in device tree. + + +Sysfs attributes +---------------- + +======================= ==================================== +curr[1-2]_label "iin[12]" +curr[1-2]_input Measured input current +curr[1-2]_max Maximum input current +curr[1-2]_max_alarm Current high alarm + +curr[3-4]_label "iout[1-2]" +curr[3-4]_input Measured output current +curr[3-4]_highest Highest measured output current +curr[3-4]_max Maximum output current +curr[3-4]_max_alarm Output current high alarm + +in[1-2]_label "vin[12]" +in[1-2]_input Measured input voltage +in[1-2]_highest Highest measured input voltage +in[1-2]_crit Critical maximum input voltage +in[1-2]_crit_alarm Input voltage critical high alarm +in[1-2]_min Minimum input voltage +in[1-2]_min_alarm Input voltage low alarm +in[1-2]_rated_min Rated minimum input voltage +in[1-2]_rated_max Rated maximum input voltage +in1_reset_history Write to reset history for all attributes + +in[3-5]_label "vmon[1-3]" +in[3-5]_input Measured voltage on ITH1/ITH2/EXTVCC pins + Only available if enabled with MFR_ADC_CONTROL_LT7182S + command. + +in[3-4|6-7]_label "vout[1-2]" +in[3-4|6-7]_input Measured output voltage +in[3-4|6-7]_highest Highest measured output voltage +in[3-4|6-7]_lcrit Critical minimum output voltage +in[3-4|6-7]_lcrit_alarm Output voltage critical low alarm +in[3-4|6-7]_min Minimum output voltage +in[3-4|6-7]_max_alarm Output voltage low alarm +in[3-4|6-7]_max Maximum output voltage +in[3-4|6-7]_max_alarm Output voltage high alarm +in[3-4|6-7]_crit Critical maximum output voltage +in[3-4|6-7]_crit_alarm Output voltage critical high alarm + +power[1-2]_label "pout[1-2]" +power[1-2]_input Measured output power + +temp1_input Measured temperature +temp1_crit Critical high temperature +temp1_crit_alarm Chip temperature critical high alarm +temp1_max Maximum temperature +temp1_max_alarm Chip temperature high alarm +======================= ==================================== diff --git a/Documentation/hwmon/pmbus-core.rst b/Documentation/hwmon/pmbus-core.rst index e7e0c9ef10bec..84c5a4e40c409 100644 --- a/Documentation/hwmon/pmbus-core.rst +++ b/Documentation/hwmon/pmbus-core.rst @@ -121,6 +121,15 @@ Specifically, it provides the following information. non-standard PMBus commands to standard commands, or to augment standard command return values with device specific information. +PEC Support +=========== + +Many PMBus devices support SMBus PEC (Packet Error Checking). If supported +by both the I2C adapter and by the PMBus chip, it is by default enabled. +If PEC is supported, the PMBus core driver adds an attribute named 'pec' to +the I2C device. This attribute can be used to control PEC support in the +communication with the PMBus chip. + API functions ============= diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 590d3d550acba..e70d9614bec2c 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -100,6 +100,7 @@ config SENSORS_AD7418 config SENSORS_ADM1021 tristate "Analog Devices ADM1021 and compatibles" depends on I2C + depends on SENSORS_LM90=n help If you say yes here you get support for Analog Devices ADM1021 and ADM1023 sensor chips and clones: Maxim MAX1617 and MAX1617A, @@ -256,13 +257,13 @@ config SENSORS_AHT10 will be called aht10. config SENSORS_AQUACOMPUTER_D5NEXT - tristate "Aquacomputer D5 Next, Octo, Farbwerk, and Farbwerk 360" + tristate "Aquacomputer D5 Next, Octo, Quadro, Farbwerk, and Farbwerk 360" depends on USB_HID select CRC16 help If you say yes here you get support for sensors and fans of - the Aquacomputer D5 Next watercooling pump, Octo fan - controller, Farbwerk and Farbwerk 360 RGB controllers, where + the Aquacomputer D5 Next watercooling pump, Octo and Quadro fan + controllers, Farbwerk and Farbwerk 360 RGB controllers, where available. This driver can also be built as a module. If so, the module @@ -381,7 +382,7 @@ config SENSORS_ARM_SCPI config SENSORS_ASB100 tristate "Asus ASB100 Bach" - depends on X86 && I2C + depends on (X86 || COMPILE_TEST) && I2C select HWMON_VID help If you say yes here you get support for the ASB100 Bach sensor @@ -626,7 +627,7 @@ config SENSORS_MC13783_ADC config SENSORS_FSCHMD tristate "Fujitsu Siemens Computers sensor chips" - depends on X86 && I2C + depends on (X86 || COMPILE_TEST) && I2C help If you say yes here you get support for the following Fujitsu Siemens Computers (FSC) sensor chips: Poseidon, Scylla, Hermes, @@ -1102,6 +1103,7 @@ config SENSORS_MAX6639 config SENSORS_MAX6642 tristate "Maxim MAX6642 sensor chip" depends on I2C + depends on SENSORS_LM90=n help If you say yes here you get support for MAX6642 sensor chip. MAX6642 is a SMBus-Compatible Remote/Local Temperature Sensor @@ -1357,12 +1359,15 @@ config SENSORS_LM90 tristate "National Semiconductor LM90 and compatibles" depends on I2C help - If you say yes here you get support for National Semiconductor LM90, - LM86, LM89 and LM99, Analog Devices ADM1032, ADT7461, and ADT7461A, - Maxim MAX6646, MAX6647, MAX6648, MAX6649, MAX6654, MAX6657, MAX6658, - MAX6659, MAX6680, MAX6681, MAX6692, MAX6695, MAX6696, - ON Semiconductor NCT1008, Winbond/Nuvoton W83L771W/G/AWG/ASG, - Philips SA56004, GMT G781, Texas Instruments TMP451 and TMP461 + If you say yes here you get support for National Semiconductor LM84, + LM90, LM86, LM89 and LM99, Analog Devices ADM1020, ADM2021, ADM1021A, + ADM1023, ADM1032, ADT7461, ADT7461A, ADT7481, ADT7482, and ADT7483A, + Maxim MAX1617, MAX6642, MAX6646, MAX6647, MAX6648, MAX6649, MAX6654, + MAX6657, MAX6658, MAX6659, MAX6680, MAX6681, MAX6692, MAX6695, + MAX6696, + ON Semiconductor NCT1008, NCT210, NCT72, NCT214, NCT218, + Winbond/Nuvoton W83L771W/G/AWG/ASG, + Philips NE1618, SA56004, GMT G781, Texas Instruments TMP451 and TMP461 sensor chips. This driver can also be built as a module. If so, the module diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c index a0e69f7ece36e..66430553cc451 100644 --- a/drivers/hwmon/aquacomputer_d5next.c +++ b/drivers/hwmon/aquacomputer_d5next.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ /* - * hwmon driver for Aquacomputer devices (D5 Next, Farbwerk, Farbwerk 360, Octo) + * hwmon driver for Aquacomputer devices (D5 Next, Farbwerk, Farbwerk 360, Octo, + * Quadro) * * Aquacomputer devices send HID reports (with ID 0x01) every second to report * sensor values. @@ -21,17 +22,19 @@ #define USB_VENDOR_ID_AQUACOMPUTER 0x0c70 #define USB_PRODUCT_ID_FARBWERK 0xf00a +#define USB_PRODUCT_ID_QUADRO 0xf00d #define USB_PRODUCT_ID_D5NEXT 0xf00e #define USB_PRODUCT_ID_FARBWERK360 0xf010 #define USB_PRODUCT_ID_OCTO 0xf011 -enum kinds { d5next, farbwerk, farbwerk360, octo }; +enum kinds { d5next, farbwerk, farbwerk360, octo, quadro }; static const char *const aqc_device_names[] = { [d5next] = "d5next", [farbwerk] = "farbwerk", [farbwerk360] = "farbwerk360", - [octo] = "octo" + [octo] = "octo", + [quadro] = "quadro" }; #define DRIVER_NAME "aquacomputer_d5next" @@ -54,60 +57,61 @@ static u8 secondary_ctrl_report[] = { 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x34, 0xC6 }; -/* Register offsets for the D5 Next pump */ -#define D5NEXT_POWER_CYCLES 24 - -#define D5NEXT_COOLANT_TEMP 87 - -#define D5NEXT_PUMP_SPEED 116 -#define D5NEXT_FAN_SPEED 103 - -#define D5NEXT_PUMP_POWER 114 -#define D5NEXT_FAN_POWER 101 - -#define D5NEXT_PUMP_VOLTAGE 110 -#define D5NEXT_FAN_VOLTAGE 97 -#define D5NEXT_5V_VOLTAGE 57 +/* Register offsets for all Aquacomputer devices */ +#define AQC_TEMP_SENSOR_SIZE 0x02 +#define AQC_TEMP_SENSOR_DISCONNECTED 0x7FFF +#define AQC_FAN_PERCENT_OFFSET 0x00 +#define AQC_FAN_VOLTAGE_OFFSET 0x02 +#define AQC_FAN_CURRENT_OFFSET 0x04 +#define AQC_FAN_POWER_OFFSET 0x06 +#define AQC_FAN_SPEED_OFFSET 0x08 -#define D5NEXT_PUMP_CURRENT 112 -#define D5NEXT_FAN_CURRENT 99 +/* Register offsets for the D5 Next pump */ +#define D5NEXT_POWER_CYCLES 0x18 +#define D5NEXT_COOLANT_TEMP 0x57 +#define D5NEXT_NUM_FANS 2 +#define D5NEXT_NUM_SENSORS 1 +#define D5NEXT_PUMP_OFFSET 0x6c +#define D5NEXT_FAN_OFFSET 0x5f +#define D5NEXT_5V_VOLTAGE 0x39 +#define D5NEXT_12V_VOLTAGE 0x37 +#define D5NEXT_CTRL_REPORT_SIZE 0x329 +static u8 d5next_sensor_fan_offsets[] = { D5NEXT_PUMP_OFFSET, D5NEXT_FAN_OFFSET }; + +/* Pump and fan speed registers in D5 Next control report (from 0-100%) */ +static u16 d5next_ctrl_fan_offsets[] = { 0x97, 0x42 }; /* Register offsets for the Farbwerk RGB controller */ #define FARBWERK_NUM_SENSORS 4 #define FARBWERK_SENSOR_START 0x2f -#define FARBWERK_SENSOR_SIZE 0x02 -#define FARBWERK_SENSOR_DISCONNECTED 0x7FFF /* Register offsets for the Farbwerk 360 RGB controller */ #define FARBWERK360_NUM_SENSORS 4 #define FARBWERK360_SENSOR_START 0x32 -#define FARBWERK360_SENSOR_SIZE 0x02 -#define FARBWERK360_SENSOR_DISCONNECTED 0x7FFF /* Register offsets for the Octo fan controller */ #define OCTO_POWER_CYCLES 0x18 #define OCTO_NUM_FANS 8 -#define OCTO_FAN_PERCENT_OFFSET 0x00 -#define OCTO_FAN_VOLTAGE_OFFSET 0x02 -#define OCTO_FAN_CURRENT_OFFSET 0x04 -#define OCTO_FAN_POWER_OFFSET 0x06 -#define OCTO_FAN_SPEED_OFFSET 0x08 - -static u8 octo_sensor_fan_offsets[] = { 0x7D, 0x8A, 0x97, 0xA4, 0xB1, 0xBE, 0xCB, 0xD8 }; - #define OCTO_NUM_SENSORS 4 #define OCTO_SENSOR_START 0x3D -#define OCTO_SENSOR_SIZE 0x02 -#define OCTO_SENSOR_DISCONNECTED 0x7FFF - -#define OCTO_CTRL_REPORT_SIZE 0x65F -#define OCTO_CTRL_REPORT_CHECKSUM_OFFSET 0x65D -#define OCTO_CTRL_REPORT_CHECKSUM_START 0x01 -#define OCTO_CTRL_REPORT_CHECKSUM_LENGTH 0x65C +#define OCTO_CTRL_REPORT_SIZE 0x65F +static u8 octo_sensor_fan_offsets[] = { 0x7D, 0x8A, 0x97, 0xA4, 0xB1, 0xBE, 0xCB, 0xD8 }; /* Fan speed registers in Octo control report (from 0-100%) */ static u16 octo_ctrl_fan_offsets[] = { 0x5B, 0xB0, 0x105, 0x15A, 0x1AF, 0x204, 0x259, 0x2AE }; +/* Register offsets for the Quadro fan controller */ +#define QUADRO_POWER_CYCLES 0x18 +#define QUADRO_NUM_FANS 4 +#define QUADRO_NUM_SENSORS 4 +#define QUADRO_SENSOR_START 0x34 +#define QUADRO_CTRL_REPORT_SIZE 0x3c1 +#define QUADRO_FLOW_SENSOR_OFFSET 0x6e +static u8 quadro_sensor_fan_offsets[] = { 0x70, 0x7D, 0x8A, 0x97 }; + +/* Fan speed registers in Quadro control report (from 0-100%) */ +static u16 quadro_ctrl_fan_offsets[] = { 0x36, 0x8b, 0xe0, 0x135 }; + /* Labels for D5 Next */ static const char *const label_d5next_temp[] = { "Coolant temp" @@ -126,7 +130,8 @@ static const char *const label_d5next_power[] = { static const char *const label_d5next_voltages[] = { "Pump voltage", "Fan voltage", - "+5V voltage" + "+5V voltage", + "+12V voltage" }; static const char *const label_d5next_current[] = { @@ -134,7 +139,7 @@ static const char *const label_d5next_current[] = { "Fan current" }; -/* Labels for Farbwerk, Farbwerk 360 and Octo temperature sensors */ +/* Labels for Farbwerk, Farbwerk 360 and Octo and Quadro temperature sensors */ static const char *const label_temp_sensors[] = { "Sensor 1", "Sensor 2", @@ -142,7 +147,7 @@ static const char *const label_temp_sensors[] = { "Sensor 4" }; -/* Labels for Octo */ +/* Labels for Octo and Quadro (except speed) */ static const char *const label_fan_speed[] = { "Fan 1 speed", "Fan 2 speed", @@ -187,6 +192,15 @@ static const char *const label_fan_current[] = { "Fan 8 current" }; +/* Labels for Quadro fan speeds */ +static const char *const label_quadro_speeds[] = { + "Fan 1 speed", + "Fan 2 speed", + "Fan 3 speed", + "Fan 4 speed", + "Flow speed [dL/h]" +}; + struct aqc_data { struct hid_device *hdev; struct device *hwmon_dev; @@ -201,11 +215,19 @@ struct aqc_data { int checksum_length; int checksum_offset; + int num_fans; + u8 *fan_sensor_offsets; + u16 *fan_ctrl_offsets; + int num_temp_sensors; + int temp_sensor_start_offset; + u16 power_cycle_count_offset; + u8 flow_sensor_offset; + /* General info, same across all devices */ u32 serial_number[2]; u16 firmware_version; - /* How many times the device was powered on */ + /* How many times the device was powered on, if available */ u32 power_cycles; /* Sensor values */ @@ -323,56 +345,47 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3 switch (type) { case hwmon_temp: - switch (priv->kind) { - case d5next: - if (channel == 0) - return 0444; - break; - case farbwerk: - case farbwerk360: - case octo: + if (channel < priv->num_temp_sensors) return 0444; - default: - break; - } break; case hwmon_pwm: - switch (priv->kind) { - case octo: + if (priv->fan_ctrl_offsets && channel < priv->num_fans) { switch (attr) { case hwmon_pwm_input: return 0644; default: break; } - break; - default: - break; } break; case hwmon_fan: - case hwmon_power: - case hwmon_curr: switch (priv->kind) { - case d5next: - if (channel < 2) + case quadro: + /* Special case to support flow sensor */ + if (channel < priv->num_fans + 1) return 0444; break; - case octo: - return 0444; default: + if (channel < priv->num_fans) + return 0444; break; } break; + case hwmon_power: + case hwmon_curr: + if (channel < priv->num_fans) + return 0444; + break; case hwmon_in: switch (priv->kind) { case d5next: - if (channel < 3) + /* Special case to support +5V and +12V voltage sensors */ + if (channel < priv->num_fans + 2) return 0444; break; - case octo: - return 0444; default: + if (channel < priv->num_fans) + return 0444; break; } break; @@ -406,16 +419,12 @@ static int aqc_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, *val = priv->power_input[channel]; break; case hwmon_pwm: - switch (priv->kind) { - case octo: - ret = aqc_get_ctrl_val(priv, octo_ctrl_fan_offsets[channel]); + if (priv->fan_ctrl_offsets) { + ret = aqc_get_ctrl_val(priv, priv->fan_ctrl_offsets[channel]); if (ret < 0) return ret; *val = aqc_percent_to_pwm(ret); - break; - default: - break; } break; case hwmon_in: @@ -469,19 +478,15 @@ static int aqc_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, case hwmon_pwm: switch (attr) { case hwmon_pwm_input: - switch (priv->kind) { - case octo: + if (priv->fan_ctrl_offsets) { pwm_value = aqc_pwm_to_percent(val); if (pwm_value < 0) return pwm_value; - ret = aqc_set_ctrl_val(priv, octo_ctrl_fan_offsets[channel], + ret = aqc_set_ctrl_val(priv, priv->fan_ctrl_offsets[channel], pwm_value); if (ret < 0) return ret; - break; - default: - break; } break; default: @@ -576,76 +581,42 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 priv->serial_number[1] = get_unaligned_be16(data + SERIAL_SECOND_PART); priv->firmware_version = get_unaligned_be16(data + FIRMWARE_VERSION); - /* Sensor readings */ - switch (priv->kind) { - case d5next: - priv->power_cycles = get_unaligned_be32(data + D5NEXT_POWER_CYCLES); - - priv->temp_input[0] = get_unaligned_be16(data + D5NEXT_COOLANT_TEMP) * 10; + /* Temperature sensor readings */ + for (i = 0; i < priv->num_temp_sensors; i++) { + sensor_value = get_unaligned_be16(data + + priv->temp_sensor_start_offset + + i * AQC_TEMP_SENSOR_SIZE); + if (sensor_value == AQC_TEMP_SENSOR_DISCONNECTED) + priv->temp_input[i] = -ENODATA; + else + priv->temp_input[i] = sensor_value * 10; + } - priv->speed_input[0] = get_unaligned_be16(data + D5NEXT_PUMP_SPEED); - priv->speed_input[1] = get_unaligned_be16(data + D5NEXT_FAN_SPEED); + /* Fan speed and related readings */ + for (i = 0; i < priv->num_fans; i++) { + priv->speed_input[i] = + get_unaligned_be16(data + priv->fan_sensor_offsets[i] + AQC_FAN_SPEED_OFFSET); + priv->power_input[i] = + get_unaligned_be16(data + priv->fan_sensor_offsets[i] + + AQC_FAN_POWER_OFFSET) * 10000; + priv->voltage_input[i] = + get_unaligned_be16(data + priv->fan_sensor_offsets[i] + + AQC_FAN_VOLTAGE_OFFSET) * 10; + priv->current_input[i] = + get_unaligned_be16(data + priv->fan_sensor_offsets[i] + AQC_FAN_CURRENT_OFFSET); + } - priv->power_input[0] = get_unaligned_be16(data + D5NEXT_PUMP_POWER) * 10000; - priv->power_input[1] = get_unaligned_be16(data + D5NEXT_FAN_POWER) * 10000; + if (priv->power_cycle_count_offset != 0) + priv->power_cycles = get_unaligned_be32(data + priv->power_cycle_count_offset); - priv->voltage_input[0] = get_unaligned_be16(data + D5NEXT_PUMP_VOLTAGE) * 10; - priv->voltage_input[1] = get_unaligned_be16(data + D5NEXT_FAN_VOLTAGE) * 10; + /* Special-case sensor readings */ + switch (priv->kind) { + case d5next: priv->voltage_input[2] = get_unaligned_be16(data + D5NEXT_5V_VOLTAGE) * 10; - - priv->current_input[0] = get_unaligned_be16(data + D5NEXT_PUMP_CURRENT); - priv->current_input[1] = get_unaligned_be16(data + D5NEXT_FAN_CURRENT); - break; - case farbwerk: - /* Temperature sensor readings */ - for (i = 0; i < FARBWERK_NUM_SENSORS; i++) { - sensor_value = get_unaligned_be16(data + FARBWERK_SENSOR_START + - i * FARBWERK_SENSOR_SIZE); - if (sensor_value == FARBWERK_SENSOR_DISCONNECTED) - priv->temp_input[i] = -ENODATA; - else - priv->temp_input[i] = sensor_value * 10; - } - break; - case farbwerk360: - /* Temperature sensor readings */ - for (i = 0; i < FARBWERK360_NUM_SENSORS; i++) { - sensor_value = get_unaligned_be16(data + FARBWERK360_SENSOR_START + - i * FARBWERK360_SENSOR_SIZE); - if (sensor_value == FARBWERK360_SENSOR_DISCONNECTED) - priv->temp_input[i] = -ENODATA; - else - priv->temp_input[i] = sensor_value * 10; - } + priv->voltage_input[3] = get_unaligned_be16(data + D5NEXT_12V_VOLTAGE) * 10; break; - case octo: - priv->power_cycles = get_unaligned_be32(data + OCTO_POWER_CYCLES); - - /* Fan speed and related readings */ - for (i = 0; i < OCTO_NUM_FANS; i++) { - priv->speed_input[i] = - get_unaligned_be16(data + octo_sensor_fan_offsets[i] + - OCTO_FAN_SPEED_OFFSET); - priv->power_input[i] = - get_unaligned_be16(data + octo_sensor_fan_offsets[i] + - OCTO_FAN_POWER_OFFSET) * 10000; - priv->voltage_input[i] = - get_unaligned_be16(data + octo_sensor_fan_offsets[i] + - OCTO_FAN_VOLTAGE_OFFSET) * 10; - priv->current_input[i] = - get_unaligned_be16(data + octo_sensor_fan_offsets[i] + - OCTO_FAN_CURRENT_OFFSET); - } - - /* Temperature sensor readings */ - for (i = 0; i < OCTO_NUM_SENSORS; i++) { - sensor_value = get_unaligned_be16(data + OCTO_SENSOR_START + - i * OCTO_SENSOR_SIZE); - if (sensor_value == OCTO_SENSOR_DISCONNECTED) - priv->temp_input[i] = -ENODATA; - else - priv->temp_input[i] = sensor_value * 10; - } + case quadro: + priv->speed_input[4] = get_unaligned_be16(data + priv->flow_sensor_offset); break; default: break; @@ -699,14 +670,8 @@ static void aqc_debugfs_init(struct aqc_data *priv) debugfs_create_file("serial_number", 0444, priv->debugfs, priv, &serial_number_fops); debugfs_create_file("firmware_version", 0444, priv->debugfs, priv, &firmware_version_fops); - switch (priv->kind) { - case d5next: - case octo: + if (priv->power_cycle_count_offset != 0) debugfs_create_file("power_cycles", 0444, priv->debugfs, priv, &power_cycles_fops); - break; - default: - break; - } } #else @@ -747,6 +712,14 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) case USB_PRODUCT_ID_D5NEXT: priv->kind = d5next; + priv->num_fans = D5NEXT_NUM_FANS; + priv->fan_sensor_offsets = d5next_sensor_fan_offsets; + priv->fan_ctrl_offsets = d5next_ctrl_fan_offsets; + priv->num_temp_sensors = D5NEXT_NUM_SENSORS; + priv->temp_sensor_start_offset = D5NEXT_COOLANT_TEMP; + priv->power_cycle_count_offset = D5NEXT_POWER_CYCLES; + priv->buffer_size = D5NEXT_CTRL_REPORT_SIZE; + priv->temp_label = label_d5next_temp; priv->speed_label = label_d5next_speeds; priv->power_label = label_d5next_power; @@ -756,19 +729,29 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) case USB_PRODUCT_ID_FARBWERK: priv->kind = farbwerk; + priv->num_fans = 0; + priv->num_temp_sensors = FARBWERK_NUM_SENSORS; + priv->temp_sensor_start_offset = FARBWERK_SENSOR_START; priv->temp_label = label_temp_sensors; break; case USB_PRODUCT_ID_FARBWERK360: priv->kind = farbwerk360; + priv->num_fans = 0; + priv->num_temp_sensors = FARBWERK360_NUM_SENSORS; + priv->temp_sensor_start_offset = FARBWERK360_SENSOR_START; priv->temp_label = label_temp_sensors; break; case USB_PRODUCT_ID_OCTO: priv->kind = octo; + + priv->num_fans = OCTO_NUM_FANS; + priv->fan_sensor_offsets = octo_sensor_fan_offsets; + priv->fan_ctrl_offsets = octo_ctrl_fan_offsets; + priv->num_temp_sensors = OCTO_NUM_SENSORS; + priv->temp_sensor_start_offset = OCTO_SENSOR_START; + priv->power_cycle_count_offset = OCTO_POWER_CYCLES; priv->buffer_size = OCTO_CTRL_REPORT_SIZE; - priv->checksum_start = OCTO_CTRL_REPORT_CHECKSUM_START; - priv->checksum_length = OCTO_CTRL_REPORT_CHECKSUM_LENGTH; - priv->checksum_offset = OCTO_CTRL_REPORT_CHECKSUM_OFFSET; priv->temp_label = label_temp_sensors; priv->speed_label = label_fan_speed; @@ -776,10 +759,34 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) priv->voltage_label = label_fan_voltage; priv->current_label = label_fan_current; break; + case USB_PRODUCT_ID_QUADRO: + priv->kind = quadro; + + priv->num_fans = QUADRO_NUM_FANS; + priv->fan_sensor_offsets = quadro_sensor_fan_offsets; + priv->fan_ctrl_offsets = quadro_ctrl_fan_offsets; + priv->num_temp_sensors = QUADRO_NUM_SENSORS; + priv->temp_sensor_start_offset = QUADRO_SENSOR_START; + priv->power_cycle_count_offset = QUADRO_POWER_CYCLES; + priv->buffer_size = QUADRO_CTRL_REPORT_SIZE; + priv->flow_sensor_offset = QUADRO_FLOW_SENSOR_OFFSET; + + priv->temp_label = label_temp_sensors; + priv->speed_label = label_quadro_speeds; + priv->power_label = label_fan_power; + priv->voltage_label = label_fan_voltage; + priv->current_label = label_fan_current; + break; default: break; } + if (priv->buffer_size != 0) { + priv->checksum_start = 0x01; + priv->checksum_length = priv->buffer_size - 3; + priv->checksum_offset = priv->buffer_size - 2; + } + priv->name = aqc_device_names[priv->kind]; priv->buffer = devm_kzalloc(&hdev->dev, priv->buffer_size, GFP_KERNEL); @@ -825,6 +832,7 @@ static const struct hid_device_id aqc_table[] = { { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_FARBWERK) }, { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_FARBWERK360) }, { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_OCTO) }, + { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_QUADRO) }, { } }; diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c index 3cb88d6fbec04..d11f674e3dc37 100644 --- a/drivers/hwmon/aspeed-pwm-tacho.c +++ b/drivers/hwmon/aspeed-pwm-tacho.c @@ -159,7 +159,7 @@ * 11: reserved. */ #define M_TACH_MODE 0x02 /* 10b */ -#define M_TACH_UNIT 0x0210 +#define M_TACH_UNIT 0x0420 #define INIT_FAN_CTRL 0xFF /* How long we sleep in us while waiting for an RPM result. */ diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 3633ab691662b..61a4684fc020e 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -54,6 +54,10 @@ static char *mutex_path_override; /* ACPI mutex for locking access to the EC for the firmware */ #define ASUS_HW_ACCESS_MUTEX_ASMX "\\AMW0.ASMX" +#define ASUS_HW_ACCESS_MUTEX_RMTW_ASMX "\\RMTW.ASMX" + +#define ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0 "\\_SB_.PCI0.SBRG.SIO1.MUT0" + #define MAX_IDENTICAL_BOARD_VARIATIONS 3 /* Moniker for the ACPI global lock (':' is not allowed in ASL identifiers) */ @@ -119,6 +123,18 @@ enum ec_sensors { ec_sensor_temp_water_in, /* "Water_Out" temperature sensor reading [℃] */ ec_sensor_temp_water_out, + /* "Water_Block_In" temperature sensor reading [℃] */ + ec_sensor_temp_water_block_in, + /* "Water_Block_Out" temperature sensor reading [℃] */ + ec_sensor_temp_water_block_out, + /* "T_sensor_2" temperature sensor reading [℃] */ + ec_sensor_temp_t_sensor_2, + /* "Extra_1" temperature sensor reading [℃] */ + ec_sensor_temp_sensor_extra_1, + /* "Extra_2" temperature sensor reading [℃] */ + ec_sensor_temp_sensor_extra_2, + /* "Extra_3" temperature sensor reading [℃] */ + ec_sensor_temp_sensor_extra_3, }; #define SENSOR_TEMP_CHIPSET BIT(ec_sensor_temp_chipset) @@ -134,11 +150,19 @@ enum ec_sensors { #define SENSOR_CURR_CPU BIT(ec_sensor_curr_cpu) #define SENSOR_TEMP_WATER_IN BIT(ec_sensor_temp_water_in) #define SENSOR_TEMP_WATER_OUT BIT(ec_sensor_temp_water_out) +#define SENSOR_TEMP_WATER_BLOCK_IN BIT(ec_sensor_temp_water_block_in) +#define SENSOR_TEMP_WATER_BLOCK_OUT BIT(ec_sensor_temp_water_block_out) +#define SENSOR_TEMP_T_SENSOR_2 BIT(ec_sensor_temp_t_sensor_2) +#define SENSOR_TEMP_SENSOR_EXTRA_1 BIT(ec_sensor_temp_sensor_extra_1) +#define SENSOR_TEMP_SENSOR_EXTRA_2 BIT(ec_sensor_temp_sensor_extra_2) +#define SENSOR_TEMP_SENSOR_EXTRA_3 BIT(ec_sensor_temp_sensor_extra_3) enum board_family { family_unknown, family_amd_400_series, family_amd_500_series, + family_intel_300_series, + family_intel_600_series }; /* All the known sensors for ASUS EC controllers */ @@ -195,12 +219,53 @@ static const struct ec_sensor_info sensors_family_amd_500[] = { EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00), [ec_sensor_temp_water_out] = EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01), + [ec_sensor_temp_water_block_in] = + EC_SENSOR("Water_Block_In", hwmon_temp, 1, 0x01, 0x02), + [ec_sensor_temp_water_block_out] = + EC_SENSOR("Water_Block_Out", hwmon_temp, 1, 0x01, 0x03), + [ec_sensor_temp_sensor_extra_1] = + EC_SENSOR("Extra_1", hwmon_temp, 1, 0x01, 0x09), + [ec_sensor_temp_t_sensor_2] = + EC_SENSOR("T_sensor_2", hwmon_temp, 1, 0x01, 0x0a), + [ec_sensor_temp_sensor_extra_2] = + EC_SENSOR("Extra_2", hwmon_temp, 1, 0x01, 0x0b), + [ec_sensor_temp_sensor_extra_3] = + EC_SENSOR("Extra_3", hwmon_temp, 1, 0x01, 0x0c), +}; + +static const struct ec_sensor_info sensors_family_intel_300[] = { + [ec_sensor_temp_chipset] = + EC_SENSOR("Chipset", hwmon_temp, 1, 0x00, 0x3a), + [ec_sensor_temp_cpu] = EC_SENSOR("CPU", hwmon_temp, 1, 0x00, 0x3b), + [ec_sensor_temp_mb] = + EC_SENSOR("Motherboard", hwmon_temp, 1, 0x00, 0x3c), + [ec_sensor_temp_t_sensor] = + EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d), + [ec_sensor_temp_vrm] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e), + [ec_sensor_fan_cpu_opt] = + EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0), + [ec_sensor_fan_vrm_hs] = EC_SENSOR("VRM HS", hwmon_fan, 2, 0x00, 0xb2), + [ec_sensor_fan_water_flow] = + EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbc), + [ec_sensor_temp_water_in] = + EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00), + [ec_sensor_temp_water_out] = + EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01), +}; + +static const struct ec_sensor_info sensors_family_intel_600[] = { + [ec_sensor_temp_t_sensor] = + EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d), + [ec_sensor_temp_vrm] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e), }; /* Shortcuts for common combinations */ #define SENSOR_SET_TEMP_CHIPSET_CPU_MB \ (SENSOR_TEMP_CHIPSET | SENSOR_TEMP_CPU | SENSOR_TEMP_MB) #define SENSOR_SET_TEMP_WATER (SENSOR_TEMP_WATER_IN | SENSOR_TEMP_WATER_OUT) +#define SENSOR_SET_WATER_BLOCK \ + (SENSOR_TEMP_WATER_BLOCK_IN | SENSOR_TEMP_WATER_BLOCK_OUT) + struct ec_board_info { const char *board_names[MAX_IDENTICAL_BOARD_VARIATIONS]; @@ -272,6 +337,18 @@ static const struct ec_board_info board_info[] = { .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, .family = family_amd_500_series, }, + { + .board_names = { + "ROG MAXIMUS XI HERO", + "ROG MAXIMUS XI HERO (WI-FI)", + }, + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_intel_300_series, + }, { .board_names = {"ROG CROSSHAIR VIII IMPACT"}, .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | @@ -324,12 +401,31 @@ static const struct ec_board_info board_info[] = { }, { .board_names = {"ROG STRIX X570-I GAMING"}, - .sensors = SENSOR_TEMP_T_SENSOR | SENSOR_FAN_VRM_HS | - SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, + .sensors = SENSOR_TEMP_CHIPSET | SENSOR_TEMP_VRM | + SENSOR_TEMP_T_SENSOR | + SENSOR_FAN_VRM_HS | SENSOR_FAN_CHIPSET | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, .family = family_amd_500_series, }, + { + .board_names = {"ROG STRIX Z690-A GAMING WIFI D4"}, + .sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM, + .mutex_path = ASUS_HW_ACCESS_MUTEX_RMTW_ASMX, + .family = family_intel_600_series, + }, + { + .board_names = {"ROG ZENITH II EXTREME"}, + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET | SENSOR_FAN_VRM_HS | + SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE | + SENSOR_SET_WATER_BLOCK | + SENSOR_TEMP_T_SENSOR_2 | SENSOR_TEMP_SENSOR_EXTRA_1 | + SENSOR_TEMP_SENSOR_EXTRA_2 | SENSOR_TEMP_SENSOR_EXTRA_3, + .mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0, + .family = family_amd_500_series, + }, {} }; @@ -799,6 +895,12 @@ static int __init asus_ec_probe(struct platform_device *pdev) case family_amd_500_series: ec_data->sensors_info = sensors_family_amd_500; break; + case family_intel_300_series: + ec_data->sensors_info = sensors_family_intel_300; + break; + case family_intel_600_series: + ec_data->sensors_info = sensors_family_intel_600; + break; default: dev_err(dev, "Unknown board family: %d", ec_data->board_info->family); diff --git a/drivers/hwmon/asus_wmi_sensors.c b/drivers/hwmon/asus_wmi_sensors.c index 9e935e34c9983..6e8a908171f0a 100644 --- a/drivers/hwmon/asus_wmi_sensors.c +++ b/drivers/hwmon/asus_wmi_sensors.c @@ -514,22 +514,20 @@ static int asus_wmi_configure_sensor_setup(struct device *dev, int i, idx; int err; - temp_sensor = devm_kcalloc(dev, 1, sizeof(*temp_sensor), GFP_KERNEL); - if (!temp_sensor) - return -ENOMEM; - for (i = 0; i < sensor_data->wmi.sensor_count; i++) { - err = asus_wmi_sensor_info(i, temp_sensor); + struct asus_wmi_sensor_info sensor; + + err = asus_wmi_sensor_info(i, &sensor); if (err) return err; - switch (temp_sensor->data_type) { + switch (sensor.data_type) { case TEMPERATURE_C: case VOLTAGE: case CURRENT: case FAN_RPM: case WATER_FLOW: - type = asus_data_types[temp_sensor->data_type]; + type = asus_data_types[sensor.data_type]; if (!nr_count[type]) nr_types++; nr_count[type]++; diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 071aa6f4e109b..7f8d95dd27176 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -130,7 +130,7 @@ struct smm_regs { unsigned int edx; unsigned int esi; unsigned int edi; -} __packed; +}; static const char * const temp_labels[] = { "CPU", @@ -175,77 +175,35 @@ static int i8k_smm_func(void *par) struct smm_regs *regs = par; int eax = regs->eax; int ebx = regs->ebx; + unsigned char carry; long long duration; - int rc; /* SMM requires CPU 0 */ if (smp_processor_id() != 0) return -EBUSY; -#if defined(CONFIG_X86_64) - asm volatile("pushq %%rax\n\t" - "movl 0(%%rax),%%edx\n\t" - "pushq %%rdx\n\t" - "movl 4(%%rax),%%ebx\n\t" - "movl 8(%%rax),%%ecx\n\t" - "movl 12(%%rax),%%edx\n\t" - "movl 16(%%rax),%%esi\n\t" - "movl 20(%%rax),%%edi\n\t" - "popq %%rax\n\t" - "out %%al,$0xb2\n\t" - "out %%al,$0x84\n\t" - "xchgq %%rax,(%%rsp)\n\t" - "movl %%ebx,4(%%rax)\n\t" - "movl %%ecx,8(%%rax)\n\t" - "movl %%edx,12(%%rax)\n\t" - "movl %%esi,16(%%rax)\n\t" - "movl %%edi,20(%%rax)\n\t" - "popq %%rdx\n\t" - "movl %%edx,0(%%rax)\n\t" - "pushfq\n\t" - "popq %%rax\n\t" - "andl $1,%%eax\n" - : "=a"(rc) - : "a"(regs) - : "%ebx", "%ecx", "%edx", "%esi", "%edi", "memory"); -#else - asm volatile("pushl %%eax\n\t" - "movl 0(%%eax),%%edx\n\t" - "push %%edx\n\t" - "movl 4(%%eax),%%ebx\n\t" - "movl 8(%%eax),%%ecx\n\t" - "movl 12(%%eax),%%edx\n\t" - "movl 16(%%eax),%%esi\n\t" - "movl 20(%%eax),%%edi\n\t" - "popl %%eax\n\t" - "out %%al,$0xb2\n\t" - "out %%al,$0x84\n\t" - "xchgl %%eax,(%%esp)\n\t" - "movl %%ebx,4(%%eax)\n\t" - "movl %%ecx,8(%%eax)\n\t" - "movl %%edx,12(%%eax)\n\t" - "movl %%esi,16(%%eax)\n\t" - "movl %%edi,20(%%eax)\n\t" - "popl %%edx\n\t" - "movl %%edx,0(%%eax)\n\t" - "lahf\n\t" - "shrl $8,%%eax\n\t" - "andl $1,%%eax\n" - : "=a"(rc) - : "a"(regs) - : "%ebx", "%ecx", "%edx", "%esi", "%edi", "memory"); -#endif - if (rc != 0 || (regs->eax & 0xffff) == 0xffff || regs->eax == eax) - rc = -EINVAL; + asm volatile("out %%al,$0xb2\n\t" + "out %%al,$0x84\n\t" + "setc %0\n" + : "=mr" (carry), + "+a" (regs->eax), + "+b" (regs->ebx), + "+c" (regs->ecx), + "+d" (regs->edx), + "+S" (regs->esi), + "+D" (regs->edi)); duration = ktime_us_delta(ktime_get(), calltime); - pr_debug("smm(0x%.4x 0x%.4x) = 0x%.4x (took %7lld usecs)\n", eax, ebx, - (rc ? 0xffff : regs->eax & 0xffff), duration); + pr_debug("smm(0x%.4x 0x%.4x) = 0x%.4x carry: %d (took %7lld usecs)\n", + eax, ebx, regs->eax & 0xffff, carry, duration); if (duration > DELL_SMM_MAX_DURATION) pr_warn_once("SMM call took %lld usecs!\n", duration); - return rc; + if (carry || (regs->eax & 0xffff) == 0xffff || regs->eax == eax) + return -EINVAL; + + return 0; } /* @@ -1131,6 +1089,13 @@ static const struct i8k_config_data i8k_config_data[] __initconst = { }; static const struct dmi_system_id i8k_dmi_table[] __initconst = { + { + .ident = "Dell G5 5590", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "G5 5590"), + }, + }, { .ident = "Dell Inspiron", .matches = { @@ -1365,6 +1330,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = { }, .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3], }, + { + .ident = "Dell XPS 13 7390", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 13 7390"), + }, + .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3], + }, { } }; diff --git a/drivers/hwmon/drivetemp.c b/drivers/hwmon/drivetemp.c index 1eb37106a220b..5bac2b0fc7bb6 100644 --- a/drivers/hwmon/drivetemp.c +++ b/drivers/hwmon/drivetemp.c @@ -621,3 +621,4 @@ module_exit(drivetemp_exit); MODULE_AUTHOR("Guenter Roeck "); MODULE_DESCRIPTION("Hard drive temperature monitor"); MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:drivetemp"); diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c index 6830e029995dc..19b6c643059a7 100644 --- a/drivers/hwmon/f71882fg.c +++ b/drivers/hwmon/f71882fg.c @@ -49,6 +49,7 @@ #define SIO_F81768D_ID 0x1210 /* Chipset ID */ #define SIO_F81865_ID 0x0704 /* Chipset ID */ #define SIO_F81866_ID 0x1010 /* Chipset ID */ +#define SIO_F71858AD_ID 0x0903 /* Chipset ID */ #define SIO_F81966_ID 0x1502 /* Chipset ID */ #define REGION_LENGTH 8 @@ -2638,6 +2639,7 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) sio_data->type = f71808a; break; case SIO_F71858_ID: + case SIO_F71858AD_ID: sio_data->type = f71858fg; break; case SIO_F71862_ID: diff --git a/drivers/hwmon/gsc-hwmon.c b/drivers/hwmon/gsc-hwmon.c index 1fe37418ff46c..d64be48f1ef6c 100644 --- a/drivers/hwmon/gsc-hwmon.c +++ b/drivers/hwmon/gsc-hwmon.c @@ -269,10 +269,13 @@ gsc_hwmon_get_devtree_pdata(struct device *dev) /* fan controller base address */ fan = of_find_compatible_node(dev->parent->of_node, NULL, "gw,gsc-fan"); if (fan && of_property_read_u32(fan, "reg", &pdata->fan_base)) { + of_node_put(fan); dev_err(dev, "fan node without base\n"); return ERR_PTR(-EINVAL); } + of_node_put(fan); + /* allocate structures for channels and count instances of each type */ device_for_each_child_node(dev, child) { if (fwnode_property_read_string(child, "label", &ch->name)) { diff --git a/drivers/hwmon/lm75.h b/drivers/hwmon/lm75.h index a398171162a83..b803ada5e3c97 100644 --- a/drivers/hwmon/lm75.h +++ b/drivers/hwmon/lm75.h @@ -11,7 +11,8 @@ * which contains this code, we don't worry about the wasted space. */ -#include +#include +#include /* straight from the datasheet */ #define LM75_TEMP_MIN (-55000) diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c index 3820f0e615108..03d07da8c2dcf 100644 --- a/drivers/hwmon/lm90.c +++ b/drivers/hwmon/lm90.c @@ -42,7 +42,8 @@ * accordingly, and is done during initialization. Extended precision is only * available at conversion rates of 1 Hz and slower. Note that extended * precision is not enabled by default, as this driver initializes all chips - * to 2 Hz by design. + * to 2 Hz by design. The driver also supports MAX6690, which is practically + * identical to MAX6654. * * This driver also supports the MAX6646, MAX6647, MAX6648, MAX6649 and * MAX6692 chips made by Maxim. These are again similar to the LM86, @@ -64,6 +65,14 @@ * and extended mode. They are mostly compatible with LM90 except for a data * format difference for the temperature value registers. * + * This driver also supports ADT7481, ADT7482, and ADT7483 from Analog Devices + * / ON Semiconductor. The chips are similar to ADT7461 but support two external + * temperature sensors. + * + * This driver also supports NCT72, NCT214, and NCT218 from ON Semiconductor. + * The chips are similar to ADT7461/ADT7461A but have full PEC support + * (undocumented). + * * This driver also supports the SA56004 from Philips. This device is * pin-compatible with the LM86, the ED/EDP parts are also address-compatible. * @@ -75,23 +84,34 @@ * They are mostly compatible with ADT7461 except for local temperature * low byte register and max conversion rate. * + * This driver also supports MAX1617 and various clones such as G767 + * and NE1617. Such clones will be detected as MAX1617. + * + * This driver also supports NE1618 from Philips. It is similar to NE1617 + * but supports 11 bit external temperature values. + * * Since the LM90 was the first chipset supported by this driver, most * comments will refer to this chipset, but are actually general and * concern all supported chipsets, unless mentioned otherwise. */ -#include +#include +#include +#include +#include #include -#include +#include #include -#include #include -#include +#include #include #include -#include -#include #include +#include +#include + +/* The maximum number of channels currently supported */ +#define MAX_CHANNELS 3 /* * Addresses to scan @@ -112,119 +132,131 @@ static const unsigned short normal_i2c[] = { 0x18, 0x19, 0x1a, 0x29, 0x2a, 0x2b, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, I2C_CLIENT_END }; -enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680, - max6646, w83l771, max6696, sa56004, g781, tmp451, tmp461, max6654 }; +enum chips { adm1023, adm1032, adt7461, adt7461a, adt7481, + g781, lm84, lm90, lm99, + max1617, max6642, max6646, max6648, max6654, max6657, max6659, max6680, max6696, + nct210, nct72, ne1618, sa56004, tmp451, tmp461, w83l771, +}; /* * The LM90 registers */ -#define LM90_REG_R_MAN_ID 0xFE -#define LM90_REG_R_CHIP_ID 0xFF -#define LM90_REG_R_CONFIG1 0x03 -#define LM90_REG_W_CONFIG1 0x09 -#define LM90_REG_R_CONFIG2 0xBF -#define LM90_REG_W_CONFIG2 0xBF -#define LM90_REG_R_CONVRATE 0x04 -#define LM90_REG_W_CONVRATE 0x0A -#define LM90_REG_R_STATUS 0x02 -#define LM90_REG_R_LOCAL_TEMP 0x00 -#define LM90_REG_R_LOCAL_HIGH 0x05 -#define LM90_REG_W_LOCAL_HIGH 0x0B -#define LM90_REG_R_LOCAL_LOW 0x06 -#define LM90_REG_W_LOCAL_LOW 0x0C -#define LM90_REG_R_LOCAL_CRIT 0x20 -#define LM90_REG_W_LOCAL_CRIT 0x20 -#define LM90_REG_R_REMOTE_TEMPH 0x01 -#define LM90_REG_R_REMOTE_TEMPL 0x10 -#define LM90_REG_R_REMOTE_OFFSH 0x11 -#define LM90_REG_W_REMOTE_OFFSH 0x11 -#define LM90_REG_R_REMOTE_OFFSL 0x12 -#define LM90_REG_W_REMOTE_OFFSL 0x12 -#define LM90_REG_R_REMOTE_HIGHH 0x07 -#define LM90_REG_W_REMOTE_HIGHH 0x0D -#define LM90_REG_R_REMOTE_HIGHL 0x13 -#define LM90_REG_W_REMOTE_HIGHL 0x13 -#define LM90_REG_R_REMOTE_LOWH 0x08 -#define LM90_REG_W_REMOTE_LOWH 0x0E -#define LM90_REG_R_REMOTE_LOWL 0x14 -#define LM90_REG_W_REMOTE_LOWL 0x14 -#define LM90_REG_R_REMOTE_CRIT 0x19 -#define LM90_REG_W_REMOTE_CRIT 0x19 -#define LM90_REG_R_TCRIT_HYST 0x21 -#define LM90_REG_W_TCRIT_HYST 0x21 +#define LM90_REG_MAN_ID 0xFE +#define LM90_REG_CHIP_ID 0xFF +#define LM90_REG_CONFIG1 0x03 +#define LM90_REG_CONFIG2 0xBF +#define LM90_REG_CONVRATE 0x04 +#define LM90_REG_STATUS 0x02 +#define LM90_REG_LOCAL_TEMP 0x00 +#define LM90_REG_LOCAL_HIGH 0x05 +#define LM90_REG_LOCAL_LOW 0x06 +#define LM90_REG_LOCAL_CRIT 0x20 +#define LM90_REG_REMOTE_TEMPH 0x01 +#define LM90_REG_REMOTE_TEMPL 0x10 +#define LM90_REG_REMOTE_OFFSH 0x11 +#define LM90_REG_REMOTE_OFFSL 0x12 +#define LM90_REG_REMOTE_HIGHH 0x07 +#define LM90_REG_REMOTE_HIGHL 0x13 +#define LM90_REG_REMOTE_LOWH 0x08 +#define LM90_REG_REMOTE_LOWL 0x14 +#define LM90_REG_REMOTE_CRIT 0x19 +#define LM90_REG_TCRIT_HYST 0x21 /* MAX6646/6647/6649/6654/6657/6658/6659/6695/6696 registers */ -#define MAX6657_REG_R_LOCAL_TEMPL 0x11 -#define MAX6696_REG_R_STATUS2 0x12 -#define MAX6659_REG_R_REMOTE_EMERG 0x16 -#define MAX6659_REG_W_REMOTE_EMERG 0x16 -#define MAX6659_REG_R_LOCAL_EMERG 0x17 -#define MAX6659_REG_W_LOCAL_EMERG 0x17 +#define MAX6657_REG_LOCAL_TEMPL 0x11 +#define MAX6696_REG_STATUS2 0x12 +#define MAX6659_REG_REMOTE_EMERG 0x16 +#define MAX6659_REG_LOCAL_EMERG 0x17 /* SA56004 registers */ -#define SA56004_REG_R_LOCAL_TEMPL 0x22 +#define SA56004_REG_LOCAL_TEMPL 0x22 #define LM90_MAX_CONVRATE_MS 16000 /* Maximum conversion rate in ms */ /* TMP451/TMP461 registers */ -#define TMP451_REG_R_LOCAL_TEMPL 0x15 +#define TMP451_REG_LOCAL_TEMPL 0x15 #define TMP451_REG_CONALERT 0x22 #define TMP461_REG_CHEN 0x16 #define TMP461_REG_DFC 0x24 -/* - * Device flags - */ -#define LM90_FLAG_ADT7461_EXT (1 << 0) /* ADT7461 extended mode */ +/* ADT7481 registers */ +#define ADT7481_REG_STATUS2 0x23 +#define ADT7481_REG_CONFIG2 0x24 + +#define ADT7481_REG_MAN_ID 0x3e +#define ADT7481_REG_CHIP_ID 0x3d + /* Device features */ -#define LM90_HAVE_OFFSET (1 << 1) /* temperature offset register */ -#define LM90_HAVE_REM_LIMIT_EXT (1 << 3) /* extended remote limit */ -#define LM90_HAVE_EMERGENCY (1 << 4) /* 3rd upper (emergency) limit */ -#define LM90_HAVE_EMERGENCY_ALARM (1 << 5)/* emergency alarm */ -#define LM90_HAVE_TEMP3 (1 << 6) /* 3rd temperature sensor */ -#define LM90_HAVE_BROKEN_ALERT (1 << 7) /* Broken alert */ -#define LM90_HAVE_EXTENDED_TEMP (1 << 8) /* extended temperature support*/ -#define LM90_PAUSE_FOR_CONFIG (1 << 9) /* Pause conversion for config */ -#define LM90_HAVE_CRIT (1 << 10)/* Chip supports CRIT/OVERT register */ -#define LM90_HAVE_CRIT_ALRM_SWP (1 << 11)/* critical alarm bits swapped */ +#define LM90_HAVE_EXTENDED_TEMP BIT(0) /* extended temperature support */ +#define LM90_HAVE_OFFSET BIT(1) /* temperature offset register */ +#define LM90_HAVE_UNSIGNED_TEMP BIT(2) /* temperatures are unsigned */ +#define LM90_HAVE_REM_LIMIT_EXT BIT(3) /* extended remote limit */ +#define LM90_HAVE_EMERGENCY BIT(4) /* 3rd upper (emergency) limit */ +#define LM90_HAVE_EMERGENCY_ALARM BIT(5)/* emergency alarm */ +#define LM90_HAVE_TEMP3 BIT(6) /* 3rd temperature sensor */ +#define LM90_HAVE_BROKEN_ALERT BIT(7) /* Broken alert */ +#define LM90_PAUSE_FOR_CONFIG BIT(8) /* Pause conversion for config */ +#define LM90_HAVE_CRIT BIT(9) /* Chip supports CRIT/OVERT register */ +#define LM90_HAVE_CRIT_ALRM_SWP BIT(10) /* critical alarm bits swapped */ +#define LM90_HAVE_PEC BIT(11) /* Chip supports PEC */ +#define LM90_HAVE_PARTIAL_PEC BIT(12) /* Partial PEC support (adm1032)*/ +#define LM90_HAVE_ALARMS BIT(13) /* Create 'alarms' attribute */ +#define LM90_HAVE_EXT_UNSIGNED BIT(14) /* extended unsigned temperature*/ +#define LM90_HAVE_LOW BIT(15) /* low limits */ +#define LM90_HAVE_CONVRATE BIT(16) /* conversion rate */ +#define LM90_HAVE_REMOTE_EXT BIT(17) /* extended remote temperature */ +#define LM90_HAVE_FAULTQUEUE BIT(18) /* configurable samples count */ /* LM90 status */ -#define LM90_STATUS_LTHRM (1 << 0) /* local THERM limit tripped */ -#define LM90_STATUS_RTHRM (1 << 1) /* remote THERM limit tripped */ -#define LM90_STATUS_ROPEN (1 << 2) /* remote is an open circuit */ -#define LM90_STATUS_RLOW (1 << 3) /* remote low temp limit tripped */ -#define LM90_STATUS_RHIGH (1 << 4) /* remote high temp limit tripped */ -#define LM90_STATUS_LLOW (1 << 5) /* local low temp limit tripped */ -#define LM90_STATUS_LHIGH (1 << 6) /* local high temp limit tripped */ -#define LM90_STATUS_BUSY (1 << 7) /* conversion is ongoing */ - -#define MAX6696_STATUS2_R2THRM (1 << 1) /* remote2 THERM limit tripped */ -#define MAX6696_STATUS2_R2OPEN (1 << 2) /* remote2 is an open circuit */ -#define MAX6696_STATUS2_R2LOW (1 << 3) /* remote2 low temp limit tripped */ -#define MAX6696_STATUS2_R2HIGH (1 << 4) /* remote2 high temp limit tripped */ -#define MAX6696_STATUS2_ROT2 (1 << 5) /* remote emergency limit tripped */ -#define MAX6696_STATUS2_R2OT2 (1 << 6) /* remote2 emergency limit tripped */ -#define MAX6696_STATUS2_LOT2 (1 << 7) /* local emergency limit tripped */ +#define LM90_STATUS_LTHRM BIT(0) /* local THERM limit tripped */ +#define LM90_STATUS_RTHRM BIT(1) /* remote THERM limit tripped */ +#define LM90_STATUS_ROPEN BIT(2) /* remote is an open circuit */ +#define LM90_STATUS_RLOW BIT(3) /* remote low temp limit tripped */ +#define LM90_STATUS_RHIGH BIT(4) /* remote high temp limit tripped */ +#define LM90_STATUS_LLOW BIT(5) /* local low temp limit tripped */ +#define LM90_STATUS_LHIGH BIT(6) /* local high temp limit tripped */ +#define LM90_STATUS_BUSY BIT(7) /* conversion is ongoing */ + +/* MAX6695/6696 and ADT7481 2nd status register */ +#define MAX6696_STATUS2_R2THRM BIT(1) /* remote2 THERM limit tripped */ +#define MAX6696_STATUS2_R2OPEN BIT(2) /* remote2 is an open circuit */ +#define MAX6696_STATUS2_R2LOW BIT(3) /* remote2 low temp limit tripped */ +#define MAX6696_STATUS2_R2HIGH BIT(4) /* remote2 high temp limit tripped */ +#define MAX6696_STATUS2_ROT2 BIT(5) /* remote emergency limit tripped */ +#define MAX6696_STATUS2_R2OT2 BIT(6) /* remote2 emergency limit tripped */ +#define MAX6696_STATUS2_LOT2 BIT(7) /* local emergency limit tripped */ /* * Driver data (common to all clients) */ static const struct i2c_device_id lm90_id[] = { + { "adm1020", max1617 }, + { "adm1021", max1617 }, + { "adm1023", adm1023 }, { "adm1032", adm1032 }, + { "adt7421", adt7461a }, { "adt7461", adt7461 }, - { "adt7461a", adt7461 }, + { "adt7461a", adt7461a }, + { "adt7481", adt7481 }, + { "adt7482", adt7481 }, + { "adt7483a", adt7481 }, { "g781", g781 }, + { "gl523sm", max1617 }, + { "lm84", lm84 }, + { "lm86", lm90 }, + { "lm89", lm90 }, { "lm90", lm90 }, - { "lm86", lm86 }, - { "lm89", lm86 }, { "lm99", lm99 }, + { "max1617", max1617 }, + { "max6642", max6642 }, { "max6646", max6646 }, { "max6647", max6646 }, + { "max6648", max6648 }, { "max6649", max6646 }, { "max6654", max6654 }, { "max6657", max6657 }, @@ -232,11 +264,20 @@ static const struct i2c_device_id lm90_id[] = { { "max6659", max6659 }, { "max6680", max6680 }, { "max6681", max6680 }, + { "max6690", max6654 }, + { "max6692", max6648 }, { "max6695", max6696 }, { "max6696", max6696 }, - { "nct1008", adt7461 }, + { "mc1066", max1617 }, + { "nct1008", adt7461a }, + { "nct210", nct210 }, + { "nct214", nct72 }, + { "nct218", nct72 }, + { "nct72", nct72 }, + { "ne1618", ne1618 }, { "w83l771", w83l771 }, { "sa56004", sa56004 }, + { "thmc10", max1617 }, { "tmp451", tmp451 }, { "tmp461", tmp461 }, { } @@ -254,7 +295,11 @@ static const struct of_device_id __maybe_unused lm90_of_match[] = { }, { .compatible = "adi,adt7461a", - .data = (void *)adt7461 + .data = (void *)adt7461a + }, + { + .compatible = "adi,adt7481", + .data = (void *)adt7481 }, { .compatible = "gmt,g781", @@ -266,11 +311,11 @@ static const struct of_device_id __maybe_unused lm90_of_match[] = { }, { .compatible = "national,lm86", - .data = (void *)lm86 + .data = (void *)lm90 }, { .compatible = "national,lm89", - .data = (void *)lm86 + .data = (void *)lm90 }, { .compatible = "national,lm99", @@ -322,7 +367,19 @@ static const struct of_device_id __maybe_unused lm90_of_match[] = { }, { .compatible = "onnn,nct1008", - .data = (void *)adt7461 + .data = (void *)adt7461a + }, + { + .compatible = "onnn,nct214", + .data = (void *)nct72 + }, + { + .compatible = "onnn,nct218", + .data = (void *)nct72 + }, + { + .compatible = "onnn,nct72", + .data = (void *)nct72 }, { .compatible = "winbond,w83l771", @@ -352,115 +409,252 @@ struct lm90_params { u16 alert_alarms; /* Which alarm bits trigger ALERT# */ /* Upper 8 bits for max6695/96 */ u8 max_convrate; /* Maximum conversion rate register value */ + u8 resolution; /* 16-bit resolution (default 11 bit) */ + u8 reg_status2; /* 2nd status register (optional) */ u8 reg_local_ext; /* Extended local temp register (optional) */ + u8 faultqueue_mask; /* fault queue bit mask */ + u8 faultqueue_depth; /* fault queue depth if mask is used */ }; static const struct lm90_params lm90_params[] = { + [adm1023] = { + .flags = LM90_HAVE_ALARMS | LM90_HAVE_OFFSET | LM90_HAVE_BROKEN_ALERT + | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, + .alert_alarms = 0x7c, + .resolution = 8, + .max_convrate = 7, + }, [adm1032] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT, + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT + | LM90_HAVE_PARTIAL_PEC | LM90_HAVE_ALARMS + | LM90_HAVE_LOW | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7c, .max_convrate = 10, }, [adt7461] = { + /* + * Standard temperature range is supposed to be unsigned, + * but that does not match reality. Negative temperatures + * are always reported. + */ + .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP + | LM90_HAVE_CRIT | LM90_HAVE_PARTIAL_PEC + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, + .alert_alarms = 0x7c, + .max_convrate = 10, + .resolution = 10, + }, + [adt7461a] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP - | LM90_HAVE_CRIT, + | LM90_HAVE_CRIT | LM90_HAVE_PEC | LM90_HAVE_ALARMS + | LM90_HAVE_LOW | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7c, .max_convrate = 10, }, + [adt7481] = { + .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP + | LM90_HAVE_UNSIGNED_TEMP | LM90_HAVE_PEC + | LM90_HAVE_TEMP3 | LM90_HAVE_CRIT | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, + .alert_alarms = 0x1c7c, + .max_convrate = 11, + .resolution = 10, + .reg_status2 = ADT7481_REG_STATUS2, + }, [g781] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT, + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7c, .max_convrate = 7, }, - [lm86] = { - .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_CRIT, - .alert_alarms = 0x7b, - .max_convrate = 9, + [lm84] = { + .flags = LM90_HAVE_ALARMS, + .resolution = 8, }, [lm90] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_CRIT, + | LM90_HAVE_CRIT | LM90_HAVE_ALARMS | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7b, .max_convrate = 9, + .faultqueue_mask = BIT(0), + .faultqueue_depth = 3, }, [lm99] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_CRIT, + | LM90_HAVE_CRIT | LM90_HAVE_ALARMS | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7b, .max_convrate = 9, + .faultqueue_mask = BIT(0), + .faultqueue_depth = 3, + }, + [max1617] = { + .flags = LM90_HAVE_CONVRATE | LM90_HAVE_BROKEN_ALERT | + LM90_HAVE_LOW | LM90_HAVE_ALARMS, + .alert_alarms = 0x78, + .resolution = 8, + .max_convrate = 7, + }, + [max6642] = { + .flags = LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXT_UNSIGNED + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, + .alert_alarms = 0x50, + .resolution = 10, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, + .faultqueue_mask = BIT(4), + .faultqueue_depth = 2, }, [max6646] = { - .flags = LM90_HAVE_CRIT | LM90_HAVE_BROKEN_ALERT, + .flags = LM90_HAVE_CRIT | LM90_HAVE_BROKEN_ALERT + | LM90_HAVE_EXT_UNSIGNED | LM90_HAVE_ALARMS | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 6, - .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, + }, + [max6648] = { + .flags = LM90_HAVE_UNSIGNED_TEMP | LM90_HAVE_CRIT + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT, + .alert_alarms = 0x7c, + .max_convrate = 6, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, }, [max6654] = { - .flags = LM90_HAVE_BROKEN_ALERT, + .flags = LM90_HAVE_BROKEN_ALERT | LM90_HAVE_ALARMS | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 7, - .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, }, [max6657] = { - .flags = LM90_PAUSE_FOR_CONFIG | LM90_HAVE_CRIT, + .flags = LM90_PAUSE_FOR_CONFIG | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 8, - .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, }, [max6659] = { - .flags = LM90_HAVE_EMERGENCY | LM90_HAVE_CRIT, + .flags = LM90_HAVE_EMERGENCY | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 8, - .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, }, [max6680] = { + /* + * Apparent temperatures of 128 degrees C or higher are reported + * and treated as negative temperatures (meaning min_alarm will + * be set). + */ .flags = LM90_HAVE_OFFSET | LM90_HAVE_CRIT - | LM90_HAVE_CRIT_ALRM_SWP | LM90_HAVE_BROKEN_ALERT, + | LM90_HAVE_CRIT_ALRM_SWP | LM90_HAVE_BROKEN_ALERT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 7, }, [max6696] = { .flags = LM90_HAVE_EMERGENCY - | LM90_HAVE_EMERGENCY_ALARM | LM90_HAVE_TEMP3 | LM90_HAVE_CRIT, + | LM90_HAVE_EMERGENCY_ALARM | LM90_HAVE_TEMP3 | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x1c7c, .max_convrate = 6, - .reg_local_ext = MAX6657_REG_R_LOCAL_TEMPL, + .reg_status2 = MAX6696_REG_STATUS2, + .reg_local_ext = MAX6657_REG_LOCAL_TEMPL, + .faultqueue_mask = BIT(5), + .faultqueue_depth = 4, + }, + [nct72] = { + .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP + | LM90_HAVE_CRIT | LM90_HAVE_PEC | LM90_HAVE_UNSIGNED_TEMP + | LM90_HAVE_LOW | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT + | LM90_HAVE_FAULTQUEUE, + .alert_alarms = 0x7c, + .max_convrate = 10, + .resolution = 10, + }, + [nct210] = { + .flags = LM90_HAVE_ALARMS | LM90_HAVE_BROKEN_ALERT + | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, + .alert_alarms = 0x7c, + .resolution = 11, + .max_convrate = 7, + }, + [ne1618] = { + .flags = LM90_PAUSE_FOR_CONFIG | LM90_HAVE_BROKEN_ALERT + | LM90_HAVE_LOW | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT, + .alert_alarms = 0x7c, + .resolution = 11, + .max_convrate = 7, }, [w83l771] = { - .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT, + .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT, .alert_alarms = 0x7c, .max_convrate = 8, }, [sa56004] = { - .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT, + /* + * Apparent temperatures of 128 degrees C or higher are reported + * and treated as negative temperatures (meaning min_alarm will + * be set). + */ + .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7b, .max_convrate = 9, - .reg_local_ext = SA56004_REG_R_LOCAL_TEMPL, + .reg_local_ext = SA56004_REG_LOCAL_TEMPL, + .faultqueue_mask = BIT(0), + .faultqueue_depth = 3, }, [tmp451] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT, + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT + | LM90_HAVE_UNSIGNED_TEMP | LM90_HAVE_ALARMS | LM90_HAVE_LOW + | LM90_HAVE_CONVRATE | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7c, .max_convrate = 9, - .reg_local_ext = TMP451_REG_R_LOCAL_TEMPL, + .resolution = 12, + .reg_local_ext = TMP451_REG_LOCAL_TEMPL, }, [tmp461] = { .flags = LM90_HAVE_OFFSET | LM90_HAVE_REM_LIMIT_EXT - | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT, + | LM90_HAVE_BROKEN_ALERT | LM90_HAVE_EXTENDED_TEMP | LM90_HAVE_CRIT + | LM90_HAVE_ALARMS | LM90_HAVE_LOW | LM90_HAVE_CONVRATE + | LM90_HAVE_REMOTE_EXT | LM90_HAVE_FAULTQUEUE, .alert_alarms = 0x7c, .max_convrate = 9, - .reg_local_ext = TMP451_REG_R_LOCAL_TEMPL, + .resolution = 12, + .reg_local_ext = TMP451_REG_LOCAL_TEMPL, }, }; /* - * TEMP8 register index + * temperature register index */ -enum lm90_temp8_reg_index { +enum lm90_temp_reg_index { LOCAL_LOW = 0, LOCAL_HIGH, LOCAL_CRIT, @@ -469,14 +663,8 @@ enum lm90_temp8_reg_index { REMOTE_EMERG, /* max6659 and max6695/96 */ REMOTE2_CRIT, /* max6695/96 only */ REMOTE2_EMERG, /* max6695/96 only */ - TEMP8_REG_NUM -}; -/* - * TEMP11 register index - */ -enum lm90_temp11_reg_index { - REMOTE_TEMP = 0, + REMOTE_TEMP, REMOTE_LOW, REMOTE_HIGH, REMOTE_OFFSET, /* except max6646, max6657/58/59, and max6695/96 */ @@ -484,7 +672,9 @@ enum lm90_temp11_reg_index { REMOTE2_TEMP, /* max6695/96 only */ REMOTE2_LOW, /* max6695/96 only */ REMOTE2_HIGH, /* max6695/96 only */ - TEMP11_REG_NUM + REMOTE2_OFFSET, + + TEMP_REG_NUM }; /* @@ -494,13 +684,20 @@ enum lm90_temp11_reg_index { struct lm90_data { struct i2c_client *client; struct device *hwmon_dev; - u32 channel_config[4]; + u32 chip_config[2]; + u32 channel_config[MAX_CHANNELS + 1]; + const char *channel_label[MAX_CHANNELS]; + struct hwmon_channel_info chip_info; struct hwmon_channel_info temp_info; const struct hwmon_channel_info *info[3]; struct hwmon_chip_info chip; struct mutex update_lock; + struct delayed_work alert_work; + struct work_struct report_work; bool valid; /* true if register values are valid */ + bool alarms_valid; /* true if status register values are valid */ unsigned long last_updated; /* in jiffies */ + unsigned long alarms_updated; /* in jiffies */ int kind; u32 flags; @@ -509,16 +706,23 @@ struct lm90_data { u8 config; /* Current configuration register value */ u8 config_orig; /* Original configuration register value */ u8 convrate_orig; /* Original conversion rate register value */ + u8 resolution; /* temperature resolution in bit */ u16 alert_alarms; /* Which alarm bits trigger ALERT# */ /* Upper 8 bits for max6695/96 */ u8 max_convrate; /* Maximum conversion rate */ + u8 reg_status2; /* 2nd status register (optional) */ u8 reg_local_ext; /* local extension register offset */ + u8 reg_remote_ext; /* remote temperature low byte */ + u8 faultqueue_mask; /* fault queue mask */ + u8 faultqueue_depth; /* fault queue mask */ /* registers values */ - s8 temp8[TEMP8_REG_NUM]; - s16 temp11[TEMP11_REG_NUM]; + u16 temp[TEMP_REG_NUM]; u8 temp_hyst; - u16 alarms; /* bitvector (upper 8 bits for max6695/96) */ + u8 conalert; + u16 reported_alarms; /* alarms reported as sysfs/udev events */ + u16 current_alarms; /* current alarms, reported by chip */ + u16 alarms; /* alarms not yet reported to user */ }; /* @@ -526,10 +730,10 @@ struct lm90_data { */ /* - * The ADM1032 supports PEC but not on write byte transactions, so we need + * If the chip supports PEC but not on write byte transactions, we need * to explicitly ask for a transaction without PEC. */ -static inline s32 adm1032_write_byte(struct i2c_client *client, u8 value) +static inline s32 lm90_write_no_pec(struct i2c_client *client, u8 value) { return i2c_smbus_xfer(client->adapter, client->addr, client->flags & ~I2C_CLIENT_PEC, @@ -538,47 +742,96 @@ static inline s32 adm1032_write_byte(struct i2c_client *client, u8 value) /* * It is assumed that client->update_lock is held (unless we are in - * detection or initialization steps). This matters when PEC is enabled, - * because we don't want the address pointer to change between the write - * byte and the read byte transactions. + * detection or initialization steps). This matters when PEC is enabled + * for chips with partial PEC support, because we don't want the address + * pointer to change between the write byte and the read byte transactions. */ static int lm90_read_reg(struct i2c_client *client, u8 reg) { + struct lm90_data *data = i2c_get_clientdata(client); + bool partial_pec = (client->flags & I2C_CLIENT_PEC) && + (data->flags & LM90_HAVE_PARTIAL_PEC); int err; - if (client->flags & I2C_CLIENT_PEC) { - err = adm1032_write_byte(client, reg); - if (err >= 0) - err = i2c_smbus_read_byte(client); - } else - err = i2c_smbus_read_byte_data(client, reg); + if (partial_pec) { + err = lm90_write_no_pec(client, reg); + if (err) + return err; + return i2c_smbus_read_byte(client); + } + return i2c_smbus_read_byte_data(client, reg); +} - return err; +/* + * Return register write address + * + * The write address for registers 0x03 .. 0x08 is the read address plus 6. + * For other registers the write address matches the read address. + */ +static u8 lm90_write_reg_addr(u8 reg) +{ + if (reg >= LM90_REG_CONFIG1 && reg <= LM90_REG_REMOTE_LOWH) + return reg + 6; + return reg; +} + +/* + * Write into LM90 register. + * Convert register address to write address if needed, then execute the + * operation. + */ +static int lm90_write_reg(struct i2c_client *client, u8 reg, u8 val) +{ + return i2c_smbus_write_byte_data(client, lm90_write_reg_addr(reg), val); +} + +/* + * Write into 16-bit LM90 register. + * Convert register addresses to write address if needed, then execute the + * operation. + */ +static int lm90_write16(struct i2c_client *client, u8 regh, u8 regl, u16 val) +{ + int ret; + + ret = lm90_write_reg(client, regh, val >> 8); + if (ret < 0 || !regl) + return ret; + return lm90_write_reg(client, regl, val & 0xff); } -static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl) +static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl, + bool is_volatile) { int oldh, newh, l; - /* - * There is a trick here. We have to read two registers to have the - * sensor temperature, but we have to beware a conversion could occur - * between the readings. The datasheet says we should either use - * the one-shot conversion register, which we don't want to do - * (disables hardware monitoring) or monitor the busy bit, which is - * impossible (we can't read the values and monitor that bit at the - * exact same time). So the solution used here is to read the high - * byte once, then the low byte, then the high byte again. If the new - * high byte matches the old one, then we have a valid reading. Else - * we have to read the low byte again, and now we believe we have a - * correct reading. - */ oldh = lm90_read_reg(client, regh); if (oldh < 0) return oldh; + + if (!regl) + return oldh << 8; + l = lm90_read_reg(client, regl); if (l < 0) return l; + + if (!is_volatile) + return (oldh << 8) | l; + + /* + * For volatile registers we have to use a trick. + * We have to read two registers to have the sensor temperature, + * but we have to beware a conversion could occur between the + * readings. The datasheet says we should either use + * the one-shot conversion register, which we don't want to do + * (disables hardware monitoring) or monitor the busy bit, which is + * impossible (we can't read the values and monitor that bit at the + * exact same time). So the solution used here is to read the high + * the high byte again. If the new high byte matches the old one, + * then we have a valid reading. Otherwise we have to read the low + * byte again, and now we believe we have a correct reading. + */ newh = lm90_read_reg(client, regh); if (newh < 0) return newh; @@ -595,9 +848,7 @@ static int lm90_update_confreg(struct lm90_data *data, u8 config) if (data->config != config) { int err; - err = i2c_smbus_write_byte_data(data->client, - LM90_REG_W_CONFIG1, - config); + err = lm90_write_reg(data->client, LM90_REG_CONFIG1, config); if (err) return err; data->config = config; @@ -613,18 +864,14 @@ static int lm90_update_confreg(struct lm90_data *data, u8 config) * various registers have different meanings as a result of selecting a * non-default remote channel. */ -static int lm90_select_remote_channel(struct lm90_data *data, int channel) +static int lm90_select_remote_channel(struct lm90_data *data, bool second) { - int err = 0; + u8 config = data->config & ~0x08; - if (data->kind == max6696) { - u8 config = data->config & ~0x08; + if (second) + config |= 0x08; - if (channel) - config |= 0x08; - err = lm90_update_confreg(data, config); - } - return err; + return lm90_update_confreg(data, config); } static int lm90_write_convrate(struct lm90_data *data, int val) @@ -640,7 +887,7 @@ static int lm90_write_convrate(struct lm90_data *data, int val) } /* Set conv rate */ - err = i2c_smbus_write_byte_data(data->client, LM90_REG_W_CONVRATE, val); + err = lm90_write_reg(data->client, LM90_REG_CONVRATE, val); /* Revert change to config */ lm90_update_confreg(data, config); @@ -673,6 +920,26 @@ static int lm90_set_convrate(struct i2c_client *client, struct lm90_data *data, return err; } +static int lm90_set_faultqueue(struct i2c_client *client, + struct lm90_data *data, int val) +{ + int err; + + if (data->faultqueue_mask) { + err = lm90_update_confreg(data, val <= data->faultqueue_depth / 2 ? + data->config & ~data->faultqueue_mask : + data->config | data->faultqueue_mask); + } else { + static const u8 values[4] = {0, 2, 6, 0x0e}; + + data->conalert = (data->conalert & 0xf1) | values[val - 1]; + err = lm90_write_reg(data->client, TMP451_REG_CONALERT, + data->conalert); + } + + return err; +} + static int lm90_update_limits(struct device *dev) { struct lm90_data *data = dev_get_drvdata(dev); @@ -680,97 +947,260 @@ static int lm90_update_limits(struct device *dev) int val; if (data->flags & LM90_HAVE_CRIT) { - val = lm90_read_reg(client, LM90_REG_R_LOCAL_CRIT); + val = lm90_read_reg(client, LM90_REG_LOCAL_CRIT); if (val < 0) return val; - data->temp8[LOCAL_CRIT] = val; + data->temp[LOCAL_CRIT] = val << 8; - val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT); + val = lm90_read_reg(client, LM90_REG_REMOTE_CRIT); if (val < 0) return val; - data->temp8[REMOTE_CRIT] = val; + data->temp[REMOTE_CRIT] = val << 8; - val = lm90_read_reg(client, LM90_REG_R_TCRIT_HYST); + val = lm90_read_reg(client, LM90_REG_TCRIT_HYST); if (val < 0) return val; data->temp_hyst = val; } - - val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH); - if (val < 0) - return val; - data->temp11[REMOTE_LOW] = val << 8; - - if (data->flags & LM90_HAVE_REM_LIMIT_EXT) { - val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWL); + if ((data->flags & LM90_HAVE_FAULTQUEUE) && !data->faultqueue_mask) { + val = lm90_read_reg(client, TMP451_REG_CONALERT); if (val < 0) return val; - data->temp11[REMOTE_LOW] |= val; + data->conalert = val; } - val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH); + val = lm90_read16(client, LM90_REG_REMOTE_LOWH, + (data->flags & LM90_HAVE_REM_LIMIT_EXT) ? LM90_REG_REMOTE_LOWL : 0, + false); if (val < 0) return val; - data->temp11[REMOTE_HIGH] = val << 8; + data->temp[REMOTE_LOW] = val; - if (data->flags & LM90_HAVE_REM_LIMIT_EXT) { - val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHL); - if (val < 0) - return val; - data->temp11[REMOTE_HIGH] |= val; - } + val = lm90_read16(client, LM90_REG_REMOTE_HIGHH, + (data->flags & LM90_HAVE_REM_LIMIT_EXT) ? LM90_REG_REMOTE_HIGHL : 0, + false); + if (val < 0) + return val; + data->temp[REMOTE_HIGH] = val; if (data->flags & LM90_HAVE_OFFSET) { - val = lm90_read16(client, LM90_REG_R_REMOTE_OFFSH, - LM90_REG_R_REMOTE_OFFSL); + val = lm90_read16(client, LM90_REG_REMOTE_OFFSH, + LM90_REG_REMOTE_OFFSL, false); if (val < 0) return val; - data->temp11[REMOTE_OFFSET] = val; + data->temp[REMOTE_OFFSET] = val; } if (data->flags & LM90_HAVE_EMERGENCY) { - val = lm90_read_reg(client, MAX6659_REG_R_LOCAL_EMERG); + val = lm90_read_reg(client, MAX6659_REG_LOCAL_EMERG); if (val < 0) return val; - data->temp8[LOCAL_EMERG] = val; + data->temp[LOCAL_EMERG] = val << 8; - val = lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG); + val = lm90_read_reg(client, MAX6659_REG_REMOTE_EMERG); if (val < 0) return val; - data->temp8[REMOTE_EMERG] = val; + data->temp[REMOTE_EMERG] = val << 8; } - if (data->kind == max6696) { - val = lm90_select_remote_channel(data, 1); + if (data->flags & LM90_HAVE_TEMP3) { + val = lm90_select_remote_channel(data, true); if (val < 0) return val; - val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT); + val = lm90_read_reg(client, LM90_REG_REMOTE_CRIT); if (val < 0) return val; - data->temp8[REMOTE2_CRIT] = val; + data->temp[REMOTE2_CRIT] = val << 8; + + if (data->flags & LM90_HAVE_EMERGENCY) { + val = lm90_read_reg(client, MAX6659_REG_REMOTE_EMERG); + if (val < 0) + return val; + data->temp[REMOTE2_EMERG] = val << 8; + } - val = lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG); + val = lm90_read_reg(client, LM90_REG_REMOTE_LOWH); if (val < 0) return val; - data->temp8[REMOTE2_EMERG] = val; + data->temp[REMOTE2_LOW] = val << 8; - val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH); + val = lm90_read_reg(client, LM90_REG_REMOTE_HIGHH); if (val < 0) return val; - data->temp11[REMOTE2_LOW] = val << 8; + data->temp[REMOTE2_HIGH] = val << 8; + + if (data->flags & LM90_HAVE_OFFSET) { + val = lm90_read16(client, LM90_REG_REMOTE_OFFSH, + LM90_REG_REMOTE_OFFSL, false); + if (val < 0) + return val; + data->temp[REMOTE2_OFFSET] = val; + } + + lm90_select_remote_channel(data, false); + } + + return 0; +} + +static void lm90_report_alarms(struct work_struct *work) +{ + struct lm90_data *data = container_of(work, struct lm90_data, report_work); + u16 cleared_alarms, new_alarms, current_alarms; + struct device *hwmon_dev = data->hwmon_dev; + struct device *dev = &data->client->dev; + int st, st2; + + current_alarms = data->current_alarms; + cleared_alarms = data->reported_alarms & ~current_alarms; + new_alarms = current_alarms & ~data->reported_alarms; + + if (!cleared_alarms && !new_alarms) + return; + + st = new_alarms & 0xff; + st2 = new_alarms >> 8; + + if ((st & (LM90_STATUS_LLOW | LM90_STATUS_LHIGH | LM90_STATUS_LTHRM)) || + (st2 & MAX6696_STATUS2_LOT2)) + dev_dbg(dev, "temp%d out of range, please check!\n", 1); + if ((st & (LM90_STATUS_RLOW | LM90_STATUS_RHIGH | LM90_STATUS_RTHRM)) || + (st2 & MAX6696_STATUS2_ROT2)) + dev_dbg(dev, "temp%d out of range, please check!\n", 2); + if (st & LM90_STATUS_ROPEN) + dev_dbg(dev, "temp%d diode open, please check!\n", 2); + if (st2 & (MAX6696_STATUS2_R2LOW | MAX6696_STATUS2_R2HIGH | + MAX6696_STATUS2_R2THRM | MAX6696_STATUS2_R2OT2)) + dev_dbg(dev, "temp%d out of range, please check!\n", 3); + if (st2 & MAX6696_STATUS2_R2OPEN) + dev_dbg(dev, "temp%d diode open, please check!\n", 3); + + st |= cleared_alarms & 0xff; + st2 |= cleared_alarms >> 8; + + if (st & LM90_STATUS_LLOW) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_min_alarm, 0); + if (st & LM90_STATUS_RLOW) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_min_alarm, 1); + if (st2 & MAX6696_STATUS2_R2LOW) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_min_alarm, 2); + + if (st & LM90_STATUS_LHIGH) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_max_alarm, 0); + if (st & LM90_STATUS_RHIGH) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_max_alarm, 1); + if (st2 & MAX6696_STATUS2_R2HIGH) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_max_alarm, 2); + + if (st & LM90_STATUS_LTHRM) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_crit_alarm, 0); + if (st & LM90_STATUS_RTHRM) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_crit_alarm, 1); + if (st2 & MAX6696_STATUS2_R2THRM) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_crit_alarm, 2); + + if (st2 & MAX6696_STATUS2_LOT2) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_emergency_alarm, 0); + if (st2 & MAX6696_STATUS2_ROT2) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_emergency_alarm, 1); + if (st2 & MAX6696_STATUS2_R2OT2) + hwmon_notify_event(hwmon_dev, hwmon_temp, hwmon_temp_emergency_alarm, 2); + + data->reported_alarms = current_alarms; +} + +static int lm90_update_alarms_locked(struct lm90_data *data, bool force) +{ + if (force || !data->alarms_valid || + time_after(jiffies, data->alarms_updated + msecs_to_jiffies(data->update_interval))) { + struct i2c_client *client = data->client; + bool check_enable; + u16 alarms; + int val; + + data->alarms_valid = false; - val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH); + val = lm90_read_reg(client, LM90_REG_STATUS); if (val < 0) return val; - data->temp11[REMOTE2_HIGH] = val << 8; + alarms = val & ~LM90_STATUS_BUSY; - lm90_select_remote_channel(data, 0); - } + if (data->reg_status2) { + val = lm90_read_reg(client, data->reg_status2); + if (val < 0) + return val; + alarms |= val << 8; + } + /* + * If the update is forced (called from interrupt or alert + * handler) and alarm data is valid, the alarms may have been + * updated after the last update interval, and the status + * register may still be cleared. Only add additional alarms + * in this case. Alarms will be cleared later if appropriate. + */ + if (force && data->alarms_valid) + data->current_alarms |= alarms; + else + data->current_alarms = alarms; + data->alarms |= alarms; + + check_enable = (client->irq || !(data->config_orig & 0x80)) && + (data->config & 0x80); + if (force || check_enable) + schedule_work(&data->report_work); + + /* + * Re-enable ALERT# output if it was originally enabled, relevant + * alarms are all clear, and alerts are currently disabled. + * Otherwise (re)schedule worker if needed. + */ + if (check_enable) { + if (!(data->current_alarms & data->alert_alarms)) { + dev_dbg(&client->dev, "Re-enabling ALERT#\n"); + lm90_update_confreg(data, data->config & ~0x80); + /* + * We may have been called from the update handler. + * If so, the worker, if scheduled, is no longer + * needed. Cancel it. Don't synchronize because + * it may already be running. + */ + cancel_delayed_work(&data->alert_work); + } else { + schedule_delayed_work(&data->alert_work, + max_t(int, HZ, msecs_to_jiffies(data->update_interval))); + } + } + data->alarms_updated = jiffies; + data->alarms_valid = true; + } return 0; } +static int lm90_update_alarms(struct lm90_data *data, bool force) +{ + int err; + + mutex_lock(&data->update_lock); + err = lm90_update_alarms_locked(data, force); + mutex_unlock(&data->update_lock); + + return err; +} + +static void lm90_alert_work(struct work_struct *__work) +{ + struct delayed_work *delayed_work = container_of(__work, struct delayed_work, work); + struct lm90_data *data = container_of(delayed_work, struct lm90_data, alert_work); + + /* Nothing to do if alerts are enabled */ + if (!(data->config & 0x80)) + return; + + lm90_update_alarms(data, true); +} + static int lm90_update_device(struct device *dev) { struct lm90_data *data = dev_get_drvdata(dev); @@ -791,71 +1221,46 @@ static int lm90_update_device(struct device *dev) data->valid = false; - val = lm90_read_reg(client, LM90_REG_R_LOCAL_LOW); + val = lm90_read_reg(client, LM90_REG_LOCAL_LOW); if (val < 0) return val; - data->temp8[LOCAL_LOW] = val; + data->temp[LOCAL_LOW] = val << 8; - val = lm90_read_reg(client, LM90_REG_R_LOCAL_HIGH); + val = lm90_read_reg(client, LM90_REG_LOCAL_HIGH); if (val < 0) return val; - data->temp8[LOCAL_HIGH] = val; + data->temp[LOCAL_HIGH] = val << 8; - if (data->reg_local_ext) { - val = lm90_read16(client, LM90_REG_R_LOCAL_TEMP, - data->reg_local_ext); - if (val < 0) - return val; - data->temp11[LOCAL_TEMP] = val; - } else { - val = lm90_read_reg(client, LM90_REG_R_LOCAL_TEMP); - if (val < 0) - return val; - data->temp11[LOCAL_TEMP] = val << 8; - } - val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, - LM90_REG_R_REMOTE_TEMPL); + val = lm90_read16(client, LM90_REG_LOCAL_TEMP, + data->reg_local_ext, true); if (val < 0) return val; - data->temp11[REMOTE_TEMP] = val; - - val = lm90_read_reg(client, LM90_REG_R_STATUS); + data->temp[LOCAL_TEMP] = val; + val = lm90_read16(client, LM90_REG_REMOTE_TEMPH, + data->reg_remote_ext, true); if (val < 0) return val; - data->alarms = val & ~LM90_STATUS_BUSY; + data->temp[REMOTE_TEMP] = val; - if (data->kind == max6696) { - val = lm90_select_remote_channel(data, 1); + if (data->flags & LM90_HAVE_TEMP3) { + val = lm90_select_remote_channel(data, true); if (val < 0) return val; - val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, - LM90_REG_R_REMOTE_TEMPL); + val = lm90_read16(client, LM90_REG_REMOTE_TEMPH, + data->reg_remote_ext, true); if (val < 0) { - lm90_select_remote_channel(data, 0); + lm90_select_remote_channel(data, false); return val; } - data->temp11[REMOTE2_TEMP] = val; + data->temp[REMOTE2_TEMP] = val; - lm90_select_remote_channel(data, 0); - - val = lm90_read_reg(client, MAX6696_REG_R_STATUS2); - if (val < 0) - return val; - data->alarms |= val << 8; + lm90_select_remote_channel(data, false); } - /* - * Re-enable ALERT# output if it was originally enabled and - * relevant alarms are all clear - */ - if ((client->irq || !(data->config_orig & 0x80)) && - !(data->alarms & data->alert_alarms)) { - if (data->config & 0x80) { - dev_dbg(&client->dev, "Re-enabling ALERT#\n"); - lm90_update_confreg(data, data->config & ~0x80); - } - } + val = lm90_update_alarms_locked(data, false); + if (val < 0) + return val; data->last_updated = jiffies; data->valid = true; @@ -864,355 +1269,247 @@ static int lm90_update_device(struct device *dev) return 0; } -/* - * Conversions - * For local temperatures and limits, critical limits and the hysteresis - * value, the LM90 uses signed 8-bit values with LSB = 1 degree Celsius. - * For remote temperatures and limits, it uses signed 11-bit values with - * LSB = 0.125 degree Celsius, left-justified in 16-bit registers. Some - * Maxim chips use unsigned values. - */ - -static inline int temp_from_s8(s8 val) +/* pec used for devices with PEC support */ +static ssize_t pec_show(struct device *dev, struct device_attribute *dummy, + char *buf) { - return val * 1000; -} + struct i2c_client *client = to_i2c_client(dev); -static inline int temp_from_u8(u8 val) -{ - return val * 1000; + return sprintf(buf, "%d\n", !!(client->flags & I2C_CLIENT_PEC)); } -static inline int temp_from_s16(s16 val) +static ssize_t pec_store(struct device *dev, struct device_attribute *dummy, + const char *buf, size_t count) { - return val / 32 * 125; -} + struct i2c_client *client = to_i2c_client(dev); + long val; + int err; -static inline int temp_from_u16(u16 val) -{ - return val / 32 * 125; -} + err = kstrtol(buf, 10, &val); + if (err < 0) + return err; -static s8 temp_to_s8(long val) -{ - if (val <= -128000) - return -128; - if (val >= 127000) - return 127; - if (val < 0) - return (val - 500) / 1000; - return (val + 500) / 1000; -} + switch (val) { + case 0: + client->flags &= ~I2C_CLIENT_PEC; + break; + case 1: + client->flags |= I2C_CLIENT_PEC; + break; + default: + return -EINVAL; + } -static u8 temp_to_u8(long val) -{ - if (val <= 0) - return 0; - if (val >= 255000) - return 255; - return (val + 500) / 1000; -} - -static s16 temp_to_s16(long val) -{ - if (val <= -128000) - return 0x8000; - if (val >= 127875) - return 0x7FE0; - if (val < 0) - return (val - 62) / 125 * 32; - return (val + 62) / 125 * 32; -} - -static u8 hyst_to_reg(long val) -{ - if (val <= 0) - return 0; - if (val >= 30500) - return 31; - return (val + 500) / 1000; -} - -/* - * ADT7461 in compatibility mode is almost identical to LM90 except that - * attempts to write values that are outside the range 0 < temp < 127 are - * treated as the boundary value. - * - * ADT7461 in "extended mode" operation uses unsigned integers offset by - * 64 (e.g., 0 -> -64 degC). The range is restricted to -64..191 degC. - */ -static inline int temp_from_u8_adt7461(struct lm90_data *data, u8 val) -{ - if (data->flags & LM90_FLAG_ADT7461_EXT) - return (val - 64) * 1000; - return temp_from_s8(val); + return count; } -static inline int temp_from_u16_adt7461(struct lm90_data *data, u16 val) -{ - if (data->flags & LM90_FLAG_ADT7461_EXT) - return (val - 0x4000) / 64 * 250; - return temp_from_s16(val); -} +static DEVICE_ATTR_RW(pec); -static u8 temp_to_u8_adt7461(struct lm90_data *data, long val) +static int lm90_temp_get_resolution(struct lm90_data *data, int index) { - if (data->flags & LM90_FLAG_ADT7461_EXT) { - if (val <= -64000) - return 0; - if (val >= 191000) - return 0xFF; - return (val + 500 + 64000) / 1000; + switch (index) { + case REMOTE_TEMP: + if (data->reg_remote_ext) + return data->resolution; + return 8; + case REMOTE_OFFSET: + case REMOTE2_OFFSET: + case REMOTE2_TEMP: + return data->resolution; + case LOCAL_TEMP: + if (data->reg_local_ext) + return data->resolution; + return 8; + case REMOTE_LOW: + case REMOTE_HIGH: + case REMOTE2_LOW: + case REMOTE2_HIGH: + if (data->flags & LM90_HAVE_REM_LIMIT_EXT) + return data->resolution; + return 8; + default: + return 8; } - if (val <= 0) - return 0; - if (val >= 127000) - return 127; - return (val + 500) / 1000; } -static u16 temp_to_u16_adt7461(struct lm90_data *data, long val) +static int lm90_temp_from_reg(u32 flags, u16 regval, u8 resolution) { - if (data->flags & LM90_FLAG_ADT7461_EXT) { - if (val <= -64000) - return 0; - if (val >= 191750) - return 0xFFC0; - return (val + 64000 + 125) / 250 * 64; - } - if (val <= 0) - return 0; - if (val >= 127750) - return 0x7FC0; - return (val + 125) / 250 * 64; -} + int val; -/* pec used for ADM1032 only */ -static ssize_t pec_show(struct device *dev, struct device_attribute *dummy, - char *buf) -{ - struct i2c_client *client = to_i2c_client(dev); + if (flags & LM90_HAVE_EXTENDED_TEMP) + val = regval - 0x4000; + else if (flags & (LM90_HAVE_UNSIGNED_TEMP | LM90_HAVE_EXT_UNSIGNED)) + val = regval; + else + val = (s16)regval; - return sprintf(buf, "%d\n", !!(client->flags & I2C_CLIENT_PEC)); + return ((val >> (16 - resolution)) * 1000) >> (resolution - 8); } -static ssize_t pec_store(struct device *dev, struct device_attribute *dummy, - const char *buf, size_t count) +static int lm90_get_temp(struct lm90_data *data, int index, int channel) { - struct i2c_client *client = to_i2c_client(dev); - long val; - int err; + int temp = lm90_temp_from_reg(data->flags, data->temp[index], + lm90_temp_get_resolution(data, index)); - err = kstrtol(buf, 10, &val); - if (err < 0) - return err; - - switch (val) { - case 0: - client->flags &= ~I2C_CLIENT_PEC; - break; - case 1: - client->flags |= I2C_CLIENT_PEC; - break; - default: - return -EINVAL; - } + /* +16 degrees offset for remote temperature on LM99 */ + if (data->kind == lm99 && channel) + temp += 16000; - return count; + return temp; } -static DEVICE_ATTR_RW(pec); - -static int lm90_get_temp11(struct lm90_data *data, int index) +static u16 lm90_temp_to_reg(u32 flags, long val, u8 resolution) { - s16 temp11 = data->temp11[index]; - int temp; - - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - temp = temp_from_u16_adt7461(data, temp11); - else if (data->kind == max6646) - temp = temp_from_u16(temp11); - else - temp = temp_from_s16(temp11); - - /* +16 degrees offset for temp2 for the LM99 */ - if (data->kind == lm99 && index <= 2) - temp += 16000; + int fraction = resolution > 8 ? + 1000 - DIV_ROUND_CLOSEST(1000, BIT(resolution - 8)) : 0; + + if (flags & LM90_HAVE_EXTENDED_TEMP) { + val = clamp_val(val, -64000, 191000 + fraction); + val += 64000; + } else if (flags & LM90_HAVE_EXT_UNSIGNED) { + val = clamp_val(val, 0, 255000 + fraction); + } else if (flags & LM90_HAVE_UNSIGNED_TEMP) { + val = clamp_val(val, 0, 127000 + fraction); + } else { + val = clamp_val(val, -128000, 127000 + fraction); + } - return temp; + return DIV_ROUND_CLOSEST(val << (resolution - 8), 1000) << (16 - resolution); } -static int lm90_set_temp11(struct lm90_data *data, int index, long val) +static int lm90_set_temp(struct lm90_data *data, int index, int channel, long val) { - static struct reg { - u8 high; - u8 low; - } reg[] = { - [REMOTE_LOW] = { LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL }, - [REMOTE_HIGH] = { LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL }, - [REMOTE_OFFSET] = { LM90_REG_W_REMOTE_OFFSH, LM90_REG_W_REMOTE_OFFSL }, - [REMOTE2_LOW] = { LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL }, - [REMOTE2_HIGH] = { LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL } + static const u8 regs[] = { + [LOCAL_LOW] = LM90_REG_LOCAL_LOW, + [LOCAL_HIGH] = LM90_REG_LOCAL_HIGH, + [LOCAL_CRIT] = LM90_REG_LOCAL_CRIT, + [REMOTE_CRIT] = LM90_REG_REMOTE_CRIT, + [LOCAL_EMERG] = MAX6659_REG_LOCAL_EMERG, + [REMOTE_EMERG] = MAX6659_REG_REMOTE_EMERG, + [REMOTE2_CRIT] = LM90_REG_REMOTE_CRIT, + [REMOTE2_EMERG] = MAX6659_REG_REMOTE_EMERG, + [REMOTE_LOW] = LM90_REG_REMOTE_LOWH, + [REMOTE_HIGH] = LM90_REG_REMOTE_HIGHH, + [REMOTE2_LOW] = LM90_REG_REMOTE_LOWH, + [REMOTE2_HIGH] = LM90_REG_REMOTE_HIGHH, }; struct i2c_client *client = data->client; - struct reg *regp = ®[index]; + u8 regh = regs[index]; + u8 regl = 0; int err; - /* +16 degrees offset for temp2 for the LM99 */ - if (data->kind == lm99 && index <= 2) { + if (channel && (data->flags & LM90_HAVE_REM_LIMIT_EXT)) { + if (index == REMOTE_LOW || index == REMOTE2_LOW) + regl = LM90_REG_REMOTE_LOWL; + else if (index == REMOTE_HIGH || index == REMOTE2_HIGH) + regl = LM90_REG_REMOTE_HIGHL; + } + + /* +16 degrees offset for remote temperature on LM99 */ + if (data->kind == lm99 && channel) { /* prevent integer underflow */ val = max(val, -128000l); val -= 16000; } - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - data->temp11[index] = temp_to_u16_adt7461(data, val); - else if (data->kind == max6646) - data->temp11[index] = temp_to_u8(val) << 8; - else if (data->flags & LM90_HAVE_REM_LIMIT_EXT) - data->temp11[index] = temp_to_s16(val); - else - data->temp11[index] = temp_to_s8(val) << 8; + data->temp[index] = lm90_temp_to_reg(data->flags, val, + lm90_temp_get_resolution(data, index)); - lm90_select_remote_channel(data, index >= 3); - err = i2c_smbus_write_byte_data(client, regp->high, - data->temp11[index] >> 8); - if (err < 0) - return err; - if (data->flags & LM90_HAVE_REM_LIMIT_EXT) - err = i2c_smbus_write_byte_data(client, regp->low, - data->temp11[index] & 0xff); + if (channel > 1) + lm90_select_remote_channel(data, true); + + err = lm90_write16(client, regh, regl, data->temp[index]); + + if (channel > 1) + lm90_select_remote_channel(data, false); - lm90_select_remote_channel(data, 0); return err; } -static int lm90_get_temp8(struct lm90_data *data, int index) +static int lm90_get_temphyst(struct lm90_data *data, int index, int channel) { - s8 temp8 = data->temp8[index]; - int temp; + int temp = lm90_get_temp(data, index, channel); - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - temp = temp_from_u8_adt7461(data, temp8); - else if (data->kind == max6646) - temp = temp_from_u8(temp8); - else - temp = temp_from_s8(temp8); - - /* +16 degrees offset for temp2 for the LM99 */ - if (data->kind == lm99 && index == 3) - temp += 16000; - - return temp; + return temp - data->temp_hyst * 1000; } -static int lm90_set_temp8(struct lm90_data *data, int index, long val) +static int lm90_set_temphyst(struct lm90_data *data, long val) { - static const u8 reg[TEMP8_REG_NUM] = { - LM90_REG_W_LOCAL_LOW, - LM90_REG_W_LOCAL_HIGH, - LM90_REG_W_LOCAL_CRIT, - LM90_REG_W_REMOTE_CRIT, - MAX6659_REG_W_LOCAL_EMERG, - MAX6659_REG_W_REMOTE_EMERG, - LM90_REG_W_REMOTE_CRIT, - MAX6659_REG_W_REMOTE_EMERG, - }; - struct i2c_client *client = data->client; - int err; + int temp = lm90_get_temp(data, LOCAL_CRIT, 0); - /* +16 degrees offset for temp2 for the LM99 */ - if (data->kind == lm99 && index == 3) { - /* prevent integer underflow */ - val = max(val, -128000l); - val -= 16000; - } + /* prevent integer overflow/underflow */ + val = clamp_val(val, -128000l, 255000l); + data->temp_hyst = clamp_val(DIV_ROUND_CLOSEST(temp - val, 1000), 0, 31); - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - data->temp8[index] = temp_to_u8_adt7461(data, val); - else if (data->kind == max6646) - data->temp8[index] = temp_to_u8(val); - else - data->temp8[index] = temp_to_s8(val); + return lm90_write_reg(data->client, LM90_REG_TCRIT_HYST, data->temp_hyst); +} - lm90_select_remote_channel(data, index >= 6); - err = i2c_smbus_write_byte_data(client, reg[index], data->temp8[index]); - lm90_select_remote_channel(data, 0); +static int lm90_get_temp_offset(struct lm90_data *data, int index) +{ + int res = lm90_temp_get_resolution(data, index); - return err; + return lm90_temp_from_reg(0, data->temp[index], res); } -static int lm90_get_temphyst(struct lm90_data *data, int index) +static int lm90_set_temp_offset(struct lm90_data *data, int index, int channel, long val) { - int temp; + int err; - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - temp = temp_from_u8_adt7461(data, data->temp8[index]); - else if (data->kind == max6646) - temp = temp_from_u8(data->temp8[index]); - else - temp = temp_from_s8(data->temp8[index]); + val = lm90_temp_to_reg(0, val, lm90_temp_get_resolution(data, index)); - /* +16 degrees offset for temp2 for the LM99 */ - if (data->kind == lm99 && index == 3) - temp += 16000; + /* For ADT7481 we can use the same registers for remote channel 1 and 2 */ + if (channel > 1) + lm90_select_remote_channel(data, true); - return temp - temp_from_s8(data->temp_hyst); -} + err = lm90_write16(data->client, LM90_REG_REMOTE_OFFSH, LM90_REG_REMOTE_OFFSL, val); -static int lm90_set_temphyst(struct lm90_data *data, long val) -{ - struct i2c_client *client = data->client; - int temp; - int err; + if (channel > 1) + lm90_select_remote_channel(data, false); - if (data->flags & LM90_HAVE_EXTENDED_TEMP) - temp = temp_from_u8_adt7461(data, data->temp8[LOCAL_CRIT]); - else if (data->kind == max6646) - temp = temp_from_u8(data->temp8[LOCAL_CRIT]); - else - temp = temp_from_s8(data->temp8[LOCAL_CRIT]); + if (err) + return err; - /* prevent integer overflow/underflow */ - val = clamp_val(val, -128000l, 255000l); + data->temp[index] = val; - data->temp_hyst = hyst_to_reg(temp - val); - err = i2c_smbus_write_byte_data(client, LM90_REG_W_TCRIT_HYST, - data->temp_hyst); - return err; + return 0; } -static const u8 lm90_temp_index[3] = { +static const u8 lm90_temp_index[MAX_CHANNELS] = { LOCAL_TEMP, REMOTE_TEMP, REMOTE2_TEMP }; -static const u8 lm90_temp_min_index[3] = { +static const u8 lm90_temp_min_index[MAX_CHANNELS] = { LOCAL_LOW, REMOTE_LOW, REMOTE2_LOW }; -static const u8 lm90_temp_max_index[3] = { +static const u8 lm90_temp_max_index[MAX_CHANNELS] = { LOCAL_HIGH, REMOTE_HIGH, REMOTE2_HIGH }; -static const u8 lm90_temp_crit_index[3] = { +static const u8 lm90_temp_crit_index[MAX_CHANNELS] = { LOCAL_CRIT, REMOTE_CRIT, REMOTE2_CRIT }; -static const u8 lm90_temp_emerg_index[3] = { +static const u8 lm90_temp_emerg_index[MAX_CHANNELS] = { LOCAL_EMERG, REMOTE_EMERG, REMOTE2_EMERG }; -static const u8 lm90_min_alarm_bits[3] = { 5, 3, 11 }; -static const u8 lm90_max_alarm_bits[3] = { 6, 4, 12 }; -static const u8 lm90_crit_alarm_bits[3] = { 0, 1, 9 }; -static const u8 lm90_crit_alarm_bits_swapped[3] = { 1, 0, 9 }; -static const u8 lm90_emergency_alarm_bits[3] = { 15, 13, 14 }; -static const u8 lm90_fault_bits[3] = { 0, 2, 10 }; +static const s8 lm90_temp_offset_index[MAX_CHANNELS] = { + -1, REMOTE_OFFSET, REMOTE2_OFFSET +}; + +static const u16 lm90_min_alarm_bits[MAX_CHANNELS] = { BIT(5), BIT(3), BIT(11) }; +static const u16 lm90_max_alarm_bits[MAX_CHANNELS] = { BIT(6), BIT(4), BIT(12) }; +static const u16 lm90_crit_alarm_bits[MAX_CHANNELS] = { BIT(0), BIT(1), BIT(9) }; +static const u16 lm90_crit_alarm_bits_swapped[MAX_CHANNELS] = { BIT(1), BIT(0), BIT(9) }; +static const u16 lm90_emergency_alarm_bits[MAX_CHANNELS] = { BIT(15), BIT(13), BIT(14) }; +static const u16 lm90_fault_bits[MAX_CHANNELS] = { BIT(0), BIT(2), BIT(10) }; static int lm90_temp_read(struct device *dev, u32 attr, int channel, long *val) { struct lm90_data *data = dev_get_drvdata(dev); int err; + u16 bit; mutex_lock(&data->update_lock); err = lm90_update_device(dev); @@ -1222,56 +1519,57 @@ static int lm90_temp_read(struct device *dev, u32 attr, int channel, long *val) switch (attr) { case hwmon_temp_input: - *val = lm90_get_temp11(data, lm90_temp_index[channel]); + *val = lm90_get_temp(data, lm90_temp_index[channel], channel); break; case hwmon_temp_min_alarm: - *val = (data->alarms >> lm90_min_alarm_bits[channel]) & 1; - break; case hwmon_temp_max_alarm: - *val = (data->alarms >> lm90_max_alarm_bits[channel]) & 1; - break; case hwmon_temp_crit_alarm: - if (data->flags & LM90_HAVE_CRIT_ALRM_SWP) - *val = (data->alarms >> lm90_crit_alarm_bits_swapped[channel]) & 1; - else - *val = (data->alarms >> lm90_crit_alarm_bits[channel]) & 1; - break; case hwmon_temp_emergency_alarm: - *val = (data->alarms >> lm90_emergency_alarm_bits[channel]) & 1; - break; case hwmon_temp_fault: - *val = (data->alarms >> lm90_fault_bits[channel]) & 1; + switch (attr) { + case hwmon_temp_min_alarm: + bit = lm90_min_alarm_bits[channel]; + break; + case hwmon_temp_max_alarm: + bit = lm90_max_alarm_bits[channel]; + break; + case hwmon_temp_crit_alarm: + if (data->flags & LM90_HAVE_CRIT_ALRM_SWP) + bit = lm90_crit_alarm_bits_swapped[channel]; + else + bit = lm90_crit_alarm_bits[channel]; + break; + case hwmon_temp_emergency_alarm: + bit = lm90_emergency_alarm_bits[channel]; + break; + case hwmon_temp_fault: + bit = lm90_fault_bits[channel]; + break; + } + *val = !!(data->alarms & bit); + data->alarms &= ~bit; + data->alarms |= data->current_alarms; break; case hwmon_temp_min: - if (channel == 0) - *val = lm90_get_temp8(data, - lm90_temp_min_index[channel]); - else - *val = lm90_get_temp11(data, - lm90_temp_min_index[channel]); + *val = lm90_get_temp(data, lm90_temp_min_index[channel], channel); break; case hwmon_temp_max: - if (channel == 0) - *val = lm90_get_temp8(data, - lm90_temp_max_index[channel]); - else - *val = lm90_get_temp11(data, - lm90_temp_max_index[channel]); + *val = lm90_get_temp(data, lm90_temp_max_index[channel], channel); break; case hwmon_temp_crit: - *val = lm90_get_temp8(data, lm90_temp_crit_index[channel]); + *val = lm90_get_temp(data, lm90_temp_crit_index[channel], channel); break; case hwmon_temp_crit_hyst: - *val = lm90_get_temphyst(data, lm90_temp_crit_index[channel]); + *val = lm90_get_temphyst(data, lm90_temp_crit_index[channel], channel); break; case hwmon_temp_emergency: - *val = lm90_get_temp8(data, lm90_temp_emerg_index[channel]); + *val = lm90_get_temp(data, lm90_temp_emerg_index[channel], channel); break; case hwmon_temp_emergency_hyst: - *val = lm90_get_temphyst(data, lm90_temp_emerg_index[channel]); + *val = lm90_get_temphyst(data, lm90_temp_emerg_index[channel], channel); break; case hwmon_temp_offset: - *val = lm90_get_temp11(data, REMOTE_OFFSET); + *val = lm90_get_temp_offset(data, lm90_temp_offset_index[channel]); break; default: return -EOPNOTSUPP; @@ -1292,36 +1590,27 @@ static int lm90_temp_write(struct device *dev, u32 attr, int channel, long val) switch (attr) { case hwmon_temp_min: - if (channel == 0) - err = lm90_set_temp8(data, - lm90_temp_min_index[channel], - val); - else - err = lm90_set_temp11(data, - lm90_temp_min_index[channel], - val); + err = lm90_set_temp(data, lm90_temp_min_index[channel], + channel, val); break; case hwmon_temp_max: - if (channel == 0) - err = lm90_set_temp8(data, - lm90_temp_max_index[channel], - val); - else - err = lm90_set_temp11(data, - lm90_temp_max_index[channel], - val); + err = lm90_set_temp(data, lm90_temp_max_index[channel], + channel, val); break; case hwmon_temp_crit: - err = lm90_set_temp8(data, lm90_temp_crit_index[channel], val); + err = lm90_set_temp(data, lm90_temp_crit_index[channel], + channel, val); break; case hwmon_temp_crit_hyst: err = lm90_set_temphyst(data, val); break; case hwmon_temp_emergency: - err = lm90_set_temp8(data, lm90_temp_emerg_index[channel], val); + err = lm90_set_temp(data, lm90_temp_emerg_index[channel], + channel, val); break; case hwmon_temp_offset: - err = lm90_set_temp11(data, REMOTE_OFFSET, val); + err = lm90_set_temp_offset(data, lm90_temp_offset_index[channel], + channel, val); break; default: err = -EOPNOTSUPP; @@ -1343,6 +1632,7 @@ static umode_t lm90_temp_is_visible(const void *data, u32 attr, int channel) case hwmon_temp_emergency_alarm: case hwmon_temp_emergency_hyst: case hwmon_temp_fault: + case hwmon_temp_label: return 0444; case hwmon_temp_min: case hwmon_temp_max: @@ -1377,6 +1667,28 @@ static int lm90_chip_read(struct device *dev, u32 attr, int channel, long *val) case hwmon_chip_alarms: *val = data->alarms; break; + case hwmon_chip_temp_samples: + if (data->faultqueue_mask) { + *val = (data->config & data->faultqueue_mask) ? + data->faultqueue_depth : 1; + } else { + switch (data->conalert & 0x0e) { + case 0x0: + default: + *val = 1; + break; + case 0x2: + *val = 2; + break; + case 0x6: + *val = 3; + break; + case 0xe: + *val = 4; + break; + } + } + break; default: return -EOPNOTSUPP; } @@ -1401,6 +1713,9 @@ static int lm90_chip_write(struct device *dev, u32 attr, int channel, long val) err = lm90_set_convrate(client, data, clamp_val(val, 0, 100000)); break; + case hwmon_chip_temp_samples: + err = lm90_set_faultqueue(client, data, clamp_val(val, 1, 4)); + break; default: err = -EOPNOTSUPP; break; @@ -1415,6 +1730,7 @@ static umode_t lm90_chip_is_visible(const void *data, u32 attr, int channel) { switch (attr) { case hwmon_chip_update_interval: + case hwmon_chip_temp_samples: return 0644; case hwmon_chip_alarms: return 0444; @@ -1436,6 +1752,16 @@ static int lm90_read(struct device *dev, enum hwmon_sensor_types type, } } +static int lm90_read_string(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + struct lm90_data *data = dev_get_drvdata(dev); + + *str = data->channel_label[channel]; + + return 0; +} + static int lm90_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val) { @@ -1462,124 +1788,358 @@ static umode_t lm90_is_visible(const void *data, enum hwmon_sensor_types type, } } -/* Return 0 if detection is successful, -ENODEV otherwise */ -static int lm90_detect(struct i2c_client *client, - struct i2c_board_info *info) +static const char *lm90_detect_lm84(struct i2c_client *client) { - struct i2c_adapter *adapter = client->adapter; + static const u8 regs[] = { + LM90_REG_STATUS, LM90_REG_LOCAL_TEMP, LM90_REG_LOCAL_HIGH, + LM90_REG_REMOTE_TEMPH, LM90_REG_REMOTE_HIGHH + }; + int status = i2c_smbus_read_byte_data(client, LM90_REG_STATUS); + int reg1, reg2, reg3, reg4; + bool nonzero = false; + u8 ff = 0xff; + int i; + + if (status < 0 || (status & 0xab)) + return NULL; + + /* + * For LM84, undefined registers return the most recent value. + * Repeat several times, each time checking against a different + * (presumably) existing register. + */ + for (i = 0; i < ARRAY_SIZE(regs); i++) { + reg1 = i2c_smbus_read_byte_data(client, regs[i]); + reg2 = i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_TEMPL); + reg3 = i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_LOW); + reg4 = i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_LOWH); + + if (reg1 < 0) + return NULL; + + /* If any register has a different value, this is not an LM84 */ + if (reg2 != reg1 || reg3 != reg1 || reg4 != reg1) + return NULL; + + nonzero |= reg1 || reg2 || reg3 || reg4; + ff &= reg1; + } + /* + * If all registers always returned 0 or 0xff, all bets are off, + * and we can not make any predictions about the chip type. + */ + return nonzero && ff != 0xff ? "lm84" : NULL; +} + +static const char *lm90_detect_max1617(struct i2c_client *client, int config1) +{ + int status = i2c_smbus_read_byte_data(client, LM90_REG_STATUS); + int llo, rlo, lhi, rhi; + + if (status < 0 || (status & 0x03)) + return NULL; + + if (config1 & 0x3f) + return NULL; + + /* + * Fail if unsupported registers return anything but 0xff. + * The calling code already checked man_id and chip_id. + * A byte read operation repeats the most recent read operation + * and should also return 0xff. + */ + if (i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_TEMPL) != 0xff || + i2c_smbus_read_byte_data(client, MAX6657_REG_LOCAL_TEMPL) != 0xff || + i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_LOWL) != 0xff || + i2c_smbus_read_byte(client) != 0xff) + return NULL; + + llo = i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_LOW); + rlo = i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_LOWH); + + lhi = i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_HIGH); + rhi = i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_HIGHH); + + if (llo < 0 || rlo < 0) + return NULL; + + /* + * A byte read operation repeats the most recent read and should + * return the same value. + */ + if (i2c_smbus_read_byte(client) != rhi) + return NULL; + + /* + * The following two checks are marginal since the checked values + * are strictly speaking valid. + */ + + /* fail for negative high limits; this also catches read errors */ + if ((s8)lhi < 0 || (s8)rhi < 0) + return NULL; + + /* fail if low limits are larger than or equal to high limits */ + if ((s8)llo >= lhi || (s8)rlo >= rhi) + return NULL; + + if (i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA)) { + /* + * Word read operations return 0xff in second byte + */ + if (i2c_smbus_read_word_data(client, LM90_REG_REMOTE_TEMPL) != + 0xffff) + return NULL; + if (i2c_smbus_read_word_data(client, LM90_REG_CONFIG1) != + (config1 | 0xff00)) + return NULL; + if (i2c_smbus_read_word_data(client, LM90_REG_LOCAL_HIGH) != + (lhi | 0xff00)) + return NULL; + } + + return "max1617"; +} + +static const char *lm90_detect_national(struct i2c_client *client, int chip_id, + int config1, int convrate) +{ + int config2 = i2c_smbus_read_byte_data(client, LM90_REG_CONFIG2); int address = client->addr; const char *name = NULL; - int man_id, chip_id, config1, config2, convrate; - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) - return -ENODEV; + if (config2 < 0) + return NULL; - /* detection and identification */ - man_id = i2c_smbus_read_byte_data(client, LM90_REG_R_MAN_ID); - chip_id = i2c_smbus_read_byte_data(client, LM90_REG_R_CHIP_ID); - config1 = i2c_smbus_read_byte_data(client, LM90_REG_R_CONFIG1); - convrate = i2c_smbus_read_byte_data(client, LM90_REG_R_CONVRATE); - if (man_id < 0 || chip_id < 0 || config1 < 0 || convrate < 0) - return -ENODEV; + if ((config1 & 0x2a) || (config2 & 0xf8) || convrate > 0x09) + return NULL; - if (man_id == 0x01 || man_id == 0x5C || man_id == 0xA1) { - config2 = i2c_smbus_read_byte_data(client, LM90_REG_R_CONFIG2); - if (config2 < 0) - return -ENODEV; + if (address != 0x4c && address != 0x4d) + return NULL; + + switch (chip_id & 0xf0) { + case 0x10: /* LM86 */ + if (address == 0x4c) + name = "lm86"; + break; + case 0x20: /* LM90 */ + if (address == 0x4c) + name = "lm90"; + break; + case 0x30: /* LM89/LM99 */ + name = "lm99"; /* detect LM89 as LM99 */ + break; + default: + break; } - if ((address == 0x4C || address == 0x4D) - && man_id == 0x01) { /* National Semiconductor */ - if ((config1 & 0x2A) == 0x00 - && (config2 & 0xF8) == 0x00 - && convrate <= 0x09) { - if (address == 0x4C - && (chip_id & 0xF0) == 0x20) { /* LM90 */ - name = "lm90"; - } else - if ((chip_id & 0xF0) == 0x30) { /* LM89/LM99 */ - name = "lm99"; - dev_info(&adapter->dev, - "Assuming LM99 chip at 0x%02x\n", - address); - dev_info(&adapter->dev, - "If it is an LM89, instantiate it " - "with the new_device sysfs " - "interface\n"); - } else - if (address == 0x4C - && (chip_id & 0xF0) == 0x10) { /* LM86 */ - name = "lm86"; - } - } - } else - if ((address == 0x4C || address == 0x4D) - && man_id == 0x41) { /* Analog Devices */ - if ((chip_id & 0xF0) == 0x40 /* ADM1032 */ - && (config1 & 0x3F) == 0x00 - && convrate <= 0x0A) { + return name; +} + +static const char *lm90_detect_on(struct i2c_client *client, int chip_id, int config1, + int convrate) +{ + int address = client->addr; + const char *name = NULL; + + switch (chip_id) { + case 0xca: /* NCT218 */ + if ((address == 0x4c || address == 0x4d) && !(config1 & 0x1b) && + convrate <= 0x0a) + name = "nct218"; + break; + default: + break; + } + return name; +} + +static const char *lm90_detect_analog(struct i2c_client *client, bool common_address, + int chip_id, int config1, int convrate) +{ + int status = i2c_smbus_read_byte_data(client, LM90_REG_STATUS); + int config2 = i2c_smbus_read_byte_data(client, ADT7481_REG_CONFIG2); + int man_id2 = i2c_smbus_read_byte_data(client, ADT7481_REG_MAN_ID); + int chip_id2 = i2c_smbus_read_byte_data(client, ADT7481_REG_CHIP_ID); + int address = client->addr; + const char *name = NULL; + + if (status < 0 || config2 < 0 || man_id2 < 0 || chip_id2 < 0) + return NULL; + + /* + * The following chips should be detected by this function. Known + * register values are listed. Registers 0x3d .. 0x3e are undocumented + * for most of the chips, yet appear to return a well defined value. + * Register 0xff is undocumented for some of the chips. Register 0x3f + * is undocumented for all chips, but also returns a well defined value. + * Values are as reported from real chips unless mentioned otherwise. + * The code below checks values for registers 0x3d, 0x3e, and 0xff, + * but not for register 0x3f. + * + * Chip Register + * 3d 3e 3f fe ff Notes + * ---------------------------------------------------------- + * adm1020 00 00 00 41 39 + * adm1021 00 00 00 41 03 + * adm1021a 00 00 00 41 3c + * adm1023 00 00 00 41 3c same as adm1021a + * adm1032 00 00 00 41 42 + * + * adt7421 21 41 04 41 04 + * adt7461 00 00 00 41 51 + * adt7461a 61 41 05 41 57 + * adt7481 81 41 02 41 62 + * adt7482 - - - 41 65 datasheet + * 82 41 05 41 75 real chip + * adt7483 83 41 04 41 94 + * + * nct72 61 41 07 41 55 + * nct210 00 00 00 41 3f + * nct214 61 41 08 41 5a + * nct1008 - - - 41 57 datasheet rev. 3 + * 61 41 06 41 54 real chip + * + * nvt210 - - - 41 - datasheet + * nvt211 - - - 41 - datasheet + */ + switch (chip_id) { + case 0x00 ... 0x03: /* ADM1021 */ + case 0x05 ... 0x0f: + if (man_id2 == 0x00 && chip_id2 == 0x00 && common_address && + !(status & 0x03) && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "adm1021"; + break; + case 0x04: /* ADT7421 (undocumented) */ + if (man_id2 == 0x41 && chip_id2 == 0x21 && + (address == 0x4c || address == 0x4d) && + (config1 & 0x0b) == 0x08 && convrate <= 0x0a) + name = "adt7421"; + break; + case 0x30 ... 0x38: /* ADM1021A, ADM1023 */ + case 0x3a ... 0x3e: + /* + * ADM1021A and compatible chips will be mis-detected as + * ADM1023. Chips labeled 'ADM1021A' and 'ADM1023' were both + * found to have a Chip ID of 0x3c. + * ADM1021A does not officially support low byte registers + * (0x12 .. 0x14), but a chip labeled ADM1021A does support it. + * Official support for the temperature offset high byte + * register (0x11) was added to revision F of the ADM1021A + * datasheet. + * It is currently unknown if there is a means to distinguish + * ADM1021A from ADM1023, and/or if revisions of ADM1021A exist + * which differ in functionality from ADM1023. + */ + if (man_id2 == 0x00 && chip_id2 == 0x00 && common_address && + !(status & 0x03) && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "adm1023"; + break; + case 0x39: /* ADM1020 (undocumented) */ + if (man_id2 == 0x00 && chip_id2 == 0x00 && + (address == 0x4c || address == 0x4d || address == 0x4e) && + !(status & 0x03) && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "adm1020"; + break; + case 0x3f: /* NCT210 */ + if (man_id2 == 0x00 && chip_id2 == 0x00 && common_address && + !(status & 0x03) && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "nct210"; + break; + case 0x40 ... 0x4f: /* ADM1032 */ + if (man_id2 == 0x00 && chip_id2 == 0x00 && + (address == 0x4c || address == 0x4d) && !(config1 & 0x3f) && + convrate <= 0x0a) name = "adm1032"; - /* - * The ADM1032 supports PEC, but only if combined - * transactions are not used. - */ - if (i2c_check_functionality(adapter, - I2C_FUNC_SMBUS_BYTE)) - info->flags |= I2C_CLIENT_PEC; - } else - if (chip_id == 0x51 /* ADT7461 */ - && (config1 & 0x1B) == 0x00 - && convrate <= 0x0A) { + break; + case 0x51: /* ADT7461 */ + if (man_id2 == 0x00 && chip_id2 == 0x00 && + (address == 0x4c || address == 0x4d) && !(config1 & 0x1b) && + convrate <= 0x0a) name = "adt7461"; - } else - if (chip_id == 0x57 /* ADT7461A, NCT1008 */ - && (config1 & 0x1B) == 0x00 - && convrate <= 0x0A) { + break; + case 0x54: /* NCT1008 */ + if (man_id2 == 0x41 && chip_id2 == 0x61 && + (address == 0x4c || address == 0x4d) && !(config1 & 0x1b) && + convrate <= 0x0a) + name = "nct1008"; + break; + case 0x55: /* NCT72 */ + if (man_id2 == 0x41 && chip_id2 == 0x61 && + (address == 0x4c || address == 0x4d) && !(config1 & 0x1b) && + convrate <= 0x0a) + name = "nct72"; + break; + case 0x57: /* ADT7461A, NCT1008 (datasheet rev. 3) */ + if (man_id2 == 0x41 && chip_id2 == 0x61 && + (address == 0x4c || address == 0x4d) && !(config1 & 0x1b) && + convrate <= 0x0a) name = "adt7461a"; + break; + case 0x5a: /* NCT214 */ + if (man_id2 == 0x41 && chip_id2 == 0x61 && + common_address && !(config1 & 0x1b) && convrate <= 0x0a) + name = "nct214"; + break; + case 0x62: /* ADT7481, undocumented */ + if (man_id2 == 0x41 && chip_id2 == 0x81 && + (address == 0x4b || address == 0x4c) && !(config1 & 0x10) && + !(config2 & 0x7f) && (convrate & 0x0f) <= 0x0b) { + name = "adt7481"; } - } else - if (man_id == 0x4D) { /* Maxim */ - int emerg, emerg2, status2; + break; + case 0x65: /* ADT7482, datasheet */ + case 0x75: /* ADT7482, real chip */ + if (man_id2 == 0x41 && chip_id2 == 0x82 && + address == 0x4c && !(config1 & 0x10) && !(config2 & 0x7f) && + convrate <= 0x0a) + name = "adt7482"; + break; + case 0x94: /* ADT7483 */ + if (man_id2 == 0x41 && chip_id2 == 0x83 && + common_address && + ((address >= 0x18 && address <= 0x1a) || + (address >= 0x29 && address <= 0x2b) || + (address >= 0x4c && address <= 0x4e)) && + !(config1 & 0x10) && !(config2 & 0x7f) && convrate <= 0x0a) + name = "adt7483a"; + break; + default: + break; + } + + return name; +} + +static const char *lm90_detect_maxim(struct i2c_client *client, bool common_address, + int chip_id, int config1, int convrate) +{ + int man_id, emerg, emerg2, status2; + int address = client->addr; + const char *name = NULL; + + switch (chip_id) { + case 0x01: + if (!common_address) + break; /* - * We read MAX6659_REG_R_REMOTE_EMERG twice, and re-read - * LM90_REG_R_MAN_ID in between. If MAX6659_REG_R_REMOTE_EMERG + * We read MAX6659_REG_REMOTE_EMERG twice, and re-read + * LM90_REG_MAN_ID in between. If MAX6659_REG_REMOTE_EMERG * exists, both readings will reflect the same value. Otherwise, * the readings will be different. */ emerg = i2c_smbus_read_byte_data(client, - MAX6659_REG_R_REMOTE_EMERG); + MAX6659_REG_REMOTE_EMERG); man_id = i2c_smbus_read_byte_data(client, - LM90_REG_R_MAN_ID); + LM90_REG_MAN_ID); emerg2 = i2c_smbus_read_byte_data(client, - MAX6659_REG_R_REMOTE_EMERG); + MAX6659_REG_REMOTE_EMERG); status2 = i2c_smbus_read_byte_data(client, - MAX6696_REG_R_STATUS2); + MAX6696_REG_STATUS2); if (emerg < 0 || man_id < 0 || emerg2 < 0 || status2 < 0) - return -ENODEV; + return NULL; - /* - * The MAX6657, MAX6658 and MAX6659 do NOT have a chip_id - * register. Reading from that address will return the last - * read value, which in our case is those of the man_id - * register. Likewise, the config1 register seems to lack a - * low nibble, so the value will be those of the previous - * read, so in our case those of the man_id register. - * MAX6659 has a third set of upper temperature limit registers. - * Those registers also return values on MAX6657 and MAX6658, - * thus the only way to detect MAX6659 is by its address. - * For this reason it will be mis-detected as MAX6657 if its - * address is 0x4C. - */ - if (chip_id == man_id - && (address == 0x4C || address == 0x4D || address == 0x4E) - && (config1 & 0x1F) == (man_id & 0x0F) - && convrate <= 0x09) { - if (address == 0x4C) - name = "max6657"; - else - name = "max6659"; - } else /* * Even though MAX6695 and MAX6696 do not have a chip ID * register, reading it returns 0x01. Bit 4 of the config1 @@ -1591,90 +2151,288 @@ static int lm90_detect(struct i2c_client *client, * limit registers. We can detect those chips by checking if * one of those registers exists. */ - if (chip_id == 0x01 - && (config1 & 0x10) == 0x00 - && (status2 & 0x01) == 0x00 - && emerg == emerg2 - && convrate <= 0x07) { + if (!(config1 & 0x10) && !(status2 & 0x01) && emerg == emerg2 && + convrate <= 0x07) name = "max6696"; - } else /* * The chip_id register of the MAX6680 and MAX6681 holds the * revision of the chip. The lowest bit of the config1 register * is unused and should return zero when read, so should the - * second to last bit of config1 (software reset). + * second to last bit of config1 (software reset). Register + * address 0x12 (LM90_REG_REMOTE_OFFSL) exists for this chip and + * should differ from emerg2, and emerg2 should match man_id + * since it does not exist. */ - if (chip_id == 0x01 - && (config1 & 0x03) == 0x00 - && convrate <= 0x07) { + else if (!(config1 & 0x03) && convrate <= 0x07 && + emerg2 == man_id && emerg2 != status2) name = "max6680"; - } else /* - * The chip_id register of the MAX6646/6647/6649 holds the - * revision of the chip. The lowest 6 bits of the config1 - * register are unused and should return zero when read. + * MAX1617A does not have any extended registers (register + * address 0x10 or higher) except for manufacturer and + * device ID registers. Unlike other chips of this series, + * unsupported registers were observed to return a fixed value + * of 0x01. + * Note: Multiple chips with different markings labeled as + * "MAX1617" (no "A") were observed to report manufacturer ID + * 0x4d and device ID 0x01. It is unknown if other variants of + * MAX1617/MAX617A with different behavior exist. The detection + * code below works for those chips. */ - if (chip_id == 0x59 - && (config1 & 0x3f) == 0x00 - && convrate <= 0x07) { - name = "max6646"; - } else + else if (!(config1 & 0x03f) && convrate <= 0x07 && + emerg == 0x01 && emerg2 == 0x01 && status2 == 0x01) + name = "max1617"; + break; + case 0x08: /* * The chip_id of the MAX6654 holds the revision of the chip. * The lowest 3 bits of the config1 register are unused and * should return zero when read. */ - if (chip_id == 0x08 - && (config1 & 0x07) == 0x00 - && convrate <= 0x07) { + if (common_address && !(config1 & 0x07) && convrate <= 0x07) name = "max6654"; + break; + case 0x09: + /* + * The chip_id of the MAX6690 holds the revision of the chip. + * The lowest 3 bits of the config1 register are unused and + * should return zero when read. + * Note that MAX6654 and MAX6690 are practically the same chips. + * The only diference is the rated accuracy. Rev. 1 of the + * MAX6690 datasheet lists a chip ID of 0x08, and a chip labeled + * MAX6654 was observed to have a chip ID of 0x09. + */ + if (common_address && !(config1 & 0x07) && convrate <= 0x07) + name = "max6690"; + break; + case 0x4d: + /* + * MAX6642, MAX6657, MAX6658 and MAX6659 do NOT have a chip_id + * register. Reading from that address will return the last + * read value, which in our case is those of the man_id + * register, or 0x4d. + * MAX6642 does not have a conversion rate register, nor low + * limit registers. Reading from those registers returns the + * last read value. + * + * For MAX6657, MAX6658 and MAX6659, the config1 register lacks + * a low nibble, so the value will be those of the previous + * read, so in our case again those of the man_id register. + * MAX6659 has a third set of upper temperature limit registers. + * Those registers also return values on MAX6657 and MAX6658, + * thus the only way to detect MAX6659 is by its address. + * For this reason it will be mis-detected as MAX6657 if its + * address is 0x4c. + */ + if (address >= 0x48 && address <= 0x4f && config1 == convrate && + !(config1 & 0x0f)) { + int regval; + + /* + * We know that this is not a MAX6657/58/59 because its + * configuration register has the wrong value and it does + * not appear to have a conversion rate register. + */ + + /* re-read manufacturer ID to have a good baseline */ + if (i2c_smbus_read_byte_data(client, LM90_REG_MAN_ID) != 0x4d) + break; + + /* check various non-existing registers */ + if (i2c_smbus_read_byte_data(client, LM90_REG_CONVRATE) != 0x4d || + i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_LOW) != 0x4d || + i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_LOWH) != 0x4d) + break; + + /* check for unused status register bits */ + regval = i2c_smbus_read_byte_data(client, LM90_REG_STATUS); + if (regval < 0 || (regval & 0x2b)) + break; + + /* re-check unsupported registers */ + if (i2c_smbus_read_byte_data(client, LM90_REG_CONVRATE) != regval || + i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_LOW) != regval || + i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_LOWH) != regval) + break; + + name = "max6642"; + } else if ((address == 0x4c || address == 0x4d || address == 0x4e) && + (config1 & 0x1f) == 0x0d && convrate <= 0x09) { + if (address == 0x4c) + name = "max6657"; + else + name = "max6659"; } - } else - if (address == 0x4C - && man_id == 0x5C) { /* Winbond/Nuvoton */ - if ((config1 & 0x2A) == 0x00 - && (config2 & 0xF8) == 0x00) { - if (chip_id == 0x01 /* W83L771W/G */ - && convrate <= 0x09) { - name = "w83l771"; - } else - if ((chip_id & 0xFE) == 0x10 /* W83L771AWG/ASG */ - && convrate <= 0x08) { - name = "w83l771"; + break; + case 0x59: + /* + * The chip_id register of the MAX6646/6647/6649 holds the + * revision of the chip. The lowest 6 bits of the config1 + * register are unused and should return zero when read. + * The I2C address of MAX6648/6692 is fixed at 0x4c. + * MAX6646 is at address 0x4d, MAX6647 is at address 0x4e, + * and MAX6649 is at address 0x4c. A slight difference between + * the two sets of chips is that the remote temperature register + * reports different values if the DXP pin is open or shorted. + * We can use that information to help distinguish between the + * chips. MAX6648 will be mis-detected as MAX6649 if the remote + * diode is connected, but there isn't really anything we can + * do about that. + */ + if (!(config1 & 0x3f) && convrate <= 0x07) { + int temp; + + switch (address) { + case 0x4c: + /* + * MAX6649 reports an external temperature + * value of 0xff if DXP is open or shorted. + * MAX6648 reports 0x80 in that case. + */ + temp = i2c_smbus_read_byte_data(client, + LM90_REG_REMOTE_TEMPH); + if (temp == 0x80) + name = "max6648"; + else + name = "max6649"; + break; + case 0x4d: + name = "max6646"; + break; + case 0x4e: + name = "max6647"; + break; + default: + break; } } - } else - if (address >= 0x48 && address <= 0x4F - && man_id == 0xA1) { /* NXP Semiconductor/Philips */ - if (chip_id == 0x00 - && (config1 & 0x2A) == 0x00 - && (config2 & 0xFE) == 0x00 - && convrate <= 0x09) { - name = "sa56004"; + break; + default: + break; + } + + return name; +} + +static const char *lm90_detect_nuvoton(struct i2c_client *client, int chip_id, + int config1, int convrate) +{ + int config2 = i2c_smbus_read_byte_data(client, LM90_REG_CONFIG2); + int address = client->addr; + const char *name = NULL; + + if (config2 < 0) + return ERR_PTR(-ENODEV); + + if (address == 0x4c && !(config1 & 0x2a) && !(config2 & 0xf8)) { + if (chip_id == 0x01 && convrate <= 0x09) { + /* W83L771W/G */ + name = "w83l771"; + } else if ((chip_id & 0xfe) == 0x10 && convrate <= 0x08) { + /* W83L771AWG/ASG */ + name = "w83l771"; } - } else - if ((address == 0x4C || address == 0x4D) - && man_id == 0x47) { /* GMT */ - if (chip_id == 0x01 /* G781 */ - && (config1 & 0x3F) == 0x00 - && convrate <= 0x08) - name = "g781"; - } else - if (man_id == 0x55 && chip_id == 0x00 && - (config1 & 0x1B) == 0x00 && convrate <= 0x09) { + } + return name; +} + +static const char *lm90_detect_nxp(struct i2c_client *client, bool common_address, + int chip_id, int config1, int convrate) +{ + int address = client->addr; + const char *name = NULL; + int config2; + + switch (chip_id) { + case 0x00: + config2 = i2c_smbus_read_byte_data(client, LM90_REG_CONFIG2); + if (config2 < 0) + return NULL; + if (address >= 0x48 && address <= 0x4f && + !(config1 & 0x2a) && !(config2 & 0xfe) && convrate <= 0x09) + name = "sa56004"; + break; + case 0x80: + if (common_address && !(config1 & 0x3f) && convrate <= 0x07) + name = "ne1618"; + break; + default: + break; + } + return name; +} + +static const char *lm90_detect_gmt(struct i2c_client *client, int chip_id, + int config1, int convrate) +{ + int address = client->addr; + + /* + * According to the datasheet, G781 is supposed to be at I2C Address + * 0x4c and have a chip ID of 0x01. G781-1 is supposed to be at I2C + * address 0x4d and have a chip ID of 0x03. However, when support + * for G781 was added, chips at 0x4c and 0x4d were found to have a + * chip ID of 0x01. A G781-1 at I2C address 0x4d was now found with + * chip ID 0x03. + * To avoid detection failures, accept chip ID 0x01 and 0x03 at both + * addresses. + * G784 reports manufacturer ID 0x47 and chip ID 0x01. A public + * datasheet is not available. Extensive testing suggests that + * the chip appears to be fully compatible with G781. + * Available register dumps show that G751 also reports manufacturer + * ID 0x47 and chip ID 0x01 even though that chip does not officially + * support those registers. This makes chip detection somewhat + * vulnerable. To improve detection quality, read the offset low byte + * and alert fault queue registers and verify that only expected bits + * are set. + */ + if ((chip_id == 0x01 || chip_id == 0x03) && + (address == 0x4c || address == 0x4d) && + !(config1 & 0x3f) && convrate <= 0x08) { + int reg; + + reg = i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_OFFSL); + if (reg < 0 || reg & 0x1f) + return NULL; + reg = i2c_smbus_read_byte_data(client, TMP451_REG_CONALERT); + if (reg < 0 || reg & 0xf1) + return NULL; + + return "g781"; + } + + return NULL; +} + +static const char *lm90_detect_ti49(struct i2c_client *client, bool common_address, + int chip_id, int config1, int convrate) +{ + if (common_address && chip_id == 0x00 && !(config1 & 0x3f) && !(convrate & 0xf8)) { + /* THMC10: Unsupported registers return 0xff */ + if (i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_TEMPL) == 0xff && + i2c_smbus_read_byte_data(client, LM90_REG_REMOTE_CRIT) == 0xff) + return "thmc10"; + } + return NULL; +} + +static const char *lm90_detect_ti(struct i2c_client *client, int chip_id, + int config1, int convrate) +{ + int address = client->addr; + const char *name = NULL; + + if (chip_id == 0x00 && !(config1 & 0x1b) && convrate <= 0x09) { int local_ext, conalert, chen, dfc; local_ext = i2c_smbus_read_byte_data(client, - TMP451_REG_R_LOCAL_TEMPL); + TMP451_REG_LOCAL_TEMPL); conalert = i2c_smbus_read_byte_data(client, TMP451_REG_CONALERT); chen = i2c_smbus_read_byte_data(client, TMP461_REG_CHEN); dfc = i2c_smbus_read_byte_data(client, TMP461_REG_DFC); - if ((local_ext & 0x0F) == 0x00 && - (conalert & 0xf1) == 0x01 && - (chen & 0xfc) == 0x00 && - (dfc & 0xfc) == 0x00) { + if (!(local_ext & 0x0f) && (conalert & 0xf1) == 0x01 && + (chen & 0xfc) == 0x00 && (dfc & 0xfc) == 0x00) { if (address == 0x4c && !(chen & 0x03)) name = "tmp451"; else if (address >= 0x48 && address <= 0x4f) @@ -1682,10 +2440,110 @@ static int lm90_detect(struct i2c_client *client, } } - if (!name) { /* identification failed */ + return name; +} + +/* Return 0 if detection is successful, -ENODEV otherwise */ +static int lm90_detect(struct i2c_client *client, struct i2c_board_info *info) +{ + struct i2c_adapter *adapter = client->adapter; + int man_id, chip_id, config1, convrate, lhigh; + const char *name = NULL; + int address = client->addr; + bool common_address = + (address >= 0x18 && address <= 0x1a) || + (address >= 0x29 && address <= 0x2b) || + (address >= 0x4c && address <= 0x4e); + + if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) + return -ENODEV; + + /* + * Get well defined register value for chips with neither man_id nor + * chip_id registers. + */ + lhigh = i2c_smbus_read_byte_data(client, LM90_REG_LOCAL_HIGH); + + /* detection and identification */ + man_id = i2c_smbus_read_byte_data(client, LM90_REG_MAN_ID); + chip_id = i2c_smbus_read_byte_data(client, LM90_REG_CHIP_ID); + config1 = i2c_smbus_read_byte_data(client, LM90_REG_CONFIG1); + convrate = i2c_smbus_read_byte_data(client, LM90_REG_CONVRATE); + if (man_id < 0 || chip_id < 0 || config1 < 0 || convrate < 0 || lhigh < 0) + return -ENODEV; + + /* Bail out immediately if all register report the same value */ + if (lhigh == man_id && lhigh == chip_id && lhigh == config1 && lhigh == convrate) + return -ENODEV; + + /* + * If reading man_id and chip_id both return the same value as lhigh, + * the chip may not support those registers and return the most recent read + * value. Check again with a different register and handle accordingly. + */ + if (man_id == lhigh && chip_id == lhigh) { + convrate = i2c_smbus_read_byte_data(client, LM90_REG_CONVRATE); + man_id = i2c_smbus_read_byte_data(client, LM90_REG_MAN_ID); + chip_id = i2c_smbus_read_byte_data(client, LM90_REG_CHIP_ID); + if (convrate < 0 || man_id < 0 || chip_id < 0) + return -ENODEV; + if (man_id == convrate && chip_id == convrate) + man_id = -1; + } + switch (man_id) { + case -1: /* Chip does not support man_id / chip_id */ + if (common_address && !convrate && !(config1 & 0x7f)) + name = lm90_detect_lm84(client); + break; + case 0x01: /* National Semiconductor */ + name = lm90_detect_national(client, chip_id, config1, convrate); + break; + case 0x1a: /* ON */ + name = lm90_detect_on(client, chip_id, config1, convrate); + break; + case 0x23: /* Genesys Logic */ + if (common_address && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "gl523sm"; + break; + case 0x41: /* Analog Devices */ + name = lm90_detect_analog(client, common_address, chip_id, config1, + convrate); + break; + case 0x47: /* GMT */ + name = lm90_detect_gmt(client, chip_id, config1, convrate); + break; + case 0x49: /* TI */ + name = lm90_detect_ti49(client, common_address, chip_id, config1, convrate); + break; + case 0x4d: /* Maxim Integrated */ + name = lm90_detect_maxim(client, common_address, chip_id, + config1, convrate); + break; + case 0x54: /* ON MC1066, Microchip TC1068, TCM1617 (originally TelCom) */ + if (common_address && !(config1 & 0x3f) && !(convrate & 0xf8)) + name = "mc1066"; + break; + case 0x55: /* TI */ + name = lm90_detect_ti(client, chip_id, config1, convrate); + break; + case 0x5c: /* Winbond/Nuvoton */ + name = lm90_detect_nuvoton(client, chip_id, config1, convrate); + break; + case 0xa1: /* NXP Semiconductor/Philips */ + name = lm90_detect_nxp(client, common_address, chip_id, config1, convrate); + break; + case 0xff: /* MAX1617, G767, NE1617 */ + if (common_address && chip_id == 0xff && convrate < 8) + name = lm90_detect_max1617(client, config1); + break; + default: + break; + } + + if (!name) { /* identification failed */ dev_dbg(&adapter->dev, - "Unsupported chip at 0x%02x (man_id=0x%02X, " - "chip_id=0x%02X)\n", address, man_id, chip_id); + "Unsupported chip at 0x%02x (man_id=0x%02X, chip_id=0x%02X)\n", + client->addr, man_id, chip_id); return -ENODEV; } @@ -1699,10 +2557,13 @@ static void lm90_restore_conf(void *_data) struct lm90_data *data = _data; struct i2c_client *client = data->client; + cancel_delayed_work_sync(&data->alert_work); + cancel_work_sync(&data->report_work); + /* Restore initial configuration */ - lm90_write_convrate(data, data->convrate_orig); - i2c_smbus_write_byte_data(client, LM90_REG_W_CONFIG1, - data->config_orig); + if (data->flags & LM90_HAVE_CONVRATE) + lm90_write_convrate(data, data->convrate_orig); + lm90_write_reg(client, LM90_REG_CONFIG1, data->config_orig); } static int lm90_init_client(struct i2c_client *client, struct lm90_data *data) @@ -1710,35 +2571,39 @@ static int lm90_init_client(struct i2c_client *client, struct lm90_data *data) struct device_node *np = client->dev.of_node; int config, convrate; - convrate = lm90_read_reg(client, LM90_REG_R_CONVRATE); - if (convrate < 0) - return convrate; - data->convrate_orig = convrate; + if (data->flags & LM90_HAVE_CONVRATE) { + convrate = lm90_read_reg(client, LM90_REG_CONVRATE); + if (convrate < 0) + return convrate; + data->convrate_orig = convrate; + lm90_set_convrate(client, data, 500); /* 500ms; 2Hz conversion rate */ + } else { + data->update_interval = 500; + } /* * Start the conversions. */ - config = lm90_read_reg(client, LM90_REG_R_CONFIG1); + config = lm90_read_reg(client, LM90_REG_CONFIG1); if (config < 0) return config; data->config_orig = config; data->config = config; - lm90_set_convrate(client, data, 500); /* 500ms; 2Hz conversion rate */ - /* Check Temperature Range Select */ if (data->flags & LM90_HAVE_EXTENDED_TEMP) { if (of_property_read_bool(np, "ti,extended-range-enable")) config |= 0x04; - - if (config & 0x04) - data->flags |= LM90_FLAG_ADT7461_EXT; + if (!(config & 0x04)) + data->flags &= ~LM90_HAVE_EXTENDED_TEMP; } /* * Put MAX6680/MAX8881 into extended resolution (bit 0x10, * 0.125 degree resolution) and range (0x08, extend range * to -64 degree) mode for the remote temperature sensor. + * Note that expeciments with an actual chip do not show a difference + * if bit 3 is set or not. */ if (data->kind == max6680) config |= 0x18; @@ -1753,9 +2618,9 @@ static int lm90_init_client(struct i2c_client *client, struct lm90_data *data) config |= 0x20; /* - * Select external channel 0 for max6695/96 + * Select external channel 0 for devices with three sensors */ - if (data->kind == max6696) + if (data->flags & LM90_HAVE_TEMP3) config &= ~0x08; /* @@ -1771,73 +2636,23 @@ static int lm90_init_client(struct i2c_client *client, struct lm90_data *data) return devm_add_action_or_reset(&client->dev, lm90_restore_conf, data); } -static bool lm90_is_tripped(struct i2c_client *client, u16 *status) +static bool lm90_is_tripped(struct i2c_client *client) { struct lm90_data *data = i2c_get_clientdata(client); - int st, st2 = 0; + int ret; - st = lm90_read_reg(client, LM90_REG_R_STATUS); - if (st < 0) + ret = lm90_update_alarms(data, true); + if (ret < 0) return false; - if (data->kind == max6696) { - st2 = lm90_read_reg(client, MAX6696_REG_R_STATUS2); - if (st2 < 0) - return false; - } - - *status = st | (st2 << 8); - - if ((st & 0x7f) == 0 && (st2 & 0xfe) == 0) - return false; - - if ((st & (LM90_STATUS_LLOW | LM90_STATUS_LHIGH | LM90_STATUS_LTHRM)) || - (st2 & MAX6696_STATUS2_LOT2)) - dev_dbg(&client->dev, - "temp%d out of range, please check!\n", 1); - if ((st & (LM90_STATUS_RLOW | LM90_STATUS_RHIGH | LM90_STATUS_RTHRM)) || - (st2 & MAX6696_STATUS2_ROT2)) - dev_dbg(&client->dev, - "temp%d out of range, please check!\n", 2); - if (st & LM90_STATUS_ROPEN) - dev_dbg(&client->dev, - "temp%d diode open, please check!\n", 2); - if (st2 & (MAX6696_STATUS2_R2LOW | MAX6696_STATUS2_R2HIGH | - MAX6696_STATUS2_R2THRM | MAX6696_STATUS2_R2OT2)) - dev_dbg(&client->dev, - "temp%d out of range, please check!\n", 3); - if (st2 & MAX6696_STATUS2_R2OPEN) - dev_dbg(&client->dev, - "temp%d diode open, please check!\n", 3); - - if (st & LM90_STATUS_LLOW) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_min_alarm, 0); - if (st & LM90_STATUS_RLOW) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_min_alarm, 1); - if (st2 & MAX6696_STATUS2_R2LOW) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_min_alarm, 2); - if (st & LM90_STATUS_LHIGH) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_max_alarm, 0); - if (st & LM90_STATUS_RHIGH) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_max_alarm, 1); - if (st2 & MAX6696_STATUS2_R2HIGH) - hwmon_notify_event(data->hwmon_dev, hwmon_temp, - hwmon_temp_max_alarm, 2); - - return true; + return !!data->current_alarms; } static irqreturn_t lm90_irq_thread(int irq, void *dev_id) { struct i2c_client *client = dev_id; - u16 status; - if (lm90_is_tripped(client, &status)) + if (lm90_is_tripped(client)) return IRQ_HANDLED; else return IRQ_NONE; @@ -1853,10 +2668,79 @@ static void lm90_regulator_disable(void *regulator) regulator_disable(regulator); } +static int lm90_probe_channel_from_dt(struct i2c_client *client, + struct device_node *child, + struct lm90_data *data) +{ + u32 id; + s32 val; + int err; + struct device *dev = &client->dev; + + err = of_property_read_u32(child, "reg", &id); + if (err) { + dev_err(dev, "missing reg property of %pOFn\n", child); + return err; + } + + if (id >= MAX_CHANNELS) { + dev_err(dev, "invalid reg property value %d in %pOFn\n", id, child); + return -EINVAL; + } + + err = of_property_read_string(child, "label", &data->channel_label[id]); + if (err == -ENODATA || err == -EILSEQ) { + dev_err(dev, "invalid label property in %pOFn\n", child); + return err; + } + + if (data->channel_label[id]) + data->channel_config[id] |= HWMON_T_LABEL; + + err = of_property_read_s32(child, "temperature-offset-millicelsius", &val); + if (!err) { + if (id == 0) { + dev_err(dev, "temperature-offset-millicelsius can't be set for internal channel\n"); + return -EINVAL; + } + + err = lm90_set_temp_offset(data, lm90_temp_offset_index[id], id, val); + if (err) { + dev_err(dev, "can't set temperature offset %d for channel %d (%d)\n", + val, id, err); + return err; + } + } + + return 0; +} + +static int lm90_parse_dt_channel_info(struct i2c_client *client, + struct lm90_data *data) +{ + int err; + struct device_node *child; + struct device *dev = &client->dev; + const struct device_node *np = dev->of_node; + + for_each_child_of_node(np, child) { + if (strcmp(child->name, "channel")) + continue; + + err = lm90_probe_channel_from_dt(client, child, data); + if (err) { + of_node_put(child); + return err; + } + } + + return 0; +} static const struct hwmon_ops lm90_ops = { .is_visible = lm90_is_visible, .read = lm90_read, + .read_string = lm90_read_string, .write = lm90_write, }; @@ -1891,41 +2775,63 @@ static int lm90_probe(struct i2c_client *client) data->client = client; i2c_set_clientdata(client, data); mutex_init(&data->update_lock); + INIT_DELAYED_WORK(&data->alert_work, lm90_alert_work); + INIT_WORK(&data->report_work, lm90_report_alarms); /* Set the device type */ if (client->dev.of_node) data->kind = (enum chips)of_device_get_match_data(&client->dev); else data->kind = i2c_match_id(lm90_id, client)->driver_data; - if (data->kind == adm1032) { - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE)) - client->flags &= ~I2C_CLIENT_PEC; - } /* * Different devices have different alarm bits triggering the * ALERT# output */ data->alert_alarms = lm90_params[data->kind].alert_alarms; + data->resolution = lm90_params[data->kind].resolution ? : 11; /* Set chip capabilities */ data->flags = lm90_params[data->kind].flags; + if ((data->flags & (LM90_HAVE_PEC | LM90_HAVE_PARTIAL_PEC)) && + !i2c_check_functionality(adapter, I2C_FUNC_SMBUS_PEC)) + data->flags &= ~(LM90_HAVE_PEC | LM90_HAVE_PARTIAL_PEC); + + if ((data->flags & LM90_HAVE_PARTIAL_PEC) && + !i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE)) + data->flags &= ~LM90_HAVE_PARTIAL_PEC; + data->chip.ops = &lm90_ops; data->chip.info = data->info; - data->info[0] = HWMON_CHANNEL_INFO(chip, - HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL | HWMON_C_ALARMS); + data->info[0] = &data->chip_info; + info = &data->chip_info; + info->type = hwmon_chip; + info->config = data->chip_config; + + data->chip_config[0] = HWMON_C_REGISTER_TZ; + if (data->flags & LM90_HAVE_ALARMS) + data->chip_config[0] |= HWMON_C_ALARMS; + if (data->flags & LM90_HAVE_CONVRATE) + data->chip_config[0] |= HWMON_C_UPDATE_INTERVAL; + if (data->flags & LM90_HAVE_FAULTQUEUE) + data->chip_config[0] |= HWMON_C_TEMP_SAMPLES; data->info[1] = &data->temp_info; info = &data->temp_info; info->type = hwmon_temp; info->config = data->channel_config; - data->channel_config[0] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM; - data->channel_config[1] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_FAULT; + data->channel_config[0] = HWMON_T_INPUT | HWMON_T_MAX | + HWMON_T_MAX_ALARM; + data->channel_config[1] = HWMON_T_INPUT | HWMON_T_MAX | + HWMON_T_MAX_ALARM | HWMON_T_FAULT; + + if (data->flags & LM90_HAVE_LOW) { + data->channel_config[0] |= HWMON_T_MIN | HWMON_T_MIN_ALARM; + data->channel_config[1] |= HWMON_T_MIN | HWMON_T_MIN_ALARM; + } if (data->flags & LM90_HAVE_CRIT) { data->channel_config[0] |= HWMON_T_CRIT | HWMON_T_CRIT_ALARM | HWMON_T_CRIT_HYST; @@ -1951,17 +2857,35 @@ static int lm90_probe(struct i2c_client *client) data->channel_config[2] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_CRIT_HYST | - HWMON_T_EMERGENCY | HWMON_T_EMERGENCY_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | - HWMON_T_CRIT_ALARM | HWMON_T_EMERGENCY_ALARM | - HWMON_T_FAULT; + HWMON_T_CRIT_ALARM | HWMON_T_FAULT; + if (data->flags & LM90_HAVE_EMERGENCY) { + data->channel_config[2] |= HWMON_T_EMERGENCY | + HWMON_T_EMERGENCY_HYST; + } + if (data->flags & LM90_HAVE_EMERGENCY_ALARM) + data->channel_config[2] |= HWMON_T_EMERGENCY_ALARM; + if (data->flags & LM90_HAVE_OFFSET) + data->channel_config[2] |= HWMON_T_OFFSET; } + data->faultqueue_mask = lm90_params[data->kind].faultqueue_mask; + data->faultqueue_depth = lm90_params[data->kind].faultqueue_depth; data->reg_local_ext = lm90_params[data->kind].reg_local_ext; + if (data->flags & LM90_HAVE_REMOTE_EXT) + data->reg_remote_ext = LM90_REG_REMOTE_TEMPL; + data->reg_status2 = lm90_params[data->kind].reg_status2; /* Set maximum conversion rate */ data->max_convrate = lm90_params[data->kind].max_convrate; + /* Parse device-tree channel information */ + if (client->dev.of_node) { + err = lm90_parse_dt_channel_info(client, data); + if (err) + return err; + } + /* Initialize the LM90 chip */ err = lm90_init_client(client, data); if (err < 0) { @@ -1973,7 +2897,7 @@ static int lm90_probe(struct i2c_client *client) * The 'pec' attribute is attached to the i2c device and thus created * separately. */ - if (client->flags & I2C_CLIENT_PEC) { + if (data->flags & (LM90_HAVE_PEC | LM90_HAVE_PARTIAL_PEC)) { err = device_create_file(dev, &dev_attr_pec); if (err) return err; @@ -2007,12 +2931,10 @@ static int lm90_probe(struct i2c_client *client) static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type, unsigned int flag) { - u16 alarms; - if (type != I2C_PROTOCOL_SMBUS_ALERT) return; - if (lm90_is_tripped(client, &alarms)) { + if (lm90_is_tripped(client)) { /* * Disable ALERT# output, because these chips don't implement * SMBus alert correctly; they should only hold the alert line @@ -2021,9 +2943,13 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type, struct lm90_data *data = i2c_get_clientdata(client); if ((data->flags & LM90_HAVE_BROKEN_ALERT) && - (alarms & data->alert_alarms)) { - dev_dbg(&client->dev, "Disabling ALERT#\n"); - lm90_update_confreg(data, data->config | 0x80); + (data->current_alarms & data->alert_alarms)) { + if (!(data->config & 0x80)) { + dev_dbg(&client->dev, "Disabling ALERT#\n"); + lm90_update_confreg(data, data->config | 0x80); + } + schedule_delayed_work(&data->alert_work, + max_t(int, HZ, msecs_to_jiffies(data->update_interval))); } } else { dev_dbg(&client->dev, "Everything OK\n"); diff --git a/drivers/hwmon/mcp3021.c b/drivers/hwmon/mcp3021.c index ce2780768074c..e093b19982968 100644 --- a/drivers/hwmon/mcp3021.c +++ b/drivers/hwmon/mcp3021.c @@ -7,7 +7,7 @@ * Reworked by Sven Schuchmann * DT support added by Clemens Gruber * - * This driver export the value of analog input voltage to sysfs, the + * This driver exports the value of analog input voltage to sysfs, the * voltage unit is mV. Through the sysfs interface, lm-sensors tool * can also display the input voltage. */ @@ -45,19 +45,29 @@ enum chips { * Client data (each client gets its own) */ struct mcp3021_data { - struct device *hwmon_dev; + struct i2c_client *client; u32 vdd; /* supply and reference voltage in millivolt */ u16 sar_shift; u16 sar_mask; u8 output_res; }; -static int mcp3021_read16(struct i2c_client *client) +static inline u16 volts_from_reg(struct mcp3021_data *data, u16 val) { - struct mcp3021_data *data = i2c_get_clientdata(client); - int ret; - u16 reg; + return DIV_ROUND_CLOSEST(data->vdd * val, 1 << data->output_res); +} + +static int mcp3021_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct mcp3021_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; __be16 buf; + u16 reg; + int ret; + + if (type != hwmon_in) + return -EOPNOTSUPP; ret = i2c_master_recv(client, (char *)&buf, 2); if (ret < 0) @@ -74,39 +84,46 @@ static int mcp3021_read16(struct i2c_client *client) */ reg = (reg >> data->sar_shift) & data->sar_mask; - return reg; -} + *val = volts_from_reg(data, reg); -static inline u16 volts_from_reg(struct mcp3021_data *data, u16 val) -{ - return DIV_ROUND_CLOSEST(data->vdd * val, 1 << data->output_res); + return 0; } -static ssize_t in0_input_show(struct device *dev, - struct device_attribute *attr, char *buf) +static umode_t mcp3021_is_visible(const void *_data, + enum hwmon_sensor_types type, + u32 attr, int channel) { - struct i2c_client *client = to_i2c_client(dev); - struct mcp3021_data *data = i2c_get_clientdata(client); - int reg, in_input; + if (type != hwmon_in) + return 0; - reg = mcp3021_read16(client); - if (reg < 0) - return reg; + if (attr != hwmon_in_input) + return 0; - in_input = volts_from_reg(data, reg); - - return sprintf(buf, "%d\n", in_input); + return 0444; } -static DEVICE_ATTR_RO(in0_input); +static const struct hwmon_channel_info *mcp3021_info[] = { + HWMON_CHANNEL_INFO(in, HWMON_I_INPUT), + NULL +}; + +static const struct hwmon_ops mcp3021_hwmon_ops = { + .is_visible = mcp3021_is_visible, + .read = mcp3021_read, +}; + +static const struct hwmon_chip_info mcp3021_chip_info = { + .ops = &mcp3021_hwmon_ops, + .info = mcp3021_info, +}; static const struct i2c_device_id mcp3021_id[]; static int mcp3021_probe(struct i2c_client *client) { - int err; struct mcp3021_data *data = NULL; struct device_node *np = client->dev.of_node; + struct device *hwmon_dev; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) return -ENODEV; @@ -147,34 +164,17 @@ static int mcp3021_probe(struct i2c_client *client) break; } + data->client = client; + if (data->vdd > MCP3021_VDD_REF_MAX || data->vdd < MCP3021_VDD_REF_MIN) return -EINVAL; - err = sysfs_create_file(&client->dev.kobj, &dev_attr_in0_input.attr); - if (err) - return err; - - data->hwmon_dev = hwmon_device_register(&client->dev); - if (IS_ERR(data->hwmon_dev)) { - err = PTR_ERR(data->hwmon_dev); - goto exit_remove; - } - - return 0; - -exit_remove: - sysfs_remove_file(&client->dev.kobj, &dev_attr_in0_input.attr); - return err; -} - -static int mcp3021_remove(struct i2c_client *client) -{ - struct mcp3021_data *data = i2c_get_clientdata(client); - - hwmon_device_unregister(data->hwmon_dev); - sysfs_remove_file(&client->dev.kobj, &dev_attr_in0_input.attr); - - return 0; + hwmon_dev = devm_hwmon_device_register_with_info(&client->dev, + client->name, + data, + &mcp3021_chip_info, + NULL); + return PTR_ERR_OR_ZERO(hwmon_dev); } static const struct i2c_device_id mcp3021_id[] = { @@ -199,7 +199,6 @@ static struct i2c_driver mcp3021_driver = { .of_match_table = of_match_ptr(of_mcp3021_match), }, .probe_new = mcp3021_probe, - .remove = mcp3021_remove, .id_table = mcp3021_id, }; diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c index 6d46c94018984..ab30437221ce3 100644 --- a/drivers/hwmon/nct6775-platform.c +++ b/drivers/hwmon/nct6775-platform.c @@ -1083,6 +1083,7 @@ static const char * const asus_wmi_boards[] = { "TUF GAMING B550M-PLUS", "TUF GAMING B550M-PLUS (WI-FI)", "TUF GAMING B550-PLUS", + "TUF GAMING B550-PLUS WIFI II", "TUF GAMING B550-PRO", "TUF GAMING X570-PLUS", "TUF GAMING X570-PLUS (WI-FI)", @@ -1200,10 +1201,8 @@ static int __init sensors_nct6775_platform_init(void) exit_device_put: platform_device_put(pdev[i]); exit_device_unregister: - while (--i >= 0) { - if (pdev[i]) - platform_device_unregister(pdev[i]); - } + while (i--) + platform_device_unregister(pdev[i]); exit_unregister: platform_driver_unregister(&nct6775_driver); return err; @@ -1213,10 +1212,8 @@ static void __exit sensors_nct6775_platform_exit(void) { int i; - for (i = 0; i < ARRAY_SIZE(pdev); i++) { - if (pdev[i]) - platform_device_unregister(pdev[i]); - } + for (i = 0; i < ARRAY_SIZE(pdev); i++) + platform_device_unregister(pdev[i]); platform_driver_unregister(&nct6775_driver); } diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index 157b73a3da29f..45407b12db4b4 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -729,18 +729,14 @@ static ssize_t occ_show_extended(struct device *dev, rc = sysfs_emit(buf, "%u", get_unaligned_be32(&extn->sensor_id)); } else { - rc = sysfs_emit(buf, "%02x%02x%02x%02x\n", - extn->name[0], extn->name[1], - extn->name[2], extn->name[3]); + rc = sysfs_emit(buf, "%4phN\n", extn->name); } break; case 1: rc = sysfs_emit(buf, "%02x\n", extn->flags); break; case 2: - rc = sysfs_emit(buf, "%02x%02x%02x%02x%02x%02x\n", - extn->data[0], extn->data[1], extn->data[2], - extn->data[3], extn->data[4], extn->data[5]); + rc = sysfs_emit(buf, "%6phN\n", extn->data); break; default: return -EINVAL; diff --git a/drivers/hwmon/occ/p9_sbe.c b/drivers/hwmon/occ/p9_sbe.c index a91937e28e12b..c1e0a1d96cd47 100644 --- a/drivers/hwmon/occ/p9_sbe.c +++ b/drivers/hwmon/occ/p9_sbe.c @@ -55,8 +55,7 @@ static bool p9_sbe_occ_save_ffdc(struct p9_sbe_occ *ctx, const void *resp, mutex_lock(&ctx->sbe_error_lock); if (!ctx->sbe_error) { if (resp_len > ctx->ffdc_size) { - if (ctx->ffdc) - kvfree(ctx->ffdc); + kvfree(ctx->ffdc); ctx->ffdc = kvmalloc(resp_len, GFP_KERNEL); if (!ctx->ffdc) { ctx->ffdc_len = 0; @@ -170,8 +169,7 @@ static int p9_sbe_occ_remove(struct platform_device *pdev) ctx->sbe = NULL; occ_shutdown(occ); - if (ctx->ffdc) - kvfree(ctx->ffdc); + kvfree(ctx->ffdc); return 0; } diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig index dfae76db65aef..951e4a9ff2d6b 100644 --- a/drivers/hwmon/pmbus/Kconfig +++ b/drivers/hwmon/pmbus/Kconfig @@ -181,6 +181,15 @@ config SENSORS_LM25066_REGULATOR If you say yes here you get regulator support for National Semiconductor LM25066, LM5064, and LM5066. +config SENSORS_LT7182S + tristate "Analog Devices LT7182S" + help + If you say yes here you get hardware monitoring support for Analog + Devices LT7182S. + + This driver can also be built as a module. If so, the module will + be called lt7182s. + config SENSORS_LTC2978 tristate "Linear Technologies LTC2978 and compatibles" help diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile index 4678fba5012c7..e2fe86f989650 100644 --- a/drivers/hwmon/pmbus/Makefile +++ b/drivers/hwmon/pmbus/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_SENSORS_IR38064) += ir38064.o obj-$(CONFIG_SENSORS_IRPS5401) += irps5401.o obj-$(CONFIG_SENSORS_ISL68137) += isl68137.o obj-$(CONFIG_SENSORS_LM25066) += lm25066.o +obj-$(CONFIG_SENSORS_LT7182S) += lt7182s.o obj-$(CONFIG_SENSORS_LTC2978) += ltc2978.o obj-$(CONFIG_SENSORS_LTC3815) += ltc3815.o obj-$(CONFIG_SENSORS_MAX15301) += max15301.o diff --git a/drivers/hwmon/pmbus/lt7182s.c b/drivers/hwmon/pmbus/lt7182s.c new file mode 100644 index 0000000000000..4cfe476fc92d2 --- /dev/null +++ b/drivers/hwmon/pmbus/lt7182s.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Hardware monitoring driver for Analog Devices LT7182S + * + * Copyright (c) 2022 Guenter Roeck + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "pmbus.h" + +#define LT7182S_NUM_PAGES 2 + +#define MFR_READ_EXTVCC 0xcd +#define MFR_READ_ITH 0xce +#define MFR_CONFIG_ALL_LT7182S 0xd1 +#define MFR_IOUT_PEAK 0xd7 +#define MFR_ADC_CONTROL_LT7182S 0xd8 + +#define MFR_DEBUG_TELEMETRY BIT(0) + +#define MFR_VOUT_PEAK 0xdd +#define MFR_VIN_PEAK 0xde +#define MFR_TEMPERATURE_1_PEAK 0xdf +#define MFR_CLEAR_PEAKS 0xe3 + +#define MFR_CONFIG_IEEE BIT(8) + +static int lt7182s_read_word_data(struct i2c_client *client, int page, int phase, int reg) +{ + int ret; + + switch (reg) { + case PMBUS_VIRT_READ_VMON: + if (page == 0 || page == 1) + ret = pmbus_read_word_data(client, page, phase, MFR_READ_ITH); + else + ret = pmbus_read_word_data(client, 0, phase, MFR_READ_EXTVCC); + break; + case PMBUS_VIRT_READ_IOUT_MAX: + ret = pmbus_read_word_data(client, page, phase, MFR_IOUT_PEAK); + break; + case PMBUS_VIRT_READ_VOUT_MAX: + ret = pmbus_read_word_data(client, page, phase, MFR_VOUT_PEAK); + break; + case PMBUS_VIRT_READ_VIN_MAX: + ret = pmbus_read_word_data(client, page, phase, MFR_VIN_PEAK); + break; + case PMBUS_VIRT_READ_TEMP_MAX: + ret = pmbus_read_word_data(client, page, phase, MFR_TEMPERATURE_1_PEAK); + break; + case PMBUS_VIRT_RESET_VIN_HISTORY: + ret = (page == 0) ? 0 : -ENODATA; + break; + default: + ret = -ENODATA; + break; + } + return ret; +} + +static int lt7182s_write_word_data(struct i2c_client *client, int page, int reg, u16 word) +{ + int ret; + + switch (reg) { + case PMBUS_VIRT_RESET_VIN_HISTORY: + ret = pmbus_write_byte(client, 0, MFR_CLEAR_PEAKS); + break; + default: + ret = -ENODATA; + break; + } + return ret; +} + +static struct pmbus_driver_info lt7182s_info = { + .pages = LT7182S_NUM_PAGES, + .format[PSC_VOLTAGE_IN] = linear, + .format[PSC_VOLTAGE_OUT] = linear, + .format[PSC_CURRENT_IN] = linear, + .format[PSC_CURRENT_OUT] = linear, + .format[PSC_TEMPERATURE] = linear, + .format[PSC_POWER] = linear, + .func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | + PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_POUT | + PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT | + PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP, + .func[1] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | + PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_POUT | + PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT | + PMBUS_HAVE_STATUS_INPUT, + .read_word_data = lt7182s_read_word_data, + .write_word_data = lt7182s_write_word_data, +}; + +static int lt7182s_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct pmbus_driver_info *info; + u8 buf[I2C_SMBUS_BLOCK_MAX + 1]; + int ret; + + if (!i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_READ_BYTE_DATA | + I2C_FUNC_SMBUS_READ_WORD_DATA | + I2C_FUNC_SMBUS_READ_BLOCK_DATA)) + return -ENODEV; + + ret = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf); + if (ret < 0) { + dev_err(dev, "Failed to read PMBUS_MFR_ID\n"); + return ret; + } + if (ret != 3 || strncmp(buf, "ADI", 3)) { + buf[ret] = '\0'; + dev_err(dev, "Manufacturer '%s' not supported\n", buf); + return -ENODEV; + } + + ret = i2c_smbus_read_block_data(client, PMBUS_MFR_MODEL, buf); + if (ret < 0) { + dev_err(dev, "Failed to read PMBUS_MFR_MODEL\n"); + return ret; + } + if (ret != 7 || strncmp(buf, "LT7182S", 7)) { + buf[ret] = '\0'; + dev_err(dev, "Model '%s' not supported\n", buf); + return -ENODEV; + } + + info = devm_kmemdup(dev, <7182s_info, + sizeof(struct pmbus_driver_info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + /* Set data format to IEEE754 if configured */ + ret = i2c_smbus_read_word_data(client, MFR_CONFIG_ALL_LT7182S); + if (ret < 0) + return ret; + if (ret & MFR_CONFIG_IEEE) { + info->format[PSC_VOLTAGE_IN] = ieee754; + info->format[PSC_VOLTAGE_OUT] = ieee754; + info->format[PSC_CURRENT_IN] = ieee754; + info->format[PSC_CURRENT_OUT] = ieee754; + info->format[PSC_TEMPERATURE] = ieee754; + info->format[PSC_POWER] = ieee754; + } + + /* Enable VMON output if configured */ + ret = i2c_smbus_read_byte_data(client, MFR_ADC_CONTROL_LT7182S); + if (ret < 0) + return ret; + if (ret & MFR_DEBUG_TELEMETRY) { + info->pages = 3; + info->func[0] |= PMBUS_HAVE_VMON; + info->func[1] |= PMBUS_HAVE_VMON; + info->func[2] = PMBUS_HAVE_VMON; + } + + return pmbus_do_probe(client, info); +} + +static const struct i2c_device_id lt7182s_id[] = { + { "lt7182s", 0 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, lt7182s_id); + +static const struct of_device_id __maybe_unused lt7182s_of_match[] = { + { .compatible = "adi,lt7182s" }, + {} +}; + +static struct i2c_driver lt7182s_driver = { + .driver = { + .name = "lt7182s", + .of_match_table = of_match_ptr(lt7182s_of_match), + }, + .probe_new = lt7182s_probe, + .id_table = lt7182s_id, +}; + +module_i2c_driver(lt7182s_driver); + +MODULE_AUTHOR("Guenter Roeck "); +MODULE_DESCRIPTION("PMBus driver for Analog Devices LT7182S"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(PMBUS); diff --git a/drivers/hwmon/pmbus/ltc2978.c b/drivers/hwmon/pmbus/ltc2978.c index 531aa674a9283..6d2592731ba3d 100644 --- a/drivers/hwmon/pmbus/ltc2978.c +++ b/drivers/hwmon/pmbus/ltc2978.c @@ -562,7 +562,24 @@ static const struct i2c_device_id ltc2978_id[] = { MODULE_DEVICE_TABLE(i2c, ltc2978_id); #if IS_ENABLED(CONFIG_SENSORS_LTC2978_REGULATOR) +#define LTC2978_ADC_RES 0xFFFF +#define LTC2978_N_ADC 122 +#define LTC2978_MAX_UV (LTC2978_ADC_RES * LTC2978_N_ADC) +#define LTC2978_UV_STEP 1000 +#define LTC2978_N_VOLTAGES ((LTC2978_MAX_UV / LTC2978_UV_STEP) + 1) + static const struct regulator_desc ltc2978_reg_desc[] = { + PMBUS_REGULATOR_STEP("vout", 0, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 1, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 2, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 3, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 4, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 5, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 6, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), + PMBUS_REGULATOR_STEP("vout", 7, LTC2978_N_VOLTAGES, LTC2978_UV_STEP), +}; + +static const struct regulator_desc ltc2978_reg_desc_default[] = { PMBUS_REGULATOR("vout", 0), PMBUS_REGULATOR("vout", 1), PMBUS_REGULATOR("vout", 2), @@ -839,10 +856,29 @@ static int ltc2978_probe(struct i2c_client *client) #if IS_ENABLED(CONFIG_SENSORS_LTC2978_REGULATOR) info->num_regulators = info->pages; - info->reg_desc = ltc2978_reg_desc; - if (info->num_regulators > ARRAY_SIZE(ltc2978_reg_desc)) { - dev_err(&client->dev, "num_regulators too large!"); - info->num_regulators = ARRAY_SIZE(ltc2978_reg_desc); + switch (data->id) { + case ltc2972: + case ltc2974: + case ltc2975: + case ltc2977: + case ltc2978: + case ltc2979: + case ltc2980: + case ltm2987: + info->reg_desc = ltc2978_reg_desc; + if (info->num_regulators > ARRAY_SIZE(ltc2978_reg_desc)) { + dev_warn(&client->dev, "num_regulators too large!"); + info->num_regulators = ARRAY_SIZE(ltc2978_reg_desc); + } + break; + default: + info->reg_desc = ltc2978_reg_desc_default; + if (info->num_regulators > ARRAY_SIZE(ltc2978_reg_desc_default)) { + dev_warn(&client->dev, "num_regulators too large!"); + info->num_regulators = + ARRAY_SIZE(ltc2978_reg_desc_default); + } + break; } #endif diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h index c031a9700ace9..7daaf0caf4d30 100644 --- a/drivers/hwmon/pmbus/pmbus.h +++ b/drivers/hwmon/pmbus/pmbus.h @@ -406,7 +406,7 @@ enum pmbus_sensor_classes { #define PMBUS_PHASE_VIRTUAL BIT(30) /* Phases on this page are virtual */ #define PMBUS_PAGE_VIRTUAL BIT(31) /* Page is virtual */ -enum pmbus_data_format { linear = 0, direct, vid }; +enum pmbus_data_format { linear = 0, ieee754, direct, vid }; enum vrm_version { vr11 = 0, vr12, vr13, imvp9, amd625mv }; struct pmbus_driver_info { @@ -463,8 +463,8 @@ struct pmbus_driver_info { extern const struct regulator_ops pmbus_regulator_ops; -/* Macro for filling in array of struct regulator_desc */ -#define PMBUS_REGULATOR(_name, _id) \ +/* Macros for filling in array of struct regulator_desc */ +#define PMBUS_REGULATOR_STEP(_name, _id, _voltages, _step) \ [_id] = { \ .name = (_name # _id), \ .supply_name = "vin", \ @@ -474,8 +474,12 @@ extern const struct regulator_ops pmbus_regulator_ops; .ops = &pmbus_regulator_ops, \ .type = REGULATOR_VOLTAGE, \ .owner = THIS_MODULE, \ + .n_voltages = _voltages, \ + .uV_step = _step, \ } +#define PMBUS_REGULATOR(_name, _id) PMBUS_REGULATOR_STEP(_name, _id, 0, 0) + /* Function declarations */ void pmbus_clear_cache(struct i2c_client *client); diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index 02912022853d8..f10bac8860fce 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -104,6 +104,9 @@ struct pmbus_data { s16 currpage; /* current page, -1 for unknown/unset */ s16 currphase; /* current phase, 0xff for all, -1 for unknown/unset */ + + int vout_low[PMBUS_PAGES]; /* voltage low margin */ + int vout_high[PMBUS_PAGES]; /* voltage high margin */ }; struct pmbus_debugfs_entry { @@ -441,6 +444,18 @@ int pmbus_update_byte_data(struct i2c_client *client, int page, u8 reg, } EXPORT_SYMBOL_NS_GPL(pmbus_update_byte_data, PMBUS); +static int pmbus_read_block_data(struct i2c_client *client, int page, u8 reg, + char *data_buf) +{ + int rv; + + rv = pmbus_set_page(client, page, 0xff); + if (rv < 0) + return rv; + + return i2c_smbus_read_block_data(client, reg, data_buf); +} + static struct pmbus_sensor *pmbus_find_sensor(struct pmbus_data *data, int page, int reg) { @@ -578,6 +593,22 @@ bool pmbus_check_word_register(struct i2c_client *client, int page, int reg) } EXPORT_SYMBOL_NS_GPL(pmbus_check_word_register, PMBUS); +static bool __maybe_unused pmbus_check_block_register(struct i2c_client *client, + int page, int reg) +{ + int rv; + struct pmbus_data *data = i2c_get_clientdata(client); + char data_buf[I2C_SMBUS_BLOCK_MAX + 2]; + + rv = pmbus_read_block_data(client, page, reg, data_buf); + if (rv >= 0 && !(data->flags & PMBUS_SKIP_STATUS_CHECK)) + rv = pmbus_check_status_cml(client); + if (rv < 0 && (data->flags & PMBUS_READ_STATUS_AFTER_FAILED_CHECK)) + data->read_status(client, -1); + pmbus_clear_fault_page(client, -1); + return rv >= 0; +} + const struct pmbus_driver_info *pmbus_get_driver_info(struct i2c_client *client) { struct pmbus_data *data = i2c_get_clientdata(client); @@ -611,6 +642,66 @@ static void pmbus_update_sensor_data(struct i2c_client *client, struct pmbus_sen sensor->phase, sensor->reg); } +/* + * Convert ieee754 sensor values to milli- or micro-units + * depending on sensor type. + * + * ieee754 data format: + * bit 15: sign + * bit 10..14: exponent + * bit 0..9: mantissa + * exponent=0: + * v=(−1)^signbit * 2^(−14) * 0.significantbits + * exponent=1..30: + * v=(−1)^signbit * 2^(exponent - 15) * 1.significantbits + * exponent=31: + * v=NaN + * + * Add the number mantissa bits into the calculations for simplicity. + * To do that, add '10' to the exponent. By doing that, we can just add + * 0x400 to normal values and get the expected result. + */ +static long pmbus_reg2data_ieee754(struct pmbus_data *data, + struct pmbus_sensor *sensor) +{ + int exponent; + bool sign; + long val; + + /* only support half precision for now */ + sign = sensor->data & 0x8000; + exponent = (sensor->data >> 10) & 0x1f; + val = sensor->data & 0x3ff; + + if (exponent == 0) { /* subnormal */ + exponent = -(14 + 10); + } else if (exponent == 0x1f) { /* NaN, convert to min/max */ + exponent = 0; + val = 65504; + } else { + exponent -= (15 + 10); /* normal */ + val |= 0x400; + } + + /* scale result to milli-units for all sensors except fans */ + if (sensor->class != PSC_FAN) + val = val * 1000L; + + /* scale result to micro-units for power sensors */ + if (sensor->class == PSC_POWER) + val = val * 1000L; + + if (exponent >= 0) + val <<= exponent; + else + val >>= -exponent; + + if (sign) + val = -val; + + return val; +} + /* * Convert linear sensor values to milli- or micro-units * depending on sensor type. @@ -741,6 +832,9 @@ static s64 pmbus_reg2data(struct pmbus_data *data, struct pmbus_sensor *sensor) case vid: val = pmbus_reg2data_vid(data, sensor); break; + case ieee754: + val = pmbus_reg2data_ieee754(data, sensor); + break; case linear: default: val = pmbus_reg2data_linear(data, sensor); @@ -749,8 +843,72 @@ static s64 pmbus_reg2data(struct pmbus_data *data, struct pmbus_sensor *sensor) return val; } -#define MAX_MANTISSA (1023 * 1000) -#define MIN_MANTISSA (511 * 1000) +#define MAX_IEEE_MANTISSA (0x7ff * 1000) +#define MIN_IEEE_MANTISSA (0x400 * 1000) + +static u16 pmbus_data2reg_ieee754(struct pmbus_data *data, + struct pmbus_sensor *sensor, long val) +{ + u16 exponent = (15 + 10); + long mantissa; + u16 sign = 0; + + /* simple case */ + if (val == 0) + return 0; + + if (val < 0) { + sign = 0x8000; + val = -val; + } + + /* Power is in uW. Convert to mW before converting. */ + if (sensor->class == PSC_POWER) + val = DIV_ROUND_CLOSEST(val, 1000L); + + /* + * For simplicity, convert fan data to milli-units + * before calculating the exponent. + */ + if (sensor->class == PSC_FAN) + val = val * 1000; + + /* Reduce large mantissa until it fits into 10 bit */ + while (val > MAX_IEEE_MANTISSA && exponent < 30) { + exponent++; + val >>= 1; + } + /* + * Increase small mantissa to generate valid 'normal' + * number + */ + while (val < MIN_IEEE_MANTISSA && exponent > 1) { + exponent--; + val <<= 1; + } + + /* Convert mantissa from milli-units to units */ + mantissa = DIV_ROUND_CLOSEST(val, 1000); + + /* + * Ensure that the resulting number is within range. + * Valid range is 0x400..0x7ff, where bit 10 reflects + * the implied high bit in normalized ieee754 numbers. + * Set the range to 0x400..0x7ff to reflect this. + * The upper bit is then removed by the mask against + * 0x3ff in the final assignment. + */ + if (mantissa > 0x7ff) + mantissa = 0x7ff; + else if (mantissa < 0x400) + mantissa = 0x400; + + /* Convert to sign, 5 bit exponent, 10 bit mantissa */ + return sign | (mantissa & 0x3ff) | ((exponent << 10) & 0x7c00); +} + +#define MAX_LIN_MANTISSA (1023 * 1000) +#define MIN_LIN_MANTISSA (511 * 1000) static u16 pmbus_data2reg_linear(struct pmbus_data *data, struct pmbus_sensor *sensor, s64 val) @@ -796,12 +954,12 @@ static u16 pmbus_data2reg_linear(struct pmbus_data *data, val = val * 1000LL; /* Reduce large mantissa until it fits into 10 bit */ - while (val >= MAX_MANTISSA && exponent < 15) { + while (val >= MAX_LIN_MANTISSA && exponent < 15) { exponent++; val >>= 1; } /* Increase small mantissa to improve precision */ - while (val < MIN_MANTISSA && exponent > -15) { + while (val < MIN_LIN_MANTISSA && exponent > -15) { exponent--; val <<= 1; } @@ -875,6 +1033,9 @@ static u16 pmbus_data2reg(struct pmbus_data *data, case vid: regval = pmbus_data2reg_vid(data, sensor, val); break; + case ieee754: + regval = pmbus_data2reg_ieee754(data, sensor, val); + break; case linear: default: regval = pmbus_data2reg_linear(data, sensor, val); @@ -2369,6 +2530,10 @@ static int pmbus_identify_common(struct i2c_client *client, if (data->info->format[PSC_VOLTAGE_OUT] != direct) return -ENODEV; break; + case 3: /* ieee 754 half precision */ + if (data->info->format[PSC_VOLTAGE_OUT] != ieee754) + return -ENODEV; + break; default: return -ENODEV; } @@ -2388,6 +2553,42 @@ static int pmbus_read_status_word(struct i2c_client *client, int page) return _pmbus_read_word_data(client, page, 0xff, PMBUS_STATUS_WORD); } +/* PEC attribute support */ + +static ssize_t pec_show(struct device *dev, struct device_attribute *dummy, + char *buf) +{ + struct i2c_client *client = to_i2c_client(dev); + + return sysfs_emit(buf, "%d\n", !!(client->flags & I2C_CLIENT_PEC)); +} + +static ssize_t pec_store(struct device *dev, struct device_attribute *dummy, + const char *buf, size_t count) +{ + struct i2c_client *client = to_i2c_client(dev); + bool enable; + int err; + + err = kstrtobool(buf, &enable); + if (err < 0) + return err; + + if (enable) + client->flags |= I2C_CLIENT_PEC; + else + client->flags &= ~I2C_CLIENT_PEC; + + return count; +} + +static DEVICE_ATTR_RW(pec); + +static void pmbus_remove_pec(void *dev) +{ + device_remove_file(dev, &dev_attr_pec); +} + static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data, struct pmbus_driver_info *info) { @@ -2474,6 +2675,20 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data, return ret; } + if (client->flags & I2C_CLIENT_PEC) { + /* + * If I2C_CLIENT_PEC is set here, both the I2C adapter and the + * chip support PEC. Add 'pec' attribute to client device to let + * the user control it. + */ + ret = device_create_file(dev, &dev_attr_pec); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, pmbus_remove_pec, dev); + if (ret) + return ret; + } + return 0; } @@ -2636,6 +2851,58 @@ static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned return 0; } +static int pmbus_regulator_get_low_margin(struct i2c_client *client, int page) +{ + struct pmbus_data *data = i2c_get_clientdata(client); + struct pmbus_sensor s = { + .page = page, + .class = PSC_VOLTAGE_OUT, + .convert = true, + .data = -1, + }; + + if (!data->vout_low[page]) { + if (pmbus_check_word_register(client, page, PMBUS_MFR_VOUT_MIN)) + s.data = _pmbus_read_word_data(client, page, 0xff, + PMBUS_MFR_VOUT_MIN); + if (s.data < 0) { + s.data = _pmbus_read_word_data(client, page, 0xff, + PMBUS_VOUT_MARGIN_LOW); + if (s.data < 0) + return s.data; + } + data->vout_low[page] = pmbus_reg2data(data, &s); + } + + return data->vout_low[page]; +} + +static int pmbus_regulator_get_high_margin(struct i2c_client *client, int page) +{ + struct pmbus_data *data = i2c_get_clientdata(client); + struct pmbus_sensor s = { + .page = page, + .class = PSC_VOLTAGE_OUT, + .convert = true, + .data = -1, + }; + + if (!data->vout_high[page]) { + if (pmbus_check_word_register(client, page, PMBUS_MFR_VOUT_MAX)) + s.data = _pmbus_read_word_data(client, page, 0xff, + PMBUS_MFR_VOUT_MAX); + if (s.data < 0) { + s.data = _pmbus_read_word_data(client, page, 0xff, + PMBUS_VOUT_MARGIN_HIGH); + if (s.data < 0) + return s.data; + } + data->vout_high[page] = pmbus_reg2data(data, &s); + } + + return data->vout_high[page]; +} + static int pmbus_regulator_get_voltage(struct regulator_dev *rdev) { struct device *dev = rdev_get_dev(rdev); @@ -2671,24 +2938,13 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, *selector = 0; - if (pmbus_check_word_register(client, s.page, PMBUS_MFR_VOUT_MIN)) - s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_MFR_VOUT_MIN); - if (s.data < 0) { - s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_VOUT_MARGIN_LOW); - if (s.data < 0) - return s.data; - } - low = pmbus_reg2data(data, &s); + low = pmbus_regulator_get_low_margin(client, s.page); + if (low < 0) + return low; - s.data = -1; - if (pmbus_check_word_register(client, s.page, PMBUS_MFR_VOUT_MAX)) - s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_MFR_VOUT_MAX); - if (s.data < 0) { - s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_VOUT_MARGIN_HIGH); - if (s.data < 0) - return s.data; - } - high = pmbus_reg2data(data, &s); + high = pmbus_regulator_get_high_margin(client, s.page); + if (high < 0) + return high; /* Make sure we are within margins */ if (low > val) @@ -2701,6 +2957,35 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, return _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); } +static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, + unsigned int selector) +{ + struct device *dev = rdev_get_dev(rdev); + struct i2c_client *client = to_i2c_client(dev->parent); + int val, low, high; + + if (selector >= rdev->desc->n_voltages || + selector < rdev->desc->linear_min_sel) + return -EINVAL; + + selector -= rdev->desc->linear_min_sel; + val = DIV_ROUND_CLOSEST(rdev->desc->min_uV + + (rdev->desc->uV_step * selector), 1000); /* convert to mV */ + + low = pmbus_regulator_get_low_margin(client, rdev_get_id(rdev)); + if (low < 0) + return low; + + high = pmbus_regulator_get_high_margin(client, rdev_get_id(rdev)); + if (high < 0) + return high; + + if (val >= low && val <= high) + return val * 1000; /* unit is uV */ + + return 0; +} + const struct regulator_ops pmbus_regulator_ops = { .enable = pmbus_regulator_enable, .disable = pmbus_regulator_disable, @@ -2708,6 +2993,7 @@ const struct regulator_ops pmbus_regulator_ops = { .get_error_flags = pmbus_regulator_get_error_flags, .get_voltage = pmbus_regulator_get_voltage, .set_voltage = pmbus_regulator_set_voltage, + .list_voltage = pmbus_regulator_list_voltage, }; EXPORT_SYMBOL_NS_GPL(pmbus_regulator_ops, PMBUS); @@ -2782,41 +3068,33 @@ static int pmbus_debugfs_get_status(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops_status, pmbus_debugfs_get_status, NULL, "0x%04llx\n"); -static int pmbus_debugfs_get_pec(void *data, u64 *val) -{ - struct i2c_client *client = data; - - *val = !!(client->flags & I2C_CLIENT_PEC); - - return 0; -} - -static int pmbus_debugfs_set_pec(void *data, u64 val) +static ssize_t pmbus_debugfs_mfr_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) { int rc; - struct i2c_client *client = data; - - if (!val) { - client->flags &= ~I2C_CLIENT_PEC; - return 0; - } - - if (val != 1) - return -EINVAL; + struct pmbus_debugfs_entry *entry = file->private_data; + char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; - rc = i2c_smbus_read_byte_data(client, PMBUS_CAPABILITY); + rc = pmbus_read_block_data(entry->client, entry->page, entry->reg, + data); if (rc < 0) return rc; - if (!(rc & PB_CAPABILITY_ERROR_CHECK)) - return -EOPNOTSUPP; + /* Add newline at the end of a read data */ + data[rc] = '\n'; - client->flags |= I2C_CLIENT_PEC; + /* Include newline into the length */ + rc += 1; - return 0; + return simple_read_from_buffer(buf, count, ppos, data, rc); } -DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops_pec, pmbus_debugfs_get_pec, - pmbus_debugfs_set_pec, "%llu\n"); + +static const struct file_operations pmbus_debugfs_ops_mfr = { + .llseek = noop_llseek, + .read = pmbus_debugfs_mfr_read, + .write = NULL, + .open = simple_open, +}; static void pmbus_remove_debugfs(void *data) { @@ -2846,16 +3124,80 @@ static int pmbus_init_debugfs(struct i2c_client *client, return -ENODEV; } - /* Allocate the max possible entries we need. */ + /* + * Allocate the max possible entries we need. + * 6 entries device-specific + * 10 entries page-specific + */ entries = devm_kcalloc(data->dev, - data->info->pages * 10, sizeof(*entries), + 6 + data->info->pages * 10, sizeof(*entries), GFP_KERNEL); if (!entries) return -ENOMEM; - debugfs_create_file("pec", 0664, data->debugfs, client, - &pmbus_debugfs_ops_pec); - + /* + * Add device-specific entries. + * Please note that the PMBUS standard allows all registers to be + * page-specific. + * To reduce the number of debugfs entries for devices with many pages + * assume that values of the following registers are the same for all + * pages and report values only for page 0. + */ + if (pmbus_check_block_register(client, 0, PMBUS_MFR_ID)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_ID; + debugfs_create_file("mfr_id", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + if (pmbus_check_block_register(client, 0, PMBUS_MFR_MODEL)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_MODEL; + debugfs_create_file("mfr_model", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + if (pmbus_check_block_register(client, 0, PMBUS_MFR_REVISION)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_REVISION; + debugfs_create_file("mfr_revision", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + if (pmbus_check_block_register(client, 0, PMBUS_MFR_LOCATION)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_LOCATION; + debugfs_create_file("mfr_location", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + if (pmbus_check_block_register(client, 0, PMBUS_MFR_DATE)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_DATE; + debugfs_create_file("mfr_date", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + if (pmbus_check_block_register(client, 0, PMBUS_MFR_SERIAL)) { + entries[idx].client = client; + entries[idx].page = 0; + entries[idx].reg = PMBUS_MFR_SERIAL; + debugfs_create_file("mfr_serial", 0444, data->debugfs, + &entries[idx++], + &pmbus_debugfs_ops_mfr); + } + + /* Add page specific entries */ for (i = 0; i < data->info->pages; ++i) { /* Check accessibility of status register if it's not page 0 */ if (!i || pmbus_check_status_register(client, i)) { diff --git a/drivers/hwmon/sch56xx-common.c b/drivers/hwmon/sch56xx-common.c index 3ece53adabd62..de3a0886c2f72 100644 --- a/drivers/hwmon/sch56xx-common.c +++ b/drivers/hwmon/sch56xx-common.c @@ -523,6 +523,28 @@ static int __init sch56xx_device_add(int address, const char *name) return PTR_ERR_OR_ZERO(sch56xx_pdev); } +static const struct dmi_system_id sch56xx_dmi_override_table[] __initconst = { + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU"), + DMI_MATCH(DMI_PRODUCT_NAME, "CELSIUS W380"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU"), + DMI_MATCH(DMI_PRODUCT_NAME, "ESPRIMO P710"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU"), + DMI_MATCH(DMI_PRODUCT_NAME, "ESPRIMO E9900"), + }, + }, + { } +}; + /* For autoloading only */ static const struct dmi_system_id sch56xx_dmi_table[] __initconst = { { @@ -543,16 +565,18 @@ static int __init sch56xx_init(void) if (!dmi_check_system(sch56xx_dmi_table)) return -ENODEV; - /* - * Some machines like the Esprimo P720 and Esprimo C700 have - * onboard devices named " Antiope"/" Theseus" instead of - * "Antiope"/"Theseus", so we need to check for both. - */ - if (!dmi_find_device(DMI_DEV_TYPE_OTHER, "Antiope", NULL) && - !dmi_find_device(DMI_DEV_TYPE_OTHER, " Antiope", NULL) && - !dmi_find_device(DMI_DEV_TYPE_OTHER, "Theseus", NULL) && - !dmi_find_device(DMI_DEV_TYPE_OTHER, " Theseus", NULL)) - return -ENODEV; + if (!dmi_check_system(sch56xx_dmi_override_table)) { + /* + * Some machines like the Esprimo P720 and Esprimo C700 have + * onboard devices named " Antiope"/" Theseus" instead of + * "Antiope"/"Theseus", so we need to check for both. + */ + if (!dmi_find_device(DMI_DEV_TYPE_OTHER, "Antiope", NULL) && + !dmi_find_device(DMI_DEV_TYPE_OTHER, " Antiope", NULL) && + !dmi_find_device(DMI_DEV_TYPE_OTHER, "Theseus", NULL) && + !dmi_find_device(DMI_DEV_TYPE_OTHER, " Theseus", NULL)) + return -ENODEV; + } } /* diff --git a/drivers/hwmon/sht15.c b/drivers/hwmon/sht15.c index 7f4a639597306..ae4d14257a11d 100644 --- a/drivers/hwmon/sht15.c +++ b/drivers/hwmon/sht15.c @@ -1020,25 +1020,20 @@ static int sht15_probe(struct platform_device *pdev) static int sht15_remove(struct platform_device *pdev) { struct sht15_data *data = platform_get_drvdata(pdev); + int ret; - /* - * Make sure any reads from the device are done and - * prevent new ones beginning - */ - mutex_lock(&data->read_lock); - if (sht15_soft_reset(data)) { - mutex_unlock(&data->read_lock); - return -EFAULT; - } hwmon_device_unregister(data->hwmon_dev); sysfs_remove_group(&pdev->dev.kobj, &sht15_attr_group); + + ret = sht15_soft_reset(data); + if (ret) + dev_err(&pdev->dev, "Failed to reset device (%pe)\n", ERR_PTR(ret)); + if (!IS_ERR(data->reg)) { regulator_unregister_notifier(data->reg, &data->nb); regulator_disable(data->reg); } - mutex_unlock(&data->read_lock); - return 0; } diff --git a/drivers/hwmon/tps23861.c b/drivers/hwmon/tps23861.c index 8bd6435c13e82..42762e87b0147 100644 --- a/drivers/hwmon/tps23861.c +++ b/drivers/hwmon/tps23861.c @@ -140,7 +140,8 @@ static int tps23861_read_temp(struct tps23861_data *data, long *val) static int tps23861_read_voltage(struct tps23861_data *data, int channel, long *val) { - unsigned int regval; + __le16 regval; + long raw_val; int err; if (channel < TPS23861_NUM_PORTS) { @@ -155,7 +156,8 @@ static int tps23861_read_voltage(struct tps23861_data *data, int channel, if (err < 0) return err; - *val = (FIELD_GET(VOLTAGE_CURRENT_MASK, regval) * VOLTAGE_LSB) / 1000; + raw_val = le16_to_cpu(regval); + *val = (FIELD_GET(VOLTAGE_CURRENT_MASK, raw_val) * VOLTAGE_LSB) / 1000; return 0; } @@ -163,8 +165,9 @@ static int tps23861_read_voltage(struct tps23861_data *data, int channel, static int tps23861_read_current(struct tps23861_data *data, int channel, long *val) { - unsigned int current_lsb; - unsigned int regval; + long raw_val, current_lsb; + __le16 regval; + int err; if (data->shunt_resistor == SHUNT_RESISTOR_DEFAULT) @@ -178,7 +181,8 @@ static int tps23861_read_current(struct tps23861_data *data, int channel, if (err < 0) return err; - *val = (FIELD_GET(VOLTAGE_CURRENT_MASK, regval) * current_lsb) / 1000000; + raw_val = le16_to_cpu(regval); + *val = (FIELD_GET(VOLTAGE_CURRENT_MASK, raw_val) * current_lsb) / 1000000; return 0; } -- 2.37.1 From b829145f163717b2f033d2e57f44de6c8a0ef67d Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:33:47 +0200 Subject: [PATCH 09/16] Introducing-OpenVPN-Data-Channel-Offload Signed-off-by: Peter Jung --- MAINTAINERS | 8 + drivers/net/Kconfig | 19 + drivers/net/Makefile | 1 + drivers/net/ovpn-dco/Makefile | 21 + drivers/net/ovpn-dco/addr.h | 41 + drivers/net/ovpn-dco/bind.c | 62 ++ drivers/net/ovpn-dco/bind.h | 67 ++ drivers/net/ovpn-dco/crypto.c | 154 ++++ drivers/net/ovpn-dco/crypto.h | 144 ++++ drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ drivers/net/ovpn-dco/crypto_aead.h | 27 + drivers/net/ovpn-dco/main.c | 271 +++++++ drivers/net/ovpn-dco/main.h | 32 + drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ drivers/net/ovpn-dco/netlink.h | 22 + drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ drivers/net/ovpn-dco/ovpn.h | 43 ++ drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ drivers/net/ovpn-dco/peer.h | 168 ++++ drivers/net/ovpn-dco/pktid.c | 127 ++++ drivers/net/ovpn-dco/pktid.h | 116 +++ drivers/net/ovpn-dco/proto.h | 101 +++ drivers/net/ovpn-dco/rcu.h | 21 + drivers/net/ovpn-dco/skb.h | 54 ++ drivers/net/ovpn-dco/sock.c | 134 ++++ drivers/net/ovpn-dco/sock.h | 54 ++ drivers/net/ovpn-dco/stats.c | 20 + drivers/net/ovpn-dco/stats.h | 67 ++ drivers/net/ovpn-dco/tcp.c | 326 ++++++++ drivers/net/ovpn-dco/tcp.h | 38 + drivers/net/ovpn-dco/udp.c | 343 +++++++++ drivers/net/ovpn-dco/udp.h | 25 + include/net/netlink.h | 1 + include/uapi/linux/ovpn_dco.h | 265 +++++++ include/uapi/linux/udp.h | 1 + 36 files changed, 5848 insertions(+) create mode 100644 drivers/net/ovpn-dco/Makefile create mode 100644 drivers/net/ovpn-dco/addr.h create mode 100644 drivers/net/ovpn-dco/bind.c create mode 100644 drivers/net/ovpn-dco/bind.h create mode 100644 drivers/net/ovpn-dco/crypto.c create mode 100644 drivers/net/ovpn-dco/crypto.h create mode 100644 drivers/net/ovpn-dco/crypto_aead.c create mode 100644 drivers/net/ovpn-dco/crypto_aead.h create mode 100644 drivers/net/ovpn-dco/main.c create mode 100644 drivers/net/ovpn-dco/main.h create mode 100644 drivers/net/ovpn-dco/netlink.c create mode 100644 drivers/net/ovpn-dco/netlink.h create mode 100644 drivers/net/ovpn-dco/ovpn.c create mode 100644 drivers/net/ovpn-dco/ovpn.h create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h create mode 100644 drivers/net/ovpn-dco/peer.c create mode 100644 drivers/net/ovpn-dco/peer.h create mode 100644 drivers/net/ovpn-dco/pktid.c create mode 100644 drivers/net/ovpn-dco/pktid.h create mode 100644 drivers/net/ovpn-dco/proto.h create mode 100644 drivers/net/ovpn-dco/rcu.h create mode 100644 drivers/net/ovpn-dco/skb.h create mode 100644 drivers/net/ovpn-dco/sock.c create mode 100644 drivers/net/ovpn-dco/sock.h create mode 100644 drivers/net/ovpn-dco/stats.c create mode 100644 drivers/net/ovpn-dco/stats.h create mode 100644 drivers/net/ovpn-dco/tcp.c create mode 100644 drivers/net/ovpn-dco/tcp.h create mode 100644 drivers/net/ovpn-dco/udp.c create mode 100644 drivers/net/ovpn-dco/udp.h create mode 100644 include/uapi/linux/ovpn_dco.h diff --git a/MAINTAINERS b/MAINTAINERS index d09396ec20737..3a57d70a98da1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15091,6 +15091,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git F: Documentation/filesystems/overlayfs.rst F: fs/overlayfs/ +OVPN-DCO NETWORK DRIVER +M: Antonio Quartulli +L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) +L: netdev@vger.kernel.org +S: Maintained +F: drivers/net/ovpn-dco/ +F: include/uapi/linux/ovpn_dco.h + P54 WIRELESS DRIVER M: Christian Lamparter L: linux-wireless@vger.kernel.org diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 8c1eeb5a8db83..9b20bd2319417 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -116,6 +116,25 @@ config WIREGUARD_DEBUG Say N here unless you know what you're doing. +config OVPN_DCO + tristate "OpenVPN data channel offload" + depends on NET && INET + select NET_UDP_TUNNEL + select DST_CACHE + select CRYPTO + select CRYPTO_AES + select CRYPTO_GCM + select CRYPTO_CHACHA20POLY1305 + help + This module enhances the performance of an OpenVPN connection by + allowing the user to offload the data channel processing to + kernelspace. + Connection handshake, parameters negotiation and other non-data + related mechanisms are still performed in userspace. + + The OpenVPN userspace software at version 2.6 or higher is required + to use this functionality. + config EQUALIZER tristate "EQL (serial line load balancing) support" help diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 3f1192d3c52d3..8ed151e8d2335 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ obj-$(CONFIG_IPVTAP) += ipvlan/ obj-$(CONFIG_DUMMY) += dummy.o obj-$(CONFIG_WIREGUARD) += wireguard/ +obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ obj-$(CONFIG_EQUALIZER) += eql.o obj-$(CONFIG_IFB) += ifb.o obj-$(CONFIG_MACSEC) += macsec.o diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile new file mode 100644 index 0000000000000..7efefe8f13a9c --- /dev/null +++ b/drivers/net/ovpn-dco/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# ovpn-dco -- OpenVPN data channel offload +# +# Copyright (C) 2020-2022 OpenVPN, Inc. +# +# Author: Antonio Quartulli + +obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o +ovpn-dco-y += main.o +ovpn-dco-y += bind.o +ovpn-dco-y += crypto.o +ovpn-dco-y += ovpn.o +ovpn-dco-y += peer.o +ovpn-dco-y += sock.o +ovpn-dco-y += stats.o +ovpn-dco-y += netlink.o +ovpn-dco-y += crypto_aead.o +ovpn-dco-y += pktid.o +ovpn-dco-y += tcp.o +ovpn-dco-y += udp.o diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h new file mode 100644 index 0000000000000..3d6ad0fc15af8 --- /dev/null +++ b/drivers/net/ovpn-dco/addr.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNADDR_H_ +#define _NET_OVPN_DCO_OVPNADDR_H_ + +#include "crypto.h" + +#include +#include +#include +#include + +/* our basic transport layer address */ +struct ovpn_sockaddr { + union { + struct sockaddr_in in4; + struct sockaddr_in6 in6; + }; +}; + +/* Translate skb->protocol value to AF_INET or AF_INET6 */ +static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) +{ + switch (skb->protocol) { + case htons(ETH_P_IP): + return AF_INET; + case htons(ETH_P_IPV6): + return AF_INET6; + default: + return 0; + } +} + +#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c new file mode 100644 index 0000000000000..107697ea983e3 --- /dev/null +++ b/drivers/net/ovpn-dco/bind.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2012-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "ovpn.h" +#include "bind.h" +#include "peer.h" + +#include +#include +#include +#include + +/* Given a remote sockaddr, compute the skb hash + * and get a dst_entry so we can send packets to the remote. + * Called from process context or softirq (must be indicated with + * process_context bool). + */ +struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) +{ + struct ovpn_bind *bind; + size_t sa_len; + + if (ss->ss_family == AF_INET) + sa_len = sizeof(struct sockaddr_in); + else if (ss->ss_family == AF_INET6) + sa_len = sizeof(struct sockaddr_in6); + else + return ERR_PTR(-EAFNOSUPPORT); + + bind = kzalloc(sizeof(*bind), GFP_ATOMIC); + if (unlikely(!bind)) + return ERR_PTR(-ENOMEM); + + memcpy(&bind->sa, ss, sa_len); + + return bind; +} + +static void ovpn_bind_release_rcu(struct rcu_head *head) +{ + struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); + + kfree(bind); +} + +void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) +{ + struct ovpn_bind *old; + + spin_lock_bh(&peer->lock); + old = rcu_replace_pointer(peer->bind, new, true); + spin_unlock_bh(&peer->lock); + + if (old) + call_rcu(&old->rcu, ovpn_bind_release_rcu); +} diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h new file mode 100644 index 0000000000000..a562e471acae5 --- /dev/null +++ b/drivers/net/ovpn-dco/bind.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OVPN -- OpenVPN protocol accelerator for Linux + * Copyright (C) 2012-2022 OpenVPN, Inc. + * All rights reserved. + * Author: James Yonan + */ + +#ifndef _NET_OVPN_DCO_OVPNBIND_H_ +#define _NET_OVPN_DCO_OVPNBIND_H_ + +#include "addr.h" +#include "rcu.h" + +#include +#include +#include + +struct ovpn_peer; + +struct ovpn_bind { + struct ovpn_sockaddr sa; /* remote sockaddr */ + + union { + struct in_addr ipv4; + struct in6_addr ipv6; + } local; + + struct rcu_head rcu; +}; + +static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) +{ + const unsigned short family = skb_protocol_to_family(skb); + const struct ovpn_sockaddr *sa = &bind->sa; + + if (unlikely(!bind)) + return false; + + if (unlikely(sa->in4.sin_family != family)) + return false; + + switch (family) { + case AF_INET: + if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) + return false; + + if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) + return false; + break; + case AF_INET6: + if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) + return false; + + if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) + return false; + break; + default: + return false; + } + + return true; +} + +struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); +void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); + +#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c new file mode 100644 index 0000000000000..fcc3a351ba9d6 --- /dev/null +++ b/drivers/net/ovpn-dco/crypto.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "main.h" +#include "crypto_aead.h" +#include "crypto.h" + +#include + +static void ovpn_ks_destroy_rcu(struct rcu_head *head) +{ + struct ovpn_crypto_key_slot *ks; + + ks = container_of(head, struct ovpn_crypto_key_slot, rcu); + ovpn_aead_crypto_key_slot_destroy(ks); +} + +void ovpn_crypto_key_slot_release(struct kref *kref) +{ + struct ovpn_crypto_key_slot *ks; + + ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); + call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); +} + +/* can only be invoked when all peer references have been dropped (i.e. RCU + * release routine) + */ +void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) +{ + struct ovpn_crypto_key_slot *ks; + + ks = rcu_access_pointer(cs->primary); + if (ks) { + RCU_INIT_POINTER(cs->primary, NULL); + ovpn_crypto_key_slot_put(ks); + } + + ks = rcu_access_pointer(cs->secondary); + if (ks) { + RCU_INIT_POINTER(cs->secondary, NULL); + ovpn_crypto_key_slot_put(ks); + } + + mutex_destroy(&cs->mutex); +} + +/* removes the primary key from the crypto context */ +void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) +{ + struct ovpn_crypto_key_slot *ks; + + mutex_lock(&cs->mutex); + ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); + ovpn_crypto_key_slot_put(ks); + mutex_unlock(&cs->mutex); +} + +/* Reset the ovpn_crypto_state object in a way that is atomic + * to RCU readers. + */ +int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, + const struct ovpn_peer_key_reset *pkr) + __must_hold(cs->mutex) +{ + struct ovpn_crypto_key_slot *old = NULL; + struct ovpn_crypto_key_slot *new; + + lockdep_assert_held(&cs->mutex); + + new = ovpn_aead_crypto_key_slot_new(&pkr->key); + if (IS_ERR(new)) + return PTR_ERR(new); + + switch (pkr->slot) { + case OVPN_KEY_SLOT_PRIMARY: + old = rcu_replace_pointer(cs->primary, new, + lockdep_is_held(&cs->mutex)); + break; + case OVPN_KEY_SLOT_SECONDARY: + old = rcu_replace_pointer(cs->secondary, new, + lockdep_is_held(&cs->mutex)); + break; + default: + goto free_key; + } + + if (old) + ovpn_crypto_key_slot_put(old); + + return 0; +free_key: + ovpn_crypto_key_slot_put(new); + return -EINVAL; +} + +void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, + enum ovpn_key_slot slot) +{ + struct ovpn_crypto_key_slot *ks = NULL; + + mutex_lock(&cs->mutex); + switch (slot) { + case OVPN_KEY_SLOT_PRIMARY: + ks = rcu_replace_pointer(cs->primary, NULL, + lockdep_is_held(&cs->mutex)); + break; + case OVPN_KEY_SLOT_SECONDARY: + ks = rcu_replace_pointer(cs->secondary, NULL, + lockdep_is_held(&cs->mutex)); + break; + default: + pr_warn("Invalid slot to release: %u\n", slot); + break; + } + mutex_unlock(&cs->mutex); + + if (!ks) { + pr_debug("Key slot already released: %u\n", slot); + return; + } + pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); + + ovpn_crypto_key_slot_put(ks); +} + +/* this swap is not atomic, but there will be a very short time frame where the + * old_secondary key won't be available. This should not be a big deal as most + * likely both peers are already using the new primary at this point. + */ +void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) +{ + const struct ovpn_crypto_key_slot *old_primary, *old_secondary; + + mutex_lock(&cs->mutex); + + old_secondary = rcu_dereference_protected(cs->secondary, + lockdep_is_held(&cs->mutex)); + old_primary = rcu_replace_pointer(cs->primary, old_secondary, + lockdep_is_held(&cs->mutex)); + rcu_assign_pointer(cs->secondary, old_primary); + + pr_debug("key swapped: %u <-> %u\n", + old_primary ? old_primary->key_id : 0, + old_secondary ? old_secondary->key_id : 0); + + mutex_unlock(&cs->mutex); +} diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h new file mode 100644 index 0000000000000..79f580e54a63e --- /dev/null +++ b/drivers/net/ovpn-dco/crypto.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ +#define _NET_OVPN_DCO_OVPNCRYPTO_H_ + +#include "main.h" +#include "pktid.h" + +#include +#include + +struct ovpn_peer; +struct ovpn_crypto_key_slot; + +/* info needed for both encrypt and decrypt directions */ +struct ovpn_key_direction { + const u8 *cipher_key; + size_t cipher_key_size; + const u8 *nonce_tail; /* only needed for GCM modes */ + size_t nonce_tail_size; /* only needed for GCM modes */ +}; + +/* all info for a particular symmetric key (primary or secondary) */ +struct ovpn_key_config { + enum ovpn_cipher_alg cipher_alg; + u8 key_id; + struct ovpn_key_direction encrypt; + struct ovpn_key_direction decrypt; +}; + +/* used to pass settings from netlink to the crypto engine */ +struct ovpn_peer_key_reset { + enum ovpn_key_slot slot; + struct ovpn_key_config key; +}; + +struct ovpn_crypto_key_slot { + u8 key_id; + + struct crypto_aead *encrypt; + struct crypto_aead *decrypt; + struct ovpn_nonce_tail nonce_tail_xmit; + struct ovpn_nonce_tail nonce_tail_recv; + + struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; + struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; + struct kref refcount; + struct rcu_head rcu; +}; + +struct ovpn_crypto_state { + struct ovpn_crypto_key_slot __rcu *primary; + struct ovpn_crypto_key_slot __rcu *secondary; + + /* protects primary and secondary slots */ + struct mutex mutex; +}; + +static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) +{ + return kref_get_unless_zero(&ks->refcount); +} + +static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) +{ + RCU_INIT_POINTER(cs->primary, NULL); + RCU_INIT_POINTER(cs->secondary, NULL); + mutex_init(&cs->mutex); +} + +static inline struct ovpn_crypto_key_slot * +ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) +{ + struct ovpn_crypto_key_slot *ks; + + if (unlikely(!cs)) + return NULL; + + rcu_read_lock(); + ks = rcu_dereference(cs->primary); + if (ks && ks->key_id == key_id) { + if (unlikely(!ovpn_crypto_key_slot_hold(ks))) + ks = NULL; + goto out; + } + + ks = rcu_dereference(cs->secondary); + if (ks && ks->key_id == key_id) { + if (unlikely(!ovpn_crypto_key_slot_hold(ks))) + ks = NULL; + goto out; + } + + /* when both key slots are occupied but no matching key ID is found, ks has to be reset to + * NULL to avoid carrying a stale pointer + */ + ks = NULL; +out: + rcu_read_unlock(); + + return ks; +} + +static inline struct ovpn_crypto_key_slot * +ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) +{ + struct ovpn_crypto_key_slot *ks; + + rcu_read_lock(); + ks = rcu_dereference(cs->primary); + if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) + ks = NULL; + rcu_read_unlock(); + + return ks; +} + +void ovpn_crypto_key_slot_release(struct kref *kref); + +static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) +{ + kref_put(&ks->refcount, ovpn_crypto_key_slot_release); +} + +int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, + const struct ovpn_peer_key_reset *pkr); + +void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, + enum ovpn_key_slot slot); + +void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); + +void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); + +void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); + +#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c new file mode 100644 index 0000000000000..c21bff90d7489 --- /dev/null +++ b/drivers/net/ovpn-dco/crypto_aead.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "crypto_aead.h" +#include "crypto.h" +#include "pktid.h" +#include "proto.h" +#include "skb.h" + +#include +#include +#include + +#define AUTH_TAG_SIZE 16 + +static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) +{ + return OVPN_OP_SIZE_V2 + /* OP header size */ + 4 + /* Packet ID */ + crypto_aead_authsize(ks->encrypt); /* Auth Tag */ +} + +int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) +{ + const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); + const unsigned int head_size = ovpn_aead_encap_overhead(ks); + struct scatterlist sg[MAX_SKB_FRAGS + 2]; + DECLARE_CRYPTO_WAIT(wait); + struct aead_request *req; + struct sk_buff *trailer; + u8 iv[NONCE_SIZE]; + int nfrags, ret; + u32 pktid, op; + + /* Sample AEAD header format: + * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... + * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] + * [4-byte + * IV head] + */ + + /* check that there's enough headroom in the skb for packet + * encapsulation, after adding network header and encryption overhead + */ + if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) + return -ENOBUFS; + + /* get number of skb frags and ensure that packet data is writable */ + nfrags = skb_cow_data(skb, 0, &trailer); + if (unlikely(nfrags < 0)) + return nfrags; + + if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) + return -ENOSPC; + + req = aead_request_alloc(ks->encrypt, GFP_KERNEL); + if (unlikely(!req)) + return -ENOMEM; + + /* sg table: + * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), + * 1, 2, 3, ..., n: payload, + * n+1: auth_tag (len=tag_size) + */ + sg_init_table(sg, nfrags + 2); + + /* build scatterlist to encrypt packet payload */ + ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); + if (unlikely(nfrags != ret)) { + ret = -EINVAL; + goto free_req; + } + + /* append auth_tag onto scatterlist */ + __skb_push(skb, tag_size); + sg_set_buf(sg + nfrags + 1, skb->data, tag_size); + + /* obtain packet ID, which is used both as a first + * 4 bytes of nonce and last 4 bytes of associated data. + */ + ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); + if (unlikely(ret < 0)) + goto free_req; + + /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ + ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); + + /* make space for packet id and push it to the front */ + __skb_push(skb, NONCE_WIRE_SIZE); + memcpy(skb->data, iv, NONCE_WIRE_SIZE); + + /* add packet op as head of additional data */ + op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); + __skb_push(skb, OVPN_OP_SIZE_V2); + BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); + *((__force __be32 *)skb->data) = htonl(op); + + /* AEAD Additional data */ + sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); + + /* setup async crypto operation */ + aead_request_set_tfm(req, ks->encrypt); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); + aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); + + /* encrypt it */ + ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); + if (ret < 0) + net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); + +free_req: + aead_request_free(req); + return ret; +} + +int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) +{ + const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); + struct scatterlist sg[MAX_SKB_FRAGS + 2]; + int ret, payload_len, nfrags; + u8 *sg_data, iv[NONCE_SIZE]; + unsigned int payload_offset; + DECLARE_CRYPTO_WAIT(wait); + struct aead_request *req; + struct sk_buff *trailer; + unsigned int sg_len; + __be32 *pid; + + payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; + payload_len = skb->len - payload_offset; + + /* sanity check on packet size, payload size must be >= 0 */ + if (unlikely(payload_len < 0)) + return -EINVAL; + + /* Prepare the skb data buffer to be accessed up until the auth tag. + * This is required because this area is directly mapped into the sg list. + */ + if (unlikely(!pskb_may_pull(skb, payload_offset))) + return -ENODATA; + + /* get number of skb frags and ensure that packet data is writable */ + nfrags = skb_cow_data(skb, 0, &trailer); + if (unlikely(nfrags < 0)) + return nfrags; + + if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) + return -ENOSPC; + + req = aead_request_alloc(ks->decrypt, GFP_KERNEL); + if (unlikely(!req)) + return -ENOMEM; + + /* sg table: + * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), + * 1, 2, 3, ..., n: payload, + * n+1: auth_tag (len=tag_size) + */ + sg_init_table(sg, nfrags + 2); + + /* packet op is head of additional data */ + sg_data = skb->data; + sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; + sg_set_buf(sg, sg_data, sg_len); + + /* build scatterlist to decrypt packet payload */ + ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); + if (unlikely(nfrags != ret)) { + ret = -EINVAL; + goto free_req; + } + + /* append auth_tag onto scatterlist */ + sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); + + /* copy nonce into IV buffer */ + memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); + memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, + sizeof(struct ovpn_nonce_tail)); + + /* setup async crypto operation */ + aead_request_set_tfm(req, ks->decrypt); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); + + aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); + + /* decrypt it */ + ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); + if (ret < 0) { + net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); + goto free_req; + } + + /* PID sits after the op */ + pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); + ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); + if (unlikely(ret < 0)) + goto free_req; + + /* point to encapsulated IP packet */ + __skb_pull(skb, payload_offset); + +free_req: + aead_request_free(req); + return ret; +} + +/* Initialize a struct crypto_aead object */ +struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, + const unsigned char *key, unsigned int keylen) +{ + struct crypto_aead *aead; + int ret; + + aead = crypto_alloc_aead(alg_name, 0, 0); + if (IS_ERR(aead)) { + ret = PTR_ERR(aead); + pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); + aead = NULL; + goto error; + } + + ret = crypto_aead_setkey(aead, key, keylen); + if (ret) { + pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); + goto error; + } + + ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); + if (ret) { + pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); + goto error; + } + + /* basic AEAD assumption */ + if (crypto_aead_ivsize(aead) != NONCE_SIZE) { + pr_err("%s IV size must be %d\n", title, NONCE_SIZE); + ret = -EINVAL; + goto error; + } + + pr_debug("********* Cipher %s (%s)\n", alg_name, title); + pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); + pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); + pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); + pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); + pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); + + return aead; + +error: + crypto_free_aead(aead); + return ERR_PTR(ret); +} + +void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) +{ + if (!ks) + return; + + crypto_free_aead(ks->encrypt); + crypto_free_aead(ks->decrypt); + kfree(ks); +} + +static struct ovpn_crypto_key_slot * +ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, + const unsigned char *encrypt_key, + unsigned int encrypt_keylen, + const unsigned char *decrypt_key, + unsigned int decrypt_keylen, + const unsigned char *encrypt_nonce_tail, + unsigned int encrypt_nonce_tail_len, + const unsigned char *decrypt_nonce_tail, + unsigned int decrypt_nonce_tail_len, + u16 key_id) +{ + struct ovpn_crypto_key_slot *ks = NULL; + const char *alg_name; + int ret; + + /* validate crypto alg */ + switch (alg) { + case OVPN_CIPHER_ALG_AES_GCM: + alg_name = "gcm(aes)"; + break; + case OVPN_CIPHER_ALG_CHACHA20_POLY1305: + alg_name = "rfc7539(chacha20,poly1305)"; + break; + default: + return ERR_PTR(-EOPNOTSUPP); + } + + /* build the key slot */ + ks = kmalloc(sizeof(*ks), GFP_KERNEL); + if (!ks) + return ERR_PTR(-ENOMEM); + + ks->encrypt = NULL; + ks->decrypt = NULL; + kref_init(&ks->refcount); + ks->key_id = key_id; + + ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, + encrypt_keylen); + if (IS_ERR(ks->encrypt)) { + ret = PTR_ERR(ks->encrypt); + ks->encrypt = NULL; + goto destroy_ks; + } + + ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, + decrypt_keylen); + if (IS_ERR(ks->decrypt)) { + ret = PTR_ERR(ks->decrypt); + ks->decrypt = NULL; + goto destroy_ks; + } + + if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || + sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { + ret = -EINVAL; + goto destroy_ks; + } + + memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, + sizeof(struct ovpn_nonce_tail)); + memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, + sizeof(struct ovpn_nonce_tail)); + + /* init packet ID generation/validation */ + ovpn_pktid_xmit_init(&ks->pid_xmit); + ovpn_pktid_recv_init(&ks->pid_recv); + + return ks; + +destroy_ks: + ovpn_aead_crypto_key_slot_destroy(ks); + return ERR_PTR(ret); +} + +struct ovpn_crypto_key_slot * +ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) +{ + return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, + kc->encrypt.cipher_key, + kc->encrypt.cipher_key_size, + kc->decrypt.cipher_key, + kc->decrypt.cipher_key_size, + kc->encrypt.nonce_tail, + kc->encrypt.nonce_tail_size, + kc->decrypt.nonce_tail, + kc->decrypt.nonce_tail_size, + kc->key_id); +} diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h new file mode 100644 index 0000000000000..1e3054e7d5a41 --- /dev/null +++ b/drivers/net/ovpn-dco/crypto_aead.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ +#define _NET_OVPN_DCO_OVPNAEAD_H_ + +#include "crypto.h" + +#include +#include + +struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, + const unsigned char *key, unsigned int keylen); + +int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); +int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); + +struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); +void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); + +#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c new file mode 100644 index 0000000000000..4eb90ea7a500d --- /dev/null +++ b/drivers/net/ovpn-dco/main.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + * James Yonan + */ + +#include "main.h" + +#include "ovpn.h" +#include "ovpnstruct.h" +#include "netlink.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* Driver info */ +#define DRV_NAME "ovpn-dco" +#define DRV_VERSION OVPN_DCO_VERSION +#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" +#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." + +static void ovpn_struct_free(struct net_device *net) +{ + struct ovpn_struct *ovpn = netdev_priv(net); + + security_tun_dev_free_security(ovpn->security); + free_percpu(net->tstats); + flush_workqueue(ovpn->crypto_wq); + flush_workqueue(ovpn->events_wq); + destroy_workqueue(ovpn->crypto_wq); + destroy_workqueue(ovpn->events_wq); + rcu_barrier(); +} + +/* Net device open */ +static int ovpn_net_open(struct net_device *dev) +{ + struct in_device *dev_v4 = __in_dev_get_rtnl(dev); + + if (dev_v4) { + /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } + + netif_tx_start_all_queues(dev); + return 0; +} + +/* Net device stop -- called prior to device unload */ +static int ovpn_net_stop(struct net_device *dev) +{ + netif_tx_stop_all_queues(dev); + return 0; +} + +/******************************************* + * ovpn ethtool ops + *******************************************/ + +static int ovpn_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); + cmd->base.speed = SPEED_1000; + cmd->base.duplex = DUPLEX_FULL; + cmd->base.port = PORT_TP; + cmd->base.phy_address = 0; + cmd->base.transceiver = XCVR_INTERNAL; + cmd->base.autoneg = AUTONEG_DISABLE; + + return 0; +} + +static void ovpn_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); +} + +bool ovpn_dev_is_valid(const struct net_device *dev) +{ + return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; +} + +/******************************************* + * ovpn exported methods + *******************************************/ + +static const struct net_device_ops ovpn_netdev_ops = { + .ndo_open = ovpn_net_open, + .ndo_stop = ovpn_net_stop, + .ndo_start_xmit = ovpn_net_xmit, + .ndo_get_stats64 = dev_get_tstats64, +}; + +static const struct ethtool_ops ovpn_ethtool_ops = { + .get_link_ksettings = ovpn_get_link_ksettings, + .get_drvinfo = ovpn_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_ts_info = ethtool_op_get_ts_info, +}; + +static void ovpn_setup(struct net_device *dev) +{ + /* compute the overhead considering AEAD encryption */ + const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + + netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | + NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | + NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; + + dev->ethtool_ops = &ovpn_ethtool_ops; + dev->needs_free_netdev = true; + + dev->netdev_ops = &ovpn_netdev_ops; + + dev->priv_destructor = ovpn_struct_free; + + /* Point-to-Point TUN Device */ + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->mtu = ETH_DATA_LEN - overhead; + dev->min_mtu = IPV4_MIN_MTU; + dev->max_mtu = IP_MAX_MTU - overhead; + + /* Zero header length */ + dev->type = ARPHRD_NONE; + dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; + + dev->features |= feat; + dev->hw_features |= feat; + dev->hw_enc_features |= feat; + + dev->needed_headroom = OVPN_HEAD_ROOM; + dev->needed_tailroom = OVPN_MAX_PADDING; +} + +static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { + [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, + __OVPN_MODE_AFTER_LAST - 1), +}; + +static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], + struct nlattr *data[], struct netlink_ext_ack *extack) +{ + struct ovpn_struct *ovpn = netdev_priv(dev); + int ret; + + ret = security_tun_dev_create(); + if (ret < 0) + return ret; + + ret = ovpn_struct_init(dev); + if (ret < 0) + return ret; + + ovpn->mode = OVPN_MODE_P2P; + if (data && data[IFLA_OVPN_MODE]) { + ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); + netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, + ovpn->mode); + } + + return register_netdevice(dev); +} + +static void ovpn_dellink(struct net_device *dev, struct list_head *head) +{ + struct ovpn_struct *ovpn = netdev_priv(dev); + + switch (ovpn->mode) { + case OVPN_MODE_P2P: + ovpn_peer_release_p2p(ovpn); + break; + default: + ovpn_peers_free(ovpn); + break; + } + + unregister_netdevice_queue(dev, head); +} + +/** + * ovpn_num_queues - define number of queues to allocate per device + * + * The value returned by this function is used to decide how many RX and TX + * queues to allocate when creating the netdev object + * + * Return the number of queues to allocate + */ +static unsigned int ovpn_num_queues(void) +{ + return num_online_cpus(); +} + +static struct rtnl_link_ops ovpn_link_ops __read_mostly = { + .kind = DRV_NAME, + .priv_size = sizeof(struct ovpn_struct), + .setup = ovpn_setup, + .policy = ovpn_policy, + .maxtype = IFLA_OVPN_MAX, + .newlink = ovpn_newlink, + .dellink = ovpn_dellink, + .get_num_tx_queues = ovpn_num_queues, + .get_num_rx_queues = ovpn_num_queues, +}; + +static int __init ovpn_init(void) +{ + int err = 0; + + pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); + + /* init RTNL link ops */ + err = rtnl_link_register(&ovpn_link_ops); + if (err) { + pr_err("ovpn: can't register RTNL link ops\n"); + goto err; + } + + err = ovpn_netlink_register(); + if (err) { + pr_err("ovpn: can't register netlink family\n"); + goto err_rtnl_unregister; + } + + return 0; + +err_rtnl_unregister: + rtnl_link_unregister(&ovpn_link_ops); +err: + pr_err("ovpn: initialization failed, error status=%d\n", err); + return err; +} + +static __exit void ovpn_cleanup(void) +{ + rtnl_link_unregister(&ovpn_link_ops); + ovpn_netlink_unregister(); + rcu_barrier(); /* because we use call_rcu */ +} + +module_init(ovpn_init); +module_exit(ovpn_cleanup); + +MODULE_DESCRIPTION(DRV_DESCRIPTION); +MODULE_AUTHOR(DRV_COPYRIGHT); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); +MODULE_ALIAS_RTNL_LINK(DRV_NAME); +MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h new file mode 100644 index 0000000000000..c4ef200b30f45 --- /dev/null +++ b/drivers/net/ovpn-dco/main.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_MAIN_H_ +#define _NET_OVPN_DCO_MAIN_H_ + +#include +#include +#include +#include + +#define OVPN_DCO_VERSION "2.0.0" + +struct net_device; +bool ovpn_dev_is_valid(const struct net_device *dev); + +#define SKB_HEADER_LEN \ + (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ + sizeof(struct udphdr) + NET_SKB_PAD) + +#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) +#define OVPN_MAX_PADDING 16 +#define OVPN_QUEUE_LEN 1024 +#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 + +#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c new file mode 100644 index 0000000000000..ee5c943e7db42 --- /dev/null +++ b/drivers/net/ovpn-dco/netlink.c @@ -0,0 +1,1143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#include "main.h" +#include "ovpn.h" +#include "peer.h" +#include "proto.h" +#include "netlink.h" +#include "ovpnstruct.h" +#include "udp.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** The ovpn-dco netlink family */ +static struct genl_family ovpn_netlink_family; + +enum ovpn_netlink_multicast_groups { + OVPN_MCGRP_PEERS, +}; + +static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { + [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, +}; + +/** Key direction policy. Can be used for configuring an encryption and a decryption key */ +static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { + [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), + [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), +}; + +/** CMD_NEW_KEY policy */ +static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { + [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, + [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, + __OVPN_KEY_SLOT_AFTER_LAST - 1), + [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, + [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, + [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), + [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), +}; + +/** CMD_DEL_KEY policy */ +static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { + [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, + [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, + __OVPN_KEY_SLOT_AFTER_LAST - 1), +}; + +/** CMD_SWAP_KEYS policy */ +static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { + [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, +}; + +/** CMD_NEW_PEER policy */ +static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { + [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, + [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), + [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, + [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, + [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), + [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), +}; + +/** CMD_SET_PEER policy */ +static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { + [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, + [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, + [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, +}; + +/** CMD_DEL_PEER policy */ +static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { + [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, + __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), + [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, +}; + +/** CMD_GET_PEER policy */ +static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { + [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, +}; + +/** CMD_PACKET polocy */ +static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { + [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, + [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), +}; + +/** Generic message container policy */ +static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { + [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, + [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), + [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), + [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), + [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), + [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), + [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), + [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), + [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), +}; + +static struct net_device * +ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) +{ + struct net_device *dev; + int ifindex; + + if (!attrs[OVPN_ATTR_IFINDEX]) + return ERR_PTR(-EINVAL); + + ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); + + dev = dev_get_by_index(net, ifindex); + if (!dev) + return ERR_PTR(-ENODEV); + + if (!ovpn_dev_is_valid(dev)) + goto err_put_dev; + + return dev; + +err_put_dev: + dev_put(dev); + + return ERR_PTR(-EINVAL); +} + +/** + * ovpn_pre_doit() - Prepare ovpn genl doit request + * @ops: requested netlink operation + * @skb: Netlink message with request data + * @info: receiver information + * + * Return: 0 on success or negative error number in case of failure + */ +static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, + struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct net_device *dev; + + dev = ovpn_get_dev_from_attrs(net, info->attrs); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + info->user_ptr[0] = netdev_priv(dev); + + return 0; +} + +/** + * ovpn_post_doit() - complete ovpn genl doit request + * @ops: requested netlink operation + * @skb: Netlink message with request data + * @info: receiver information + */ +static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, + struct genl_info *info) +{ + struct ovpn_struct *ovpn; + + ovpn = info->user_ptr[0]; + dev_put(ovpn->dev); +} + +static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, + enum ovpn_cipher_alg cipher, + struct ovpn_key_direction *dir) +{ + struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; + int ret; + + ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); + if (ret) + return ret; + + switch (cipher) { + case OVPN_CIPHER_ALG_AES_GCM: + case OVPN_CIPHER_ALG_CHACHA20_POLY1305: + attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; + if (!attr) + return -EINVAL; + + dir->cipher_key = nla_data(attr); + dir->cipher_key_size = nla_len(attr); + + attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; + /* These algorithms require a 96bit nonce, + * Construct it by combining 4-bytes packet id and + * 8-bytes nonce-tail from userspace + */ + if (!attr) + return -EINVAL; + + dir->nonce_tail = nla_data(attr); + dir->nonce_tail_size = nla_len(attr); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + struct ovpn_peer_key_reset pkr; + struct ovpn_peer *peer; + u32 peer_id; + int ret; + + if (!info->attrs[OVPN_ATTR_NEW_KEY]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], + NULL, info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || + !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || + !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || + !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || + !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || + !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); + pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); + pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); + + pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); + + ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], + pkr.key.cipher_alg, &pkr.key.encrypt); + if (ret < 0) + return ret; + + ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], + pkr.key.cipher_alg, &pkr.key.decrypt); + if (ret < 0) + return ret; + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) { + netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); + return -ENOENT; + } + + mutex_lock(&peer->crypto.mutex); + ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); + if (ret < 0) { + netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, + peer_id); + goto unlock; + } + + netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, + pkr.key.key_id, peer_id); +unlock: + mutex_unlock(&peer->crypto.mutex); + ovpn_peer_put(peer); + return ret; +} + +static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + enum ovpn_key_slot slot; + struct ovpn_peer *peer; + u32 peer_id; + int ret; + + if (!info->attrs[OVPN_ATTR_DEL_KEY]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, + info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); + slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) + return -ENOENT; + + ovpn_crypto_key_slot_delete(&peer->crypto, slot); + ovpn_peer_put(peer); + + return 0; +} + +static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + struct ovpn_peer *peer; + u32 peer_id; + int ret; + + if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], + NULL, info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) + return -ENOENT; + + ovpn_crypto_key_slots_swap(&peer->crypto); + ovpn_peer_put(peer); + + return 0; +} + +static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + struct sockaddr_storage *ss = NULL; + struct sockaddr_in mapped; + struct sockaddr_in6 *in6; + struct ovpn_peer *peer; + size_t sa_len, ip_len; + struct socket *sock; + u8 *local_ip = NULL; + u32 sockfd, id; + int ret; + + if (!info->attrs[OVPN_ATTR_NEW_PEER]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, + info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || + (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { + netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); + return -EINVAL; + } + + /* lookup the fd in the kernel table and extract the socket object */ + sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); + /* sockfd_lookup() increases sock's refcounter */ + sock = sockfd_lookup(sockfd, &ret); + if (!sock) { + netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, + sockfd, ret); + return -ENOTSOCK; + } + + /* Only when using UDP as transport protocol the remote endpoint must be configured + * so that ovpn-dco knows where to send packets to. + * + * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes + * over it, without the need to specify a destination. + */ + if (sock->sk->sk_protocol == IPPROTO_UDP) { + ret = -EINVAL; + + if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { + netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", + __func__); + goto sockfd_release; + } + + ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); + sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); + switch (sa_len) { + case sizeof(struct sockaddr_in): + if (ss->ss_family == AF_INET) + /* valid sockaddr */ + break; + + netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", + __func__); + goto sockfd_release; + case sizeof(struct sockaddr_in6): + if (ss->ss_family == AF_INET6) + /* valid sockaddr */ + break; + + netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", + __func__); + goto sockfd_release; + default: + netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); + goto sockfd_release; + } + + if (ss->ss_family == AF_INET6) { + in6 = (struct sockaddr_in6 *)ss; + + if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { + mapped.sin_family = AF_INET; + mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; + mapped.sin_port = in6->sin6_port; + ss = (struct sockaddr_storage *)&mapped; + } + } + + /* When using UDP we may be talking over socket bound to 0.0.0.0/::. + * In this case, if the host has multiple IPs, we need to make sure + * that outgoing traffic has as source IP the same address that the + * peer is using to reach us. + * + * Since early control packets were all forwarded to userspace, we + * need the latter to tell us what IP has to be used. + */ + if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { + ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); + local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); + + if (ip_len == sizeof(struct in_addr)) { + if (ss->ss_family != AF_INET) { + netdev_dbg(ovpn->dev, + "%s: the specified local IP is IPv4, but the peer endpoint is not\n", + __func__); + goto sockfd_release; + } + } else if (ip_len == sizeof(struct in6_addr)) { + bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & + IPV6_ADDR_MAPPED; + + if (ss->ss_family != AF_INET6 && !is_mapped) { + netdev_dbg(ovpn->dev, + "%s: the specified local IP is IPv6, but the peer endpoint is not\n", + __func__); + goto sockfd_release; + } + + if (is_mapped) + /* this is an IPv6-mapped IPv4 address, therefore extract + * the actual v4 address from the last 4 bytes + */ + local_ip += 12; + } else { + netdev_dbg(ovpn->dev, + "%s: invalid length %zu for local IP\n", __func__, + ip_len); + goto sockfd_release; + } + } + + /* sanity checks passed */ + ret = 0; + } + + id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); + peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); + if (IS_ERR(peer)) { + netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", + __func__, id, ss); + ret = PTR_ERR(peer); + goto sockfd_release; + } + + if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { + if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { + ret = -EINVAL; + goto peer_release; + } + + peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); + } + + if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { + if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { + ret = -EINVAL; + goto peer_release; + } + + memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), + sizeof(struct in6_addr)); + } + + netdev_dbg(ovpn->dev, + "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", + __func__, ss, sock->sk->sk_prot_creator->name, peer->id, + &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); + + ret = ovpn_peer_add(ovpn, peer); + if (ret < 0) { + netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", + __func__, peer->id, ret); + goto peer_release; + } + + return 0; + +peer_release: + /* release right away because peer is not really used in any context */ + ovpn_peer_release(peer); + return ret; + +sockfd_release: + sockfd_put(sock); + return ret; +} + +static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + u32 peer_id, interv, timeout; + bool keepalive_set = false; + struct ovpn_peer *peer; + int ret; + + if (!info->attrs[OVPN_ATTR_SET_PEER]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, + info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) + return -ENOENT; + + /* when setting the keepalive, both parameters have to be configured */ + if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && + attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { + keepalive_set = true; + interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); + timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); + } + + if (keepalive_set) + ovpn_peer_keepalive_set(peer, interv, timeout); + + ovpn_peer_put(peer); + return 0; +} + +static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, + u32 seq, int flags) +{ + const struct ovpn_bind *bind; + struct nlattr *attr; + void *hdr; + + hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); + if (!hdr) { + netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); + return -EMSGSIZE; + } + + attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); + if (!attr) { + netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); + goto err; + } + + if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) + goto err; + + if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) + if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), + &peer->vpn_addrs.ipv4)) + goto err; + + if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) + if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), + &peer->vpn_addrs.ipv6)) + goto err; + + if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, + peer->keepalive_interval) || + nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, + peer->keepalive_timeout)) + goto err; + + rcu_read_lock(); + bind = rcu_dereference(peer->bind); + if (bind) { + if (bind->sa.in4.sin_family == AF_INET) { + if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, + sizeof(bind->sa.in4), &bind->sa.in4) || + nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, + sizeof(bind->local.ipv4), &bind->local.ipv4)) + goto err_unlock; + } else if (bind->sa.in4.sin_family == AF_INET6) { + if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, + sizeof(bind->sa.in6), &bind->sa.in6) || + nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, + sizeof(bind->local.ipv6), &bind->local.ipv6)) + goto err_unlock; + } + } + rcu_read_unlock(); + + if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, + inet_sk(peer->sock->sock->sk)->inet_sport) || + /* RX stats */ + nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, + atomic64_read(&peer->stats.rx.bytes), + OVPN_GET_PEER_RESP_ATTR_UNSPEC) || + nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, + atomic_read(&peer->stats.rx.packets)) || + /* TX stats */ + nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, + atomic64_read(&peer->stats.tx.bytes), + OVPN_GET_PEER_RESP_ATTR_UNSPEC) || + nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, + atomic_read(&peer->stats.tx.packets))) + goto err; + + nla_nest_end(skb, attr); + genlmsg_end(skb, hdr); + + return 0; +err_unlock: + rcu_read_unlock(); +err: + genlmsg_cancel(skb, hdr); + return -EMSGSIZE; +} + +static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + struct ovpn_peer *peer; + struct sk_buff *msg; + u32 peer_id; + int ret; + + if (!info->attrs[OVPN_ATTR_GET_PEER]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, + info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) + return -ENOENT; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); + if (ret < 0) { + nlmsg_free(msg); + goto err; + } + + ret = genlmsg_reply(msg, info); +err: + ovpn_peer_put(peer); + return ret; +} + +static int ovpn_netlink_dump_done(struct netlink_callback *cb) +{ + struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; + + dev_put(ovpn->dev); + return 0; +} + +static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) +{ + struct net *netns = sock_net(cb->skb->sk); + struct nlattr **attrbuf; + struct net_device *dev; + int ret; + + attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); + if (!attrbuf) + return -ENOMEM; + + ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, + ovpn_netlink_policy, NULL); + if (ret < 0) + goto err; + + dev = ovpn_get_dev_from_attrs(netns, attrbuf); + if (IS_ERR(dev)) { + ret = PTR_ERR(dev); + goto err; + } + + cb->args[0] = (long)netdev_priv(dev); + ret = 0; +err: + kfree(attrbuf); + return ret; +} + +static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; + int ret, bkt, last_idx = cb->args[1], dumped = 0; + struct ovpn_peer *peer; + + if (!ovpn) { + ret = ovpn_netlink_dump_prepare(cb); + if (ret < 0) { + netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); + return ret; + } + + ovpn = (struct ovpn_struct *)cb->args[0]; + } + + rcu_read_lock(); + hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { + /* skip already dumped peers that were dumped by previous invocations */ + if (last_idx > 0) { + last_idx--; + continue; + } + + if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) + break; + + /* count peers being dumped during this invocation */ + dumped++; + } + rcu_read_unlock(); + + /* sum up peers dumped in this message, so that at the next invocation + * we can continue from where we left + */ + cb->args[1] += dumped; + + return skb->len; +} + +static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + struct ovpn_peer *peer; + u32 peer_id; + int ret; + + if (!info->attrs[OVPN_ATTR_DEL_PEER]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, + info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) + return -EINVAL; + + peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) + return -ENOENT; + + netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); + ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); + ovpn_peer_put(peer); + + return ret; +} + +static int ovpn_netlink_register_packet(struct sk_buff *skb, + struct genl_info *info) +{ + struct ovpn_struct *ovpn = info->user_ptr[0]; + + /* only one registered process per interface is allowed for now */ + if (ovpn->registered_nl_portid_set) { + netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); + return -EBUSY; + } + + netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); + + ovpn->registered_nl_portid = info->snd_portid; + ovpn->registered_nl_portid_set = true; + + return 0; +} + +static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) +{ + struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; + struct ovpn_struct *ovpn = info->user_ptr[0]; + const u8 *packet; + u32 peer_id; + size_t len; + u8 opcode; + int ret; + + if (!info->attrs[OVPN_ATTR_PACKET]) + return -EINVAL; + + ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], + NULL, info->extack); + if (ret) + return ret; + + if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { + netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); + return -EINVAL; + } + + peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); + + len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); + + if (len < 4 || len > ovpn->dev->mtu) { + netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", + __func__, len, ovpn->dev->mtu); + return -EINVAL; + } + + packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); + opcode = ovpn_opcode_from_byte(packet[0]); + + /* reject data packets from userspace as they could lead to IV reuse */ + if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { + netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", + __func__, opcode); + return -EINVAL; + } + + netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); + + return ovpn_send_data(ovpn, peer_id, packet, len); +} + +static const struct genl_ops ovpn_netlink_ops[] = { + { + .cmd = OVPN_CMD_NEW_PEER, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_new_peer, + }, + { + .cmd = OVPN_CMD_SET_PEER, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_set_peer, + }, + { + .cmd = OVPN_CMD_DEL_PEER, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_del_peer, + }, + { + .cmd = OVPN_CMD_GET_PEER, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_get_peer, + .dumpit = ovpn_netlink_dump_peers, + .done = ovpn_netlink_dump_done, + }, + { + .cmd = OVPN_CMD_NEW_KEY, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_new_key, + }, + { + .cmd = OVPN_CMD_DEL_KEY, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_del_key, + }, + { + .cmd = OVPN_CMD_SWAP_KEYS, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_swap_keys, + }, + { + .cmd = OVPN_CMD_REGISTER_PACKET, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_register_packet, + }, + { + .cmd = OVPN_CMD_PACKET, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .flags = GENL_ADMIN_PERM, + .doit = ovpn_netlink_packet, + }, +}; + +static struct genl_family ovpn_netlink_family __ro_after_init = { + .hdrsize = 0, + .name = OVPN_NL_NAME, + .version = 1, + .maxattr = OVPN_ATTR_MAX, + .policy = ovpn_netlink_policy, + .netnsok = true, + .pre_doit = ovpn_pre_doit, + .post_doit = ovpn_post_doit, + .module = THIS_MODULE, + .ops = ovpn_netlink_ops, + .n_ops = ARRAY_SIZE(ovpn_netlink_ops), + .mcgrps = ovpn_netlink_mcgrps, + .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), +}; + +int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) +{ + struct sk_buff *msg; + struct nlattr *attr; + void *hdr; + int ret; + + netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", + peer->ovpn->dev->name, peer->id, peer->delete_reason); + + msg = nlmsg_new(100, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, + OVPN_CMD_DEL_PEER); + if (!hdr) { + ret = -ENOBUFS; + goto err_free_msg; + } + + if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); + if (!attr) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + nla_nest_end(msg, attr); + + genlmsg_end(msg, hdr); + + genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), + msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); + + return 0; + +err_free_msg: + nlmsg_free(msg); + return ret; +} + +int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, + const u8 *buf, size_t len) +{ + struct nlattr *attr; + struct sk_buff *msg; + void *hdr; + int ret; + + if (!ovpn->registered_nl_portid_set) { + net_warn_ratelimited("%s: no userspace listener\n", __func__); + return 0; + } + + netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); + + msg = nlmsg_new(100 + len, GFP_ATOMIC); + if (!msg) + return -ENOMEM; + + hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, + OVPN_CMD_PACKET); + if (!hdr) { + ret = -ENOBUFS; + goto err_free_msg; + } + + if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + attr = nla_nest_start(msg, OVPN_ATTR_PACKET); + if (!attr) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { + ret = -EMSGSIZE; + goto err_free_msg; + } + + nla_nest_end(msg, attr); + + genlmsg_end(msg, hdr); + + return genlmsg_unicast(dev_net(ovpn->dev), msg, + ovpn->registered_nl_portid); + +err_free_msg: + nlmsg_free(msg); + return ret; +} + +static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, + void *_notify) +{ + struct netlink_notify *notify = _notify; + struct ovpn_struct *ovpn; + struct net_device *dev; + struct net *netns; + bool found = false; + + if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) + return NOTIFY_DONE; + + rcu_read_lock(); + for_each_net_rcu(netns) { + for_each_netdev_rcu(netns, dev) { + if (!ovpn_dev_is_valid(dev)) + continue; + + ovpn = netdev_priv(dev); + if (notify->portid != ovpn->registered_nl_portid) + continue; + + found = true; + netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); + ovpn->registered_nl_portid_set = false; + break; + } + } + rcu_read_unlock(); + + /* if no interface matched our purposes, pass the notification along */ + if (!found) + return NOTIFY_DONE; + + return NOTIFY_OK; +} + +static struct notifier_block ovpn_netlink_notifier = { + .notifier_call = ovpn_netlink_notify, +}; + +int ovpn_netlink_init(struct ovpn_struct *ovpn) +{ + ovpn->registered_nl_portid_set = false; + + return 0; +} + +/** + * ovpn_netlink_register() - register the ovpn genl netlink family + */ +int __init ovpn_netlink_register(void) +{ + int ret; + + ret = genl_register_family(&ovpn_netlink_family); + if (ret) + return ret; + + ret = netlink_register_notifier(&ovpn_netlink_notifier); + if (ret) + goto err; + + return 0; +err: + genl_unregister_family(&ovpn_netlink_family); + return ret; +} + +/** + * ovpn_netlink_unregister() - unregister the ovpn genl netlink family + */ +void __exit ovpn_netlink_unregister(void) +{ + netlink_unregister_notifier(&ovpn_netlink_notifier); + genl_unregister_family(&ovpn_netlink_family); +} diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h new file mode 100644 index 0000000000000..843daf052c039 --- /dev/null +++ b/drivers/net/ovpn-dco/netlink.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_NETLINK_H_ +#define _NET_OVPN_DCO_NETLINK_H_ + +struct ovpn_struct; +struct ovpn_peer; + +int ovpn_netlink_init(struct ovpn_struct *ovpn); +int ovpn_netlink_register(void); +void ovpn_netlink_unregister(void); +int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, + const u8 *buf, size_t len); +int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); + +#endif /* _NET_OVPN_DCO_NETLINK_H_ */ diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c new file mode 100644 index 0000000000000..66c019174f5e4 --- /dev/null +++ b/drivers/net/ovpn-dco/ovpn.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "main.h" +#include "bind.h" +#include "netlink.h" +#include "ovpn.h" +#include "sock.h" +#include "peer.h" +#include "stats.h" +#include "proto.h" +#include "crypto.h" +#include "crypto_aead.h" +#include "skb.h" +#include "tcp.h" +#include "udp.h" + +#include +#include + +static const unsigned char ovpn_keepalive_message[] = { + 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, + 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 +}; + +static const unsigned char ovpn_explicit_exit_notify_message[] = { + 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, + 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, + 6 // OCC_EXIT +}; + +/* Is keepalive message? + * Assumes that single byte at skb->data is defined. + */ +static bool ovpn_is_keepalive(struct sk_buff *skb) +{ + if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) + return false; + + if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) + return false; + + return !memcmp(skb->data, ovpn_keepalive_message, + sizeof(ovpn_keepalive_message)); +} + +int ovpn_struct_init(struct net_device *dev) +{ + struct ovpn_struct *ovpn = netdev_priv(dev); + int err; + + memset(ovpn, 0, sizeof(*ovpn)); + + ovpn->dev = dev; + + err = ovpn_netlink_init(ovpn); + if (err < 0) + return err; + + spin_lock_init(&ovpn->lock); + spin_lock_init(&ovpn->peers.lock); + + ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, + dev->name); + if (!ovpn->crypto_wq) + return -ENOMEM; + + ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); + if (!ovpn->events_wq) + return -ENOMEM; + + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + if (!dev->tstats) + return -ENOMEM; + + err = security_tun_dev_alloc_security(&ovpn->security); + if (err < 0) + return err; + + /* kernel -> userspace tun queue length */ + ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; + + return 0; +} + +/* Called after decrypt to write IP packet to tun netdev. + * This method is expected to manage/free skb. + */ +static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) +{ + /* packet integrity was verified on the VPN layer - no need to perform + * any additional check along the stack + */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = ~0; + + /* skb hash for transport packet no longer valid after decapsulation */ + skb_clear_hash(skb); + + /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun + * interface, based on __skb_tunnel_rx() in dst.h + */ + skb->dev = peer->ovpn->dev; + skb_set_queue_mapping(skb, 0); + skb_scrub_packet(skb, true); + + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb_probe_transport_header(skb); + skb_reset_inner_headers(skb); + + /* update per-cpu RX stats with the stored size of encrypted packet */ + + /* we are in softirq context - hence no locking nor disable preemption needed */ + dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); + + /* cause packet to be "received" by tun interface */ + napi_gro_receive(&peer->napi, skb); +} + +int ovpn_napi_poll(struct napi_struct *napi, int budget) +{ + struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); + struct sk_buff *skb; + int work_done = 0; + + if (unlikely(budget <= 0)) + return 0; + /* this function should schedule at most 'budget' number of + * packets for delivery to the tun interface. + * If in the queue we have more packets than what allowed by the + * budget, the next polling will take care of those + */ + while ((work_done < budget) && + (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { + tun_netdev_write(peer, skb); + work_done++; + } + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; +} + +static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, + struct sk_buff *skb) +{ + int ret; + + ret = skb_linearize(skb); + if (ret < 0) + return ret; + + ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); + if (ret < 0) + return ret; + + consume_skb(skb); + return 0; +} + +/* Entry point for processing an incoming packet (in skb form) + * + * Enqueue the packet and schedule RX consumer. + * Reference to peer is dropped only in case of success. + * + * Return 0 if the packet was handled (and consumed) + * Return <0 in case of error (return value is error code) + */ +int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) +{ + int ret; + + /* At this point we know the packet is from a configured peer. + * DATA_V2 packets are handled in kernel space, the rest goes to user space. + * + * Packets are sent to userspace via netlink API in order to be consistenbt across + * UDP and TCP. + */ + if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { + ret = ovpn_transport_to_userspace(ovpn, peer, skb); + if (ret < 0) + return ret; + + ovpn_peer_put(peer); + return 0; + } + + ret = ptr_ring_produce_bh(&peer->rx_ring, skb); + if (unlikely(ret < 0)) + return -ENOSPC; + + if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) + ovpn_peer_put(peer); + + return 0; +} + +static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) +{ + struct ovpn_peer *allowed_peer = NULL; + struct ovpn_crypto_key_slot *ks; + unsigned int rx_stats_size; + __be16 proto; + int ret = -1; + u8 key_id; + + /* save original packet size for stats accounting */ + OVPN_SKB_CB(skb)->rx_stats_size = skb->len; + + /* get the key slot matching the key Id in the received packet */ + key_id = ovpn_key_id_from_skb(skb); + ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); + if (unlikely(!ks)) { + net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, + peer->id, key_id); + goto drop; + } + + /* decrypt */ + ret = ovpn_aead_decrypt(ks, skb); + + ovpn_crypto_key_slot_put(ks); + + if (unlikely(ret < 0)) { + net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", + __func__, peer->id, key_id, ret); + goto drop; + } + + /* note event of authenticated packet received for keepalive */ + ovpn_peer_keepalive_recv_reset(peer); + + /* update source and destination endpoint for this peer */ + if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) + ovpn_peer_update_local_endpoint(peer, skb); + + /* increment RX stats */ + rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; + ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); + + /* check if this is a valid datapacket that has to be delivered to the + * tun interface + */ + skb_reset_network_header(skb); + proto = ovpn_ip_check_protocol(skb); + if (unlikely(!proto)) { + /* check if null packet */ + if (unlikely(!pskb_may_pull(skb, 1))) { + ret = -EINVAL; + goto drop; + } + + /* check if special OpenVPN message */ + if (ovpn_is_keepalive(skb)) { + netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", + __func__, peer->id); + /* not an error */ + consume_skb(skb); + /* inform the caller that NAPI should not be scheduled + * for this packet + */ + return -1; + } + + ret = -EPROTONOSUPPORT; + goto drop; + } + skb->protocol = proto; + + /* perform Reverse Path Filtering (RPF) */ + allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); + if (unlikely(allowed_peer != peer)) { + ret = -EPERM; + goto drop; + } + + ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); +drop: + if (likely(allowed_peer)) + ovpn_peer_put(allowed_peer); + + if (unlikely(ret < 0)) + kfree_skb(skb); + + return ret; +} + +/* pick packet from RX queue, decrypt and forward it to the tun device */ +void ovpn_decrypt_work(struct work_struct *work) +{ + struct ovpn_peer *peer; + struct sk_buff *skb; + + peer = container_of(work, struct ovpn_peer, decrypt_work); + while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { + if (likely(ovpn_decrypt_one(peer, skb) == 0)) { + /* if a packet has been enqueued for NAPI, signal + * availability to the networking stack + */ + local_bh_disable(); + napi_schedule(&peer->napi); + local_bh_enable(); + } + + /* give a chance to be rescheduled if needed */ + cond_resched(); + } + ovpn_peer_put(peer); +} + +static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) +{ + struct ovpn_crypto_key_slot *ks; + bool success = false; + int ret; + + /* get primary key to be used for encrypting data */ + ks = ovpn_crypto_key_slot_primary(&peer->crypto); + if (unlikely(!ks)) { + net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); + return false; + } + + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_help(skb))) { + net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); + goto err; + } + + ovpn_peer_stats_increment_tx(&peer->stats, skb->len); + + /* encrypt */ + ret = ovpn_aead_encrypt(ks, skb, peer->id); + if (unlikely(ret < 0)) { + /* if we ran out of IVs we must kill the key as it can't be used anymore */ + if (ret == -ERANGE) { + netdev_warn(peer->ovpn->dev, + "%s: killing primary key as we ran out of IVs\n", __func__); + ovpn_crypto_kill_primary(&peer->crypto); + goto err; + } + net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", + __func__, peer->id, ks->key_id, ret); + goto err; + } + + success = true; +err: + ovpn_crypto_key_slot_put(ks); + return success; +} + +/* Process packets in TX queue in a transport-specific way. + * + * UDP transport - encrypt and send across the tunnel. + * TCP transport - encrypt and put into TCP TX queue. + */ +void ovpn_encrypt_work(struct work_struct *work) +{ + struct sk_buff *skb, *curr, *next; + struct ovpn_peer *peer; + + peer = container_of(work, struct ovpn_peer, encrypt_work); + while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { + /* this might be a GSO-segmented skb list: process each skb + * independently + */ + skb_list_walk_safe(skb, curr, next) { + /* if one segment fails encryption, we drop the entire + * packet, because it does not really make sense to send + * only part of it at this point + */ + if (unlikely(!ovpn_encrypt_one(peer, curr))) { + kfree_skb_list(skb); + skb = NULL; + break; + } + } + + /* successful encryption */ + if (skb) { + skb_list_walk_safe(skb, curr, next) { + skb_mark_not_on_list(curr); + + switch (peer->sock->sock->sk->sk_protocol) { + case IPPROTO_UDP: + ovpn_udp_send_skb(peer->ovpn, peer, curr); + break; + case IPPROTO_TCP: + ovpn_tcp_send_skb(peer, curr); + break; + default: + /* no transport configured yet */ + consume_skb(skb); + break; + } + } + + /* note event of authenticated packet xmit for keepalive */ + ovpn_peer_keepalive_xmit_reset(peer); + } + + /* give a chance to be rescheduled if needed */ + cond_resched(); + } + ovpn_peer_put(peer); +} + +/* Put skb into TX queue and schedule a consumer */ +static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) +{ + int ret; + + if (likely(!peer)) + peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); + if (unlikely(!peer)) { + net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); + goto drop; + } + + ret = ptr_ring_produce_bh(&peer->tx_ring, skb); + if (unlikely(ret < 0)) { + net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); + goto drop; + } + + if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) + ovpn_peer_put(peer); + + return; +drop: + if (peer) + ovpn_peer_put(peer); + kfree_skb_list(skb); +} + +/* Net device start xmit + */ +netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct ovpn_struct *ovpn = netdev_priv(dev); + struct sk_buff *segments, *tmp, *curr, *next; + struct sk_buff_head skb_list; + __be16 proto; + int ret; + + /* reset netfilter state */ + nf_reset_ct(skb); + + /* verify IP header size in network packet */ + proto = ovpn_ip_check_protocol(skb); + if (unlikely(!proto || skb->protocol != proto)) { + net_dbg_ratelimited("%s: dropping malformed payload packet\n", + dev->name); + goto drop; + } + + if (skb_is_gso(skb)) { + segments = skb_gso_segment(skb, 0); + if (IS_ERR(segments)) { + ret = PTR_ERR(segments); + net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); + goto drop; + } + + consume_skb(skb); + skb = segments; + } + + /* from this moment on, "skb" might be a list */ + + __skb_queue_head_init(&skb_list); + skb_list_walk_safe(skb, curr, next) { + skb_mark_not_on_list(curr); + + tmp = skb_share_check(curr, GFP_ATOMIC); + if (unlikely(!tmp)) { + kfree_skb_list(next); + net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); + goto drop_list; + } + + __skb_queue_tail(&skb_list, tmp); + } + skb_list.prev->next = NULL; + + ovpn_queue_skb(ovpn, skb_list.next, NULL); + + return NETDEV_TX_OK; + +drop_list: + skb_queue_walk_safe(&skb_list, curr, next) + kfree_skb(curr); +drop: + skb_tx_error(skb); + kfree_skb_list(skb); + return NET_XMIT_DROP; +} + +/* Encrypt and transmit a special message to peer, such as keepalive + * or explicit-exit-notify. Called from softirq context. + * Assumes that caller holds a reference to peer. + */ +static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, + const unsigned int len) +{ + struct ovpn_struct *ovpn; + struct sk_buff *skb; + + ovpn = peer->ovpn; + if (unlikely(!ovpn)) + return; + + skb = alloc_skb(256 + len, GFP_ATOMIC); + if (unlikely(!skb)) + return; + + skb_reserve(skb, 128); + skb->priority = TC_PRIO_BESTEFFORT; + memcpy(__skb_put(skb, len), data, len); + + /* increase reference counter when passing peer to sending queue */ + if (!ovpn_peer_hold(peer)) { + netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", + __func__); + kfree_skb(skb); + return; + } + + ovpn_queue_skb(ovpn, skb, peer); +} + +void ovpn_keepalive_xmit(struct ovpn_peer *peer) +{ + ovpn_xmit_special(peer, ovpn_keepalive_message, + sizeof(ovpn_keepalive_message)); +} + +/* Transmit explicit exit notification. + * Called from process context. + */ +void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) +{ + ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, + sizeof(ovpn_explicit_exit_notify_message)); +} + +/* Copy buffer into skb and send it across the tunnel. + * + * For UDP transport: just sent the skb to peer + * For TCP transport: put skb into TX queue + */ +int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) +{ + u16 skb_len = SKB_HEADER_LEN + len; + struct ovpn_peer *peer; + struct sk_buff *skb; + bool tcp = false; + int ret = 0; + + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (unlikely(!peer)) { + netdev_dbg(ovpn->dev, "no peer to send data to\n"); + return -EHOSTUNREACH; + } + + if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { + skb_len += sizeof(u16); + tcp = true; + } + + skb = alloc_skb(skb_len, GFP_ATOMIC); + if (unlikely(!skb)) { + ret = -ENOMEM; + goto out; + } + + skb_reserve(skb, SKB_HEADER_LEN); + skb_put_data(skb, data, len); + + /* prepend TCP packet with size, as required by OpenVPN protocol */ + if (tcp) { + *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); + ovpn_queue_tcp_skb(peer, skb); + } else { + ovpn_udp_send_skb(ovpn, peer, skb); + } +out: + ovpn_peer_put(peer); + return ret; +} diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h new file mode 100644 index 0000000000000..9364fd5dd309e --- /dev/null +++ b/drivers/net/ovpn-dco/ovpn.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPN_H_ +#define _NET_OVPN_DCO_OVPN_H_ + +#include "main.h" +#include "peer.h" +#include "sock.h" +#include "ovpnstruct.h" + +#include +#include +#include + +struct ovpn_struct; +struct net_device; + +int ovpn_struct_init(struct net_device *dev); + +u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev); + +void ovpn_keepalive_xmit(struct ovpn_peer *peer); +void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); + +netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); + +int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); + +void ovpn_encrypt_work(struct work_struct *work); +void ovpn_decrypt_work(struct work_struct *work); +int ovpn_napi_poll(struct napi_struct *napi, int budget); + +int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); + +#endif /* _NET_OVPN_DCO_OVPN_H_ */ diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h new file mode 100644 index 0000000000000..f9bc559609cd2 --- /dev/null +++ b/drivers/net/ovpn-dco/ovpnstruct.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ +#define _NET_OVPN_DCO_OVPNSTRUCT_H_ + +#include "peer.h" + +#include +#include +#include + +/* Our state per ovpn interface */ +struct ovpn_struct { + /* read-mostly objects in this section */ + struct net_device *dev; + + /* device operation mode (i.e. P2P, MP) */ + enum ovpn_mode mode; + + /* protect writing to the ovpn_struct object */ + spinlock_t lock; + + /* workqueue used to schedule crypto work that may sleep */ + struct workqueue_struct *crypto_wq; + /* workqueue used to schedule generic event that may sleep or that need + * to be performed out of softirq context + */ + struct workqueue_struct *events_wq; + + /* list of known peers */ + struct { + DECLARE_HASHTABLE(by_id, 12); + DECLARE_HASHTABLE(by_transp_addr, 12); + DECLARE_HASHTABLE(by_vpn_addr, 12); + /* protects write access to any of the hashtables above */ + spinlock_t lock; + } peers; + + /* for p2p mode */ + struct ovpn_peer __rcu *peer; + + unsigned int max_tun_queue_len; + + netdev_features_t set_features; + + void *security; + + u32 registered_nl_portid; + bool registered_nl_portid_set; +}; + +#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c new file mode 100644 index 0000000000000..87d3f1b34c4dc --- /dev/null +++ b/drivers/net/ovpn-dco/peer.c @@ -0,0 +1,906 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "ovpn.h" +#include "bind.h" +#include "crypto.h" +#include "peer.h" +#include "netlink.h" +#include "tcp.h" + +#include +#include +#include +#include +#include + +static void ovpn_peer_ping(struct timer_list *t) +{ + struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); + + netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); + ovpn_keepalive_xmit(peer); +} + +static void ovpn_peer_expire(struct timer_list *t) +{ + struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); + + netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); + ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); +} + +/* Construct a new peer */ +static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) +{ + struct ovpn_peer *peer; + int ret; + + /* alloc and init peer object */ + peer = kzalloc(sizeof(*peer), GFP_KERNEL); + if (!peer) + return ERR_PTR(-ENOMEM); + + peer->id = id; + peer->halt = false; + peer->ovpn = ovpn; + + peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); + peer->vpn_addrs.ipv6 = in6addr_any; + + RCU_INIT_POINTER(peer->bind, NULL); + ovpn_crypto_state_init(&peer->crypto); + spin_lock_init(&peer->lock); + kref_init(&peer->refcount); + ovpn_peer_stats_init(&peer->stats); + + INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); + INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); + + ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); + if (ret < 0) { + netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); + goto err; + } + + ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); + if (ret < 0) { + netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); + goto err_dst_cache; + } + + ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); + if (ret < 0) { + netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); + goto err_tx_ring; + } + + ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); + if (ret < 0) { + netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); + goto err_rx_ring; + } + + /* configure and start NAPI */ + netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, + NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); + + dev_hold(ovpn->dev); + + timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); + timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); + + return peer; +err_rx_ring: + ptr_ring_cleanup(&peer->rx_ring, NULL); +err_tx_ring: + ptr_ring_cleanup(&peer->tx_ring, NULL); +err_dst_cache: + dst_cache_destroy(&peer->dst_cache); +err: + kfree(peer); + return ERR_PTR(ret); +} + +/* Reset the ovpn_sockaddr associated with a peer */ +static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, + const u8 *local_ip) +{ + struct ovpn_bind *bind; + size_t ip_len; + + /* create new ovpn_bind object */ + bind = ovpn_bind_from_sockaddr(ss); + if (IS_ERR(bind)) + return PTR_ERR(bind); + + if (local_ip) { + if (ss->ss_family == AF_INET) { + ip_len = sizeof(struct in_addr); + } else if (ss->ss_family == AF_INET6) { + ip_len = sizeof(struct in6_addr); + } else { + netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", + __func__); + kfree(bind); + return -EINVAL; + } + + memcpy(&bind->local, local_ip, ip_len); + } + + /* set binding */ + ovpn_bind_reset(peer, bind); + + return 0; +} + +void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) +{ + struct sockaddr_storage ss; + const u8 *local_ip = NULL; + struct sockaddr_in6 *sa6; + struct sockaddr_in *sa; + struct ovpn_bind *bind; + sa_family_t family; + + rcu_read_lock(); + bind = rcu_dereference(peer->bind); + if (unlikely(!bind)) + goto unlock; + + if (likely(ovpn_bind_skb_src_match(bind, skb))) + goto unlock; + + family = skb_protocol_to_family(skb); + + if (bind->sa.in4.sin_family == family) + local_ip = (u8 *)&bind->local; + + switch (family) { + case AF_INET: + sa = (struct sockaddr_in *)&ss; + sa->sin_family = AF_INET; + sa->sin_addr.s_addr = ip_hdr(skb)->saddr; + sa->sin_port = udp_hdr(skb)->source; + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)&ss; + sa6->sin6_family = AF_INET6; + sa6->sin6_addr = ipv6_hdr(skb)->saddr; + sa6->sin6_port = udp_hdr(skb)->source; + sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); + break; + default: + goto unlock; + } + + netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); + ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); +unlock: + rcu_read_unlock(); +} + +static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) +{ + del_timer_sync(&peer->keepalive_xmit); + del_timer_sync(&peer->keepalive_recv); +} + +static void ovpn_peer_free(struct ovpn_peer *peer) +{ + ovpn_bind_reset(peer, NULL); + ovpn_peer_timer_delete_all(peer); + + WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); + ptr_ring_cleanup(&peer->tx_ring, NULL); + WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); + ptr_ring_cleanup(&peer->rx_ring, NULL); + WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); + ptr_ring_cleanup(&peer->netif_rx_ring, NULL); + + dst_cache_destroy(&peer->dst_cache); + + dev_put(peer->ovpn->dev); + + kfree(peer); +} + +static void ovpn_peer_release_rcu(struct rcu_head *head) +{ + struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); + + ovpn_crypto_state_release(&peer->crypto); + ovpn_peer_free(peer); +} + +void ovpn_peer_release(struct ovpn_peer *peer) +{ + napi_disable(&peer->napi); + netif_napi_del(&peer->napi); + + if (peer->sock) + ovpn_socket_put(peer->sock); + + call_rcu(&peer->rcu, ovpn_peer_release_rcu); +} + +static void ovpn_peer_delete_work(struct work_struct *work) +{ + struct ovpn_peer *peer = container_of(work, struct ovpn_peer, + delete_work); + ovpn_peer_release(peer); + ovpn_netlink_notify_del_peer(peer); +} + +/* Use with kref_put calls, when releasing refcount + * on ovpn_peer objects. This method should only + * be called from process context with config_mutex held. + */ +void ovpn_peer_release_kref(struct kref *kref) +{ + struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); + + INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); + queue_work(peer->ovpn->events_wq, &peer->delete_work); +} + +struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, + struct socket *sock, u32 id, uint8_t *local_ip) +{ + struct ovpn_peer *peer; + int ret; + + /* create new peer */ + peer = ovpn_peer_create(ovpn, id); + if (IS_ERR(peer)) + return peer; + + if (sock->sk->sk_protocol == IPPROTO_UDP) { + /* a UDP peer must have a remote endpoint */ + if (!sa) { + ovpn_peer_release(peer); + return ERR_PTR(-EINVAL); + } + + /* set peer sockaddr */ + ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); + if (ret < 0) { + ovpn_peer_release(peer); + return ERR_PTR(ret); + } + } + + peer->sock = ovpn_socket_new(sock, peer); + if (IS_ERR(peer->sock)) { + peer->sock = NULL; + ovpn_peer_release(peer); + return ERR_PTR(-ENOTSOCK); + } + + /* schedule initial TCP RX work only after having assigned peer->sock */ + if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) + queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); + + return peer; +} + +/* Configure keepalive parameters */ +void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) +{ + u32 delta; + + netdev_dbg(peer->ovpn->dev, + "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, + peer->id, interval, timeout); + + peer->keepalive_interval = interval; + if (interval > 0) { + delta = msecs_to_jiffies(interval * MSEC_PER_SEC); + mod_timer(&peer->keepalive_xmit, jiffies + delta); + } else { + del_timer(&peer->keepalive_xmit); + } + + peer->keepalive_timeout = timeout; + if (timeout) { + delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); + mod_timer(&peer->keepalive_recv, jiffies + delta); + } else { + del_timer(&peer->keepalive_recv); + } +} + +#define ovpn_peer_index(_tbl, _key, _key_len) \ + (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ + +static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) +{ + struct ovpn_peer *tmp, *peer = NULL; + + rcu_read_lock(); + hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { + if (*addr != tmp->vpn_addrs.ipv4.s_addr) + continue; + + if (!ovpn_peer_hold(tmp)) + continue; + + peer = tmp; + break; + } + rcu_read_unlock(); + + return peer; +} + +static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) +{ + struct ovpn_peer *tmp, *peer = NULL; + int i; + + rcu_read_lock(); + hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { + for (i = 0; i < 4; i++) { + if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) + continue; + } + + if (!ovpn_peer_hold(tmp)) + continue; + + peer = tmp; + break; + } + rcu_read_unlock(); + + return peer; +} + +/** + * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination + * + * Looks up in the IPv4 system routing table the IO of the nexthop to be used + * to reach the destination passed as argument. IF no nexthop can be found, the + * destination itself is returned as it probably has to be used as nexthop. + * + * @ovpn: the private data representing the current VPN session + * @dst: the destination to be looked up + * + * Return the IP of the next hop if found or the dst itself otherwise + */ +static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) +{ + struct rtable *rt; + struct flowi4 fl = { + .daddr = dst + }; + + rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); + if (IS_ERR(rt)) { + net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); + /* if we end up here this packet is probably going to be thrown away later */ + return dst; + } + + if (!rt->rt_uses_gateway) + goto out; + + dst = rt->rt_gw4; +out: + ip_rt_put(rt); + return dst; +} + +/** + * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination + * + * Looks up in the IPv6 system routing table the IO of the nexthop to be used + * to reach the destination passed as argument. IF no nexthop can be found, the + * destination itself is returned as it probably has to be used as nexthop. + * + * @ovpn: the private data representing the current VPN session + * @dst: the destination to be looked up + * + * Return the IP of the next hop if found or the dst itself otherwise + */ +static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct rt6_info *rt; + struct flowi6 fl = { + .daddr = dst, + }; + + rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, + NULL); + if (IS_ERR(rt)) { + net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); + /* if we end up here this packet is probably going to be thrown away later */ + return dst; + } + + if (!(rt->rt6i_flags & RTF_GATEWAY)) + goto out; + + dst = rt->rt6i_gateway; +out: + dst_release((struct dst_entry *)rt); +#endif + return dst; +} + +/** + * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to + * + * This function takes a tunnel packet and looks up the peer to send it to + * after encapsulation. The skb is expected to be the in-tunnel packet, without + * any OpenVPN related header. + * + * Assume that the IP header is accessible in the skb data. + * + * @ovpn: the private data representing the current VPN session + * @skb: the skb to extract the destination address from + * + * Return the peer if found or NULL otherwise. + */ +struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, + bool use_src) +{ + struct ovpn_peer *tmp, *peer = NULL; + struct hlist_head *head; + struct rt6_info *rt6i = NULL; + struct rtable *rt = NULL; + sa_family_t sa_fam; + struct in6_addr addr6; + __be32 addr4; + u32 index; + + /* in P2P mode, no matter the destination, packets are always sent to the single peer + * listening on the other side + */ + if (ovpn->mode == OVPN_MODE_P2P) { + rcu_read_lock(); + tmp = rcu_dereference(ovpn->peer); + if (likely(tmp && ovpn_peer_hold(tmp))) + peer = tmp; + rcu_read_unlock(); + return peer; + } + + sa_fam = skb_protocol_to_family(skb); + + switch (sa_fam) { + case AF_INET: + if (use_src) + addr4 = ip_hdr(skb)->saddr; + else + addr4 = ip_hdr(skb)->daddr; + addr4 = ovpn_nexthop4(ovpn, addr4); + + index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); + head = &ovpn->peers.by_vpn_addr[index]; + + peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); + break; + case AF_INET6: + if (use_src) + addr6 = ipv6_hdr(skb)->saddr; + else + addr6 = ipv6_hdr(skb)->daddr; + addr6 = ovpn_nexthop6(ovpn, addr6); + + index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); + head = &ovpn->peers.by_vpn_addr[index]; + + peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); + break; + } + + if (rt) + ip_rt_put(rt); + if (rt6i) + dst_release((struct dst_entry *)rt6i); + + return peer; +} + +static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) +{ + struct ovpn_bind *bind = rcu_dereference(peer->bind); + struct sockaddr_in6 *sa6; + struct sockaddr_in *sa4; + + if (unlikely(!bind)) + return false; + + if (ss->ss_family != bind->sa.in4.sin_family) + return false; + + switch (ss->ss_family) { + case AF_INET: + sa4 = (struct sockaddr_in *)ss; + if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) + return false; + if (sa4->sin_port != bind->sa.in4.sin_port) + return false; + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)ss; + if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) + return false; + if (sa6->sin6_port != bind->sa.in6.sin6_port) + return false; + break; + default: + return false; + } + + return true; +} + +static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) +{ + struct sockaddr_in6 *sa6; + struct sockaddr_in *sa4; + + ss->ss_family = skb_protocol_to_family(skb); + switch (ss->ss_family) { + case AF_INET: + sa4 = (struct sockaddr_in *)ss; + sa4->sin_family = AF_INET; + sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; + sa4->sin_port = udp_hdr(skb)->source; + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)ss; + sa6->sin6_family = AF_INET6; + sa6->sin6_addr = ipv6_hdr(skb)->saddr; + sa6->sin6_port = udp_hdr(skb)->source; + break; + default: + return false; + } + + return true; +} + +static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, + struct sockaddr_storage *ss) +{ + struct ovpn_peer *tmp, *peer = NULL; + + rcu_read_lock(); + tmp = rcu_dereference(ovpn->peer); + if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) + peer = tmp; + rcu_read_unlock(); + + return peer; +} + +struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) +{ + struct ovpn_peer *peer = NULL, *tmp; + struct sockaddr_storage ss = { 0 }; + struct hlist_head *head; + size_t sa_len; + bool found; + u32 index; + + if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) + return NULL; + + if (ovpn->mode == OVPN_MODE_P2P) + return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); + + switch (ss.ss_family) { + case AF_INET: + sa_len = sizeof(struct sockaddr_in); + break; + case AF_INET6: + sa_len = sizeof(struct sockaddr_in6); + break; + default: + return NULL; + } + + index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); + head = &ovpn->peers.by_transp_addr[index]; + + rcu_read_lock(); + hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { + found = ovpn_peer_transp_match(tmp, &ss); + if (!found) + continue; + + if (!ovpn_peer_hold(tmp)) + continue; + + peer = tmp; + break; + } + rcu_read_unlock(); + + return peer; +} + +static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) +{ + struct ovpn_peer *tmp, *peer = NULL; + + rcu_read_lock(); + tmp = rcu_dereference(ovpn->peer); + if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) + peer = tmp; + rcu_read_unlock(); + + return peer; +} + +struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) +{ + struct ovpn_peer *tmp, *peer = NULL; + struct hlist_head *head; + u32 index; + + if (ovpn->mode == OVPN_MODE_P2P) + return ovpn_peer_lookup_id_p2p(ovpn, peer_id); + + index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); + head = &ovpn->peers.by_id[index]; + + rcu_read_lock(); + hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { + if (tmp->id != peer_id) + continue; + + if (!ovpn_peer_hold(tmp)) + continue; + + peer = tmp; + break; + } + rcu_read_unlock(); + + return peer; +} + +void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) +{ + struct ovpn_bind *bind; + + rcu_read_lock(); + bind = rcu_dereference(peer->bind); + if (unlikely(!bind)) + goto unlock; + + switch (skb_protocol_to_family(skb)) { + case AF_INET: + if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { + netdev_dbg(peer->ovpn->dev, + "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, + peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); + bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; + } + break; + case AF_INET6: + if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, + sizeof(bind->local.ipv6)))) { + netdev_dbg(peer->ovpn->dev, + "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", + __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); + bind->local.ipv6 = ipv6_hdr(skb)->daddr; + } + break; + default: + break; + } +unlock: + rcu_read_unlock(); +} + +static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ + struct sockaddr_storage sa = { 0 }; + struct sockaddr_in6 *sa6; + struct sockaddr_in *sa4; + struct ovpn_bind *bind; + struct ovpn_peer *tmp; + size_t salen; + int ret = 0; + u32 index; + + spin_lock_bh(&ovpn->peers.lock); + /* do not add duplicates */ + tmp = ovpn_peer_lookup_id(ovpn, peer->id); + if (tmp) { + ovpn_peer_put(tmp); + ret = -EEXIST; + goto unlock; + } + + hlist_del_init_rcu(&peer->hash_entry_transp_addr); + bind = rcu_dereference_protected(peer->bind, true); + /* peers connected via UDP have bind == NULL */ + if (bind) { + switch (bind->sa.in4.sin_family) { + case AF_INET: + sa4 = (struct sockaddr_in *)&sa; + + sa4->sin_family = AF_INET; + sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; + sa4->sin_port = bind->sa.in4.sin_port; + salen = sizeof(*sa4); + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)&sa; + + sa6->sin6_family = AF_INET6; + sa6->sin6_addr = bind->sa.in6.sin6_addr; + sa6->sin6_port = bind->sa.in6.sin6_port; + salen = sizeof(*sa6); + break; + default: + ret = -EPROTONOSUPPORT; + goto unlock; + } + + index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); + hlist_add_head_rcu(&peer->hash_entry_transp_addr, + &ovpn->peers.by_transp_addr[index]); + } + + index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); + hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); + + if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { + index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, + sizeof(peer->vpn_addrs.ipv4)); + hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); + } + + hlist_del_init_rcu(&peer->hash_entry_addr6); + if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { + index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, + sizeof(peer->vpn_addrs.ipv6)); + hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); + } + +unlock: + spin_unlock_bh(&ovpn->peers.lock); + + return ret; +} + +static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ + struct ovpn_peer *tmp; + + spin_lock_bh(&ovpn->lock); + /* in p2p mode it is possible to have a single peer only, therefore the + * old one is released and substituted by the new one + */ + tmp = rcu_dereference(ovpn->peer); + if (tmp) { + tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; + ovpn_peer_put(tmp); + } + + rcu_assign_pointer(ovpn->peer, peer); + spin_unlock_bh(&ovpn->lock); + + return 0; +} + +/* assume refcounter was increased by caller */ +int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ + switch (ovpn->mode) { + case OVPN_MODE_MP: + return ovpn_peer_add_mp(ovpn, peer); + case OVPN_MODE_P2P: + return ovpn_peer_add_p2p(ovpn, peer); + default: + return -EOPNOTSUPP; + } +} + +static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ + hlist_del_init_rcu(&peer->hash_entry_id); + hlist_del_init_rcu(&peer->hash_entry_addr4); + hlist_del_init_rcu(&peer->hash_entry_addr6); + hlist_del_init_rcu(&peer->hash_entry_transp_addr); + + ovpn_peer_put(peer); + peer->delete_reason = reason; +} + +static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ + struct ovpn_peer *tmp; + int ret = 0; + + spin_lock_bh(&peer->ovpn->peers.lock); + tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); + if (tmp != peer) { + ret = -ENOENT; + goto unlock; + } + ovpn_peer_unhash(peer, reason); + +unlock: + spin_unlock_bh(&peer->ovpn->peers.lock); + + if (tmp) + ovpn_peer_put(tmp); + + return ret; +} + +static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ + struct ovpn_peer *tmp; + int ret = -ENOENT; + + spin_lock_bh(&peer->ovpn->lock); + tmp = rcu_dereference(peer->ovpn->peer); + if (tmp != peer) + goto unlock; + + ovpn_peer_put(tmp); + tmp->delete_reason = reason; + RCU_INIT_POINTER(peer->ovpn->peer, NULL); + ret = 0; + +unlock: + spin_unlock_bh(&peer->ovpn->lock); + + return ret; +} + +void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) +{ + struct ovpn_peer *tmp; + + rcu_read_lock(); + tmp = rcu_dereference(ovpn->peer); + if (!tmp) + goto unlock; + + ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); +unlock: + rcu_read_unlock(); +} + +int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ + switch (peer->ovpn->mode) { + case OVPN_MODE_MP: + return ovpn_peer_del_mp(peer, reason); + case OVPN_MODE_P2P: + return ovpn_peer_del_p2p(peer, reason); + default: + return -EOPNOTSUPP; + } +} + +void ovpn_peers_free(struct ovpn_struct *ovpn) +{ + struct hlist_node *tmp; + struct ovpn_peer *peer; + int bkt; + + spin_lock_bh(&ovpn->peers.lock); + hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) + ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); + spin_unlock_bh(&ovpn->peers.lock); +} diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h new file mode 100644 index 0000000000000..b759c2c9da48e --- /dev/null +++ b/drivers/net/ovpn-dco/peer.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNPEER_H_ +#define _NET_OVPN_DCO_OVPNPEER_H_ + +#include "addr.h" +#include "bind.h" +#include "sock.h" +#include "stats.h" + +#include +#include +#include + +struct ovpn_peer { + struct ovpn_struct *ovpn; + + u32 id; + + struct { + struct in_addr ipv4; + struct in6_addr ipv6; + } vpn_addrs; + + struct hlist_node hash_entry_id; + struct hlist_node hash_entry_addr4; + struct hlist_node hash_entry_addr6; + struct hlist_node hash_entry_transp_addr; + + /* work objects to handle encryption/decryption of packets. + * these works are queued on the ovpn->crypt_wq workqueue. + */ + struct work_struct encrypt_work; + struct work_struct decrypt_work; + + struct ptr_ring tx_ring; + struct ptr_ring rx_ring; + struct ptr_ring netif_rx_ring; + + struct napi_struct napi; + + struct ovpn_socket *sock; + + /* state of the TCP reading. Needed to keep track of how much of a single packet has already + * been read from the stream and how much is missing + */ + struct { + struct ptr_ring tx_ring; + struct work_struct tx_work; + struct work_struct rx_work; + + u8 raw_len[sizeof(u16)]; + struct sk_buff *skb; + u16 offset; + u16 data_len; + struct { + void (*sk_state_change)(struct sock *sk); + void (*sk_data_ready)(struct sock *sk); + void (*sk_write_space)(struct sock *sk); + } sk_cb; + } tcp; + + struct dst_cache dst_cache; + + /* our crypto state */ + struct ovpn_crypto_state crypto; + + /* our binding to peer, protected by spinlock */ + struct ovpn_bind __rcu *bind; + + /* timer used to send periodic ping messages to the other peer, if no + * other data was sent within the past keepalive_interval seconds + */ + struct timer_list keepalive_xmit; + /* keepalive interval in seconds */ + unsigned long keepalive_interval; + + /* timer used to mark a peer as expired when no data is received for + * keepalive_timeout seconds + */ + struct timer_list keepalive_recv; + /* keepalive timeout in seconds */ + unsigned long keepalive_timeout; + + /* true if ovpn_peer_mark_delete was called */ + bool halt; + + /* per-peer rx/tx stats */ + struct ovpn_peer_stats stats; + + /* why peer was deleted - keepalive timeout, module removed etc */ + enum ovpn_del_peer_reason delete_reason; + + /* protects binding to peer (bind) and timers + * (keepalive_xmit, keepalive_expire) + */ + spinlock_t lock; + + /* needed because crypto methods can go async */ + struct kref refcount; + + /* needed to free a peer in an RCU safe way */ + struct rcu_head rcu; + + /* needed to notify userspace about deletion */ + struct work_struct delete_work; +}; + +void ovpn_peer_release_kref(struct kref *kref); +void ovpn_peer_release(struct ovpn_peer *peer); + +static inline bool ovpn_peer_hold(struct ovpn_peer *peer) +{ + return kref_get_unless_zero(&peer->refcount); +} + +static inline void ovpn_peer_put(struct ovpn_peer *peer) +{ + kref_put(&peer->refcount, ovpn_peer_release_kref); +} + +static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) +{ + u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); + + if (unlikely(!delta)) + return; + + mod_timer(&peer->keepalive_recv, jiffies + delta); +} + +static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) +{ + u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); + + if (unlikely(!delta)) + return; + + mod_timer(&peer->keepalive_xmit, jiffies + delta); +} + +struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, + struct socket *sock, u32 id, uint8_t *local_ip); + +void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); + +int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); +int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); +struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); +void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); +void ovpn_peers_free(struct ovpn_struct *ovpn); + +struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); +struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, + bool use_src); +struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); + +void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); +void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); + +#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c new file mode 100644 index 0000000000000..fcde8fba5156c --- /dev/null +++ b/drivers/net/ovpn-dco/pktid.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + * James Yonan + */ + +#include "pktid.h" + +#include +#include + +void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) +{ + atomic64_set(&pid->seq_num, 1); + pid->tcp_linear = NULL; +} + +void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) +{ + memset(pr, 0, sizeof(*pr)); + spin_lock_init(&pr->lock); +} + +/* Packet replay detection. + * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. + */ +int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) +{ + const unsigned long now = jiffies; + int ret; + + spin_lock(&pr->lock); + + /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ + if (unlikely(time_after_eq(now, pr->expire))) + pr->id_floor = pr->id; + + /* ID must not be zero */ + if (unlikely(pkt_id == 0)) { + ret = -EINVAL; + goto out; + } + + /* time changed? */ + if (unlikely(pkt_time != pr->time)) { + if (pkt_time > pr->time) { + /* time moved forward, accept */ + pr->base = 0; + pr->extent = 0; + pr->id = 0; + pr->time = pkt_time; + pr->id_floor = 0; + } else { + /* time moved backward, reject */ + ret = -ETIME; + goto out; + } + } + + if (likely(pkt_id == pr->id + 1)) { + /* well-formed ID sequence (incremented by 1) */ + pr->base = REPLAY_INDEX(pr->base, -1); + pr->history[pr->base / 8] |= (1 << (pr->base % 8)); + if (pr->extent < REPLAY_WINDOW_SIZE) + ++pr->extent; + pr->id = pkt_id; + } else if (pkt_id > pr->id) { + /* ID jumped forward by more than one */ + const unsigned int delta = pkt_id - pr->id; + + if (delta < REPLAY_WINDOW_SIZE) { + unsigned int i; + + pr->base = REPLAY_INDEX(pr->base, -delta); + pr->history[pr->base / 8] |= (1 << (pr->base % 8)); + pr->extent += delta; + if (pr->extent > REPLAY_WINDOW_SIZE) + pr->extent = REPLAY_WINDOW_SIZE; + for (i = 1; i < delta; ++i) { + unsigned int newb = REPLAY_INDEX(pr->base, i); + + pr->history[newb / 8] &= ~BIT(newb % 8); + } + } else { + pr->base = 0; + pr->extent = REPLAY_WINDOW_SIZE; + memset(pr->history, 0, sizeof(pr->history)); + pr->history[0] = 1; + } + pr->id = pkt_id; + } else { + /* ID backtrack */ + const unsigned int delta = pr->id - pkt_id; + + if (delta > pr->max_backtrack) + pr->max_backtrack = delta; + if (delta < pr->extent) { + if (pkt_id > pr->id_floor) { + const unsigned int ri = REPLAY_INDEX(pr->base, + delta); + u8 *p = &pr->history[ri / 8]; + const u8 mask = (1 << (ri % 8)); + + if (*p & mask) { + ret = -EINVAL; + goto out; + } + *p |= mask; + } else { + ret = -EINVAL; + goto out; + } + } else { + ret = -EINVAL; + goto out; + } + } + + pr->expire = now + PKTID_RECV_EXPIRE; + ret = 0; +out: + spin_unlock(&pr->lock); + return ret; +} diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h new file mode 100644 index 0000000000000..2447bb37ba552 --- /dev/null +++ b/drivers/net/ovpn-dco/pktid.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + * James Yonan + */ + +#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ +#define _NET_OVPN_DCO_OVPNPKTID_H_ + +#include "main.h" + +/* When the OpenVPN protocol is run in AEAD mode, use + * the OpenVPN packet ID as the AEAD nonce: + * + * 00000005 521c3b01 4308c041 + * [seq # ] [ nonce_tail ] + * [ 12-byte full IV ] -> NONCE_SIZE + * [4-bytes -> NONCE_WIRE_SIZE + * on wire] + */ + +/* OpenVPN nonce size */ +#define NONCE_SIZE 12 +/* amount of bytes of the nonce received from user space */ +#define NONCE_TAIL_SIZE 8 + +/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the + * size of the AEAD Associated Data (AD) sent over the wire + * and is normally the head of the IV + */ +#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) + +/* If no packets received for this length of time, set a backtrack floor + * at highest received packet ID thus far. + */ +#define PKTID_RECV_EXPIRE (30 * HZ) + +/* Last 8 bytes of AEAD nonce + * Provided by userspace and usually derived from + * key material generated during TLS handshake + */ +struct ovpn_nonce_tail { + u8 u8[NONCE_TAIL_SIZE]; +}; + +/* Packet-ID state for transmitter */ +struct ovpn_pktid_xmit { + atomic64_t seq_num; + struct ovpn_tcp_linear *tcp_linear; +}; + +/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ +#define REPLAY_WINDOW_ORDER 8 + +#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) +#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) +#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) + +/* Packet-ID state for receiver. + * Other than lock member, can be zeroed to initialize. + */ +struct ovpn_pktid_recv { + /* "sliding window" bitmask of recent packet IDs received */ + u8 history[REPLAY_WINDOW_BYTES]; + /* bit position of deque base in history */ + unsigned int base; + /* extent (in bits) of deque in history */ + unsigned int extent; + /* expiration of history in jiffies */ + unsigned long expire; + /* highest sequence number received */ + u32 id; + /* highest time stamp received */ + u32 time; + /* we will only accept backtrack IDs > id_floor */ + u32 id_floor; + unsigned int max_backtrack; + /* protects entire pktd ID state */ + spinlock_t lock; +}; + +/* Get the next packet ID for xmit */ +static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) +{ + const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, + 0x100000000LL); + /* when the 32bit space is over, we return an error because the packet ID is used to create + * the cipher IV and we do not want to re-use the same value more than once + */ + if (unlikely(seq_num == 0x100000000LL)) + return -ERANGE; + + *pktid = (u32)seq_num; + + return 0; +} + +/* Write 12-byte AEAD IV to dest */ +static inline void ovpn_pktid_aead_write(const u32 pktid, + const struct ovpn_nonce_tail *nt, + unsigned char *dest) +{ + *(__force __be32 *)(dest) = htonl(pktid); + BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); + memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); +} + +void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); +void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); + +int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); + +#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h new file mode 100644 index 0000000000000..875529021c1b2 --- /dev/null +++ b/drivers/net/ovpn-dco/proto.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + * James Yonan + */ + +#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ +#define _NET_OVPN_DCO_OVPNPROTO_H_ + +#include "main.h" + +#include + +/* Methods for operating on the initial command + * byte of the OpenVPN protocol. + */ + +/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in + * one byte + */ +#define OVPN_KEY_ID_MASK 0x07 +#define OVPN_OPCODE_SHIFT 3 +#define OVPN_OPCODE_MASK 0x1F +/* upper bounds on opcode and key ID */ +#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) +#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) +/* packet opcodes of interest to us */ +#define OVPN_DATA_V1 6 /* data channel V1 packet */ +#define OVPN_DATA_V2 9 /* data channel V2 packet */ +/* size of initial packet opcode */ +#define OVPN_OP_SIZE_V1 1 +#define OVPN_OP_SIZE_V2 4 +#define OVPN_PEER_ID_MASK 0x00FFFFFF +#define OVPN_PEER_ID_UNDEF 0x00FFFFFF +/* first byte of keepalive message */ +#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a +/* first byte of exit message */ +#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 + +/** + * Extract the OP code from the specified byte + * + * Return the OP code + */ +static inline u8 ovpn_opcode_from_byte(u8 byte) +{ + return byte >> OVPN_OPCODE_SHIFT; +} + +/** + * Extract the OP code from the skb head. + * + * Note: this function assumes that the skb head was pulled enough + * to access the first byte. + * + * Return the OP code + */ +static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) +{ + return ovpn_opcode_from_byte(*(skb->data + offset)); +} + +/** + * Extract the key ID from the skb head. + * + * Note: this function assumes that the skb head was pulled enough + * to access the first byte. + * + * Return the key ID + */ + +static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) +{ + return *skb->data & OVPN_KEY_ID_MASK; +} + +/** + * Extract the peer ID from the skb head. + * + * Note: this function assumes that the skb head was pulled enough + * to access the first 4 bytes. + * + * Return the peer ID. + */ + +static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) +{ + return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; +} + +static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) +{ + const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); + + return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); +} + +#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h new file mode 100644 index 0000000000000..02a50f49ba2ea --- /dev/null +++ b/drivers/net/ovpn-dco/rcu.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_OVPNRCU_H_ +#define _NET_OVPN_DCO_OVPNRCU_H_ + +static inline void ovpn_rcu_lockdep_assert_held(void) +{ +#ifdef CONFIG_PROVE_RCU + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ovpn-dco RCU read lock not held"); +#endif +} + +#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h new file mode 100644 index 0000000000000..d38dc2da01dff --- /dev/null +++ b/drivers/net/ovpn-dco/skb.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + * James Yonan + */ + +#ifndef _NET_OVPN_DCO_SKB_H_ +#define _NET_OVPN_DCO_SKB_H_ + +#include +#include +#include +#include +#include +#include + +#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) + +struct ovpn_skb_cb { + /* original recv packet size for stats accounting */ + unsigned int rx_stats_size; + + union { + struct in_addr ipv4; + struct in6_addr ipv6; + } local; + sa_family_t sa_fam; +}; + +/* Return IP protocol version from skb header. + * Return 0 if protocol is not IPv4/IPv6 or cannot be read. + */ +static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) +{ + __be16 proto = 0; + + /* skb could be non-linear, + * make sure IP header is in non-fragmented part + */ + if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) + return 0; + + if (ip_hdr(skb)->version == 4) + proto = htons(ETH_P_IP); + else if (ip_hdr(skb)->version == 6) + proto = htons(ETH_P_IPV6); + + return proto; +} + +#endif /* _NET_OVPN_DCO_SKB_H_ */ diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c new file mode 100644 index 0000000000000..e92a4a9b952e6 --- /dev/null +++ b/drivers/net/ovpn-dco/sock.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "main.h" +#include "ovpn.h" +#include "peer.h" +#include "sock.h" +#include "rcu.h" +#include "tcp.h" +#include "udp.h" + +#include +#include + +/* Finalize release of socket, called after RCU grace period */ +static void ovpn_socket_detach(struct socket *sock) +{ + if (!sock) + return; + + if (sock->sk->sk_protocol == IPPROTO_UDP) + ovpn_udp_socket_detach(sock); + else if (sock->sk->sk_protocol == IPPROTO_TCP) + ovpn_tcp_socket_detach(sock); + + sockfd_put(sock); +} + +void ovpn_socket_release_kref(struct kref *kref) +{ + struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); + + ovpn_socket_detach(sock->sock); + kfree_rcu(sock, rcu); +} + +static bool ovpn_socket_hold(struct ovpn_socket *sock) +{ + return kref_get_unless_zero(&sock->refcount); +} + +static struct ovpn_socket *ovpn_socket_get(struct socket *sock) +{ + struct ovpn_socket *ovpn_sock; + + rcu_read_lock(); + ovpn_sock = rcu_dereference_sk_user_data(sock->sk); + if (!ovpn_socket_hold(ovpn_sock)) { + pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); + ovpn_sock = NULL; + } + rcu_read_unlock(); + + return ovpn_sock; +} + +/* Finalize release of socket, called after RCU grace period */ +static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) +{ + int ret = -EOPNOTSUPP; + + if (!sock || !peer) + return -EINVAL; + + if (sock->sk->sk_protocol == IPPROTO_UDP) + ret = ovpn_udp_socket_attach(sock, peer->ovpn); + else if (sock->sk->sk_protocol == IPPROTO_TCP) + ret = ovpn_tcp_socket_attach(sock, peer); + + return ret; +} + +struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) +{ + struct ovpn_socket *ovpn_sock; + + ovpn_rcu_lockdep_assert_held(); + + if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) + return NULL; + + ovpn_sock = rcu_dereference_sk_user_data(sk); + if (unlikely(!ovpn_sock)) + return NULL; + + /* make sure that sk matches our stored transport socket */ + if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) + return NULL; + + return ovpn_sock->ovpn; +} + +struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) +{ + struct ovpn_socket *ovpn_sock; + int ret; + + ret = ovpn_socket_attach(sock, peer); + if (ret < 0 && ret != -EALREADY) + return ERR_PTR(ret); + + /* if this socket is already owned by this interface, just increase the refcounter */ + if (ret == -EALREADY) { + /* caller is expected to increase the sock refcounter before passing it to this + * function. For this reason we drop it if not needed, like when this socket is + * already owned. + */ + ovpn_sock = ovpn_socket_get(sock); + sockfd_put(sock); + return ovpn_sock; + } + + ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); + if (!ovpn_sock) + return ERR_PTR(-ENOMEM); + + ovpn_sock->ovpn = peer->ovpn; + ovpn_sock->sock = sock; + kref_init(&ovpn_sock->refcount); + + /* TCP sockets are per-peer, therefore they are linked to their unique peer */ + if (sock->sk->sk_protocol == IPPROTO_TCP) + ovpn_sock->peer = peer; + + rcu_assign_sk_user_data(sock->sk, ovpn_sock); + + return ovpn_sock; +} diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h new file mode 100644 index 0000000000000..9e79c1b5fe049 --- /dev/null +++ b/drivers/net/ovpn-dco/sock.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_SOCK_H_ +#define _NET_OVPN_DCO_SOCK_H_ + +#include +#include +#include + +#include "peer.h" + +struct ovpn_struct; + +/** + * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code + */ +struct ovpn_socket { + union { + /** @ovpn: the VPN session object owning this socket (UDP only) */ + struct ovpn_struct *ovpn; + + /** @peer: the unique peer transmitting over this socket (TCP only) */ + struct ovpn_peer *peer; + }; + + /** @sock: the kernel socket */ + struct socket *sock; + + /** @refcount: amount of contexts currently referencing this object */ + struct kref refcount; + + /** @rcu: member used to schedule RCU destructor callback */ + struct rcu_head rcu; +}; + +struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); + +void ovpn_socket_release_kref(struct kref *kref); + +static inline void ovpn_socket_put(struct ovpn_socket *sock) +{ + kref_put(&sock->refcount, ovpn_socket_release_kref); +} + +struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); + +#endif /* _NET_OVPN_DCO_SOCK_H_ */ diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c new file mode 100644 index 0000000000000..ee000b2a21772 --- /dev/null +++ b/drivers/net/ovpn-dco/stats.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#include "main.h" +#include "stats.h" + +void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) +{ + atomic64_set(&ps->rx.bytes, 0); + atomic_set(&ps->rx.packets, 0); + + atomic64_set(&ps->tx.bytes, 0); + atomic_set(&ps->tx.packets, 0); +} diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h new file mode 100644 index 0000000000000..3aa6bdc049c69 --- /dev/null +++ b/drivers/net/ovpn-dco/stats.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2020-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + * Lev Stipakov + */ + +#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ +#define _NET_OVPN_DCO_OVPNSTATS_H_ + +#include +#include + +struct ovpn_struct; + +/* per-peer stats, measured on transport layer */ + +/* one stat */ +struct ovpn_peer_stat { + atomic64_t bytes; + atomic_t packets; +}; + +/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ +struct ovpn_peer_stats { + struct ovpn_peer_stat rx; + struct ovpn_peer_stat tx; +}; + +/* struct for OVPN_ERR_STATS */ + +struct ovpn_err_stat { + unsigned int category; + int errcode; + u64 count; +}; + +struct ovpn_err_stats { + /* total stats, returned by kovpn */ + unsigned int total_stats; + /* number of stats dimensioned below */ + unsigned int n_stats; + struct ovpn_err_stat stats[]; +}; + +void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); + +static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) +{ + atomic64_add(n, &stat->bytes); + atomic_inc(&stat->packets); +} + +static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) +{ + ovpn_peer_stats_increment(&stats->rx, n); +} + +static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) +{ + ovpn_peer_stats_increment(&stats->tx, n); +} + +#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c new file mode 100644 index 0000000000000..7e6690fee6e7b --- /dev/null +++ b/drivers/net/ovpn-dco/tcp.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#include "main.h" +#include "ovpnstruct.h" +#include "ovpn.h" +#include "peer.h" +#include "skb.h" +#include "tcp.h" + +#include +#include +#include + +static void ovpn_tcp_state_change(struct sock *sk) +{ +} + +static void ovpn_tcp_data_ready(struct sock *sk) +{ + struct ovpn_socket *sock; + + rcu_read_lock(); + sock = rcu_dereference_sk_user_data(sk); + rcu_read_unlock(); + + if (!sock || !sock->peer) + return; + + queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); +} + +static void ovpn_tcp_write_space(struct sock *sk) +{ + struct ovpn_socket *sock; + + rcu_read_lock(); + sock = rcu_dereference_sk_user_data(sk); + rcu_read_unlock(); + + if (!sock || !sock->peer) + return; + + queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); +} + +static void ovpn_destroy_skb(void *skb) +{ + consume_skb(skb); +} + +void ovpn_tcp_socket_detach(struct socket *sock) +{ + struct ovpn_socket *ovpn_sock; + struct ovpn_peer *peer; + + if (!sock) + return; + + rcu_read_lock(); + ovpn_sock = rcu_dereference_sk_user_data(sock->sk); + rcu_read_unlock(); + + if (!ovpn_sock->peer) + return; + + peer = ovpn_sock->peer; + + /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ + write_lock_bh(&sock->sk->sk_callback_lock); + sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; + sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; + sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; + rcu_assign_sk_user_data(sock->sk, NULL); + write_unlock_bh(&sock->sk->sk_callback_lock); + + /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be + * re-armed + */ + cancel_work_sync(&peer->tcp.tx_work); + cancel_work_sync(&peer->tcp.rx_work); + + ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); +} + +/* Try to send one skb (or part of it) over the TCP stream. + * + * Return 0 on success or a negative error code otherwise. + * + * Note that the skb is modified by putting away the data being sent, therefore + * the caller should check if skb->len is zero to understand if the full skb was + * sent or not. + */ +static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) +{ + struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; + struct kvec iv = { 0 }; + int ret; + + if (skb_linearize(skb) < 0) { + net_err_ratelimited("%s: can't linearize packet\n", __func__); + return -ENOMEM; + } + + /* initialize iv structure now as skb_linearize() may have changed skb->data */ + iv.iov_base = skb->data; + iv.iov_len = skb->len; + + ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); + if (ret > 0) { + __skb_pull(skb, ret); + + /* since we update per-cpu stats in process context, + * we need to disable softirqs + */ + local_bh_disable(); + dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); + local_bh_enable(); + + return 0; + } + + return ret; +} + +/* Process packets in TCP TX queue */ +static void ovpn_tcp_tx_work(struct work_struct *work) +{ + struct ovpn_peer *peer; + struct sk_buff *skb; + int ret; + + peer = container_of(work, struct ovpn_peer, tcp.tx_work); + while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { + ret = ovpn_tcp_send_one(peer, skb); + if (ret < 0 && ret != -EAGAIN) { + net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, + peer->id, ret); + /* in case of TCP error stop sending loop and delete peer */ + ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); + break; + } else if (!skb->len) { + /* skb was entirely consumed and can now be removed from the ring */ + __ptr_ring_discard_one(&peer->tcp.tx_ring); + consume_skb(skb); + } + + /* give a chance to be rescheduled if needed */ + cond_resched(); + } +} + +static int ovpn_tcp_rx_one(struct ovpn_peer *peer) +{ + struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; + struct ovpn_skb_cb *cb; + int status, ret; + + /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix + * containing the actual packet size. + */ + if (!peer->tcp.skb) { + struct kvec iv = { + .iov_base = peer->tcp.raw_len + peer->tcp.offset, + .iov_len = sizeof(u16) - peer->tcp.offset, + }; + + ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); + if (ret <= 0) + return ret; + + peer->tcp.offset += ret; + /* the entire packet size was read, prepare skb for reading data */ + if (peer->tcp.offset == sizeof(u16)) { + u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); + /* invalid packet length: this is a fatal TCP error */ + if (!len) { + netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", + __func__); + return -EINVAL; + } + + peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); + peer->tcp.offset = 0; + peer->tcp.data_len = len; + } + } else { + struct kvec iv = { + .iov_base = peer->tcp.skb->data + peer->tcp.offset, + .iov_len = peer->tcp.data_len - peer->tcp.offset, + }; + + ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); + if (ret <= 0) + return ret; + + peer->tcp.offset += ret; + /* full packet received, send it up for processing */ + if (peer->tcp.offset == peer->tcp.data_len) { + /* update the skb data structure with the amount of data written by + * kernel_recvmsg() + */ + skb_put(peer->tcp.skb, peer->tcp.data_len); + + /* do not perform IP caching for TCP connections */ + cb = OVPN_SKB_CB(peer->tcp.skb); + cb->sa_fam = AF_UNSPEC; + + /* hold reference to peer as requird by ovpn_recv() */ + ovpn_peer_hold(peer); + status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); + /* skb not consumed - free it now */ + if (unlikely(status < 0)) + kfree_skb(peer->tcp.skb); + + peer->tcp.skb = NULL; + peer->tcp.offset = 0; + peer->tcp.data_len = 0; + } + } + + return ret; +} + +static void ovpn_tcp_rx_work(struct work_struct *work) +{ + struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); + int ret; + + while (true) { + /* give a chance to be rescheduled if needed */ + cond_resched(); + + ret = ovpn_tcp_rx_one(peer); + if (ret <= 0) + break; + } + + if (ret < 0 && ret != -EAGAIN) + netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); +} + +/* Put packet into TCP TX queue and schedule a consumer */ +void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) +{ + int ret; + + ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); + if (ret < 0) { + kfree_skb_list(skb); + return; + } + + queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); +} + +/* Set TCP encapsulation callbacks */ +int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) +{ + void *old_data; + int ret; + + INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); + INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); + + ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); + if (ret < 0) { + netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); + return ret; + } + + peer->tcp.skb = NULL; + peer->tcp.offset = 0; + peer->tcp.data_len = 0; + + write_lock_bh(&sock->sk->sk_callback_lock); + + /* make sure no pre-existing encapsulation handler exists */ + rcu_read_lock(); + old_data = rcu_dereference_sk_user_data(sock->sk); + rcu_read_unlock(); + if (old_data) { + netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); + ret = -EBUSY; + goto err; + } + + /* sanity check */ + if (sock->sk->sk_protocol != IPPROTO_TCP) { + netdev_err(peer->ovpn->dev, "expected TCP socket\n"); + ret = -EINVAL; + goto err; + } + + /* only a fully connected socket are expected. Connection should be handled in userspace */ + if (sock->sk->sk_state != TCP_ESTABLISHED) { + netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", + sock->sk->sk_state); + ret = -EINVAL; + goto err; + } + + /* save current CBs so that they can be restored upon socket release */ + peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; + peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; + peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; + + /* assign our static CBs */ + sock->sk->sk_state_change = ovpn_tcp_state_change; + sock->sk->sk_data_ready = ovpn_tcp_data_ready; + sock->sk->sk_write_space = ovpn_tcp_write_space; + + write_unlock_bh(&sock->sk->sk_callback_lock); + + return 0; +err: + write_unlock_bh(&sock->sk->sk_callback_lock); + ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); + + return ret; +} diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h new file mode 100644 index 0000000000000..d243a8e1c34e4 --- /dev/null +++ b/drivers/net/ovpn-dco/tcp.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_TCP_H_ +#define _NET_OVPN_DCO_TCP_H_ + +#include "peer.h" + +#include +#include +#include +#include + +void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); + +int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); +void ovpn_tcp_socket_detach(struct socket *sock); + +/* Prepare skb and enqueue it for sending to peer. + * + * Preparation consist in prepending the skb payload with its size. + * Required by the OpenVPN protocol in order to extract packets from + * the TCP stream on the receiver side. + */ +static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) +{ + u16 len = skb->len; + + *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); + ovpn_queue_tcp_skb(peer, skb); +} + +#endif /* _NET_OVPN_DCO_TCP_H_ */ diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c new file mode 100644 index 0000000000000..afa236d1f15c9 --- /dev/null +++ b/drivers/net/ovpn-dco/udp.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#include "main.h" +#include "bind.h" +#include "ovpn.h" +#include "ovpnstruct.h" +#include "peer.h" +#include "proto.h" +#include "skb.h" +#include "udp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * ovpn_udp_encap_recv() - Start processing a received UDP packet. + * If the first byte of the payload is DATA_V2, the packet is further processed, + * otherwise it is forwarded to the UDP stack for delivery to user space. + * + * @sk: the socket the packet was received on + * @skb: the sk_buff containing the actual packet + * + * Return codes: + * 0 : we consumed or dropped packet + * >0 : skb should be passed up to userspace as UDP (packet not consumed) + * <0 : skb should be resubmitted as proto -N (packet not consumed) + */ +static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) +{ + struct ovpn_peer *peer = NULL; + struct ovpn_struct *ovpn; + u32 peer_id; + u8 opcode; + int ret; + + ovpn = ovpn_from_udp_sock(sk); + if (unlikely(!ovpn)) { + net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); + goto drop; + } + + /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. + * They are required to fetch the OP code, the key ID and the peer ID. + */ + if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { + net_dbg_ratelimited("%s: packet too small\n", __func__); + goto drop; + } + + opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); + if (likely(opcode == OVPN_DATA_V2)) { + peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); + /* some OpenVPN server implementations send data packets with the peer-id set to + * undef. In this case we skip the peer lookup by peer-id and we try with the + * transport address + */ + if (peer_id != OVPN_PEER_ID_UNDEF) { + peer = ovpn_peer_lookup_id(ovpn, peer_id); + if (!peer) { + net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", + __func__, peer_id); + goto drop; + } + + /* check if this peer changed it's IP address and update state */ + ovpn_peer_float(peer, skb); + } + } + + if (!peer) { + /* might be a control packet or a data packet with undef peer-id */ + peer = ovpn_peer_lookup_transp_addr(ovpn, skb); + if (unlikely(!peer)) { + if (opcode != OVPN_DATA_V2) { + netdev_dbg(ovpn->dev, + "%s: control packet from unknown peer, sending to userspace", + __func__); + return 1; + } + + netdev_dbg(ovpn->dev, + "%s: received data with undef peer-id from unknown source\n", + __func__); + goto drop; + } + } + + /* pop off outer UDP header */ + __skb_pull(skb, sizeof(struct udphdr)); + + ret = ovpn_recv(ovpn, peer, skb); + if (unlikely(ret < 0)) { + net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); + goto drop; + } + + /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP + * stack to continue processing this packet as usual (i.e. deliver to user space) + */ + return ret; + +drop: + if (peer) + ovpn_peer_put(peer); + kfree_skb(skb); + return 0; +} + +static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, + struct dst_cache *cache, struct sock *sk, + struct sk_buff *skb) +{ + struct rtable *rt; + struct flowi4 fl = { + .saddr = bind->local.ipv4.s_addr, + .daddr = bind->sa.in4.sin_addr.s_addr, + .fl4_sport = inet_sk(sk)->inet_sport, + .fl4_dport = bind->sa.in4.sin_port, + .flowi4_proto = sk->sk_protocol, + .flowi4_mark = sk->sk_mark, + }; + int ret; + + local_bh_disable(); + rt = dst_cache_get_ip4(cache, &fl.saddr); + if (rt) + goto transmit; + + if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { + /* we may end up here when the cached address is not usable anymore. + * In this case we reset address/cache and perform a new look up + */ + fl.saddr = 0; + bind->local.ipv4.s_addr = 0; + dst_cache_reset(cache); + } + + rt = ip_route_output_flow(sock_net(sk), &fl, sk); + if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { + fl.saddr = 0; + bind->local.ipv4.s_addr = 0; + dst_cache_reset(cache); + + rt = ip_route_output_flow(sock_net(sk), &fl, sk); + } + + if (IS_ERR(rt)) { + ret = PTR_ERR(rt); + net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, + &bind->sa.in4, ret); + goto err; + } + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + +transmit: + udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, sk->sk_no_check_tx); + ret = 0; +err: + local_bh_enable(); + return ret; +} + +#if IS_ENABLED(CONFIG_IPV6) +static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, + struct dst_cache *cache, struct sock *sk, + struct sk_buff *skb) +{ + struct dst_entry *dst; + int ret; + + struct flowi6 fl = { + .saddr = bind->local.ipv6, + .daddr = bind->sa.in6.sin6_addr, + .fl6_sport = inet_sk(sk)->inet_sport, + .fl6_dport = bind->sa.in6.sin6_port, + .flowi6_proto = sk->sk_protocol, + .flowi6_mark = sk->sk_mark, + .flowi6_oif = bind->sa.in6.sin6_scope_id, + }; + + local_bh_disable(); + dst = dst_cache_get_ip6(cache, &fl.saddr); + if (dst) + goto transmit; + + if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { + /* we may end up here when the cached address is not usable anymore. + * In this case we reset address/cache and perform a new look up + */ + fl.saddr = in6addr_any; + bind->local.ipv6 = in6addr_any; + dst_cache_reset(cache); + } + + dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); + if (IS_ERR(dst)) { + ret = PTR_ERR(dst); + net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, + &bind->sa.in6, ret); + goto err; + } + dst_cache_set_ip6(cache, dst, &fl.saddr); + +transmit: + udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, udp_get_no_check6_tx(sk)); + ret = 0; +err: + local_bh_enable(); + return ret; +} +#endif + +/* Transmit skb utilizing kernel-provided UDP tunneling framework. + * + * rcu_read_lock should be held on entry. + * On return, the skb is consumed. + */ +static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, + struct dst_cache *cache, struct sock *sk, + struct sk_buff *skb) +{ + int ret; + + ovpn_rcu_lockdep_assert_held(); + + /* set sk to null if skb is already orphaned */ + if (!skb->destructor) + skb->sk = NULL; + + switch (bind->sa.in4.sin_family) { + case AF_INET: + ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); + break; +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); + break; +#endif + default: + ret = -EAFNOSUPPORT; + break; + } + + return ret; +} + +void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, + struct sk_buff *skb) +{ + struct ovpn_bind *bind; + struct socket *sock; + int ret = -1; + + skb->dev = ovpn->dev; + /* no checksum performed at this layer */ + skb->ip_summed = CHECKSUM_NONE; + + /* get socket info */ + sock = peer->sock->sock; + if (unlikely(!sock)) { + net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); + goto out; + } + + rcu_read_lock(); + /* get binding */ + bind = rcu_dereference(peer->bind); + if (unlikely(!bind)) { + net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); + goto out_unlock; + } + + /* crypto layer -> transport (UDP) */ + ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); + +out_unlock: + rcu_read_unlock(); +out: + if (ret < 0) + kfree_skb(skb); +} + +/* Set UDP encapsulation callbacks */ +int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) +{ + struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = ovpn, + .encap_type = UDP_ENCAP_OVPNINUDP, + .encap_rcv = ovpn_udp_encap_recv, + }; + struct ovpn_socket *old_data; + + /* sanity check */ + if (sock->sk->sk_protocol != IPPROTO_UDP) { + netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); + return -EINVAL; + } + + /* make sure no pre-existing encapsulation handler exists */ + rcu_read_lock(); + old_data = rcu_dereference_sk_user_data(sock->sk); + rcu_read_unlock(); + if (old_data) { + if (old_data->ovpn == ovpn) { + netdev_dbg(ovpn->dev, + "%s: provided socket already owned by this interface\n", + __func__); + return -EALREADY; + } + + netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", + __func__); + return -EBUSY; + } + + setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); + + return 0; +} + +/* Detach socket from encapsulation handler and/or other callbacks */ +void ovpn_udp_socket_detach(struct socket *sock) +{ + struct udp_tunnel_sock_cfg cfg = { }; + + setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); +} diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h new file mode 100644 index 0000000000000..be94fb74669ba --- /dev/null +++ b/drivers/net/ovpn-dco/udp.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: Antonio Quartulli + */ + +#ifndef _NET_OVPN_DCO_UDP_H_ +#define _NET_OVPN_DCO_UDP_H_ + +#include "peer.h" +#include "ovpnstruct.h" + +#include +#include +#include +#include + +int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); +void ovpn_udp_socket_detach(struct socket *sock); +void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, + struct sk_buff *skb); + +#endif /* _NET_OVPN_DCO_UDP_H_ */ diff --git a/include/net/netlink.h b/include/net/netlink.h index 7a2a9d3144ba6..335f44871529f 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -441,6 +441,7 @@ struct nla_policy { .max = _len \ } #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) +#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) /** * struct nl_info - netlink source information diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h new file mode 100644 index 0000000000000..6afee8b3fedd3 --- /dev/null +++ b/include/uapi/linux/ovpn_dco.h @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * OpenVPN data channel accelerator + * + * Copyright (C) 2019-2022 OpenVPN, Inc. + * + * Author: James Yonan + * Antonio Quartulli + */ + +#ifndef _UAPI_LINUX_OVPN_DCO_H_ +#define _UAPI_LINUX_OVPN_DCO_H_ + +#define OVPN_NL_NAME "ovpn-dco" + +#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" + +/** + * enum ovpn_nl_commands - supported netlink commands + */ +enum ovpn_nl_commands { + /** + * @OVPN_CMD_UNSPEC: unspecified command to catch errors + */ + OVPN_CMD_UNSPEC = 0, + + /** + * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys + */ + OVPN_CMD_NEW_PEER, + + /** + * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer + */ + OVPN_CMD_SET_PEER, + + /** + * @OVPN_CMD_DEL_PEER: Remove peer from internal table + */ + OVPN_CMD_DEL_PEER, + + OVPN_CMD_NEW_KEY, + + OVPN_CMD_SWAP_KEYS, + + OVPN_CMD_DEL_KEY, + + /** + * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be + * forwarded to userspace + */ + OVPN_CMD_REGISTER_PACKET, + + /** + * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also + * used to send to userspace packets for which a process had registered + * with OVPN_CMD_REGISTER_PACKET + */ + OVPN_CMD_PACKET, + + /** + * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers + */ + OVPN_CMD_GET_PEER, +}; + +enum ovpn_cipher_alg { + /** + * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only + */ + OVPN_CIPHER_ALG_NONE = 0, + /** + * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size + */ + OVPN_CIPHER_ALG_AES_GCM, + /** + * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher + */ + OVPN_CIPHER_ALG_CHACHA20_POLY1305, +}; + +enum ovpn_del_peer_reason { + __OVPN_DEL_PEER_REASON_FIRST, + OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, + OVPN_DEL_PEER_REASON_USERSPACE, + OVPN_DEL_PEER_REASON_EXPIRED, + OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, + __OVPN_DEL_PEER_REASON_AFTER_LAST +}; + +enum ovpn_key_slot { + __OVPN_KEY_SLOT_FIRST, + OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, + OVPN_KEY_SLOT_SECONDARY, + __OVPN_KEY_SLOT_AFTER_LAST, +}; + +enum ovpn_netlink_attrs { + OVPN_ATTR_UNSPEC = 0, + OVPN_ATTR_IFINDEX, + OVPN_ATTR_NEW_PEER, + OVPN_ATTR_SET_PEER, + OVPN_ATTR_DEL_PEER, + OVPN_ATTR_NEW_KEY, + OVPN_ATTR_SWAP_KEYS, + OVPN_ATTR_DEL_KEY, + OVPN_ATTR_PACKET, + OVPN_ATTR_GET_PEER, + + __OVPN_ATTR_AFTER_LAST, + OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_key_dir_attrs { + OVPN_KEY_DIR_ATTR_UNSPEC = 0, + OVPN_KEY_DIR_ATTR_CIPHER_KEY, + OVPN_KEY_DIR_ATTR_NONCE_TAIL, + + __OVPN_KEY_DIR_ATTR_AFTER_LAST, + OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_new_key_attrs { + OVPN_NEW_KEY_ATTR_UNSPEC = 0, + OVPN_NEW_KEY_ATTR_PEER_ID, + OVPN_NEW_KEY_ATTR_KEY_SLOT, + OVPN_NEW_KEY_ATTR_KEY_ID, + OVPN_NEW_KEY_ATTR_CIPHER_ALG, + OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, + OVPN_NEW_KEY_ATTR_DECRYPT_KEY, + + __OVPN_NEW_KEY_ATTR_AFTER_LAST, + OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_del_key_attrs { + OVPN_DEL_KEY_ATTR_UNSPEC = 0, + OVPN_DEL_KEY_ATTR_PEER_ID, + OVPN_DEL_KEY_ATTR_KEY_SLOT, + + __OVPN_DEL_KEY_ATTR_AFTER_LAST, + OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_swap_keys_attrs { + OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, + OVPN_SWAP_KEYS_ATTR_PEER_ID, + + __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, + OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, + +}; + +enum ovpn_netlink_new_peer_attrs { + OVPN_NEW_PEER_ATTR_UNSPEC = 0, + OVPN_NEW_PEER_ATTR_PEER_ID, + OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, + OVPN_NEW_PEER_ATTR_SOCKET, + OVPN_NEW_PEER_ATTR_IPV4, + OVPN_NEW_PEER_ATTR_IPV6, + OVPN_NEW_PEER_ATTR_LOCAL_IP, + + __OVPN_NEW_PEER_ATTR_AFTER_LAST, + OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_set_peer_attrs { + OVPN_SET_PEER_ATTR_UNSPEC = 0, + OVPN_SET_PEER_ATTR_PEER_ID, + OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, + OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, + + __OVPN_SET_PEER_ATTR_AFTER_LAST, + OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_del_peer_attrs { + OVPN_DEL_PEER_ATTR_UNSPEC = 0, + OVPN_DEL_PEER_ATTR_REASON, + OVPN_DEL_PEER_ATTR_PEER_ID, + + __OVPN_DEL_PEER_ATTR_AFTER_LAST, + OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_get_peer_attrs { + OVPN_GET_PEER_ATTR_UNSPEC = 0, + OVPN_GET_PEER_ATTR_PEER_ID, + + __OVPN_GET_PEER_ATTR_AFTER_LAST, + OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_get_peer_response_attrs { + OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, + OVPN_GET_PEER_RESP_ATTR_PEER_ID, + OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, + OVPN_GET_PEER_RESP_ATTR_IPV4, + OVPN_GET_PEER_RESP_ATTR_IPV6, + OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, + OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, + OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, + OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, + OVPN_GET_PEER_RESP_ATTR_RX_BYTES, + OVPN_GET_PEER_RESP_ATTR_TX_BYTES, + OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, + OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, + + __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, + OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_peer_stats_attrs { + OVPN_PEER_STATS_ATTR_UNSPEC = 0, + OVPN_PEER_STATS_BYTES, + OVPN_PEER_STATS_PACKETS, + + __OVPN_PEER_STATS_ATTR_AFTER_LAST, + OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_peer_attrs { + OVPN_PEER_ATTR_UNSPEC = 0, + OVPN_PEER_ATTR_PEER_ID, + OVPN_PEER_ATTR_SOCKADDR_REMOTE, + OVPN_PEER_ATTR_IPV4, + OVPN_PEER_ATTR_IPV6, + OVPN_PEER_ATTR_LOCAL_IP, + OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, + OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, + OVPN_PEER_ATTR_ENCRYPT_KEY, + OVPN_PEER_ATTR_DECRYPT_KEY, + OVPN_PEER_ATTR_RX_STATS, + OVPN_PEER_ATTR_TX_STATS, + + __OVPN_PEER_ATTR_AFTER_LAST, + OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_netlink_packet_attrs { + OVPN_PACKET_ATTR_UNSPEC = 0, + OVPN_PACKET_ATTR_PACKET, + OVPN_PACKET_ATTR_PEER_ID, + + __OVPN_PACKET_ATTR_AFTER_LAST, + OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, +}; + +enum ovpn_ifla_attrs { + IFLA_OVPN_UNSPEC = 0, + IFLA_OVPN_MODE, + + __IFLA_OVPN_AFTER_LAST, + IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, +}; + +enum ovpn_mode { + __OVPN_MODE_FIRST = 0, + OVPN_MODE_P2P = __OVPN_MODE_FIRST, + OVPN_MODE_MP, + + __OVPN_MODE_AFTER_LAST, +}; + +#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h index 4828794efcf86..8008c762e6b8d 100644 --- a/include/uapi/linux/udp.h +++ b/include/uapi/linux/udp.h @@ -43,5 +43,6 @@ struct udphdr { #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ #define UDP_ENCAP_RXRPC 6 #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ +#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ #endif /* _UAPI_LINUX_UDP_H */ -- 2.37.1 From 88b8092df8c1625e2fcb294183a88837caba15a4 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:34:05 +0200 Subject: [PATCH 10/16] kbuild-module Signed-off-by: Peter Jung --- .../admin-guide/kernel-parameters.txt | 17 +- Documentation/kbuild/kconfig-language.rst | 2 +- MAINTAINERS | 2 +- Makefile | 3 + arch/arc/configs/axs101_defconfig | 1 - arch/arc/configs/axs103_defconfig | 1 - arch/arc/configs/axs103_smp_defconfig | 1 - arch/arc/configs/haps_hs_defconfig | 1 - arch/arc/configs/haps_hs_smp_defconfig | 1 - arch/arc/configs/hsdk_defconfig | 1 - arch/arc/configs/nsim_700_defconfig | 1 - arch/arc/configs/nsimosci_defconfig | 1 - arch/arc/configs/nsimosci_hs_defconfig | 1 - arch/arc/configs/nsimosci_hs_smp_defconfig | 1 - arch/arc/configs/tb10x_defconfig | 1 - arch/arc/configs/vdk_hs38_defconfig | 1 - arch/arc/configs/vdk_hs38_smp_defconfig | 1 - certs/Makefile | 14 +- certs/blacklist_hashes.c | 1 - certs/blacklist_nohashes.c | 6 - {scripts => certs}/check-blacklist-hashes.awk | 0 init/Kconfig | 302 +----------------- kernel/module/Kconfig | 293 +++++++++++++++++ kernel/module/internal.h | 2 +- kernel/module/kallsyms.c | 41 ++- kernel/module/main.c | 43 ++- kernel/module/procfs.c | 2 +- scripts/Kconfig.include | 2 +- scripts/Makefile.build | 5 +- scripts/Makefile.modinst | 3 + scripts/Makefile.package | 4 +- .../dummy-plugin-dir/include/plugin-version.h | 0 scripts/dummy-tools/gcc | 8 +- scripts/headers_install.sh | 2 - scripts/kconfig/qconf-cfg.sh | 1 + scripts/mod/file2alias.c | 4 +- scripts/mod/modpost.c | 36 ++- scripts/mod/modpost.h | 1 + scripts/module.lds.S | 2 + scripts/package/mkspec | 3 + 40 files changed, 424 insertions(+), 388 deletions(-) delete mode 100644 certs/blacklist_nohashes.c rename {scripts => certs}/check-blacklist-hashes.awk (100%) create mode 100644 kernel/module/Kconfig create mode 100644 scripts/dummy-tools/dummy-plugin-dir/include/plugin-version.h diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 877a057d71c36..9d96da88c04f1 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1152,8 +1152,12 @@ nopku [X86] Disable Memory Protection Keys CPU feature found in some Intel CPUs. - .async_probe [KNL] - Enable asynchronous probe on this module. + .async_probe[=] [KNL] + If no value is specified or if the value + specified is not a valid , enable asynchronous + probe on this module. Otherwise, enable/disable + asynchronous probe on this module as indicated by the + value. See also: module.async_probe early_ioremap_debug [KNL] Enable debug messages in early_ioremap support. This @@ -3243,6 +3247,15 @@ For details see: Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst + module.async_probe= + [KNL] When set to true, modules will use async probing + by default. To enable/disable async probing for a + specific module, use the module specific control that + is documented under .async_probe. When both + module.async_probe and .async_probe are + specified, .async_probe takes precedence for + the specific module. + module.sig_enforce [KNL] When CONFIG_MODULE_SIG is set, this means that modules without (valid) signatures will fail to load. diff --git a/Documentation/kbuild/kconfig-language.rst b/Documentation/kbuild/kconfig-language.rst index a7173843a2940..7fb398649f510 100644 --- a/Documentation/kbuild/kconfig-language.rst +++ b/Documentation/kbuild/kconfig-language.rst @@ -672,7 +672,7 @@ Future kconfig work Work on kconfig is welcomed on both areas of clarifying semantics and on evaluating the use of a full SAT solver for it. A full SAT solver can be desirable to enable more complex dependency mappings and / or queries, -for instance on possible use case for a SAT solver could be that of handling +for instance one possible use case for a SAT solver could be that of handling the current known recursive dependency issues. It is not known if this would address such issues but such evaluation is desirable. If support for a full SAT solver proves too complex or that it cannot address recursive dependency issues diff --git a/MAINTAINERS b/MAINTAINERS index 3a57d70a98da1..173d7200a8cee 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4717,7 +4717,6 @@ L: keyrings@vger.kernel.org S: Maintained F: Documentation/admin-guide/module-signing.rst F: certs/ -F: scripts/check-blacklist-hashes.awk F: scripts/sign-file.c F: tools/certs/ @@ -13543,6 +13542,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git modules-next F: include/linux/module.h F: kernel/module/ +F: scripts/module* MONOLITHIC POWER SYSTEM PMIC DRIVER M: Saravanan Sekar diff --git a/Makefile b/Makefile index 1a4593dd3d3be..507e4b8ddbe67 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,9 @@ endif $(if $(word 2, $(KBUILD_EXTMOD)), \ $(error building multiple external modules is not supported)) +$(foreach x, % :, $(if $(findstring $x, $(KBUILD_EXTMOD)), \ + $(error module directory path cannot contain '$x'))) + # Remove trailing slashes ifneq ($(filter %/, $(KBUILD_EXTMOD)),) KBUILD_EXTMOD := $(shell dirname $(KBUILD_EXTMOD).) diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig index 0016149f95830..e31a8ebc3eccb 100644 --- a/arch/arc/configs/axs101_defconfig +++ b/arch/arc/configs/axs101_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig index 5b031582a1cf8..e0e8567f0d758 100644 --- a/arch/arc/configs/axs103_defconfig +++ b/arch/arc/configs/axs103_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index d4eec39e0112c..fcbc952bc75bb 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig index 7337cdf4ffdda..d87ad7e88d624 100644 --- a/arch/arc/configs/haps_hs_defconfig +++ b/arch/arc/configs/haps_hs_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EXPERT=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig index bc927221afc0e..8d82cdb7f86a6 100644 --- a/arch/arc/configs/haps_hs_smp_defconfig +++ b/arch/arc/configs/haps_hs_smp_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig index aa000075a5757..f856b03e0fb5c 100644 --- a/arch/arc/configs/hsdk_defconfig +++ b/arch/arc/configs/hsdk_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y CONFIG_BLK_DEV_RAM=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig index 326f6cde78268..a1ce12bf5b165 100644 --- a/arch/arc/configs/nsim_700_defconfig +++ b/arch/arc/configs/nsim_700_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig index bf39a0091679c..ca10f4a2c823f 100644 --- a/arch/arc/configs/nsimosci_defconfig +++ b/arch/arc/configs/nsimosci_defconfig @@ -10,7 +10,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig index 7121bd71c543a..31b6ec3683c65 100644 --- a/arch/arc/configs/nsimosci_hs_defconfig +++ b/arch/arc/configs/nsimosci_hs_defconfig @@ -10,7 +10,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig index f9863b294a707..41a0037f48a58 100644 --- a/arch/arc/configs/nsimosci_hs_smp_defconfig +++ b/arch/arc/configs/nsimosci_hs_smp_defconfig @@ -8,7 +8,6 @@ CONFIG_IKCONFIG_PROC=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set CONFIG_KPROBES=y diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index a12656ec00720..d93b65008d4af 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -14,7 +14,6 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" CONFIG_INITRAMFS_ROOT_UID=2100 CONFIG_INITRAMFS_ROOT_GID=501 # CONFIG_RD_GZIP is not set -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_KALLSYMS_ALL=y # CONFIG_AIO is not set CONFIG_EMBEDDED=y diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig index d7c858df520cc..0c3b214168197 100644 --- a/arch/arc/configs/vdk_hs38_defconfig +++ b/arch/arc/configs/vdk_hs38_defconfig @@ -4,7 +4,6 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig index 015c1d43889e6..f9ad9d3ee702d 100644 --- a/arch/arc/configs/vdk_hs38_smp_defconfig +++ b/arch/arc/configs/vdk_hs38_smp_defconfig @@ -4,7 +4,6 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/certs/Makefile b/certs/Makefile index 88a73b28d254a..9486ed924731b 100644 --- a/certs/Makefile +++ b/certs/Makefile @@ -4,24 +4,22 @@ # obj-$(CONFIG_SYSTEM_TRUSTED_KEYRING) += system_keyring.o system_certificates.o -obj-$(CONFIG_SYSTEM_BLACKLIST_KEYRING) += blacklist.o +obj-$(CONFIG_SYSTEM_BLACKLIST_KEYRING) += blacklist.o blacklist_hashes.o obj-$(CONFIG_SYSTEM_REVOCATION_LIST) += revocation_certificates.o -ifneq ($(CONFIG_SYSTEM_BLACKLIST_HASH_LIST),) $(obj)/blacklist_hashes.o: $(obj)/blacklist_hash_list CFLAGS_blacklist_hashes.o := -I $(obj) quiet_cmd_check_and_copy_blacklist_hash_list = GEN $@ cmd_check_and_copy_blacklist_hash_list = \ - $(AWK) -f $(srctree)/scripts/check-blacklist-hashes.awk $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) >&2; \ - cat $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) > $@ + $(if $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST), \ + $(AWK) -f $(srctree)/$(src)/check-blacklist-hashes.awk $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) >&2; \ + { cat $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST); echo $(comma) NULL; } > $@, \ + echo NULL > $@) $(obj)/blacklist_hash_list: $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) FORCE $(call if_changed,check_and_copy_blacklist_hash_list) -obj-$(CONFIG_SYSTEM_BLACKLIST_KEYRING) += blacklist_hashes.o -else -obj-$(CONFIG_SYSTEM_BLACKLIST_KEYRING) += blacklist_nohashes.o -endif + targets += blacklist_hash_list quiet_cmd_extract_certs = CERT $@ diff --git a/certs/blacklist_hashes.c b/certs/blacklist_hashes.c index 86d66fe113489..0c5476abebd96 100644 --- a/certs/blacklist_hashes.c +++ b/certs/blacklist_hashes.c @@ -3,5 +3,4 @@ const char __initconst *const blacklist_hashes[] = { #include "blacklist_hash_list" - , NULL }; diff --git a/certs/blacklist_nohashes.c b/certs/blacklist_nohashes.c deleted file mode 100644 index 753b703ef0ef8..0000000000000 --- a/certs/blacklist_nohashes.c +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "blacklist.h" - -const char __initconst *const blacklist_hashes[] = { - NULL -}; diff --git a/scripts/check-blacklist-hashes.awk b/certs/check-blacklist-hashes.awk similarity index 100% rename from scripts/check-blacklist-hashes.awk rename to certs/check-blacklist-hashes.awk diff --git a/init/Kconfig b/init/Kconfig index 11ec35b657793..da0119e5c15af 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1744,16 +1744,17 @@ config KALLSYMS_ALL help Normally kallsyms only contains the symbols of functions for nicer OOPS messages and backtraces (i.e., symbols from the text and inittext - sections). This is sufficient for most cases. And only in very rare - cases (e.g., when a debugger is used) all symbols are required (e.g., - names of variables from the data sections, etc). + sections). This is sufficient for most cases. And only if you want to + enable kernel live patching, or other less common use cases (e.g., + when a debugger is used) all symbols are required (i.e., names of + variables from the data sections, etc). This option makes sure that all symbols are loaded into the kernel image (i.e., symbols from all sections) in cost of increased kernel size (depending on the kernel configuration, it may be 300KiB or something like this). - Say N unless you really need all symbols. + Say N unless you really need all symbols, or kernel live patching. config KALLSYMS_ABSOLUTE_PERCPU bool @@ -1944,298 +1945,7 @@ config MODULE_SIG_FORMAT def_bool n select SYSTEM_DATA_VERIFICATION -menuconfig MODULES - bool "Enable loadable module support" - modules - help - Kernel modules are small pieces of compiled code which can - be inserted in the running kernel, rather than being - permanently built into the kernel. You use the "modprobe" - tool to add (and sometimes remove) them. If you say Y here, - many parts of the kernel can be built as modules (by - answering M instead of Y where indicated): this is most - useful for infrequently used options which are not required - for booting. For more information, see the man pages for - modprobe, lsmod, modinfo, insmod and rmmod. - - If you say Y here, you will need to run "make - modules_install" to put the modules under /lib/modules/ - where modprobe can find them (you may need to be root to do - this). - - If unsure, say Y. - -if MODULES - -config MODULE_FORCE_LOAD - bool "Forced module loading" - default n - help - Allow loading of modules without version information (ie. modprobe - --force). Forced module loading sets the 'F' (forced) taint flag and - is usually a really bad idea. - -config MODULE_UNLOAD - bool "Module unloading" - help - Without this option you will not be able to unload any - modules (note that some modules may not be unloadable - anyway), which makes your kernel smaller, faster - and simpler. If unsure, say Y. - -config MODULE_FORCE_UNLOAD - bool "Forced module unloading" - depends on MODULE_UNLOAD - help - This option allows you to force a module to unload, even if the - kernel believes it is unsafe: the kernel will remove the module - without waiting for anyone to stop using it (using the -f option to - rmmod). This is mainly for kernel developers and desperate users. - If unsure, say N. - -config MODULE_UNLOAD_TAINT_TRACKING - bool "Tainted module unload tracking" - depends on MODULE_UNLOAD - default n - help - This option allows you to maintain a record of each unloaded - module that tainted the kernel. In addition to displaying a - list of linked (or loaded) modules e.g. on detection of a bad - page (see bad_page()), the aforementioned details are also - shown. If unsure, say N. - -config MODVERSIONS - bool "Module versioning support" - help - Usually, you have to use modules compiled with your kernel. - Saying Y here makes it sometimes possible to use modules - compiled for different kernels, by adding enough information - to the modules to (hopefully) spot any changes which would - make them incompatible with the kernel you are running. If - unsure, say N. - -config ASM_MODVERSIONS - bool - default HAVE_ASM_MODVERSIONS && MODVERSIONS - help - This enables module versioning for exported symbols also from - assembly. This can be enabled only when the target architecture - supports it. - -config MODULE_SRCVERSION_ALL - bool "Source checksum for all modules" - help - Modules which contain a MODULE_VERSION get an extra "srcversion" - field inserted into their modinfo section, which contains a - sum of the source files which made it. This helps maintainers - see exactly which source was used to build a module (since - others sometimes change the module source without updating - the version). With this option, such a "srcversion" field - will be created for all modules. If unsure, say N. - -config MODULE_SIG - bool "Module signature verification" - select MODULE_SIG_FORMAT - help - Check modules for valid signatures upon load: the signature - is simply appended to the module. For more information see - . - - Note that this option adds the OpenSSL development packages as a - kernel build dependency so that the signing tool can use its crypto - library. - - You should enable this option if you wish to use either - CONFIG_SECURITY_LOCKDOWN_LSM or lockdown functionality imposed via - another LSM - otherwise unsigned modules will be loadable regardless - of the lockdown policy. - - !!!WARNING!!! If you enable this option, you MUST make sure that the - module DOES NOT get stripped after being signed. This includes the - debuginfo strip done by some packagers (such as rpmbuild) and - inclusion into an initramfs that wants the module size reduced. - -config MODULE_SIG_FORCE - bool "Require modules to be validly signed" - depends on MODULE_SIG - help - Reject unsigned modules or signed modules for which we don't have a - key. Without this, such modules will simply taint the kernel. - -config MODULE_SIG_ALL - bool "Automatically sign all modules" - default y - depends on MODULE_SIG || IMA_APPRAISE_MODSIG - help - Sign all modules during make modules_install. Without this option, - modules must be signed manually, using the scripts/sign-file tool. - -comment "Do not forget to sign required modules with scripts/sign-file" - depends on MODULE_SIG_FORCE && !MODULE_SIG_ALL - -choice - prompt "Which hash algorithm should modules be signed with?" - depends on MODULE_SIG || IMA_APPRAISE_MODSIG - help - This determines which sort of hashing algorithm will be used during - signature generation. This algorithm _must_ be built into the kernel - directly so that signature verification can take place. It is not - possible to load a signed module containing the algorithm to check - the signature on that module. - -config MODULE_SIG_SHA1 - bool "Sign modules with SHA-1" - select CRYPTO_SHA1 - -config MODULE_SIG_SHA224 - bool "Sign modules with SHA-224" - select CRYPTO_SHA256 - -config MODULE_SIG_SHA256 - bool "Sign modules with SHA-256" - select CRYPTO_SHA256 - -config MODULE_SIG_SHA384 - bool "Sign modules with SHA-384" - select CRYPTO_SHA512 - -config MODULE_SIG_SHA512 - bool "Sign modules with SHA-512" - select CRYPTO_SHA512 - -endchoice - -config MODULE_SIG_HASH - string - depends on MODULE_SIG || IMA_APPRAISE_MODSIG - default "sha1" if MODULE_SIG_SHA1 - default "sha224" if MODULE_SIG_SHA224 - default "sha256" if MODULE_SIG_SHA256 - default "sha384" if MODULE_SIG_SHA384 - default "sha512" if MODULE_SIG_SHA512 - -choice - prompt "Module compression mode" - help - This option allows you to choose the algorithm which will be used to - compress modules when 'make modules_install' is run. (or, you can - choose to not compress modules at all.) - - External modules will also be compressed in the same way during the - installation. - - For modules inside an initrd or initramfs, it's more efficient to - compress the whole initrd or initramfs instead. - - This is fully compatible with signed modules. - - Please note that the tool used to load modules needs to support the - corresponding algorithm. module-init-tools MAY support gzip, and kmod - MAY support gzip, xz and zstd. - - Your build system needs to provide the appropriate compression tool - to compress the modules. - - If in doubt, select 'None'. - -config MODULE_COMPRESS_NONE - bool "None" - help - Do not compress modules. The installed modules are suffixed - with .ko. - -config MODULE_COMPRESS_GZIP - bool "GZIP" - help - Compress modules with GZIP. The installed modules are suffixed - with .ko.gz. - -config MODULE_COMPRESS_XZ - bool "XZ" - help - Compress modules with XZ. The installed modules are suffixed - with .ko.xz. - -config MODULE_COMPRESS_ZSTD - bool "ZSTD" - help - Compress modules with ZSTD. The installed modules are suffixed - with .ko.zst. - -endchoice - -config MODULE_DECOMPRESS - bool "Support in-kernel module decompression" - depends on MODULE_COMPRESS_GZIP || MODULE_COMPRESS_XZ - select ZLIB_INFLATE if MODULE_COMPRESS_GZIP - select XZ_DEC if MODULE_COMPRESS_XZ - help - - Support for decompressing kernel modules by the kernel itself - instead of relying on userspace to perform this task. Useful when - load pinning security policy is enabled. - - If unsure, say N. - -config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS - bool "Allow loading of modules with missing namespace imports" - help - Symbols exported with EXPORT_SYMBOL_NS*() are considered exported in - a namespace. A module that makes use of a symbol exported with such a - namespace is required to import the namespace via MODULE_IMPORT_NS(). - There is no technical reason to enforce correct namespace imports, - but it creates consistency between symbols defining namespaces and - users importing namespaces they make use of. This option relaxes this - requirement and lifts the enforcement when loading a module. - - If unsure, say N. - -config MODPROBE_PATH - string "Path to modprobe binary" - default "/sbin/modprobe" - help - When kernel code requests a module, it does so by calling - the "modprobe" userspace utility. This option allows you to - set the path where that binary is found. This can be changed - at runtime via the sysctl file - /proc/sys/kernel/modprobe. Setting this to the empty string - removes the kernel's ability to request modules (but - userspace can still load modules explicitly). - -config TRIM_UNUSED_KSYMS - bool "Trim unused exported kernel symbols" if EXPERT - depends on !COMPILE_TEST - help - The kernel and some modules make many symbols available for - other modules to use via EXPORT_SYMBOL() and variants. Depending - on the set of modules being selected in your kernel configuration, - many of those exported symbols might never be used. - - This option allows for unused exported symbols to be dropped from - the build. In turn, this provides the compiler more opportunities - (especially when using LTO) for optimizing the code and reducing - binary size. This might have some security advantages as well. - - If unsure, or if you need to build out-of-tree modules, say N. - -config UNUSED_KSYMS_WHITELIST - string "Whitelist of symbols to keep in ksymtab" - depends on TRIM_UNUSED_KSYMS - help - By default, all unused exported symbols will be un-exported from the - build when TRIM_UNUSED_KSYMS is selected. - - UNUSED_KSYMS_WHITELIST allows to whitelist symbols that must be kept - exported at all times, even in absence of in-tree users. The value to - set here is the path to a text file containing the list of symbols, - one per line. The path can be absolute, or relative to the kernel - source tree. - -endif # MODULES - -config MODULES_TREE_LOOKUP - def_bool y - depends on PERF_EVENTS || TRACING || CFI_CLANG +source "kernel/module/Kconfig" config INIT_ALL_POSSIBLE bool diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig new file mode 100644 index 0000000000000..26ea5d04f56c2 --- /dev/null +++ b/kernel/module/Kconfig @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig MODULES + bool "Enable loadable module support" + modules + help + Kernel modules are small pieces of compiled code which can + be inserted in the running kernel, rather than being + permanently built into the kernel. You use the "modprobe" + tool to add (and sometimes remove) them. If you say Y here, + many parts of the kernel can be built as modules (by + answering M instead of Y where indicated): this is most + useful for infrequently used options which are not required + for booting. For more information, see the man pages for + modprobe, lsmod, modinfo, insmod and rmmod. + + If you say Y here, you will need to run "make + modules_install" to put the modules under /lib/modules/ + where modprobe can find them (you may need to be root to do + this). + + If unsure, say Y. + +if MODULES + +config MODULE_FORCE_LOAD + bool "Forced module loading" + default n + help + Allow loading of modules without version information (ie. modprobe + --force). Forced module loading sets the 'F' (forced) taint flag and + is usually a really bad idea. + +config MODULE_UNLOAD + bool "Module unloading" + help + Without this option you will not be able to unload any + modules (note that some modules may not be unloadable + anyway), which makes your kernel smaller, faster + and simpler. If unsure, say Y. + +config MODULE_FORCE_UNLOAD + bool "Forced module unloading" + depends on MODULE_UNLOAD + help + This option allows you to force a module to unload, even if the + kernel believes it is unsafe: the kernel will remove the module + without waiting for anyone to stop using it (using the -f option to + rmmod). This is mainly for kernel developers and desperate users. + If unsure, say N. + +config MODULE_UNLOAD_TAINT_TRACKING + bool "Tainted module unload tracking" + depends on MODULE_UNLOAD + default n + help + This option allows you to maintain a record of each unloaded + module that tainted the kernel. In addition to displaying a + list of linked (or loaded) modules e.g. on detection of a bad + page (see bad_page()), the aforementioned details are also + shown. If unsure, say N. + +config MODVERSIONS + bool "Module versioning support" + help + Usually, you have to use modules compiled with your kernel. + Saying Y here makes it sometimes possible to use modules + compiled for different kernels, by adding enough information + to the modules to (hopefully) spot any changes which would + make them incompatible with the kernel you are running. If + unsure, say N. + +config ASM_MODVERSIONS + bool + default HAVE_ASM_MODVERSIONS && MODVERSIONS + help + This enables module versioning for exported symbols also from + assembly. This can be enabled only when the target architecture + supports it. + +config MODULE_SRCVERSION_ALL + bool "Source checksum for all modules" + help + Modules which contain a MODULE_VERSION get an extra "srcversion" + field inserted into their modinfo section, which contains a + sum of the source files which made it. This helps maintainers + see exactly which source was used to build a module (since + others sometimes change the module source without updating + the version). With this option, such a "srcversion" field + will be created for all modules. If unsure, say N. + +config MODULE_SIG + bool "Module signature verification" + select MODULE_SIG_FORMAT + help + Check modules for valid signatures upon load: the signature + is simply appended to the module. For more information see + . + + Note that this option adds the OpenSSL development packages as a + kernel build dependency so that the signing tool can use its crypto + library. + + You should enable this option if you wish to use either + CONFIG_SECURITY_LOCKDOWN_LSM or lockdown functionality imposed via + another LSM - otherwise unsigned modules will be loadable regardless + of the lockdown policy. + + !!!WARNING!!! If you enable this option, you MUST make sure that the + module DOES NOT get stripped after being signed. This includes the + debuginfo strip done by some packagers (such as rpmbuild) and + inclusion into an initramfs that wants the module size reduced. + +config MODULE_SIG_FORCE + bool "Require modules to be validly signed" + depends on MODULE_SIG + help + Reject unsigned modules or signed modules for which we don't have a + key. Without this, such modules will simply taint the kernel. + +config MODULE_SIG_ALL + bool "Automatically sign all modules" + default y + depends on MODULE_SIG || IMA_APPRAISE_MODSIG + help + Sign all modules during make modules_install. Without this option, + modules must be signed manually, using the scripts/sign-file tool. + +comment "Do not forget to sign required modules with scripts/sign-file" + depends on MODULE_SIG_FORCE && !MODULE_SIG_ALL + +choice + prompt "Which hash algorithm should modules be signed with?" + depends on MODULE_SIG || IMA_APPRAISE_MODSIG + help + This determines which sort of hashing algorithm will be used during + signature generation. This algorithm _must_ be built into the kernel + directly so that signature verification can take place. It is not + possible to load a signed module containing the algorithm to check + the signature on that module. + +config MODULE_SIG_SHA1 + bool "Sign modules with SHA-1" + select CRYPTO_SHA1 + +config MODULE_SIG_SHA224 + bool "Sign modules with SHA-224" + select CRYPTO_SHA256 + +config MODULE_SIG_SHA256 + bool "Sign modules with SHA-256" + select CRYPTO_SHA256 + +config MODULE_SIG_SHA384 + bool "Sign modules with SHA-384" + select CRYPTO_SHA512 + +config MODULE_SIG_SHA512 + bool "Sign modules with SHA-512" + select CRYPTO_SHA512 + +endchoice + +config MODULE_SIG_HASH + string + depends on MODULE_SIG || IMA_APPRAISE_MODSIG + default "sha1" if MODULE_SIG_SHA1 + default "sha224" if MODULE_SIG_SHA224 + default "sha256" if MODULE_SIG_SHA256 + default "sha384" if MODULE_SIG_SHA384 + default "sha512" if MODULE_SIG_SHA512 + +choice + prompt "Module compression mode" + help + This option allows you to choose the algorithm which will be used to + compress modules when 'make modules_install' is run. (or, you can + choose to not compress modules at all.) + + External modules will also be compressed in the same way during the + installation. + + For modules inside an initrd or initramfs, it's more efficient to + compress the whole initrd or initramfs instead. + + This is fully compatible with signed modules. + + Please note that the tool used to load modules needs to support the + corresponding algorithm. module-init-tools MAY support gzip, and kmod + MAY support gzip, xz and zstd. + + Your build system needs to provide the appropriate compression tool + to compress the modules. + + If in doubt, select 'None'. + +config MODULE_COMPRESS_NONE + bool "None" + help + Do not compress modules. The installed modules are suffixed + with .ko. + +config MODULE_COMPRESS_GZIP + bool "GZIP" + help + Compress modules with GZIP. The installed modules are suffixed + with .ko.gz. + +config MODULE_COMPRESS_XZ + bool "XZ" + help + Compress modules with XZ. The installed modules are suffixed + with .ko.xz. + +config MODULE_COMPRESS_ZSTD + bool "ZSTD" + help + Compress modules with ZSTD. The installed modules are suffixed + with .ko.zst. + +endchoice + +config MODULE_DECOMPRESS + bool "Support in-kernel module decompression" + depends on MODULE_COMPRESS_GZIP || MODULE_COMPRESS_XZ + select ZLIB_INFLATE if MODULE_COMPRESS_GZIP + select XZ_DEC if MODULE_COMPRESS_XZ + help + + Support for decompressing kernel modules by the kernel itself + instead of relying on userspace to perform this task. Useful when + load pinning security policy is enabled. + + If unsure, say N. + +config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS + bool "Allow loading of modules with missing namespace imports" + help + Symbols exported with EXPORT_SYMBOL_NS*() are considered exported in + a namespace. A module that makes use of a symbol exported with such a + namespace is required to import the namespace via MODULE_IMPORT_NS(). + There is no technical reason to enforce correct namespace imports, + but it creates consistency between symbols defining namespaces and + users importing namespaces they make use of. This option relaxes this + requirement and lifts the enforcement when loading a module. + + If unsure, say N. + +config MODPROBE_PATH + string "Path to modprobe binary" + default "/sbin/modprobe" + help + When kernel code requests a module, it does so by calling + the "modprobe" userspace utility. This option allows you to + set the path where that binary is found. This can be changed + at runtime via the sysctl file + /proc/sys/kernel/modprobe. Setting this to the empty string + removes the kernel's ability to request modules (but + userspace can still load modules explicitly). + +config TRIM_UNUSED_KSYMS + bool "Trim unused exported kernel symbols" if EXPERT + depends on !COMPILE_TEST + help + The kernel and some modules make many symbols available for + other modules to use via EXPORT_SYMBOL() and variants. Depending + on the set of modules being selected in your kernel configuration, + many of those exported symbols might never be used. + + This option allows for unused exported symbols to be dropped from + the build. In turn, this provides the compiler more opportunities + (especially when using LTO) for optimizing the code and reducing + binary size. This might have some security advantages as well. + + If unsure, or if you need to build out-of-tree modules, say N. + +config UNUSED_KSYMS_WHITELIST + string "Whitelist of symbols to keep in ksymtab" + depends on TRIM_UNUSED_KSYMS + help + By default, all unused exported symbols will be un-exported from the + build when TRIM_UNUSED_KSYMS is selected. + + UNUSED_KSYMS_WHITELIST allows to whitelist symbols that must be kept + exported at all times, even in absence of in-tree users. The value to + set here is the path to a text file containing the list of symbols, + one per line. The path can be absolute, or relative to the kernel + source tree. + +config MODULES_TREE_LOOKUP + def_bool y + depends on PERF_EVENTS || TRACING || CFI_CLANG + +endif # MODULES diff --git a/kernel/module/internal.h b/kernel/module/internal.h index 851ea72471a0e..8a3abfff9fe98 100644 --- a/kernel/module/internal.h +++ b/kernel/module/internal.h @@ -105,7 +105,7 @@ struct module *find_module_all(const char *name, size_t len, bool even_unformed) int cmp_name(const void *name, const void *sym); long module_get_offset(struct module *mod, unsigned int *size, Elf_Shdr *sechdr, unsigned int section); -char *module_flags(struct module *mod, char *buf); +char *module_flags(struct module *mod, char *buf, bool show_state); size_t module_flags_taint(unsigned long taints, char *buf); static inline void module_assert_mutex_or_preempt(void) diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 77e75bead5698..f5c5c9175333d 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -457,26 +457,39 @@ unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name) return 0; } -/* Look for this name: can be of form module:name. */ -unsigned long module_kallsyms_lookup_name(const char *name) +static unsigned long __module_kallsyms_lookup_name(const char *name) { struct module *mod; char *colon; - unsigned long ret = 0; + + colon = strnchr(name, MODULE_NAME_LEN, ':'); + if (colon) { + mod = find_module_all(name, colon - name, false); + if (mod) + return find_kallsyms_symbol_value(mod, colon + 1); + return 0; + } + + list_for_each_entry_rcu(mod, &modules, list) { + unsigned long ret; + + if (mod->state == MODULE_STATE_UNFORMED) + continue; + ret = find_kallsyms_symbol_value(mod, name); + if (ret) + return ret; + } + return 0; +} + +/* Look for this name: can be of form module:name. */ +unsigned long module_kallsyms_lookup_name(const char *name) +{ + unsigned long ret; /* Don't lock: we're in enough trouble already. */ preempt_disable(); - if ((colon = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) { - if ((mod = find_module_all(name, colon - name, false)) != NULL) - ret = find_kallsyms_symbol_value(mod, colon + 1); - } else { - list_for_each_entry_rcu(mod, &modules, list) { - if (mod->state == MODULE_STATE_UNFORMED) - continue; - if ((ret = find_kallsyms_symbol_value(mod, name)) != 0) - break; - } - } + ret = __module_kallsyms_lookup_name(name); preempt_enable(); return ret; } diff --git a/kernel/module/main.c b/kernel/module/main.c index f9212da570f62..f58b5bc1fe862 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -120,7 +120,7 @@ static void mod_update_bounds(struct module *mod) } /* Block module loading/unloading? */ -int modules_disabled = 0; +int modules_disabled; core_param(nomodule, modules_disabled, bint, 0); /* Waiting for a module to finish initializing? */ @@ -525,7 +525,10 @@ static struct module_attribute modinfo_##field = { \ MODINFO_ATTR(version); MODINFO_ATTR(srcversion); -static char last_unloaded_module[MODULE_NAME_LEN+1]; +static struct { + char name[MODULE_NAME_LEN + 1]; + char taints[MODULE_FLAGS_BUF_SIZE]; +} last_unloaded_module; #ifdef CONFIG_MODULE_UNLOAD @@ -695,6 +698,7 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user, { struct module *mod; char name[MODULE_NAME_LEN]; + char buf[MODULE_FLAGS_BUF_SIZE]; int ret, forced = 0; if (!capable(CAP_SYS_MODULE) || modules_disabled) @@ -754,8 +758,9 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user, async_synchronize_full(); - /* Store the name of the last unloaded module for diagnostic purposes */ - strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); + /* Store the name and taints of the last unloaded module for diagnostic purposes */ + strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); + strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints)); free_module(mod); /* someone could wait for the module in add_unformed_module() */ @@ -2139,7 +2144,7 @@ static int move_module(struct module *mod, struct load_info *info) #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC /* Do the allocs. */ - ptr = vmalloc(mod->data_layout.size); + ptr = vzalloc(mod->data_layout.size); /* * The pointer to this block is stored in the module structure * which is inside the block. Just mark it as not being a @@ -2152,7 +2157,6 @@ static int move_module(struct module *mod, struct load_info *info) return -ENOMEM; } - memset(ptr, 0, mod->data_layout.size); mod->data_layout.base = ptr; #endif /* Transfer each section which specifies SHF_ALLOC */ @@ -2411,6 +2415,12 @@ static void do_free_init(struct work_struct *w) } } +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "module." +/* Default value for module->async_probe_requested */ +static bool async_probe; +module_param(async_probe, bool, 0644); + /* * This is where the real work happens. * @@ -2631,7 +2641,8 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname, int ret; if (strcmp(param, "async_probe") == 0) { - mod->async_probe_requested = true; + if (strtobool(val, &mod->async_probe_requested)) + mod->async_probe_requested = true; return 0; } @@ -2798,6 +2809,8 @@ static int load_module(struct load_info *info, const char __user *uargs, if (err) goto bug_cleanup; + mod->async_probe_requested = async_probe; + /* Module is ready to execute: parsing args may do that. */ after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, -32768, 32767, mod, @@ -2972,24 +2985,27 @@ static void cfi_cleanup(struct module *mod) } /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ -char *module_flags(struct module *mod, char *buf) +char *module_flags(struct module *mod, char *buf, bool show_state) { int bx = 0; BUG_ON(mod->state == MODULE_STATE_UNFORMED); + if (!mod->taints && !show_state) + goto out; if (mod->taints || mod->state == MODULE_STATE_GOING || mod->state == MODULE_STATE_COMING) { buf[bx++] = '('; bx += module_flags_taint(mod->taints, buf + bx); /* Show a - for module-is-being-unloaded */ - if (mod->state == MODULE_STATE_GOING) + if (mod->state == MODULE_STATE_GOING && show_state) buf[bx++] = '-'; /* Show a + for module-is-being-loaded */ - if (mod->state == MODULE_STATE_COMING) + if (mod->state == MODULE_STATE_COMING && show_state) buf[bx++] = '+'; buf[bx++] = ')'; } +out: buf[bx] = '\0'; return buf; @@ -3122,12 +3138,13 @@ void print_modules(void) list_for_each_entry_rcu(mod, &modules, list) { if (mod->state == MODULE_STATE_UNFORMED) continue; - pr_cont(" %s%s", mod->name, module_flags(mod, buf)); + pr_cont(" %s%s", mod->name, module_flags(mod, buf, true)); } print_unloaded_tainted_modules(); preempt_enable(); - if (last_unloaded_module[0]) - pr_cont(" [last unloaded: %s]", last_unloaded_module); + if (last_unloaded_module.name[0]) + pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name, + last_unloaded_module.taints); pr_cont("\n"); } diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c index 1b6af54d88c15..80260fa4dac51 100644 --- a/kernel/module/procfs.c +++ b/kernel/module/procfs.c @@ -91,7 +91,7 @@ static int m_show(struct seq_file *m, void *p) /* Taints info */ if (mod->taints) - seq_printf(m, " %s", module_flags(mod, buf)); + seq_printf(m, " %s", module_flags(mod, buf, true)); seq_puts(m, "\n"); return 0; diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include index 0496efd6e1179..c1f4222d223d2 100644 --- a/scripts/Kconfig.include +++ b/scripts/Kconfig.include @@ -25,7 +25,7 @@ failure = $(if-success,$(1),n,y) # $(cc-option,) # Return y if the compiler supports , n otherwise -cc-option = $(success,mkdir .tmp_$$$$; trap "rm -rf .tmp_$$$$" EXIT; $(CC) -Werror $(CLANG_FLAGS) $(1) -c -x c /dev/null -o .tmp_$$$$/tmp.o) +cc-option = $(success,mkdir .tmp_$$; trap "rm -rf .tmp_$$" EXIT; $(CC) -Werror $(CLANG_FLAGS) $(1) -c -x c /dev/null -o .tmp_$$/tmp.o) # $(ld-option,) # Return y if the linker supports , n otherwise diff --git a/scripts/Makefile.build b/scripts/Makefile.build index cac070aee7915..784f46d41959b 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -358,9 +358,8 @@ $(subdir-modorder): $(obj)/%/modules.order: $(obj)/% ; quiet_cmd_ar_builtin = AR $@ cmd_ar_builtin = rm -f $@; \ - echo $(patsubst $(obj)/%,%,$(real-prereqs)) | \ - sed -E 's:([^ ]+):$(obj)/\1:g' | \ - xargs $(AR) cDPrST $@ + $(if $(real-prereqs), printf "$(obj)/%s " $(patsubst $(obj)/%,%,$(real-prereqs)) | xargs) \ + $(AR) cDPrST $@ $(obj)/built-in.a: $(real-obj-y) FORCE $(call if_changed,ar_builtin) diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst index 16a02e9237d36..a4c987c23750f 100644 --- a/scripts/Makefile.modinst +++ b/scripts/Makefile.modinst @@ -18,6 +18,9 @@ INSTALL_MOD_DIR ?= extra dst := $(MODLIB)/$(INSTALL_MOD_DIR) endif +$(foreach x, % :, $(if $(findstring $x, $(dst)), \ + $(error module installation path cannot contain '$x'))) + suffix-y := suffix-$(CONFIG_MODULE_COMPRESS_GZIP) := .gz suffix-$(CONFIG_MODULE_COMPRESS_XZ) := .xz diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 77b612183c082..5017f6b2da809 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -56,7 +56,7 @@ rpm-pkg: $(MAKE) clean $(CONFIG_SHELL) $(MKSPEC) >$(objtree)/kernel.spec $(call cmd,src_tar,$(KERNELPATH),kernel.spec) - +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz \ + +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE)-linux -ta $(KERNELPATH).tar.gz \ --define='_smp_mflags %{nil}' # binrpm-pkg @@ -66,7 +66,7 @@ binrpm-pkg: $(MAKE) -f $(srctree)/Makefile $(CONFIG_SHELL) $(MKSPEC) prebuilt > $(objtree)/binkernel.spec +rpmbuild $(RPMOPTS) --define "_builddir $(objtree)" --target \ - $(UTS_MACHINE) -bb $(objtree)/binkernel.spec + $(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec PHONY += deb-pkg deb-pkg: diff --git a/scripts/dummy-tools/dummy-plugin-dir/include/plugin-version.h b/scripts/dummy-tools/dummy-plugin-dir/include/plugin-version.h new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/scripts/dummy-tools/gcc b/scripts/dummy-tools/gcc index b2483149bbe55..7db8258434355 100755 --- a/scripts/dummy-tools/gcc +++ b/scripts/dummy-tools/gcc @@ -96,12 +96,8 @@ fi # To set GCC_PLUGINS if arg_contain -print-file-name=plugin "$@"; then - plugin_dir=$(mktemp -d) - - mkdir -p $plugin_dir/include - touch $plugin_dir/include/plugin-version.h - - echo $plugin_dir + # Use $0 to find the in-tree dummy directory + echo "$(dirname "$(readlink -f "$0")")/dummy-plugin-dir" exit 0 fi diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh index dd554bd436cc6..4041881746ade 100755 --- a/scripts/headers_install.sh +++ b/scripts/headers_install.sh @@ -70,7 +70,6 @@ configs=$(sed -e ' # # The format is : in each line. config_leak_ignores=" -arch/alpha/include/uapi/asm/setup.h:CONFIG_ALPHA_LEGACY_START_ADDRESS arch/arc/include/uapi/asm/page.h:CONFIG_ARC_PAGE_SIZE_16K arch/arc/include/uapi/asm/page.h:CONFIG_ARC_PAGE_SIZE_4K arch/arc/include/uapi/asm/swab.h:CONFIG_ARC_HAS_SWAPE @@ -84,7 +83,6 @@ arch/nios2/include/uapi/asm/swab.h:CONFIG_NIOS2_CI_SWAB_SUPPORT arch/x86/include/uapi/asm/auxvec.h:CONFIG_IA32_EMULATION arch/x86/include/uapi/asm/auxvec.h:CONFIG_X86_64 arch/x86/include/uapi/asm/mman.h:CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS -include/uapi/asm-generic/fcntl.h:CONFIG_64BIT include/uapi/linux/atmdev.h:CONFIG_COMPAT include/uapi/linux/eventpoll.h:CONFIG_PM_SLEEP include/uapi/linux/hw_breakpoint.h:CONFIG_HAVE_MIXED_BREAKPOINTS_REGS diff --git a/scripts/kconfig/qconf-cfg.sh b/scripts/kconfig/qconf-cfg.sh index 9b695e5cd9b37..ad652cb539474 100755 --- a/scripts/kconfig/qconf-cfg.sh +++ b/scripts/kconfig/qconf-cfg.sh @@ -20,5 +20,6 @@ fi echo >&2 "*" echo >&2 "* Could not find Qt5 via ${HOSTPKG_CONFIG}." echo >&2 "* Please install Qt5 and make sure it's in PKG_CONFIG_PATH" +echo >&2 "* You need $PKG" echo >&2 "*" exit 1 diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index cbd6b0f48b4ed..80d973144fded 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1571,9 +1571,7 @@ void handle_moddevtable(struct module *mod, struct elf_info *info, zeros = calloc(1, sym->st_size); symval = zeros; } else { - symval = (void *)info->hdr - + info->sechdrs[get_secindex(info, sym)].sh_offset - + sym->st_value; + symval = sym_get_data(info, sym); } /* First handle the "special" cases */ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 620dc8c4c8140..26254e96c300b 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -321,13 +321,10 @@ static void *sym_get_data_by_offset(const struct elf_info *info, { Elf_Shdr *sechdr = &info->sechdrs[secindex]; - if (info->hdr->e_type != ET_REL) - offset -= sechdr->sh_addr; - return (void *)info->hdr + sechdr->sh_offset + offset; } -static void *sym_get_data(const struct elf_info *info, const Elf_Sym *sym) +void *sym_get_data(const struct elf_info *info, const Elf_Sym *sym) { return sym_get_data_by_offset(info, get_secindex(info, sym), sym->st_value); @@ -339,8 +336,19 @@ static const char *sech_name(const struct elf_info *info, Elf_Shdr *sechdr) sechdr->sh_name); } -static const char *sec_name(const struct elf_info *info, int secindex) +static const char *sec_name_of_symbol(const struct elf_info *info, + const Elf_Sym *sym) { + unsigned int secindex = get_secindex(info, sym); + + /* + * If sym->st_shndx is a special section index, there is no + * corresponding section header. + * Return "" if the index is out of range of info->sechdrs[] array. + */ + if (secindex >= info->num_sections) + return ""; + return sech_name(info, &info->sechdrs[secindex]); } @@ -466,6 +474,10 @@ static int parse_elf(struct elf_info *info, const char *filename) sechdrs = (void *)hdr + hdr->e_shoff; info->sechdrs = sechdrs; + /* modpost only works for relocatable objects */ + if (hdr->e_type != ET_REL) + fatal("%s: not relocatable object.", filename); + /* Check if file offset is correct */ if (hdr->e_shoff > info->size) { fatal("section header offset=%lu in file '%s' is bigger than filesize=%zu\n", @@ -649,7 +661,7 @@ static void handle_symbol(struct module *mod, struct elf_info *info, const char *name, *secname; name = symname + strlen("__ksymtab_"); - secname = sec_name(info, get_secindex(info, sym)); + secname = sec_name_of_symbol(info, sym); if (strstarts(secname, "___ksymtab_gpl+")) sym_add_exported(name, mod, true); @@ -742,7 +754,6 @@ static const char *const section_white_list[] = { ".comment*", ".debug*", - ".cranges", /* sh64 */ ".zdebug*", /* Compressed debug sections. */ ".GCC.command.line", /* record-gcc-switches */ ".mdebug*", /* alpha, score, mips etc. */ @@ -1217,7 +1228,7 @@ static Elf_Sym *find_elf_symbol2(struct elf_info *elf, Elf_Addr addr, if (is_shndx_special(sym->st_shndx)) continue; - symsec = sec_name(elf, get_secindex(elf, sym)); + symsec = sec_name_of_symbol(elf, sym); if (strcmp(symsec, sec) != 0) continue; if (!is_valid_name(elf, sym)) @@ -1457,7 +1468,7 @@ static void default_mismatch_handler(const char *modname, struct elf_info *elf, if (strstarts(fromsym, "reference___initcall")) return; - tosec = sec_name(elf, get_secindex(elf, sym)); + tosec = sec_name_of_symbol(elf, sym); to = find_elf_symbol(elf, r->r_addend, sym); tosym = sym_name(elf, to); @@ -1559,7 +1570,7 @@ static void extable_mismatch_handler(const char* modname, struct elf_info *elf, Elf_Rela* r, Elf_Sym* sym, const char *fromsec) { - const char* tosec = sec_name(elf, get_secindex(elf, sym)); + const char *tosec = sec_name_of_symbol(elf, sym); sec_mismatch_count++; @@ -1593,7 +1604,7 @@ static void extable_mismatch_handler(const char* modname, struct elf_info *elf, static void check_section_mismatch(const char *modname, struct elf_info *elf, Elf_Rela *r, Elf_Sym *sym, const char *fromsec) { - const char *tosec = sec_name(elf, get_secindex(elf, sym)); + const char *tosec = sec_name_of_symbol(elf, sym); const struct sectioncheck *mismatch = section_mismatch(fromsec, tosec); if (mismatch) { @@ -1623,9 +1634,6 @@ static int addend_386_rel(struct elf_info *elf, Elf_Shdr *sechdr, Elf_Rela *r) break; case R_386_PC32: r->r_addend = TO_NATIVE(*location) + 4; - /* For CONFIG_RELOCATABLE=y */ - if (elf->hdr->e_type == ET_EXEC) - r->r_addend += r->r_offset; break; } return 0; diff --git a/scripts/mod/modpost.h b/scripts/mod/modpost.h index 044bdfb894b72..4d8a1ae1d1e36 100644 --- a/scripts/mod/modpost.h +++ b/scripts/mod/modpost.h @@ -187,6 +187,7 @@ void get_src_version(const char *modname, char sum[], unsigned sumlen); /* from modpost.c */ char *read_text_file(const char *filename); char *get_line(char **stringp); +void *sym_get_data(const struct elf_info *info, const Elf_Sym *sym); enum loglevel { LOG_WARN, diff --git a/scripts/module.lds.S b/scripts/module.lds.S index 1d0e1e4dc3d2a..3a3aa2354ed86 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -27,6 +27,8 @@ SECTIONS { .ctors 0 : ALIGN(8) { *(SORT(.ctors.*)) *(.ctors) } .init_array 0 : ALIGN(8) { *(SORT(.init_array.*)) *(.init_array) } + .altinstructions 0 : ALIGN(8) { KEEP(*(.altinstructions)) } + __bug_table 0 : ALIGN(8) { KEEP(*(__bug_table)) } __jump_table 0 : ALIGN(8) { KEEP(*(__jump_table)) } __patchable_function_entries : { *(__patchable_function_entries) } diff --git a/scripts/package/mkspec b/scripts/package/mkspec index 7c477ca7dc982..8fa7c5b8a1a15 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -49,6 +49,9 @@ sed -e '/^DEL/d' -e 's/^\t*//' < Date: Fri, 5 Aug 2022 19:34:26 +0200 Subject: [PATCH 11/16] lrng Signed-off-by: Peter Jung --- MAINTAINERS | 7 + crypto/drbg.c | 16 +- crypto/jitterentropy-kcapi.c | 3 +- crypto/jitterentropy.c | 2 +- drivers/char/Kconfig | 2 + drivers/char/Makefile | 5 +- drivers/char/lrng/Kconfig | 907 ++++++++++++++++++ drivers/char/lrng/Makefile | 39 + drivers/char/lrng/lrng_definitions.h | 151 +++ drivers/char/lrng/lrng_drng_atomic.c | 171 ++++ drivers/char/lrng/lrng_drng_atomic.h | 23 + drivers/char/lrng/lrng_drng_chacha20.c | 195 ++++ drivers/char/lrng/lrng_drng_chacha20.h | 42 + drivers/char/lrng/lrng_drng_drbg.c | 179 ++++ drivers/char/lrng/lrng_drng_drbg.h | 13 + drivers/char/lrng/lrng_drng_kcapi.c | 208 ++++ drivers/char/lrng/lrng_drng_kcapi.h | 13 + drivers/char/lrng/lrng_drng_mgr.c | 585 +++++++++++ drivers/char/lrng/lrng_drng_mgr.h | 86 ++ drivers/char/lrng/lrng_es_aux.c | 335 +++++++ drivers/char/lrng/lrng_es_aux.h | 44 + drivers/char/lrng/lrng_es_cpu.c | 275 ++++++ drivers/char/lrng/lrng_es_cpu.h | 17 + drivers/char/lrng/lrng_es_irq.c | 727 ++++++++++++++ drivers/char/lrng/lrng_es_irq.h | 24 + drivers/char/lrng/lrng_es_jent.c | 136 +++ drivers/char/lrng/lrng_es_jent.h | 17 + drivers/char/lrng/lrng_es_krng.c | 121 +++ drivers/char/lrng/lrng_es_krng.h | 17 + drivers/char/lrng/lrng_es_mgr.c | 433 +++++++++ drivers/char/lrng/lrng_es_mgr.h | 49 + drivers/char/lrng/lrng_es_mgr_cb.h | 87 ++ drivers/char/lrng/lrng_es_sched.c | 562 +++++++++++ drivers/char/lrng/lrng_es_sched.h | 20 + drivers/char/lrng/lrng_es_timer_common.c | 144 +++ drivers/char/lrng/lrng_es_timer_common.h | 83 ++ drivers/char/lrng/lrng_hash_kcapi.c | 140 +++ drivers/char/lrng/lrng_health.c | 420 ++++++++ drivers/char/lrng/lrng_health.h | 42 + drivers/char/lrng/lrng_interface_aux.c | 126 +++ drivers/char/lrng/lrng_interface_dev.c | 35 + drivers/char/lrng/lrng_interface_dev_common.c | 297 ++++++ drivers/char/lrng/lrng_interface_dev_common.h | 50 + drivers/char/lrng/lrng_interface_hwrand.c | 68 ++ drivers/char/lrng/lrng_interface_kcapi.c | 129 +++ .../char/lrng/lrng_interface_random_kernel.c | 207 ++++ .../char/lrng/lrng_interface_random_kernel.h | 15 + .../char/lrng/lrng_interface_random_user.c | 71 ++ drivers/char/lrng/lrng_numa.c | 124 +++ drivers/char/lrng/lrng_numa.h | 15 + drivers/char/lrng/lrng_proc.c | 74 ++ drivers/char/lrng/lrng_proc.h | 15 + drivers/char/lrng/lrng_selftest.c | 397 ++++++++ drivers/char/lrng/lrng_sha.h | 14 + drivers/char/lrng/lrng_sha1.c | 88 ++ drivers/char/lrng/lrng_sha256.c | 72 ++ drivers/char/lrng/lrng_switch.c | 286 ++++++ drivers/char/lrng/lrng_sysctl.c | 140 +++ drivers/char/lrng/lrng_sysctl.h | 15 + drivers/char/lrng/lrng_testing.c | 901 +++++++++++++++++ drivers/char/lrng/lrng_testing.h | 85 ++ include/crypto/drbg.h | 7 + .../crypto/internal}/jitterentropy.h | 0 include/linux/lrng.h | 177 ++++ kernel/sched/core.c | 3 + 65 files changed, 9741 insertions(+), 10 deletions(-) create mode 100644 drivers/char/lrng/Kconfig create mode 100644 drivers/char/lrng/Makefile create mode 100644 drivers/char/lrng/lrng_definitions.h create mode 100644 drivers/char/lrng/lrng_drng_atomic.c create mode 100644 drivers/char/lrng/lrng_drng_atomic.h create mode 100644 drivers/char/lrng/lrng_drng_chacha20.c create mode 100644 drivers/char/lrng/lrng_drng_chacha20.h create mode 100644 drivers/char/lrng/lrng_drng_drbg.c create mode 100644 drivers/char/lrng/lrng_drng_drbg.h create mode 100644 drivers/char/lrng/lrng_drng_kcapi.c create mode 100644 drivers/char/lrng/lrng_drng_kcapi.h create mode 100644 drivers/char/lrng/lrng_drng_mgr.c create mode 100644 drivers/char/lrng/lrng_drng_mgr.h create mode 100644 drivers/char/lrng/lrng_es_aux.c create mode 100644 drivers/char/lrng/lrng_es_aux.h create mode 100644 drivers/char/lrng/lrng_es_cpu.c create mode 100644 drivers/char/lrng/lrng_es_cpu.h create mode 100644 drivers/char/lrng/lrng_es_irq.c create mode 100644 drivers/char/lrng/lrng_es_irq.h create mode 100644 drivers/char/lrng/lrng_es_jent.c create mode 100644 drivers/char/lrng/lrng_es_jent.h create mode 100644 drivers/char/lrng/lrng_es_krng.c create mode 100644 drivers/char/lrng/lrng_es_krng.h create mode 100644 drivers/char/lrng/lrng_es_mgr.c create mode 100644 drivers/char/lrng/lrng_es_mgr.h create mode 100644 drivers/char/lrng/lrng_es_mgr_cb.h create mode 100644 drivers/char/lrng/lrng_es_sched.c create mode 100644 drivers/char/lrng/lrng_es_sched.h create mode 100644 drivers/char/lrng/lrng_es_timer_common.c create mode 100644 drivers/char/lrng/lrng_es_timer_common.h create mode 100644 drivers/char/lrng/lrng_hash_kcapi.c create mode 100644 drivers/char/lrng/lrng_health.c create mode 100644 drivers/char/lrng/lrng_health.h create mode 100644 drivers/char/lrng/lrng_interface_aux.c create mode 100644 drivers/char/lrng/lrng_interface_dev.c create mode 100644 drivers/char/lrng/lrng_interface_dev_common.c create mode 100644 drivers/char/lrng/lrng_interface_dev_common.h create mode 100644 drivers/char/lrng/lrng_interface_hwrand.c create mode 100644 drivers/char/lrng/lrng_interface_kcapi.c create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.c create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.h create mode 100644 drivers/char/lrng/lrng_interface_random_user.c create mode 100644 drivers/char/lrng/lrng_numa.c create mode 100644 drivers/char/lrng/lrng_numa.h create mode 100644 drivers/char/lrng/lrng_proc.c create mode 100644 drivers/char/lrng/lrng_proc.h create mode 100644 drivers/char/lrng/lrng_selftest.c create mode 100644 drivers/char/lrng/lrng_sha.h create mode 100644 drivers/char/lrng/lrng_sha1.c create mode 100644 drivers/char/lrng/lrng_sha256.c create mode 100644 drivers/char/lrng/lrng_switch.c create mode 100644 drivers/char/lrng/lrng_sysctl.c create mode 100644 drivers/char/lrng/lrng_sysctl.h create mode 100644 drivers/char/lrng/lrng_testing.c create mode 100644 drivers/char/lrng/lrng_testing.h rename {crypto => include/crypto/internal}/jitterentropy.h (100%) create mode 100644 include/linux/lrng.h diff --git a/MAINTAINERS b/MAINTAINERS index 173d7200a8cee..b0149b7b885bf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11544,6 +11544,13 @@ F: Documentation/litmus-tests/ F: Documentation/memory-barriers.txt F: tools/memory-model/ +LINUX RANDOM NUMBER GENERATOR (LRNG) DRIVER +M: Stephan Mueller +S: Maintained +W: https://www.chronox.de/lrng.html +F: drivers/char/lrng/ +F: include/linux/lrng.h + LIS3LV02D ACCELEROMETER DRIVER M: Eric Piel S: Maintained diff --git a/crypto/drbg.c b/crypto/drbg.c index 177983b6ae38b..ea79a31014a40 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -115,7 +115,7 @@ * the SHA256 / AES 256 over other ciphers. Thus, the favored * DRBGs are the latest entries in this array. */ -static const struct drbg_core drbg_cores[] = { +const struct drbg_core drbg_cores[] = { #ifdef CONFIG_CRYPTO_DRBG_CTR { .flags = DRBG_CTR | DRBG_STRENGTH128, @@ -192,6 +192,7 @@ static const struct drbg_core drbg_cores[] = { }, #endif /* CONFIG_CRYPTO_DRBG_HMAC */ }; +EXPORT_SYMBOL(drbg_cores); static int drbg_uninstantiate(struct drbg_state *drbg); @@ -207,7 +208,7 @@ static int drbg_uninstantiate(struct drbg_state *drbg); * Return: normalized strength in *bytes* value or 32 as default * to counter programming errors */ -static inline unsigned short drbg_sec_strength(drbg_flag_t flags) +unsigned short drbg_sec_strength(drbg_flag_t flags) { switch (flags & DRBG_STRENGTH_MASK) { case DRBG_STRENGTH128: @@ -220,6 +221,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) return 32; } } +EXPORT_SYMBOL(drbg_sec_strength); /* * FIPS 140-2 continuous self test for the noise source @@ -1252,7 +1254,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, } /* Free all substructures in a DRBG state without the DRBG state structure */ -static inline void drbg_dealloc_state(struct drbg_state *drbg) +void drbg_dealloc_state(struct drbg_state *drbg) { if (!drbg) return; @@ -1273,12 +1275,13 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) drbg->fips_primed = false; } } +EXPORT_SYMBOL(drbg_dealloc_state); /* * Allocate all sub-structures for a DRBG state. * The DRBG state structure must already be allocated. */ -static inline int drbg_alloc_state(struct drbg_state *drbg) +int drbg_alloc_state(struct drbg_state *drbg) { int ret = -ENOMEM; unsigned int sb_size = 0; @@ -1359,6 +1362,7 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) drbg_dealloc_state(drbg); return ret; } +EXPORT_SYMBOL(drbg_alloc_state); /************************************************************************* * DRBG interface functions @@ -1895,8 +1899,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, * * return: flags */ -static inline void drbg_convert_tfm_core(const char *cra_driver_name, - int *coreref, bool *pr) +void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, bool *pr) { int i = 0; size_t start = 0; @@ -1923,6 +1926,7 @@ static inline void drbg_convert_tfm_core(const char *cra_driver_name, } } } +EXPORT_SYMBOL(drbg_convert_tfm_core); static int drbg_kcapi_init(struct crypto_tfm *tfm) { diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 2d115bec15aeb..e7dac734d2377 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -42,8 +42,7 @@ #include #include #include - -#include "jitterentropy.h" +#include /*************************************************************************** * Helper function diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 93bff32138238..81b80a4d3d3a0 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -133,7 +133,7 @@ struct rand_data { #define JENT_ENTROPY_SAFETY_FACTOR 64 #include -#include "jitterentropy.h" +#include /*************************************************************************** * Adaptive Proportion Test diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 0b6c03643ddc6..c7d6f6752aa46 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -465,4 +465,6 @@ config RANDOM_TRUST_BOOTLOADER believe its RNG facilities may be faulty. This may also be configured at boot time with "random.trust_bootloader=on/off". +source "drivers/char/lrng/Kconfig" + endmenu diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 264eb398fdd4f..c2676f5d5c964 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -3,7 +3,8 @@ # Makefile for the kernel character device drivers. # -obj-y += mem.o random.o +obj-y += mem.o +obj-$(CONFIG_RANDOM_DEFAULT_IMPL) += random.o obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o obj-y += misc.o obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o @@ -46,3 +47,5 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o obj-$(CONFIG_ADI) += adi.o + +obj-$(CONFIG_LRNG) += lrng/ diff --git a/drivers/char/lrng/Kconfig b/drivers/char/lrng/Kconfig new file mode 100644 index 0000000000000..bfd6b3ea4f1be --- /dev/null +++ b/drivers/char/lrng/Kconfig @@ -0,0 +1,907 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Linux Random Number Generator configuration +# + +config RANDOM_DEFAULT_IMPL + bool "Kernel RNG Default Implementation" + default y + help + The default random number generator as provided with + drivers/char/random.c is selected with this option. + +config LRNG + bool "Linux Random Number Generator" + default n + select CRYPTO_LIB_SHA256 if CRYPTO + help + The Linux Random Number Generator (LRNG) generates entropy + from different entropy sources. Each entropy source can + be enabled and configured independently. The interrupt + entropy source can be configured to be SP800-90B compliant. + The entire LRNG can be configured to be SP800-90C compliant. + Runtime-switchable cryptographic support is available. + The LRNG delivers significant entropy during boot. + + The LRNG also provides compliance to SP800-90A/B/C. + +menu "Linux Random Number Generator Configuration" + depends on LRNG + +if LRNG + +config LRNG_SHA256 + bool + default y if CRYPTO_LIB_SHA256 + +config LRNG_SHA1 + bool + default y if !CRYPTO_LIB_SHA256 + +config LRNG_COMMON_DEV_IF + bool + +config LRNG_DRNG_ATOMIC + bool + select LRNG_DRNG_CHACHA20 + +config LRNG_SYSCTL + bool + depends on SYSCTL + +config LRNG_RANDOM_IF + bool + default n if RANDOM_DEFAULT_IMPL + default y if !RANDOM_DEFAULT_IMPL + select LRNG_COMMON_DEV_IF + select LRNG_DRNG_ATOMIC + select LRNG_SYSCTL + +menu "LRNG Interfaces" + +config LRNG_KCAPI_IF + tristate "Interface with Kernel Crypto API" + depends on CRYPTO_RNG + help + The LRNG can be registered with the kernel crypto API's + random number generator framework. This offers a random + number generator with the name "lrng" and a priority that + is intended to be higher than the existing RNG + implementations. + +config LRNG_HWRAND_IF + tristate "Interface with Hardware Random Number Generator Framework" + depends on HW_RANDOM + select LRNG_DRNG_ATOMIC + help + The LRNG can be registered with the hardware random number + generator framework. This offers a random number generator + with the name "lrng" that is accessible via the framework. + For example it allows pulling data from the LRNG via the + /dev/hwrng file. + +config LRNG_DEV_IF + bool "Character device file interface" + select LRNG_COMMON_DEV_IF + help + The LRNG can create a character device file that operates + identically to /dev/random including IOCTL, read and write + operations. + +endmenu # "LRNG Interfaces" + +menu "Entropy Source Configuration" + +config LRNG_RUNTIME_ES_CONFIG + bool "Enable runtime configuration of entropy sources" + help + When enabling this option, the LRNG provides the mechanism + allowing to alter the entropy rate of each entropy source + during boot time and runtime. + + Each entropy source allows its entropy rate changed with + a kernel command line option. When not providing any + option, the default specified during kernel compilation + is applied. + +comment "Common Timer-based Entropy Source Configuration" + +config LRNG_IRQ_DFLT_TIMER_ES + bool + +config LRNG_SCHED_DFLT_TIMER_ES + bool + +config LRNG_TIMER_COMMON + bool + +choice + prompt "Default Timer-based Entropy Source" + default LRNG_IRQ_DFLT_TIMER_ES + depends on LRNG_TIMER_COMMON + help + Select the timer-based entropy source that is credited + with entropy. The other timer-based entropy sources may + be operational and provide data, but are credited with no + entropy. + + config LRNG_IRQ_DFLT_TIMER_ES + bool "Interrupt Entropy Source" + depends on LRNG_IRQ + help + The interrupt entropy source is selected as a timer-based + entropy source to provide entropy. + + config LRNG_SCHED_DFLT_TIMER_ES + bool "Scheduler Entropy Source" + depends on LRNG_SCHED + help + The scheduler entropy source is selected as timer-based + entropy source to provide entropy. +endchoice + +choice + prompt "LRNG Entropy Collection Pool Size" + default LRNG_COLLECTION_SIZE_1024 + depends on LRNG_TIMER_COMMON + help + Select the size of the LRNG entropy collection pool + storing data for the interrupt as well as the scheduler + entropy sources without performing a compression + operation. The larger the collection size is, the faster + the average interrupt handling will be. The collection + size represents the number of bytes of the per-CPU memory + used to batch up entropy event data. + + The default value is good for regular operations. Choose + larger sizes for servers that have no memory limitations. + If runtime memory is precious, choose a smaller size. + + The collection size is unrelated to the entropy rate + or the amount of entropy the LRNG can process. + + config LRNG_COLLECTION_SIZE_32 + depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED + depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION + depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES + bool "32 interrupt events" + + config LRNG_COLLECTION_SIZE_256 + depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES + bool "256 interrupt events" + + config LRNG_COLLECTION_SIZE_512 + bool "512 interrupt events" + + config LRNG_COLLECTION_SIZE_1024 + bool "1024 interrupt events (default)" + + config LRNG_COLLECTION_SIZE_2048 + bool "2048 interrupt events" + + config LRNG_COLLECTION_SIZE_4096 + bool "4096 interrupt events" + + config LRNG_COLLECTION_SIZE_8192 + bool "8192 interrupt events" + +endchoice + +config LRNG_COLLECTION_SIZE + int + default 32 if LRNG_COLLECTION_SIZE_32 + default 256 if LRNG_COLLECTION_SIZE_256 + default 512 if LRNG_COLLECTION_SIZE_512 + default 1024 if LRNG_COLLECTION_SIZE_1024 + default 2048 if LRNG_COLLECTION_SIZE_2048 + default 4096 if LRNG_COLLECTION_SIZE_4096 + default 8192 if LRNG_COLLECTION_SIZE_8192 + +config LRNG_HEALTH_TESTS + bool "Enable internal entropy source online health tests" + depends on LRNG_TIMER_COMMON + help + The online health tests applied to the interrupt entropy + source and to the scheduler entropy source to validate + the noise source at runtime for fatal errors. These tests + include SP800-90B compliant tests which are invoked if + the system is booted with fips=1. In case of fatal errors + during active SP800-90B tests, the issue is logged and + the noise data is discarded. These tests are required for + full compliance of the interrupt entropy source with + SP800-90B. + + If both, the scheduler and the interrupt entropy sources, + are enabled, the health tests for both are applied + independent of each other. + + If unsure, say Y. + +config LRNG_RCT_BROKEN + bool "SP800-90B RCT with dangerous low cutoff value" + depends on LRNG_HEALTH_TESTS + depends on BROKEN + default n + help + This option enables a dangerously low SP800-90B repetitive + count test (RCT) cutoff value which makes it very likely + that the RCT is triggered to raise a self test failure. + + This option is ONLY intended for developers wanting to + test the effectiveness of the SP800-90B RCT health test. + + If unsure, say N. + +config LRNG_APT_BROKEN + bool "SP800-90B APT with dangerous low cutoff value" + depends on LRNG_HEALTH_TESTS + depends on BROKEN + default n + help + This option enables a dangerously low SP800-90B adaptive + proportion test (APT) cutoff value which makes it very + likely that the APT is triggered to raise a self test + failure. + + This option is ONLY intended for developers wanting to + test the effectiveness of the SP800-90B APT health test. + + If unsure, say N. + +# Default taken from SP800-90B sec 4.4.1 - significance level 2^-30 +config LRNG_RCT_CUTOFF + int + default 31 if !LRNG_RCT_BROKEN + default 1 if LRNG_RCT_BROKEN + +# Default taken from SP800-90B sec 4.4.2 - significance level 2^-30 +config LRNG_APT_CUTOFF + int + default 325 if !LRNG_APT_BROKEN + default 32 if LRNG_APT_BROKEN + +comment "Interrupt Entropy Source" + +config LRNG_IRQ + bool "Enable Interrupt Entropy Source as LRNG Seed Source" + default y + depends on !RANDOM_DEFAULT_IMPL + select LRNG_TIMER_COMMON + help + The LRNG models an entropy source based on the timing of the + occurrence of interrupts. Enable this option to enable this + IRQ entropy source. + + The IRQ entropy source is triggered every time an interrupt + arrives and thus causes the interrupt handler to execute + slightly longer. Disabling the IRQ entropy source implies + that the performance penalty on the interrupt handler added + by the LRNG is eliminated. Yet, this entropy source is + considered to be an internal entropy source of the LRNG. + Thus, only disable it if you ensured that other entropy + sources are available that supply the LRNG with entropy. + + If you disable the IRQ entropy source, you MUST ensure + one or more entropy sources collectively have the + capability to deliver sufficient entropy with one invocation + at a rate compliant to the security strength of the DRNG + (usually 256 bits of entropy). In addition, if those + entropy sources do not deliver sufficient entropy during + first request, the reseed must be triggered from user + space or kernel space when sufficient entropy is considered + to be present. + + If unsure, say Y. + +choice + prompt "Continuous entropy compression boot time setting" + default LRNG_CONTINUOUS_COMPRESSION_ENABLED + depends on LRNG_IRQ + help + Select the default behavior of the interrupt entropy source + continuous compression operation. + + The LRNG IRQ ES collects entropy data during each interrupt. + For performance reasons, a amount of entropy data defined by + the LRNG entropy collection pool size is concatenated into + an array. When that array is filled up, a hash is calculated + to compress the entropy. That hash is calculated in + interrupt context. + + In case such hash calculation in interrupt context is deemed + too time-consuming, the continuous compression operation + can be disabled. If disabled, the collection of entropy will + not trigger a hash compression operation in interrupt context. + The compression happens only when the DRNG is reseeded which is + in process context. This implies that old entropy data + collected after the last DRNG-reseed is overwritten with newer + entropy data once the collection pool is full instead of + retaining its entropy with the compression operation. + + config LRNG_CONTINUOUS_COMPRESSION_ENABLED + bool "Enable continuous compression (default)" + + config LRNG_CONTINUOUS_COMPRESSION_DISABLED + bool "Disable continuous compression" + +endchoice + +config LRNG_ENABLE_CONTINUOUS_COMPRESSION + bool + default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED + default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED + +config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION + bool "Runtime-switchable continuous entropy compression" + depends on LRNG_IRQ + help + Per default, the interrupt entropy source continuous + compression operation behavior is hard-wired into the kernel. + Enable this option to allow it to be configurable at boot time. + + To modify the default behavior of the continuous + compression operation, use the kernel command line option + of lrng_sw_noise.lrng_pcpu_continuous_compression. + + If unsure, say N. + +config LRNG_IRQ_ENTROPY_RATE + int "Interrupt Entropy Source Entropy Rate" + depends on LRNG_IRQ + range 256 4294967295 if LRNG_IRQ_DFLT_TIMER_ES + range 4294967295 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES + default 256 if LRNG_IRQ_DFLT_TIMER_ES + default 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES + help + The LRNG will collect the configured number of interrupts to + obtain 256 bits of entropy. This value can be set to any between + 256 and 4294967295. The LRNG guarantees that this value is not + lower than 256. This lower limit implies that one interrupt event + is credited with one bit of entropy. This value is subject to the + increase by the oversampling factor, if no high-resolution timer + is found. + + In order to effectively disable the interrupt entropy source, + the option has to be set to 4294967295. In this case, the + interrupt entropy source will still deliver data but without + being credited with entropy. + +comment "Jitter RNG Entropy Source" + +config LRNG_JENT + bool "Enable Jitter RNG as LRNG Seed Source" + depends on CRYPTO + select CRYPTO_JITTERENTROPY + help + The LRNG may use the Jitter RNG as entropy source. Enabling + this option enables the use of the Jitter RNG. Its default + entropy level is 16 bits of entropy per 256 data bits delivered + by the Jitter RNG. This entropy level can be changed at boot + time or at runtime with the lrng_base.jitterrng configuration + variable. + +config LRNG_JENT_ENTROPY_RATE + int "Jitter RNG Entropy Source Entropy Rate" + depends on LRNG_JENT + range 0 256 + default 16 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the Jitter RNG entropy source. The + LRNG enforces the limit that this value must be in the range + between 0 and 256. + + When configuring this value to 0, the Jitter RNG entropy source + will provide 256 bits of data without being credited to contain + entropy. + +comment "CPU Entropy Source" + +config LRNG_CPU + bool "Enable CPU Entropy Source as LRNG Seed Source" + default y + help + Current CPUs commonly contain entropy sources which can be + used to seed the LRNG. For example, the Intel RDSEED + instruction, or the POWER DARN instruction will be sourced + to seed the LRNG if this option is enabled. + + Note, if this option is enabled and the underlying CPU + does not offer such entropy source, the LRNG will automatically + detect this and ignore the hardware. + +config LRNG_CPU_FULL_ENT_MULTIPLIER + int + default 1 if !LRNG_TEST_CPU_ES_COMPRESSION + default 123 if LRNG_TEST_CPU_ES_COMPRESSION + +config LRNG_CPU_ENTROPY_RATE + int "CPU Entropy Source Entropy Rate" + depends on LRNG_CPU + range 0 256 + default 8 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the CPU entropy source. The LRNG + enforces the limit that this value must be in the range between + 0 and 256. + + When configuring this value to 0, the CPU entropy source will + provide 256 bits of data without being credited to contain + entropy. + + Note, this option is overwritten when the option + CONFIG_RANDOM_TRUST_CPU is set. + +comment "Scheduler Entropy Source" + +config LRNG_SCHED + bool "Enable Scheduer Entropy Source as LRNG Seed Source" + select LRNG_TIMER_COMMON + help + The LRNG models an entropy source based on the timing of the + occurrence of scheduler-triggered context switches. Enable + this option to enable this scheduler entropy source. + + The scheduler entropy source is triggered every time a + context switch is triggered thus causes the scheduler to + execute slightly longer. Disabling the scheduler entropy + source implies that the performance penalty on the scheduler + added by the LRNG is eliminated. Yet, this entropy source is + considered to be an internal entropy source of the LRNG. + Thus, only disable it if you ensured that other entropy + sources are available that supply the LRNG with entropy. + + If you disable the scheduler entropy source, you MUST + ensure one or more entropy sources collectively have the + capability to deliver sufficient entropy with one invocation + at a rate compliant to the security strength of the DRNG + (usually 256 bits of entropy). In addition, if those + entropy sources do not deliver sufficient entropy during + first request, the reseed must be triggered from user + space or kernel space when sufficient entropy is considered + to be present. + + If unsure, say Y. + +config LRNG_SCHED_ENTROPY_RATE + int "Scheduler Entropy Source Entropy Rate" + depends on LRNG_SCHED + range 256 4294967295 if LRNG_SCHED_DFLT_TIMER_ES + range 4294967295 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES + default 256 if LRNG_SCHED_DFLT_TIMER_ES + default 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES + help + The LRNG will collect the configured number of context switches + triggered by the scheduler to obtain 256 bits of entropy. This + value can be set to any between 256 and 4294967295. The LRNG + guarantees that this value is not lower than 256. This lower + limit implies that one interrupt event is credited with one bit + of entropy. This value is subject to the increase by the + oversampling factor, if no high-resolution timer is found. + + In order to effectively disable the scheduler entropy source, + the option has to be set to 4294967295. In this case, the + scheduler entropy source will still deliver data but without + being credited with entropy. + +comment "Kernel RNG Entropy Source" + +config LRNG_KERNEL_RNG + bool "Enable Kernel RNG as LRNG Seed Source" + depends on RANDOM_DEFAULT_IMPL + help + The LRNG may use the kernel RNG (random.c) as entropy + source. + +config LRNG_KERNEL_RNG_ENTROPY_RATE + int "Kernel RNG Entropy Source Entropy Rate" + depends on LRNG_KERNEL_RNG + range 0 256 + default 256 + help + The option defines the amount of entropy the LRNG applies to 256 + bits of data obtained from the kernel RNG entropy source. The + LRNG enforces the limit that this value must be in the range + between 0 and 256. + + When configuring this value to 0, the kernel RNG entropy source + will provide 256 bits of data without being credited to contain + entropy. + + Note: This value is set to 0 automatically when booting the + kernel in FIPS mode (with fips=1 kernel command line option). + This is due to the fact that random.c is not SP800-90B + compliant. + +endmenu # "Entropy Source Configuration" + +config LRNG_DRNG_CHACHA20 + tristate + +config LRNG_DRBG + tristate + depends on CRYPTO + select CRYPTO_DRBG_MENU + +config LRNG_DRNG_KCAPI + tristate + depends on CRYPTO + select CRYPTO_RNG + +config LRNG_SWITCH + bool + +menuconfig LRNG_SWITCH_HASH + bool "Support conditioning hash runtime switching" + select LRNG_SWITCH + help + The LRNG uses a default message digest. With this + configuration option other message digests can be selected + and loaded at runtime. + +if LRNG_SWITCH_HASH + +config LRNG_HASH_KCAPI + tristate "Kernel crypto API hashing support for LRNG" + select CRYPTO_HASH + select CRYPTO_SHA512 + help + Enable the kernel crypto API support for entropy compression + and conditioning functions. + +endif # LRNG_SWITCH_HASH + +menuconfig LRNG_SWITCH_DRNG + bool "Support DRNG runtime switching" + select LRNG_SWITCH + help + The LRNG uses a default DRNG With this configuration + option other DRNGs or message digests can be selected and + loaded at runtime. + +if LRNG_SWITCH_DRNG + +config LRNG_SWITCH_DRNG_CHACHA20 + tristate "ChaCha20-based DRNG support for LRNG" + depends on !LRNG_DFLT_DRNG_CHACHA20 + select LRNG_DRNG_CHACHA20 + help + Enable the ChaCha20-based DRNG. This DRNG implementation + does not depend on the kernel crypto API presence. + +config LRNG_SWITCH_DRBG + tristate "SP800-90A support for the LRNG" + depends on !LRNG_DFLT_DRNG_DRBG + select LRNG_DRBG + help + Enable the SP800-90A DRBG support for the LRNG. Once the + module is loaded, output from /dev/random, /dev/urandom, + getrandom(2), or get_random_bytes_full is provided by a DRBG. + +config LRNG_SWITCH_DRNG_KCAPI + tristate "Kernel Crypto API support for the LRNG" + depends on !LRNG_DFLT_DRNG_KCAPI + depends on !LRNG_SWITCH_DRBG + select LRNG_DRNG_KCAPI + help + Enable the support for generic pseudo-random number + generators offered by the kernel crypto API with the + LRNG. Once the module is loaded, output from /dev/random, + /dev/urandom, getrandom(2), or get_random_bytes is + provided by the selected kernel crypto API RNG. + +endif # LRNG_SWITCH_DRNG + +choice + prompt "LRNG Default DRNG" + default LRNG_DFLT_DRNG_CHACHA20 + help + Select the default deterministic random number generator + that is used by the LRNG. When enabling the switchable + cryptographic mechanism support, this DRNG can be + replaced at runtime. + + config LRNG_DFLT_DRNG_CHACHA20 + bool "ChaCha20-based DRNG" + select LRNG_DRNG_CHACHA20 + + config LRNG_DFLT_DRNG_DRBG + bool "SP800-90A DRBG" + select LRNG_DRBG + + config LRNG_DFLT_DRNG_KCAPI + bool "Kernel Crypto API DRNG" + select LRNG_DRNG_KCAPI +endchoice + +menuconfig LRNG_TESTING_MENU + bool "LRNG testing interfaces" + depends on DEBUG_FS + help + Enable one or more of the following test interfaces. + + If unsure, say N. + +if LRNG_TESTING_MENU + +config LRNG_TESTING + bool + +config LRNG_TESTING_RECORDING + bool + +comment "Interrupt Entropy Source Test Interfaces" + +config LRNG_RAW_HIRES_ENTROPY + bool "Interface to obtain raw unprocessed IRQ noise source data" + default y + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned high resolution time stamp noise that + is collected by the LRNG for statistical analysis. Extracted + noise data is not used to seed the LRNG. + + The raw noise data can be obtained using the lrng_raw_hires + debugfs file. Using the option lrng_testing.boot_raw_hires_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_JIFFIES_ENTROPY + bool "Entropy test interface to Jiffies of IRQ noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned Jiffies that is collected by + the LRNG for statistical analysis. This data is used for + seeding the LRNG if a high-resolution time stamp is not + available. If a high-resolution time stamp is detected, + the Jiffies value is not collected by the LRNG and no + data is provided via the test interface. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_jiffies + debugfs file. Using the option lrng_testing.boot_raw_jiffies_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_IRQ_ENTROPY + bool "Entropy test interface to IRQ number noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned interrupt number that is collected by + the LRNG for statistical analysis. Extracted noise data is + not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_irq + debugfs file. Using the option lrng_testing.boot_raw_irq_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_RETIP_ENTROPY + bool "Entropy test interface to RETIP value of IRQ noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned return instruction pointer value + that is collected by the LRNG for statistical analysis. + Extracted noise data is not used to seed the random number + generator. + + The raw noise data can be obtained using the lrng_raw_retip + debugfs file. Using the option lrng_testing.boot_raw_retip_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_REGS_ENTROPY + bool "Entropy test interface to IRQ register value noise source" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned interrupt register value that is + collected by the LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the lrng_raw_regs + debugfs file. Using the option lrng_testing.boot_raw_regs_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_ARRAY + bool "Test interface to LRNG raw entropy IRQ storage array" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw noise data that is collected by the LRNG + in the per-CPU array for statistical analysis. The purpose + of this interface is to verify that the array handling code + truly only concatenates data and provides the same entropy + rate as the raw unconditioned noise source when assessing + the collected data byte-wise. + + The data can be obtained using the lrng_raw_array debugfs + file. Using the option lrng_testing.boot_raw_array=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_IRQ_PERF + bool "LRNG interrupt entropy source performance monitor" + depends on LRNG_IRQ + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + With this option, the performance monitor of the LRNG + interrupt handling code is enabled. The file provides + the execution time of the interrupt handler in + cycles. + + The interrupt performance data can be obtained using + the lrng_irq_perf debugfs file. Using the option + lrng_testing.boot_irq_perf=1 the performance data of + the first 1000 entropy events since boot can be sampled. + +comment "Scheduler Entropy Source Test Interfaces" + +config LRNG_RAW_SCHED_HIRES_ENTROPY + bool "Interface to obtain raw unprocessed scheduler noise source data" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned high resolution time stamp noise that + is collected by the LRNG for the Scheduler-based noise source + for statistical analysis. Extracted noise data is not used to + seed the LRNG. + + The raw noise data can be obtained using the lrng_raw_sched_hires + debugfs file. Using the option + lrng_testing.boot_raw_sched_hires_test=1 the raw noise of the + first 1000 entropy events since boot can be sampled. + +config LRNG_RAW_SCHED_PID_ENTROPY + bool "Entropy test interface to PID value" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned PID value that is collected by the + LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_pid debugfs file. Using the option + lrng_testing.boot_raw_sched_pid_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_RAW_SCHED_START_TIME_ENTROPY + bool "Entropy test interface to task start time value" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned task start time value that is collected + by the LRNG for statistical analysis. Extracted noise + data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_starttime debugfs file. Using the option + lrng_testing.boot_raw_sched_starttime_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + + +config LRNG_RAW_SCHED_NVCSW_ENTROPY + bool "Entropy test interface to task context switch numbers" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + The test interface allows a privileged process to capture + the raw unconditioned task numbers of context switches that + are collected by the LRNG for statistical analysis. Extracted + noise data is not used to seed the random number generator. + + The raw noise data can be obtained using the + lrng_raw_sched_nvcsw debugfs file. Using the option + lrng_testing.boot_raw_sched_nvcsw_test=1 + the raw noise of the first 1000 entropy events since boot + can be sampled. + +config LRNG_SCHED_PERF + bool "LRNG scheduler entropy source performance monitor" + depends on LRNG_SCHED + select LRNG_TESTING + select LRNG_TESTING_RECORDING + help + With this option, the performance monitor of the LRNG + scheduler event handling code is enabled. The file provides + the execution time of the interrupt handler in cycles. + + The scheduler performance data can be obtained using + the lrng_sched_perf debugfs file. Using the option + lrng_testing.boot_sched_perf=1 the performance data of + the first 1000 entropy events since boot can be sampled. + +comment "Auxiliary Test Interfaces" + +config LRNG_ACVT_HASH + bool "Enable LRNG ACVT Hash interface" + select LRNG_TESTING + help + With this option, the LRNG built-in hash function used for + auxiliary pool management and prior to switching the + cryptographic backends is made available for ACVT. The + interface allows writing of the data to be hashed + into the interface. The read operation triggers the hash + operation to generate message digest. + + The ACVT interface is available with the lrng_acvt_hash + debugfs file. + +config LRNG_RUNTIME_MAX_WO_RESEED_CONFIG + bool "Enable runtime configuration of max reseed threshold" + help + When enabling this option, the LRNG provides an interface + allowing the setting of the maximum number of DRNG generate + operations without a reseed that has full entropy. The + interface is lrng_drng.max_wo_reseed. + +config LRNG_TEST_CPU_ES_COMPRESSION + bool "Force CPU ES compression operation" + help + When enabling this option, the CPU ES compression operation + is forced by setting an arbitrary value > 1 for the data + multiplier even when the CPU ES would deliver full entropy. + This allows testing of the compression operation. It + therefore forces to pull more data from the CPU ES + than what may be required. + +endif #LRNG_TESTING_MENU + +config LRNG_SELFTEST + bool "Enable power-on and on-demand self-tests" + help + The power-on self-tests are executed during boot time + covering the ChaCha20 DRNG, the hash operation used for + processing the entropy pools and the auxiliary pool, and + the time stamp management of the LRNG. + + The on-demand self-tests are triggered by writing any + value into the SysFS file selftest_status. At the same + time, when reading this file, the test status is + returned. A zero indicates that all tests were executed + successfully. + + If unsure, say Y. + +if LRNG_SELFTEST + +config LRNG_SELFTEST_PANIC + bool "Panic the kernel upon self-test failure" + help + If the option is enabled, the kernel is terminated if an + LRNG power-on self-test failure is detected. + +endif # LRNG_SELFTEST + +endif # LRNG + +endmenu # LRNG diff --git a/drivers/char/lrng/Makefile b/drivers/char/lrng/Makefile new file mode 100644 index 0000000000000..f61fc40f4620d --- /dev/null +++ b/drivers/char/lrng/Makefile @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Entropy Source and DRNG Manager. +# + +obj-y += lrng_es_mgr.o lrng_drng_mgr.o \ + lrng_es_aux.o +obj-$(CONFIG_LRNG_SHA256) += lrng_sha256.o +obj-$(CONFIG_LRNG_SHA1) += lrng_sha1.o + +obj-$(CONFIG_SYSCTL) += lrng_proc.o +obj-$(CONFIG_LRNG_SYSCTL) += lrng_sysctl.o +obj-$(CONFIG_NUMA) += lrng_numa.o + +obj-$(CONFIG_LRNG_SWITCH) += lrng_switch.o +obj-$(CONFIG_LRNG_HASH_KCAPI) += lrng_hash_kcapi.o +obj-$(CONFIG_LRNG_DRNG_CHACHA20) += lrng_drng_chacha20.o +obj-$(CONFIG_LRNG_DRBG) += lrng_drng_drbg.o +obj-$(CONFIG_LRNG_DRNG_KCAPI) += lrng_drng_kcapi.o +obj-$(CONFIG_LRNG_DRNG_ATOMIC) += lrng_drng_atomic.o + +obj-$(CONFIG_LRNG_TIMER_COMMON) += lrng_es_timer_common.o +obj-$(CONFIG_LRNG_IRQ) += lrng_es_irq.o +obj-$(CONFIG_LRNG_KERNEL_RNG) += lrng_es_krng.o +obj-$(CONFIG_LRNG_SCHED) += lrng_es_sched.o +obj-$(CONFIG_LRNG_CPU) += lrng_es_cpu.o +obj-$(CONFIG_LRNG_JENT) += lrng_es_jent.o + +obj-$(CONFIG_LRNG_HEALTH_TESTS) += lrng_health.o +obj-$(CONFIG_LRNG_TESTING) += lrng_testing.o +obj-$(CONFIG_LRNG_SELFTEST) += lrng_selftest.o + +obj-$(CONFIG_LRNG_COMMON_DEV_IF) += lrng_interface_dev_common.o +obj-$(CONFIG_LRNG_RANDOM_IF) += lrng_interface_random_user.o \ + lrng_interface_random_kernel.o \ + lrng_interface_aux.o +obj-$(CONFIG_LRNG_KCAPI_IF) += lrng_interface_kcapi.o +obj-$(CONFIG_LRNG_DEV_IF) += lrng_interface_dev.o +obj-$(CONFIG_LRNG_HWRAND_IF) += lrng_interface_hwrand.o diff --git a/drivers/char/lrng/lrng_definitions.h b/drivers/char/lrng/lrng_definitions.h new file mode 100644 index 0000000000000..50f87c40f701a --- /dev/null +++ b/drivers/char/lrng/lrng_definitions.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DEFINITIONS_H +#define _LRNG_DEFINITIONS_H + +#include +#include +#include + +/*************************** General LRNG parameter ***************************/ + +/* + * Specific settings for different use cases + */ +#ifdef CONFIG_CRYPTO_FIPS +# define LRNG_OVERSAMPLE_ES_BITS 64 +# define LRNG_SEED_BUFFER_INIT_ADD_BITS 128 +#else /* CONFIG_CRYPTO_FIPS */ +# define LRNG_OVERSAMPLE_ES_BITS 0 +# define LRNG_SEED_BUFFER_INIT_ADD_BITS 0 +#endif /* CONFIG_CRYPTO_FIPS */ + +/* Security strength of LRNG -- this must match DRNG security strength */ +#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 +#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) +#define LRNG_DRNG_INIT_SEED_SIZE_BITS \ + (LRNG_DRNG_SECURITY_STRENGTH_BITS + LRNG_SEED_BUFFER_INIT_ADD_BITS) +#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) + +/* + * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is + * considered a safer margin. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_REQSIZE (1<<12) + +/* + * SP800-90A defines a maximum number of requests between reseeds of 2^48. + * The given value is considered a much safer margin, balancing requests for + * frequent reseeds with the need to conserve entropy. This value MUST NOT be + * larger than INT_MAX because it is used in an atomic_t. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_RESEED_THRESH (1<<20) + +/* + * Maximum DRNG generation operations without reseed having full entropy + * This value defines the absolute maximum value of DRNG generation operations + * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the + * threshold when a new reseed is attempted. But it is possible that this fails + * to deliver full entropy. In this case the DRNG will continue to provide data + * even though it was not reseeded with full entropy. To avoid in the extreme + * case that no reseed is performed for too long, this threshold is enforced. + * If that absolute low value is reached, the LRNG is marked as not operational. + * + * This value is allowed to be changed. + */ +#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) + +/* + * Min required seed entropy is 128 bits covering the minimum entropy + * requirement of SP800-131A and the German BSI's TR02102. + * + * This value is allowed to be changed. + */ +#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS +#define LRNG_MIN_SEED_ENTROPY_BITS 128 +#define LRNG_INIT_ENTROPY_BITS 32 + +/* + * Wakeup value + * + * This value is allowed to be changed but must not be larger than the + * digest size of the hash operation used update the aux_pool. + */ +#ifdef CONFIG_LRNG_SHA256 +# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE +#else +# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE +#endif +#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE + +/* + * If the switching support is configured, we must provide support up to + * the largest digest size. Without switching support, we know it is only + * the built-in digest size. + */ +#ifdef CONFIG_LRNG_SWITCH +# define LRNG_MAX_DIGESTSIZE 64 +#else +# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE +#endif + +/* + * Oversampling factor of timer-based events to obtain + * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a + * high-resolution time stamp is not available. In this case, jiffies and + * register contents are used to fill the entropy pool. These noise sources + * are much less entropic than the high-resolution timer. The entropy content + * is the entropy content assumed with LRNG_[IRQ|SCHED]_ENTROPY_BITS divided by + * LRNG_ES_OVERSAMPLING_FACTOR. + * + * This value is allowed to be changed. + */ +#define LRNG_ES_OVERSAMPLING_FACTOR 10 + +/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ +#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN + +/* + * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK + * as it will be casted into a struct shash_desc. + */ +#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) + +/****************************** Helper code ***********************************/ + +static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) +{ + /* Obtain entropy statement */ + ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; + /* Cap entropy to buffer size in bits */ + ent_bits = min_t(u32, ent_bits, requested_bits); + return ent_bits; +} + +/* Convert entropy in bits into nr. of events with the same entropy content. */ +static inline u32 lrng_entropy_to_data(u32 entropy_bits, u32 entropy_rate) +{ + return ((entropy_bits * entropy_rate) / + LRNG_DRNG_SECURITY_STRENGTH_BITS); +} + +/* Convert number of events into entropy value. */ +static inline u32 lrng_data_to_entropy(u32 num, u32 entropy_rate) +{ + return ((num * LRNG_DRNG_SECURITY_STRENGTH_BITS) / + entropy_rate); +} + +static inline u32 atomic_read_u32(atomic_t *v) +{ + return (u32)atomic_read(v); +} + +#endif /* _LRNG_DEFINITIONS_H */ diff --git a/drivers/char/lrng/lrng_drng_atomic.c b/drivers/char/lrng/lrng_drng_atomic.c new file mode 100644 index 0000000000000..c112366677b63 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_atomic.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG for atomic contexts + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_drng_atomic.h" +#include "lrng_drng_chacha20.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_sha.h" + +static struct chacha20_state chacha20_atomic = { + LRNG_CC20_INIT_RFC7539(.block) +}; + +/* + * DRNG usable in atomic context. This DRNG will always use the ChaCha20 + * DRNG. It will never benefit from a DRNG switch like the "regular" DRNG. If + * there was no DRNG switch, the atomic DRNG is identical to the "regular" DRNG. + * + * The reason for having this is due to the fact that DRNGs other than + * the ChaCha20 DRNG may sleep. + */ +static struct lrng_drng lrng_drng_atomic = { + LRNG_DRNG_STATE_INIT(lrng_drng_atomic, + &chacha20_atomic, NULL, + &lrng_cc20_drng_cb, &lrng_sha_hash_cb), + .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock) +}; + +void lrng_drng_atomic_reset(void) +{ + unsigned long flags; + + spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); + lrng_drng_reset(&lrng_drng_atomic); + spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); +} + +void lrng_drng_atomic_force_reseed(void) +{ + lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded; +} + +static bool lrng_drng_atomic_must_reseed(struct lrng_drng *drng) +{ + return (!drng->fully_seeded || + atomic_read(&lrng_drng_atomic.requests) == 0 || + drng->force_reseed || + time_after(jiffies, + drng->last_seeded + lrng_drng_reseed_max_time * HZ)); +} + +void lrng_drng_atomic_seed_drng(struct lrng_drng *regular_drng) +{ + u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES] + __aligned(LRNG_KCAPI_ALIGN); + int ret; + + if (!lrng_drng_atomic_must_reseed(&lrng_drng_atomic)) + return; + + /* + * Reseed atomic DRNG another DRNG "regular" while this regular DRNG + * is reseeded. Therefore, this code operates in non-atomic context and + * thus can use the lrng_drng_get function to get random numbers from + * the just freshly seeded DRNG. + */ + ret = lrng_drng_get(regular_drng, seedbuf, sizeof(seedbuf)); + + if (ret < 0) { + pr_warn("Error generating random numbers for atomic DRNG: %d\n", + ret); + } else { + unsigned long flags; + + spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); + lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret, + regular_drng->fully_seeded, "atomic"); + spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); + } + memzero_explicit(&seedbuf, sizeof(seedbuf)); +} + +void lrng_drng_atomic_seed_es(void) +{ + struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); + struct lrng_drng *drng = &lrng_drng_atomic; + unsigned long flags; + u32 requested_bits = LRNG_MIN_SEED_ENTROPY_BITS; + + if (lrng_drng_atomic.fully_seeded) + return; + + /* + * When the LRNG reached the minimally seeded level, leave 256 bits of + * entropy for the regular DRNG. In addition ensure that additional + * 256 bits are available for the atomic DRNG to get to the fully + * seeded stage of the LRNG. + */ + if (lrng_state_min_seeded()) { + u32 avail_bits = lrng_avail_entropy(); + + requested_bits = + (avail_bits >= 2 * LRNG_FULL_SEED_ENTROPY_BITS) ? + LRNG_FULL_SEED_ENTROPY_BITS : 0; + + if (!requested_bits) + return; + } + + pr_debug("atomic DRNG seeding attempt to pull %u bits of entropy directly from entropy sources\n", + requested_bits); + + lrng_fill_seed_buffer(&seedbuf, requested_bits); + spin_lock_irqsave(&drng->spin_lock, flags); + lrng_drng_inject(&lrng_drng_atomic, (u8 *)&seedbuf, sizeof(seedbuf), + lrng_fully_seeded(drng->fully_seeded, + lrng_entropy_rate_eb(&seedbuf)), + "atomic"); + spin_unlock_irqrestore(&drng->spin_lock, flags); + lrng_init_ops(&seedbuf); + + /* + * Do not call lrng_init_ops(seedbuf) here as the atomic DRNG does not + * serve common users. + */ + memzero_explicit(&seedbuf, sizeof(seedbuf)); +} + +static void lrng_drng_atomic_get(u8 *outbuf, u32 outbuflen) +{ + struct lrng_drng *drng = &lrng_drng_atomic; + unsigned long flags; + + if (!outbuf || !outbuflen) + return; + + outbuflen = min_t(size_t, outbuflen, INT_MAX); + + while (outbuflen) { + u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); + int ret; + + atomic_dec(&drng->requests); + + spin_lock_irqsave(&drng->spin_lock, flags); + ret = drng->drng_cb->drng_generate(drng->drng, outbuf, todo); + spin_unlock_irqrestore(&drng->spin_lock, flags); + if (ret <= 0) { + pr_warn("getting random data from DRNG failed (%d)\n", + ret); + return; + } + outbuflen -= ret; + outbuf += ret; + } +} + +void lrng_get_random_bytes(void *buf, int nbytes) +{ + lrng_drng_atomic_get((u8 *)buf, (u32)nbytes); + lrng_debug_report_seedlevel("lrng_get_random_bytes"); +} +EXPORT_SYMBOL(lrng_get_random_bytes); diff --git a/drivers/char/lrng/lrng_drng_atomic.h b/drivers/char/lrng/lrng_drng_atomic.h new file mode 100644 index 0000000000000..d5d24fc94f2e2 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_atomic.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRNG_ATOMIC_H +#define _LRNG_DRNG_ATOMIC_H + +#include "lrng_drng_mgr.h" + +#ifdef CONFIG_LRNG_DRNG_ATOMIC +void lrng_drng_atomic_reset(void); +void lrng_drng_atomic_seed_drng(struct lrng_drng *drng); +void lrng_drng_atomic_seed_es(void); +void lrng_drng_atomic_force_reseed(void); +#else /* CONFIG_LRNG_DRNG_ATOMIC */ +static inline void lrng_drng_atomic_reset(void) { } +static inline void lrng_drng_atomic_seed_drng(struct lrng_drng *drng) { } +static inline void lrng_drng_atomic_seed_es(void) { } +static inline void lrng_drng_atomic_force_reseed(void) { } +#endif /* CONFIG_LRNG_DRNG_ATOMIC */ + +#endif /* _LRNG_DRNG_ATOMIC_H */ diff --git a/drivers/char/lrng/lrng_drng_chacha20.c b/drivers/char/lrng/lrng_drng_chacha20.c new file mode 100644 index 0000000000000..31be102e30079 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_chacha20.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using + * ChaCha20 cipher implementations. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_drng_chacha20.h" + +/******************************* ChaCha20 DRNG *******************************/ + +#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) + +/* + * Update of the ChaCha20 state by either using an unused buffer part or by + * generating one ChaCha20 block which is half of the state of the ChaCha20. + * The block is XORed into the key part of the state. This shall ensure + * backtracking resistance as well as a proper mix of the ChaCha20 state once + * the key is injected. + */ +static void lrng_chacha20_update(struct chacha20_state *chacha20_state, + __le32 *buf, u32 used_words) +{ + struct chacha20_block *chacha20 = &chacha20_state->block; + u32 i; + __le32 tmp[CHACHA_BLOCK_WORDS]; + + BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE); + BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE); + + if (used_words > CHACHA_KEY_SIZE_WORDS) { + chacha20_block(&chacha20->constants[0], (u8 *)tmp); + for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) + chacha20->key.u[i] ^= le32_to_cpu(tmp[i]); + memzero_explicit(tmp, sizeof(tmp)); + } else { + for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) + chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]); + } + + /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */ + chacha20->nonce[0]++; + if (chacha20->nonce[0] == 0) { + chacha20->nonce[1]++; + if (chacha20->nonce[1] == 0) + chacha20->nonce[2]++; + } + + /* Leave counter untouched as it is start value is undefined in RFC */ +} + +/* + * Seed the ChaCha20 DRNG by injecting the input data into the key part of + * the ChaCha20 state. If the input data is longer than the ChaCha20 key size, + * perform a ChaCha20 operation after processing of key size input data. + * This operation shall spread out the entropy into the ChaCha20 state before + * new entropy is injected into the key part. + */ +static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + struct chacha20_block *chacha20 = &chacha20_state->block; + + while (inbuflen) { + u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE); + + for (i = 0; i < todo; i++) + chacha20->key.b[i] ^= inbuf[i]; + + /* Break potential dependencies between the inbuf key blocks */ + lrng_chacha20_update(chacha20_state, NULL, + CHACHA_BLOCK_WORDS); + inbuf += todo; + inbuflen -= todo; + } + + return 0; +} + +/* + * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20 + * is the random number. After the completion of the generation of the + * stream, the entire ChaCha20 state is updated. + * + * Note, as the ChaCha20 implements a 32 bit counter, we must ensure + * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks + * before a reseed or an update happens. This is ensured by the variable + * outbuflen which is a 32 bit integer defining the number of bytes to be + * generated by the ChaCha20 DRNG. At the end of this function, an update + * operation is invoked which implies that the 32 bit counter will never be + * overflown in this implementation. + */ +static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + struct chacha20_block *chacha20 = &chacha20_state->block; + __le32 aligned_buf[CHACHA_BLOCK_WORDS]; + u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS; + int zeroize_buf = 0; + + while (outbuflen >= CHACHA_BLOCK_SIZE) { + chacha20_block(&chacha20->constants[0], outbuf); + outbuf += CHACHA_BLOCK_SIZE; + outbuflen -= CHACHA_BLOCK_SIZE; + } + + if (outbuflen) { + chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf); + memcpy(outbuf, aligned_buf, outbuflen); + used = ((outbuflen + sizeof(aligned_buf[0]) - 1) / + sizeof(aligned_buf[0])); + zeroize_buf = 1; + } + + lrng_chacha20_update(chacha20_state, aligned_buf, used); + + if (zeroize_buf) + memzero_explicit(aligned_buf, sizeof(aligned_buf)); + + return ret; +} + +/* + * Allocation of the DRNG state + */ +static void *lrng_cc20_drng_alloc(u32 sec_strength) +{ + struct chacha20_state *state = NULL; + + if (sec_strength > CHACHA_KEY_SIZE) { + pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n", + CHACHA_KEY_SIZE * 8, sec_strength * 8); + return ERR_PTR(-EINVAL); + } + if (sec_strength < CHACHA_KEY_SIZE) + pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n", + CHACHA_KEY_SIZE * 8, sec_strength * 8); + + state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL); + if (!state) + return ERR_PTR(-ENOMEM); + pr_debug("memory for ChaCha20 core allocated\n"); + + lrng_cc20_init_rfc7539(&state->block); + + return state; +} + +static void lrng_cc20_drng_dealloc(void *drng) +{ + struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; + + pr_debug("ChaCha20 core zeroized and freed\n"); + kfree_sensitive(chacha20_state); +} + +static const char *lrng_cc20_drng_name(void) +{ + return "ChaCha20 DRNG"; +} + +const struct lrng_drng_cb lrng_cc20_drng_cb = { + .drng_name = lrng_cc20_drng_name, + .drng_alloc = lrng_cc20_drng_alloc, + .drng_dealloc = lrng_cc20_drng_dealloc, + .drng_seed = lrng_cc20_drng_seed_helper, + .drng_generate = lrng_cc20_drng_generate_helper, +}; + +#if !defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) && \ + !defined(CONFIG_LRNG_DRNG_ATOMIC) +static int __init lrng_cc20_drng_init(void) +{ + return lrng_set_drng_cb(&lrng_cc20_drng_cb); +} + +static void __exit lrng_cc20_drng_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_cc20_drng_init); +module_exit(lrng_cc20_drng_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - ChaCha20-based DRNG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_CHACHA20 */ diff --git a/drivers/char/lrng/lrng_drng_chacha20.h b/drivers/char/lrng/lrng_drng_chacha20.h new file mode 100644 index 0000000000000..fee6571281b64 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_chacha20.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG ChaCha20 definitions + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_CHACHA20_H +#define _LRNG_CHACHA20_H + +#include + +/* State according to RFC 7539 section 2.3 */ +struct chacha20_block { + u32 constants[4]; + union { +#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) + u32 u[CHACHA_KEY_SIZE_WORDS]; + u8 b[CHACHA_KEY_SIZE]; + } key; + u32 counter; + u32 nonce[3]; +}; + +struct chacha20_state { + struct chacha20_block block; +}; + +static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20) +{ + chacha_init_consts(chacha20->constants); +} + +#define LRNG_CC20_INIT_RFC7539(x) \ + x.constants[0] = 0x61707865, \ + x.constants[1] = 0x3320646e, \ + x.constants[2] = 0x79622d32, \ + x.constants[3] = 0x6b206574, + +extern const struct lrng_drng_cb lrng_cc20_drng_cb; + +#endif /* _LRNG_CHACHA20_H */ diff --git a/drivers/char/lrng/lrng_drng_drbg.c b/drivers/char/lrng/lrng_drng_drbg.c new file mode 100644 index 0000000000000..1feec2c095a5e --- /dev/null +++ b/drivers/char/lrng/lrng_drng_drbg.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using the + * kernel crypto API and its DRBG. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_drng_drbg.h" + +/* + * Define a DRBG plus a hash / MAC used to extract data from the entropy pool. + * For LRNG_HASH_NAME you can use a hash or a MAC (HMAC or CMAC) of your choice + * (Note, you should use the suggested selections below -- using SHA-1 or MD5 + * is not wise). The idea is that the used cipher primitive can be selected to + * be the same as used for the DRBG. I.e. the LRNG only uses one cipher + * primitive using the same cipher implementation with the options offered in + * the following. This means, if the CTR DRBG is selected and AES-NI is present, + * both the CTR DRBG and the selected cmac(aes) use AES-NI. + * + * The security strengths of the DRBGs are all 256 bits according to + * SP800-57 section 5.6.1. + * + * This definition is allowed to be changed. + */ +#ifdef CONFIG_CRYPTO_DRBG_CTR +static unsigned int lrng_drbg_type = 0; +#elif defined CONFIG_CRYPTO_DRBG_HMAC +static unsigned int lrng_drbg_type = 1; +#elif defined CONFIG_CRYPTO_DRBG_HASH +static unsigned int lrng_drbg_type = 2; +#else +#error "Unknown DRBG in use" +#endif + +/* The parameter must be r/o in sysfs as otherwise races appear. */ +module_param(lrng_drbg_type, uint, 0444); +MODULE_PARM_DESC(lrng_drbg_type, "DRBG type used for LRNG (0->CTR_DRBG, 1->HMAC_DRBG, 2->Hash_DRBG)"); + +struct lrng_drbg { + const char *hash_name; + const char *drbg_core; +}; + +static const struct lrng_drbg lrng_drbg_types[] = { + { /* CTR_DRBG with AES-256 using derivation function */ + .drbg_core = "drbg_nopr_ctr_aes256", + }, { /* HMAC_DRBG with SHA-512 */ + .drbg_core = "drbg_nopr_hmac_sha512", + }, { /* Hash_DRBG with SHA-512 using derivation function */ + .drbg_core = "drbg_nopr_sha512" + } +}; + +static int lrng_drbg_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + LIST_HEAD(seedlist); + struct drbg_string data; + int ret; + + drbg_string_fill(&data, inbuf, inbuflen); + list_add_tail(&data.list, &seedlist); + ret = drbg->d_ops->update(drbg, &seedlist, drbg->seeded); + + if (ret >= 0) + drbg->seeded = true; + + return ret; +} + +static int lrng_drbg_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + + return drbg->d_ops->generate(drbg, outbuf, outbuflen, NULL); +} + +static void *lrng_drbg_drng_alloc(u32 sec_strength) +{ + struct drbg_state *drbg; + int coreref = -1; + bool pr = false; + int ret; + + drbg_convert_tfm_core(lrng_drbg_types[lrng_drbg_type].drbg_core, + &coreref, &pr); + if (coreref < 0) + return ERR_PTR(-EFAULT); + + drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); + if (!drbg) + return ERR_PTR(-ENOMEM); + + drbg->core = &drbg_cores[coreref]; + drbg->seeded = false; + ret = drbg_alloc_state(drbg); + if (ret) + goto err; + + if (sec_strength > drbg_sec_strength(drbg->core->flags)) { + pr_err("Security strength of DRBG (%u bits) lower than requested by LRNG (%u bits)\n", + drbg_sec_strength(drbg->core->flags) * 8, + sec_strength * 8); + goto dealloc; + } + + if (sec_strength < drbg_sec_strength(drbg->core->flags)) + pr_warn("Security strength of DRBG (%u bits) higher than requested by LRNG (%u bits)\n", + drbg_sec_strength(drbg->core->flags) * 8, + sec_strength * 8); + + pr_info("DRBG with %s core allocated\n", drbg->core->backend_cra_name); + + return drbg; + +dealloc: + if (drbg->d_ops) + drbg->d_ops->crypto_fini(drbg); + drbg_dealloc_state(drbg); +err: + kfree(drbg); + return ERR_PTR(-EINVAL); +} + +static void lrng_drbg_drng_dealloc(void *drng) +{ + struct drbg_state *drbg = (struct drbg_state *)drng; + + if (drbg && drbg->d_ops) + drbg->d_ops->crypto_fini(drbg); + drbg_dealloc_state(drbg); + kfree_sensitive(drbg); + pr_info("DRBG deallocated\n"); +} + +static const char *lrng_drbg_name(void) +{ + return lrng_drbg_types[lrng_drbg_type].drbg_core; +} + +const struct lrng_drng_cb lrng_drbg_cb = { + .drng_name = lrng_drbg_name, + .drng_alloc = lrng_drbg_drng_alloc, + .drng_dealloc = lrng_drbg_drng_dealloc, + .drng_seed = lrng_drbg_drng_seed_helper, + .drng_generate = lrng_drbg_drng_generate_helper, +}; + +#ifndef CONFIG_LRNG_DFLT_DRNG_DRBG +static int __init lrng_drbg_init(void) +{ + if (lrng_drbg_type >= ARRAY_SIZE(lrng_drbg_types)) { + pr_err("lrng_drbg_type parameter too large (given %u - max: %lu)", + lrng_drbg_type, + (unsigned long)ARRAY_SIZE(lrng_drbg_types) - 1); + return -EAGAIN; + } + return lrng_set_drng_cb(&lrng_drbg_cb); +} + +static void __exit lrng_drbg_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_drbg_init); +module_exit(lrng_drbg_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - SP800-90A DRBG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_DRBG */ diff --git a/drivers/char/lrng/lrng_drng_drbg.h b/drivers/char/lrng/lrng_drng_drbg.h new file mode 100644 index 0000000000000..b3d556caf294e --- /dev/null +++ b/drivers/char/lrng/lrng_drng_drbg.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG SP800-90A definitions + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRBG_H +#define _LRNG_DRBG_H + +extern const struct lrng_drng_cb lrng_drbg_cb; + +#endif /* _LRNG_DRBG_H */ diff --git a/drivers/char/lrng/lrng_drng_kcapi.c b/drivers/char/lrng/lrng_drng_kcapi.c new file mode 100644 index 0000000000000..a204bcf52a9ac --- /dev/null +++ b/drivers/char/lrng/lrng_drng_kcapi.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the cryptographic primitives using the + * kernel crypto API. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include "lrng_drng_kcapi.h" + +static char *drng_name = NULL; +module_param(drng_name, charp, 0444); +MODULE_PARM_DESC(drng_name, "Kernel crypto API name of DRNG"); + +static char *seed_hash = NULL; +module_param(seed_hash, charp, 0444); +MODULE_PARM_DESC(seed_hash, + "Kernel crypto API name of hash with output size equal to seedsize of DRNG to bring seed string to the size required by the DRNG"); + +struct lrng_drng_info { + struct crypto_rng *kcapi_rng; + struct crypto_shash *hash_tfm; +}; + +static int lrng_kcapi_drng_seed_helper(void *drng, const u8 *inbuf, + u32 inbuflen) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + struct crypto_shash *hash_tfm = lrng_drng_info->hash_tfm; + SHASH_DESC_ON_STACK(shash, hash_tfm); + u32 digestsize; + u8 digest[HASH_MAX_DIGESTSIZE] __aligned(8); + int ret; + + if (!hash_tfm) + return crypto_rng_reset(kcapi_rng, inbuf, inbuflen); + + shash->tfm = hash_tfm; + digestsize = crypto_shash_digestsize(hash_tfm); + + ret = crypto_shash_digest(shash, inbuf, inbuflen, digest); + shash_desc_zero(shash); + if (ret) + return ret; + + ret = crypto_rng_reset(kcapi_rng, digest, digestsize); + if (ret) + return ret; + + memzero_explicit(digest, digestsize); + return 0; +} + +static int lrng_kcapi_drng_generate_helper(void *drng, u8 *outbuf, + u32 outbuflen) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + int ret = crypto_rng_get_bytes(kcapi_rng, outbuf, outbuflen); + + if (ret < 0) + return ret; + + return outbuflen; +} + +static void *lrng_kcapi_drng_alloc(u32 sec_strength) +{ + struct lrng_drng_info *lrng_drng_info; + struct crypto_rng *kcapi_rng; + u32 time = random_get_entropy(); + int seedsize, rv; + void *ret = ERR_PTR(-ENOMEM); + + if (!drng_name) { + pr_err("DRNG name missing\n"); + return ERR_PTR(-EINVAL); + } + + if (!memcmp(drng_name, "stdrng", 6) || + !memcmp(drng_name, "lrng", 4) || + !memcmp(drng_name, "drbg", 4) || + !memcmp(drng_name, "jitterentropy_rng", 17)) { + pr_err("Refusing to load the requested random number generator\n"); + return ERR_PTR(-EINVAL); + } + + lrng_drng_info = kzalloc(sizeof(*lrng_drng_info), GFP_KERNEL); + if (!lrng_drng_info) + return ERR_PTR(-ENOMEM); + + kcapi_rng = crypto_alloc_rng(drng_name, 0, 0); + if (IS_ERR(kcapi_rng)) { + pr_err("DRNG %s cannot be allocated\n", drng_name); + ret = ERR_CAST(kcapi_rng); + goto free; + } + + lrng_drng_info->kcapi_rng = kcapi_rng; + + seedsize = crypto_rng_seedsize(kcapi_rng); + if (seedsize) { + struct crypto_shash *hash_tfm; + + if (!seed_hash) { + switch (seedsize) { + case 32: + seed_hash = "sha256"; + break; + case 48: + seed_hash = "sha384"; + break; + case 64: + seed_hash = "sha512"; + break; + default: + pr_err("Seed size %d cannot be processed\n", + seedsize); + goto dealloc; + } + } + + hash_tfm = crypto_alloc_shash(seed_hash, 0, 0); + if (IS_ERR(hash_tfm)) { + ret = ERR_CAST(hash_tfm); + goto dealloc; + } + + if (seedsize != crypto_shash_digestsize(hash_tfm)) { + pr_err("Seed hash output size not equal to DRNG seed size\n"); + crypto_free_shash(hash_tfm); + ret = ERR_PTR(-EINVAL); + goto dealloc; + } + + lrng_drng_info->hash_tfm = hash_tfm; + + pr_info("Seed hash %s allocated\n", seed_hash); + } + + rv = lrng_kcapi_drng_seed_helper(lrng_drng_info, (u8 *)(&time), + sizeof(time)); + if (rv) { + ret = ERR_PTR(rv); + goto dealloc; + } + + pr_info("Kernel crypto API DRNG %s allocated\n", drng_name); + + return lrng_drng_info; + +dealloc: + crypto_free_rng(kcapi_rng); +free: + kfree(lrng_drng_info); + return ret; +} + +static void lrng_kcapi_drng_dealloc(void *drng) +{ + struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; + struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; + + crypto_free_rng(kcapi_rng); + if (lrng_drng_info->hash_tfm) + crypto_free_shash(lrng_drng_info->hash_tfm); + kfree(lrng_drng_info); + pr_info("DRNG %s deallocated\n", drng_name); +} + +static const char *lrng_kcapi_drng_name(void) +{ + return drng_name; +} + +const struct lrng_drng_cb lrng_kcapi_drng_cb = { + .drng_name = lrng_kcapi_drng_name, + .drng_alloc = lrng_kcapi_drng_alloc, + .drng_dealloc = lrng_kcapi_drng_dealloc, + .drng_seed = lrng_kcapi_drng_seed_helper, + .drng_generate = lrng_kcapi_drng_generate_helper, +}; + +#ifndef CONFIG_LRNG_DFLT_DRNG_KCAPI +static int __init lrng_kcapi_init(void) +{ + return lrng_set_drng_cb(&lrng_kcapi_drng_cb); +} +static void __exit lrng_kcapi_exit(void) +{ + lrng_set_drng_cb(NULL); +} + +late_initcall(lrng_kcapi_init); +module_exit(lrng_kcapi_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - kernel crypto API DRNG backend"); +#endif /* CONFIG_LRNG_DFLT_DRNG_KCAPI */ diff --git a/drivers/char/lrng/lrng_drng_kcapi.h b/drivers/char/lrng/lrng_drng_kcapi.h new file mode 100644 index 0000000000000..5db25aaf830c5 --- /dev/null +++ b/drivers/char/lrng/lrng_drng_kcapi.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG kernel crypto API DRNG definition + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_KCAPI_DRNG_H +#define _LRNG_KCAPI_DRNG_H + +extern const struct lrng_drng_cb lrng_kcapi_drng_cb; + +#endif /* _LRNG_KCAPI_DRNG_H */ diff --git a/drivers/char/lrng/lrng_drng_mgr.c b/drivers/char/lrng/lrng_drng_mgr.c new file mode 100644 index 0000000000000..0171f72c8b4bc --- /dev/null +++ b/drivers/char/lrng/lrng_drng_mgr.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG management + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include "lrng_drng_atomic.h" +#include "lrng_drng_chacha20.h" +#include "lrng_drng_drbg.h" +#include "lrng_drng_kcapi.h" +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_random_kernel.h" +#include "lrng_numa.h" +#include "lrng_sha.h" + +/* + * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note, + * this is enforced with the next request of random numbers from the + * DRNG. Setting this value to zero implies a reseeding attempt before every + * generated random number. + */ +int lrng_drng_reseed_max_time = 600; + +/* + * Is LRNG for general-purpose use (i.e. is at least the lrng_drng_init + * fully allocated)? + */ +static atomic_t lrng_avail = ATOMIC_INIT(0); + +/* Guard protecting all crypto callback update operation of all DRNGs. */ +DEFINE_MUTEX(lrng_crypto_cb_update); + +/* + * Default hash callback that provides the crypto primitive right from the + * kernel start. It must not perform any memory allocation operation, but + * simply perform the hash calculation. + */ +const struct lrng_hash_cb *lrng_default_hash_cb = &lrng_sha_hash_cb; + +/* + * Default DRNG callback that provides the crypto primitive which is + * allocated either during late kernel boot stage. So, it is permissible for + * the callback to perform memory allocation operations. + */ +const struct lrng_drng_cb *lrng_default_drng_cb = +#if defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) + &lrng_cc20_drng_cb; +#elif defined(CONFIG_LRNG_DFLT_DRNG_DRBG) + &lrng_drbg_cb; +#elif defined(CONFIG_LRNG_DFLT_DRNG_KCAPI) + &lrng_kcapi_drng_cb; +#else +#error "Unknown default DRNG selected" +#endif + +/* DRNG for non-atomic use cases */ +static struct lrng_drng lrng_drng_init = { + LRNG_DRNG_STATE_INIT(lrng_drng_init, NULL, NULL, NULL, + &lrng_sha_hash_cb), + .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock), +}; + +/* Prediction-resistance DRNG: only deliver as much data as received entropy */ +static struct lrng_drng lrng_drng_pr = { + LRNG_DRNG_STATE_INIT(lrng_drng_pr, NULL, NULL, NULL, + &lrng_sha_hash_cb), + .lock = __MUTEX_INITIALIZER(lrng_drng_pr.lock), +}; + +static u32 max_wo_reseed = LRNG_DRNG_MAX_WITHOUT_RESEED; +#ifdef CONFIG_LRNG_RUNTIME_MAX_WO_RESEED_CONFIG +module_param(max_wo_reseed, uint, 0444); +MODULE_PARM_DESC(max_wo_reseed, + "Maximum number of DRNG generate operation without full reseed\n"); +#endif + +/* Wait queue to wait until the LRNG is initialized - can freely be used */ +DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait); + +/********************************** Helper ************************************/ + +bool lrng_get_available(void) +{ + return likely(atomic_read(&lrng_avail)); +} + +struct lrng_drng *lrng_drng_init_instance(void) +{ + return &lrng_drng_init; +} + +struct lrng_drng *lrng_drng_pr_instance(void) +{ + return &lrng_drng_pr; +} + +struct lrng_drng *lrng_drng_node_instance(void) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + int node = numa_node_id(); + + if (lrng_drng && lrng_drng[node]) + return lrng_drng[node]; + + return lrng_drng_init_instance(); +} + +void lrng_drng_reset(struct lrng_drng *drng) +{ + atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); + atomic_set(&drng->requests_since_fully_seeded, 0); + drng->last_seeded = jiffies; + drng->fully_seeded = false; + drng->force_reseed = true; + pr_debug("reset DRNG\n"); +} + +/* Initialize the DRNG, except the mutex lock */ +int lrng_drng_alloc_common(struct lrng_drng *drng, + const struct lrng_drng_cb *drng_cb) +{ + if (!drng || !drng_cb) + return -EINVAL; + if (!IS_ERR_OR_NULL(drng->drng)) + return 0; + + drng->drng_cb = drng_cb; + drng->drng = drng_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); + if (IS_ERR(drng->drng)) + return PTR_ERR(drng->drng); + + lrng_drng_reset(drng); + return 0; +} + +/* Initialize the default DRNG during boot and perform its seeding */ +int lrng_drng_initalize(void) +{ + int ret; + + if (lrng_get_available()) + return 0; + + /* Catch programming error */ + WARN_ON(lrng_drng_init.hash_cb != lrng_default_hash_cb); + + mutex_lock(&lrng_drng_init.lock); + if (lrng_get_available()) { + mutex_unlock(&lrng_drng_init.lock); + return 0; + } + + /* Initialize the PR DRNG inside init lock as it guards lrng_avail. */ + mutex_lock(&lrng_drng_pr.lock); + ret = lrng_drng_alloc_common(&lrng_drng_pr, lrng_default_drng_cb); + mutex_unlock(&lrng_drng_pr.lock); + + if (!ret) { + ret = lrng_drng_alloc_common(&lrng_drng_init, + lrng_default_drng_cb); + if (!ret) + atomic_set(&lrng_avail, 1); + } + mutex_unlock(&lrng_drng_init.lock); + if (ret) + return ret; + + pr_debug("LRNG for general use is available\n"); + + /* Seed the DRNG with any entropy available */ + if (!lrng_pool_trylock()) { + pr_info("Initial DRNG initialized triggering first seeding\n"); + lrng_drng_seed_work(NULL); + } else { + pr_info("Initial DRNG initialized without seeding\n"); + } + + return 0; +} + +static int __init lrng_drng_make_available(void) +{ + return lrng_drng_initalize(); +} +late_initcall(lrng_drng_make_available); + +bool lrng_sp80090c_compliant(void) +{ + /* SP800-90C compliant oversampling is only requested in FIPS mode */ + return fips_enabled; +} + +bool lrng_ntg1_compliant(void) +{ + /* Implies using of /dev/random with O_SYNC */ + return true; +} + +/************************* Random Number Generation ***************************/ + +/* Inject a data buffer into the DRNG - caller must hold its lock */ +void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, + bool fully_seeded, const char *drng_type) +{ + BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX); + pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen); + if (drng->drng_cb->drng_seed(drng->drng, inbuf, inbuflen) < 0) { + pr_warn("seeding of %s DRNG failed\n", drng_type); + drng->force_reseed = true; + } else { + int gc = LRNG_DRNG_RESEED_THRESH - atomic_read(&drng->requests); + + pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n", + drng_type, + (time_after(jiffies, drng->last_seeded) ? + (jiffies - drng->last_seeded) : 0) / HZ, gc); + + /* Count the numbers of generate ops since last fully seeded */ + if (fully_seeded) + atomic_set(&drng->requests_since_fully_seeded, 0); + else + atomic_add(gc, &drng->requests_since_fully_seeded); + + drng->last_seeded = jiffies; + atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); + drng->force_reseed = false; + + if (!drng->fully_seeded) { + drng->fully_seeded = fully_seeded; + if (drng->fully_seeded) + pr_debug("%s DRNG fully seeded\n", drng_type); + } + } +} + +/* + * Perform the seeding of the DRNG with data from entropy source. + * The function returns the entropy injected into the DRNG in bits. + */ +static u32 lrng_drng_seed_es_nolock(struct lrng_drng *drng) +{ + struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); + u32 collected_entropy; + + lrng_fill_seed_buffer(&seedbuf, + lrng_get_seed_entropy_osr(drng->fully_seeded)); + + collected_entropy = lrng_entropy_rate_eb(&seedbuf); + lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf), + lrng_fully_seeded(drng->fully_seeded, + collected_entropy), + "regular"); + + /* Set the seeding state of the LRNG */ + lrng_init_ops(&seedbuf); + + memzero_explicit(&seedbuf, sizeof(seedbuf)); + + return collected_entropy; +} + +static void lrng_drng_seed_es(struct lrng_drng *drng) +{ + mutex_lock(&drng->lock); + lrng_drng_seed_es_nolock(drng); + mutex_unlock(&drng->lock); +} + +static void lrng_drng_seed(struct lrng_drng *drng) +{ + BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS > + LRNG_DRNG_SECURITY_STRENGTH_BITS); + + if (lrng_get_available()) { + /* (Re-)Seed DRNG */ + lrng_drng_seed_es(drng); + /* (Re-)Seed atomic DRNG from regular DRNG */ + lrng_drng_atomic_seed_drng(drng); + } else { + /* + * If no-one is waiting for the DRNG, seed the atomic DRNG + * directly from the entropy sources. + */ + if (!wq_has_sleeper(&lrng_init_wait)) + lrng_drng_atomic_seed_es(); + else + lrng_init_ops(NULL); + } +} + +static void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node) +{ + pr_debug("reseed triggered by system events for DRNG on NUMA node %d\n", + node); + lrng_drng_seed(drng); + if (drng->fully_seeded) { + /* Prevent reseed storm */ + drng->last_seeded += node * 100 * HZ; + } +} + +/* + * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work() + */ +void lrng_drng_seed_work(struct work_struct *dummy) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + u32 node; + + if (lrng_drng) { + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (drng && !drng->fully_seeded) { + _lrng_drng_seed_work(drng, node); + goto out; + } + } + } else { + if (!lrng_drng_init.fully_seeded) { + _lrng_drng_seed_work(&lrng_drng_init, 0); + goto out; + } + } + + if (!lrng_drng_pr.fully_seeded) { + _lrng_drng_seed_work(&lrng_drng_pr, 0); + goto out; + } + + lrng_pool_all_numa_nodes_seeded(true); + +out: + /* Allow the seeding operation to be called again */ + lrng_pool_unlock(); +} + +/* Force all DRNGs to reseed before next generation */ +void lrng_drng_force_reseed(void) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + u32 node; + + /* + * If the initial DRNG is over the reseed threshold, allow a forced + * reseed only for the initial DRNG as this is the fallback for all. It + * must be kept seeded before all others to keep the LRNG operational. + */ + if (!lrng_drng || + (atomic_read_u32(&lrng_drng_init.requests_since_fully_seeded) > + LRNG_DRNG_RESEED_THRESH)) { + lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded; + pr_debug("force reseed of initial DRNG\n"); + return; + } + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (!drng) + continue; + + drng->force_reseed = drng->fully_seeded; + pr_debug("force reseed of DRNG on node %u\n", node); + } + lrng_drng_atomic_force_reseed(); +} +EXPORT_SYMBOL(lrng_drng_force_reseed); + +static bool lrng_drng_must_reseed(struct lrng_drng *drng) +{ + return (atomic_dec_and_test(&drng->requests) || + drng->force_reseed || + time_after(jiffies, + drng->last_seeded + lrng_drng_reseed_max_time * HZ)); +} + +/* + * lrng_drng_get() - Get random data out of the DRNG which is reseeded + * frequently. + * + * @drng: DRNG instance + * @outbuf: buffer for storing random data + * @outbuflen: length of outbuf + * + * Return: + * * < 0 in error case (DRNG generation or update failed) + * * >=0 returning the returned number of bytes + */ +int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen) +{ + u32 processed = 0; + bool pr = (drng == &lrng_drng_pr) ? true : false; + + if (!outbuf || !outbuflen) + return 0; + + if (!lrng_get_available()) + return -EOPNOTSUPP; + + outbuflen = min_t(size_t, outbuflen, INT_MAX); + + /* If DRNG operated without proper reseed for too long, block LRNG */ + BUILD_BUG_ON(LRNG_DRNG_MAX_WITHOUT_RESEED < LRNG_DRNG_RESEED_THRESH); + if (atomic_read_u32(&drng->requests_since_fully_seeded) > max_wo_reseed) + lrng_unset_fully_seeded(drng); + + while (outbuflen) { + u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); + int ret; + + /* In normal operation, check whether to reseed */ + if (!pr && lrng_drng_must_reseed(drng)) { + if (lrng_pool_trylock()) { + drng->force_reseed = true; + } else { + lrng_drng_seed(drng); + lrng_pool_unlock(); + } + } + + mutex_lock(&drng->lock); + + if (pr) { + /* If async reseed did not deliver entropy, try now */ + if (!drng->fully_seeded) { + u32 coll_ent_bits; + + /* If we cannot get the pool lock, try again. */ + if (lrng_pool_trylock()) { + mutex_unlock(&drng->lock); + continue; + } + + coll_ent_bits = lrng_drng_seed_es_nolock(drng); + + lrng_pool_unlock(); + + /* If no new entropy was received, stop now. */ + if (!coll_ent_bits) { + mutex_unlock(&drng->lock); + goto out; + } + + /* Produce no more data than received entropy */ + todo = min_t(u32, todo, coll_ent_bits >> 3); + } + + /* Do not produce more than DRNG security strength */ + todo = min_t(u32, todo, lrng_security_strength() >> 3); + } + ret = drng->drng_cb->drng_generate(drng->drng, + outbuf + processed, todo); + + mutex_unlock(&drng->lock); + if (ret <= 0) { + pr_warn("getting random data from DRNG failed (%d)\n", + ret); + return -EFAULT; + } + processed += ret; + outbuflen -= ret; + + if (pr) { + /* Force the async reseed for PR DRNG */ + lrng_unset_fully_seeded(drng); + if (outbuflen) + cond_resched(); + } + } + +out: + return processed; +} + +int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *drng = &lrng_drng_init; + int ret, node = numa_node_id(); + + might_sleep(); + + if (pr) + drng = &lrng_drng_pr; + else if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded) + drng = lrng_drng[node]; + + ret = lrng_drng_initalize(); + if (ret) + return ret; + + return lrng_drng_get(drng, outbuf, outbuflen); +} + +/* Reset LRNG such that all existing entropy is gone */ +static void _lrng_reset(struct work_struct *work) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + + if (!lrng_drng) { + mutex_lock(&lrng_drng_init.lock); + lrng_drng_reset(&lrng_drng_init); + mutex_unlock(&lrng_drng_init.lock); + } else { + u32 node; + + for_each_online_node(node) { + struct lrng_drng *drng = lrng_drng[node]; + + if (!drng) + continue; + mutex_lock(&drng->lock); + lrng_drng_reset(drng); + mutex_unlock(&drng->lock); + } + } + + mutex_lock(&lrng_drng_pr.lock); + lrng_drng_reset(&lrng_drng_pr); + mutex_unlock(&lrng_drng_pr.lock); + + lrng_drng_atomic_reset(); + lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS); + + lrng_reset_state(); +} + +static DECLARE_WORK(lrng_reset_work, _lrng_reset); + +void lrng_reset(void) +{ + schedule_work(&lrng_reset_work); +} + +/******************* Generic LRNG kernel output interfaces ********************/ + +int lrng_drng_sleep_while_nonoperational(int nonblock) +{ + if (likely(lrng_state_operational())) + return 0; + if (nonblock) + return -EAGAIN; + return wait_event_interruptible(lrng_init_wait, + lrng_state_operational()); +} + +int lrng_drng_sleep_while_non_min_seeded(void) +{ + if (likely(lrng_state_min_seeded())) + return 0; + return wait_event_interruptible(lrng_init_wait, + lrng_state_min_seeded()); +} + +void lrng_get_random_bytes_full(void *buf, int nbytes) +{ + lrng_drng_sleep_while_nonoperational(0); + lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); +} +EXPORT_SYMBOL(lrng_get_random_bytes_full); + +void lrng_get_random_bytes_min(void *buf, int nbytes) +{ + lrng_drng_sleep_while_non_min_seeded(); + lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); +} +EXPORT_SYMBOL(lrng_get_random_bytes_min); + +int lrng_get_random_bytes_pr(void *buf, int nbytes) +{ + lrng_drng_sleep_while_nonoperational(0); + return lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, true); +} +EXPORT_SYMBOL(lrng_get_random_bytes_pr); diff --git a/drivers/char/lrng/lrng_drng_mgr.h b/drivers/char/lrng/lrng_drng_mgr.h new file mode 100644 index 0000000000000..ec120e4980b5f --- /dev/null +++ b/drivers/char/lrng/lrng_drng_mgr.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_DRNG_H +#define _LRNG_DRNG_H + +#include +#include +#include + +#include "lrng_definitions.h" + +extern struct wait_queue_head lrng_init_wait; +extern int lrng_drng_reseed_max_time; +extern struct mutex lrng_crypto_cb_update; +extern const struct lrng_drng_cb *lrng_default_drng_cb; +extern const struct lrng_hash_cb *lrng_default_hash_cb; + +/* DRNG state handle */ +struct lrng_drng { + void *drng; /* DRNG handle */ + void *hash; /* Hash handle */ + const struct lrng_drng_cb *drng_cb; /* DRNG callbacks */ + const struct lrng_hash_cb *hash_cb; /* Hash callbacks */ + atomic_t requests; /* Number of DRNG requests */ + atomic_t requests_since_fully_seeded; /* Number DRNG requests since + * last fully seeded + */ + unsigned long last_seeded; /* Last time it was seeded */ + bool fully_seeded; /* Is DRNG fully seeded? */ + bool force_reseed; /* Force a reseed */ + + rwlock_t hash_lock; /* Lock hash_cb replacement */ + /* Lock write operations on DRNG state, DRNG replacement of drng_cb */ + struct mutex lock; /* Non-atomic DRNG operation */ + spinlock_t spin_lock; /* Atomic DRNG operation */ +}; + +#define LRNG_DRNG_STATE_INIT(x, d, h, d_cb, h_cb) \ + .drng = d, \ + .hash = h, \ + .drng_cb = d_cb, \ + .hash_cb = h_cb, \ + .requests = ATOMIC_INIT(LRNG_DRNG_RESEED_THRESH),\ + .requests_since_fully_seeded = ATOMIC_INIT(0), \ + .last_seeded = 0, \ + .fully_seeded = false, \ + .force_reseed = true, \ + .hash_lock = __RW_LOCK_UNLOCKED(x.hash_lock) + +struct lrng_drng *lrng_drng_init_instance(void); +struct lrng_drng *lrng_drng_pr_instance(void); +struct lrng_drng *lrng_drng_node_instance(void); + +void lrng_reset(void); +int lrng_drng_alloc_common(struct lrng_drng *drng, + const struct lrng_drng_cb *crypto_cb); +int lrng_drng_initalize(void); +bool lrng_sp80090c_compliant(void); +bool lrng_ntg1_compliant(void); +bool lrng_get_available(void); +void lrng_drng_reset(struct lrng_drng *drng); +void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, + bool fully_seeded, const char *drng_type); +int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen); +int lrng_drng_sleep_while_nonoperational(int nonblock); +int lrng_drng_sleep_while_non_min_seeded(void); +int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr); +void lrng_drng_seed_work(struct work_struct *dummy); +void lrng_drng_force_reseed(void); + +static inline u32 lrng_compress_osr(void) +{ + return lrng_sp80090c_compliant() ? LRNG_OVERSAMPLE_ES_BITS : 0; +} + +static inline u32 lrng_reduce_by_osr(u32 entropy_bits) +{ + u32 osr_bits = lrng_compress_osr(); + + return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; +} + +#endif /* _LRNG_DRNG_H */ diff --git a/drivers/char/lrng/lrng_es_aux.c b/drivers/char/lrng/lrng_es_aux.c new file mode 100644 index 0000000000000..245bc829998be --- /dev/null +++ b/drivers/char/lrng/lrng_es_aux.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Auxiliary entropy pool + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_sysctl.h" + +/* + * This is the auxiliary pool + * + * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API + * cipher implementations of the hash functions used to read the pool: for some + * accelerated implementations, we need an alignment to avoid a realignment + * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto + * implementations. + */ +struct lrng_pool { + u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */ + atomic_t aux_entropy_bits; + atomic_t digestsize; /* Digest size of used hash */ + bool initialized; /* Aux pool initialized? */ + + /* Serialize read of entropy pool and update of aux pool */ + spinlock_t lock; +}; + +static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = { + .aux_entropy_bits = ATOMIC_INIT(0), + .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE), + .initialized = false, + .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock) +}; + +/********************************** Helper ***********************************/ + +/* Entropy in bits present in aux pool */ +static u32 lrng_aux_avail_entropy(u32 __unused) +{ + /* Cap available entropy with max entropy */ + u32 avail_bits = min_t(u32, lrng_get_digestsize(), + atomic_read_u32(&lrng_pool.aux_entropy_bits)); + + /* Consider oversampling rate due to aux pool conditioning */ + return lrng_reduce_by_osr(avail_bits); +} + +/* Set the digest size of the used hash in bytes */ +static void lrng_set_digestsize(u32 digestsize) +{ + struct lrng_pool *pool = &lrng_pool; + u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0), + old_digestsize = lrng_get_digestsize(); + + atomic_set(&lrng_pool.digestsize, digestsize); + + /* + * Update the write wakeup threshold which must not be larger + * than the digest size of the current conditioning hash. + */ + digestsize = lrng_reduce_by_osr(digestsize << 3); + lrng_sysctl_update_max_write_thresh(digestsize); + lrng_write_wakeup_bits = digestsize; + + /* + * In case the new digest is larger than the old one, cap the available + * entropy to the old message digest used to process the existing data. + */ + ent_bits = min_t(u32, ent_bits, old_digestsize); + atomic_add(ent_bits, &pool->aux_entropy_bits); +} + +static int __init lrng_init_wakeup_bits(void) +{ + u32 digestsize = lrng_reduce_by_osr(lrng_get_digestsize()); + + lrng_sysctl_update_max_write_thresh(digestsize); + lrng_write_wakeup_bits = digestsize; + return 0; +} +core_initcall(lrng_init_wakeup_bits); + +/* Obtain the digest size provided by the used hash in bits */ +u32 lrng_get_digestsize(void) +{ + return atomic_read_u32(&lrng_pool.digestsize) << 3; +} + +/* Set entropy content in user-space controllable aux pool */ +void lrng_pool_set_entropy(u32 entropy_bits) +{ + atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits); +} + +static void lrng_aux_reset(void) +{ + lrng_pool_set_entropy(0); +} + +/* + * Replace old with new hash for auxiliary pool handling + * + * Assumption: the caller must guarantee that the new_cb is available during the + * entire operation (e.g. it must hold the write lock against pointer updating). + */ +static int +lrng_aux_switch_hash(struct lrng_drng *drng, int __unused, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + struct lrng_drng *init_drng = lrng_drng_init_instance(); + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + u8 digest[LRNG_MAX_DIGESTSIZE]; + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + if (unlikely(!pool->initialized)) + return 0; + + /* We only switch if the processed DRNG is the initial DRNG. */ + if (init_drng != drng) + return 0; + + /* Get the aux pool hash with old digest ... */ + ret = old_cb->hash_final(shash, digest) ?: + /* ... re-initialize the hash with the new digest ... */ + new_cb->hash_init(shash, new_hash) ?: + /* + * ... feed the old hash into the new state. We may feed + * uninitialized memory into the new state, but this is + * considered no issue and even good as we have some more + * uncertainty here. + */ + new_cb->hash_update(shash, digest, sizeof(digest)); + if (!ret) { + lrng_set_digestsize(new_cb->hash_digestsize(new_hash)); + pr_debug("Re-initialize aux entropy pool with hash %s\n", + new_cb->hash_name()); + } + + memzero_explicit(digest, sizeof(digest)); + return ret; +} + +/* Insert data into auxiliary pool by using the hash update function. */ +static int +lrng_aux_pool_insert_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) +{ + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + struct lrng_drng *drng = lrng_drng_init_instance(); + const struct lrng_hash_cb *hash_cb; + unsigned long flags; + void *hash; + int ret; + + entropy_bits = min_t(u32, entropy_bits, inbuflen << 3); + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + if (unlikely(!pool->initialized)) { + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto out; + pool->initialized = true; + } + + ret = hash_cb->hash_update(shash, inbuf, inbuflen); + if (ret) + goto out; + + /* + * Cap the available entropy to the hash output size compliant to + * SP800-90B section 3.1.5.1 table 1. + */ + entropy_bits += atomic_read_u32(&pool->aux_entropy_bits); + atomic_set(&pool->aux_entropy_bits, + min_t(u32, entropy_bits, + hash_cb->hash_digestsize(hash) << 3)); + +out: + read_unlock_irqrestore(&drng->hash_lock, flags); + return ret; +} + +int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) +{ + struct lrng_pool *pool = &lrng_pool; + unsigned long flags; + int ret; + + spin_lock_irqsave(&pool->lock, flags); + ret = lrng_aux_pool_insert_locked(inbuf, inbuflen, entropy_bits); + spin_unlock_irqrestore(&pool->lock, flags); + + lrng_es_add_entropy(); + + return ret; +} +EXPORT_SYMBOL(lrng_pool_insert_aux); + +/************************* Get data from entropy pool *************************/ + +/* + * Get auxiliary entropy pool and its entropy content for seed buffer. + * Caller must hold lrng_pool.pool->lock. + * @outbuf: buffer to store data in with size requested_bits + * @requested_bits: Requested amount of entropy + * @return: amount of entropy in outbuf in bits. + */ +static u32 lrng_aux_get_pool(u8 *outbuf, u32 requested_bits) +{ + struct lrng_pool *pool = &lrng_pool; + struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; + struct lrng_drng *drng = lrng_drng_init_instance(); + const struct lrng_hash_cb *hash_cb; + unsigned long flags; + void *hash; + u32 collected_ent_bits, returned_ent_bits, unused_bits = 0, + digestsize, digestsize_bits, requested_bits_osr; + u8 aux_output[LRNG_MAX_DIGESTSIZE]; + + if (unlikely(!pool->initialized)) + return 0; + + read_lock_irqsave(&drng->hash_lock, flags); + + hash_cb = drng->hash_cb; + hash = drng->hash; + digestsize = hash_cb->hash_digestsize(hash); + digestsize_bits = digestsize << 3; + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(digestsize_bits, requested_bits); + + /* Ensure that no more than the size of aux_pool can be requested */ + requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3)); + requested_bits_osr = requested_bits + lrng_compress_osr(); + + /* Cap entropy with entropy counter from aux pool and the used digest */ + collected_ent_bits = min_t(u32, digestsize_bits, + atomic_xchg_relaxed(&pool->aux_entropy_bits, 0)); + + /* We collected too much entropy and put the overflow back */ + if (collected_ent_bits > requested_bits_osr) { + /* Amount of bits we collected too much */ + unused_bits = collected_ent_bits - requested_bits_osr; + /* Put entropy back */ + atomic_add(unused_bits, &pool->aux_entropy_bits); + /* Fix collected entropy */ + collected_ent_bits = requested_bits_osr; + } + + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n", + returned_ent_bits, collected_ent_bits, unused_bits); + + /* Get the digest for the aux pool to be returned to the caller ... */ + if (hash_cb->hash_final(shash, aux_output) || + /* + * ... and re-initialize the aux state. Do not add the aux pool + * digest for backward secrecy as it will be added with the + * insertion of the complete seed buffer after it has been filled. + */ + hash_cb->hash_init(shash, hash)) { + returned_ent_bits = 0; + } else { + /* + * Do not truncate the output size exactly to collected_ent_bits + * as the aux pool may contain data that is not credited with + * entropy, but we want to use them to stir the DRNG state. + */ + memcpy(outbuf, aux_output, requested_bits >> 3); + } + + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(aux_output, digestsize); + return returned_ent_bits; +} + +static void lrng_aux_get_backtrack(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + struct lrng_pool *pool = &lrng_pool; + unsigned long flags; + + /* Ensure aux pool extraction and backtracking op are atomic */ + spin_lock_irqsave(&pool->lock, flags); + + eb->e_bits[lrng_ext_es_aux] = lrng_aux_get_pool(eb->e[lrng_ext_es_aux], + requested_bits); + + /* Mix the extracted data back into pool for backtracking resistance */ + if (lrng_aux_pool_insert_locked((u8 *)eb, + sizeof(struct entropy_buf), 0)) + pr_warn("Backtracking resistance operation failed\n"); + + spin_unlock_irqrestore(&pool->lock, flags); +} + +static void lrng_aux_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_aux_avail_entropy(0)); +} + +struct lrng_es_cb lrng_es_aux = { + .name = "Auxiliary", + .get_ent = lrng_aux_get_backtrack, + .curr_entropy = lrng_aux_avail_entropy, + .max_entropy = lrng_get_digestsize, + .state = lrng_aux_es_state, + .reset = lrng_aux_reset, + .switch_hash = lrng_aux_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_aux.h b/drivers/char/lrng/lrng_es_aux.h new file mode 100644 index 0000000000000..bc41e6474aad2 --- /dev/null +++ b/drivers/char/lrng/lrng_es_aux.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_AUX_H +#define _LRNG_ES_AUX_H + +#include "lrng_drng_mgr.h" +#include "lrng_es_mgr_cb.h" + +u32 lrng_get_digestsize(void); +void lrng_pool_set_entropy(u32 entropy_bits); +int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); + +extern struct lrng_es_cb lrng_es_aux; + +/****************************** Helper code ***********************************/ + +/* Obtain the security strength of the LRNG in bits */ +static inline u32 lrng_security_strength(void) +{ + /* + * We use a hash to read the entropy in the entropy pool. According to + * SP800-90B table 1, the entropy can be at most the digest size. + * Considering this together with the last sentence in section 3.1.5.1.2 + * the security strength of a (approved) hash is equal to its output + * size. On the other hand the entropy cannot be larger than the + * security strength of the used DRBG. + */ + return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); +} + +static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) +{ + u32 requested_bits = lrng_security_strength(); + + /* Apply oversampling during initialization according to SP800-90C */ + if (lrng_sp80090c_compliant() && !fully_seeded) + requested_bits += LRNG_SEED_BUFFER_INIT_ADD_BITS; + return requested_bits; +} + +#endif /* _LRNG_ES_AUX_H */ diff --git a/drivers/char/lrng/lrng_es_cpu.c b/drivers/char/lrng/lrng_es_cpu.c new file mode 100644 index 0000000000000..3910108cfd088 --- /dev/null +++ b/drivers/char/lrng/lrng_es_cpu.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: CPU-based entropy source + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_es_aux.h" +#include "lrng_es_cpu.h" + +/* + * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS. + * As we have no ability to review the implementation of those noise sources, + * it is prudent to have a conservative estimate here. + */ +#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE +#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS +#ifdef CONFIG_RANDOM_TRUST_CPU +static u32 cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; +#else +static u32 cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; +#endif +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(cpu_entropy, uint, 0644); +MODULE_PARM_DESC(cpu_entropy, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDSEED)"); +#endif + +static int __init lrng_parse_trust_cpu(char *arg) +{ + int ret; + bool trust_cpu = false; + + ret = kstrtobool(arg, &trust_cpu); + if (ret) + return ret; + + if (trust_cpu) { + cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; + lrng_es_add_entropy(); + } else { + cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; + } + + return 0; +} +early_param("random.trust_cpu", lrng_parse_trust_cpu); + +static u32 lrng_cpu_entropylevel(u32 requested_bits) +{ + return lrng_fast_noise_entropylevel(cpu_entropy, requested_bits); +} + +static u32 lrng_cpu_poolsize(void) +{ + return lrng_cpu_entropylevel(lrng_security_strength()); +} + +static u32 lrng_get_cpu_data(u8 *outbuf, u32 requested_bits) +{ + u32 i; + + /* operate on full blocks */ + BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long)); + BUILD_BUG_ON(LRNG_SEED_BUFFER_INIT_ADD_BITS % sizeof(unsigned long)); + /* ensure we have aligned buffers */ + BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long)); + + for (i = 0; i < (requested_bits >> 3); + i += sizeof(unsigned long)) { + if (!arch_get_random_seed_long((unsigned long *)(outbuf + i)) && + !arch_get_random_long((unsigned long *)(outbuf + i))) { + cpu_entropy = 0; + return 0; + } + } + + return requested_bits; +} + +static u32 lrng_get_cpu_data_compress(u8 *outbuf, u32 requested_bits, + u32 data_multiplier) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng *drng = lrng_drng_node_instance(); + unsigned long flags; + u32 ent_bits = 0, i, partial_bits = 0, digestsize, digestsize_bits, + full_bits; + void *hash; + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + digestsize = hash_cb->hash_digestsize(hash); + digestsize_bits = digestsize << 3; + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(digestsize_bits, requested_bits); + full_bits = requested_bits * data_multiplier; + + /* Calculate oversampling for SP800-90C */ + if (lrng_sp80090c_compliant()) { + /* Complete amount of bits to be pulled */ + full_bits += LRNG_OVERSAMPLE_ES_BITS * data_multiplier; + /* Full blocks that will be pulled */ + data_multiplier = full_bits / requested_bits; + /* Partial block in bits to be pulled */ + partial_bits = full_bits - (data_multiplier * requested_bits); + } + + if (hash_cb->hash_init(shash, hash)) + goto out; + + /* Hash all data from the CPU entropy source */ + for (i = 0; i < data_multiplier; i++) { + ent_bits = lrng_get_cpu_data(outbuf, requested_bits); + if (!ent_bits) + goto out; + + if (hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) + goto err; + } + + /* Hash partial block, if applicable */ + ent_bits = lrng_get_cpu_data(outbuf, partial_bits); + if (ent_bits && + hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) + goto err; + + pr_debug("pulled %u bits from CPU RNG entropy source\n", full_bits); + ent_bits = requested_bits; + + /* Generate the compressed data to be returned to the caller */ + if (requested_bits < digestsize_bits) { + u8 digest[LRNG_MAX_DIGESTSIZE]; + + if (hash_cb->hash_final(shash, digest)) + goto err; + + /* Truncate output data to requested size */ + memcpy(outbuf, digest, requested_bits >> 3); + memzero_explicit(digest, digestsize); + } else { + if (hash_cb->hash_final(shash, outbuf)) + goto err; + } + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + return ent_bits; + +err: + ent_bits = 0; + goto out; +} + +/* + * If CPU entropy source requires does not return full entropy, return the + * multiplier of how much data shall be sampled from it. + */ +static u32 lrng_cpu_multiplier(void) +{ + static u32 data_multiplier = 0; + unsigned long v; + + if (data_multiplier > 0) + return data_multiplier; + + if (IS_ENABLED(CONFIG_X86) && !arch_get_random_seed_long(&v)) { + /* + * Intel SPEC: pulling 512 blocks from RDRAND ensures + * one reseed making it logically equivalent to RDSEED. + */ + data_multiplier = 512; + } else if (IS_ENABLED(CONFIG_PPC)) { + /* + * PowerISA defines DARN to deliver at least 0.5 bits of + * entropy per data bit. + */ + data_multiplier = 2; + } else if (IS_ENABLED(CONFIG_RISCV)) { + /* + * riscv-crypto-spec-scalar-1.0.0-rc6.pdf section 4.2 defines + * this requirement. + */ + data_multiplier = 2; + } else { + /* CPU provides full entropy */ + data_multiplier = CONFIG_LRNG_CPU_FULL_ENT_MULTIPLIER; + } + return data_multiplier; +} + +static int +lrng_cpu_switch_hash(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + u32 digestsize, multiplier; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + digestsize = lrng_get_digestsize(); + multiplier = lrng_cpu_multiplier(); + + /* + * It would be security violation if the new digestsize is smaller than + * the set CPU entropy rate. + */ + WARN_ON(multiplier > 1 && digestsize < cpu_entropy); + cpu_entropy = min_t(u32, digestsize, cpu_entropy); + return 0; +} + +/* + * lrng_get_arch() - Get CPU entropy source entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_cpu_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + u32 ent_bits, data_multiplier = lrng_cpu_multiplier(); + + if (data_multiplier <= 1) { + ent_bits = lrng_get_cpu_data(eb->e[lrng_ext_es_cpu], + requested_bits); + } else { + ent_bits = lrng_get_cpu_data_compress(eb->e[lrng_ext_es_cpu], + requested_bits, + data_multiplier); + } + + ent_bits = lrng_cpu_entropylevel(ent_bits); + pr_debug("obtained %u bits of entropy from CPU RNG entropy source\n", + ent_bits); + eb->e_bits[lrng_ext_es_cpu] = ent_bits; +} + +static void lrng_cpu_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + u32 data_multiplier = lrng_cpu_multiplier(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for compressing data: %s\n" + " Available entropy: %u\n" + " Data multiplier: %u\n", + (data_multiplier <= 1) ? + "N/A" : lrng_drng_init->hash_cb->hash_name(), + lrng_cpu_poolsize(), + data_multiplier); +} + +struct lrng_es_cb lrng_es_cpu = { + .name = "CPU", + .get_ent = lrng_cpu_get, + .curr_entropy = lrng_cpu_entropylevel, + .max_entropy = lrng_cpu_poolsize, + .state = lrng_cpu_es_state, + .reset = NULL, + .switch_hash = lrng_cpu_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_cpu.h b/drivers/char/lrng/lrng_es_cpu.h new file mode 100644 index 0000000000000..8dbb4d9a2926c --- /dev/null +++ b/drivers/char/lrng/lrng_es_cpu.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_CPU_H +#define _LRNG_ES_CPU_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_CPU + +extern struct lrng_es_cb lrng_es_cpu; + +#endif /* CONFIG_LRNG_CPU */ + +#endif /* _LRNG_ES_CPU_H */ diff --git a/drivers/char/lrng/lrng_es_irq.c b/drivers/char/lrng/lrng_es_irq.c new file mode 100644 index 0000000000000..97c5c1d264471 --- /dev/null +++ b/drivers/char/lrng/lrng_es_irq.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Interrupt data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_irq.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" +#include "lrng_numa.h" +#include "lrng_testing.h" + +/* + * Number of interrupts to be recorded to assume that DRNG security strength + * bits of entropy are received. + * Note: a value below the DRNG security strength should not be defined as this + * may imply the DRNG can never be fully seeded in case other noise + * sources are unavailable. + */ +#define LRNG_IRQ_ENTROPY_BITS LRNG_UINT32_C(CONFIG_LRNG_IRQ_ENTROPY_RATE) + + +/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ +static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS; + +static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(irq_entropy, uint, 0444); +MODULE_PARM_DESC(irq_entropy, + "How many interrupts must be collected for obtaining 256 bits of entropy\n"); +#endif + +/* Per-CPU array holding concatenated IRQ entropy events */ +static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_irq_array) + __aligned(LRNG_KCAPI_ALIGN); +static DEFINE_PER_CPU(u32, lrng_irq_array_ptr) = 0; +static DEFINE_PER_CPU(atomic_t, lrng_irq_array_irqs) = ATOMIC_INIT(0); + +/* + * The entropy collection is performed by executing the following steps: + * 1. fill up the per-CPU array holding the time stamps + * 2. once the per-CPU array is full, a compression of the data into + * the entropy pool is performed - this happens in interrupt context + * + * If step 2 is not desired in interrupt context, the following boolean + * needs to be set to false. This implies that old entropy data in the + * per-CPU array collected since the last DRNG reseed is overwritten with + * new entropy data instead of retaining the entropy with the compression + * operation. + * + * Impact on entropy: + * + * If continuous compression is enabled, the maximum entropy that is collected + * per CPU between DRNG reseeds is equal to the digest size of the used hash. + * + * If continuous compression is disabled, the maximum number of entropy events + * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount + * of events is converted into an entropy statement which then represents the + * maximum amount of entropy collectible per CPU between DRNG reseeds. + */ +static bool lrng_irq_continuous_compression __read_mostly = + IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION); + +#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION +module_param(lrng_irq_continuous_compression, bool, 0444); +MODULE_PARM_DESC(lrng_irq_continuous_compression, + "Perform entropy compression if per-CPU entropy data array is full\n"); +#endif + +/* + * Per-CPU entropy pool with compressed entropy event + * + * The per-CPU entropy pool is defined as the hash state. New data is simply + * inserted into the entropy pool by performing a hash update operation. + * To read the entropy pool, a hash final must be invoked. However, before + * the entropy pool is released again after a hash final, the hash init must + * be performed. + */ +static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_irq_pool) + __aligned(LRNG_KCAPI_ALIGN); +/* + * Lock to allow other CPUs to read the pool - as this is only done during + * reseed which is infrequent, this lock is hardly contended. + */ +static DEFINE_PER_CPU(spinlock_t, lrng_irq_lock); +static DEFINE_PER_CPU(bool, lrng_irq_lock_init) = false; + +static bool lrng_irq_pool_online(int cpu) +{ + return per_cpu(lrng_irq_lock_init, cpu); +} + +static void __init lrng_irq_check_compression_state(void) +{ + /* One pool must hold sufficient entropy for disabled compression */ + if (!lrng_irq_continuous_compression) { + u32 max_ent = min_t(u32, lrng_get_digestsize(), + lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, + lrng_irq_entropy_bits)); + if (max_ent < lrng_security_strength()) { + pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n"); + lrng_irq_continuous_compression = true; + } + } +} + +void __init lrng_irq_es_init(bool highres_timer) +{ + /* Set a minimum number of interrupts that must be collected */ + irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy); + + if (highres_timer) { + lrng_irq_entropy_bits = irq_entropy; + } else { + u32 new_entropy = irq_entropy * LRNG_ES_OVERSAMPLING_FACTOR; + + lrng_irq_entropy_bits = (irq_entropy < new_entropy) ? + new_entropy : irq_entropy; + pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n", + LRNG_ES_OVERSAMPLING_FACTOR); + } + + lrng_irq_check_compression_state(); +} + +/* + * Reset all per-CPU pools - reset entropy estimator but leave the pool data + * that may or may not have entropy unchanged. + */ +static void lrng_irq_reset(void) +{ + int cpu; + + /* Trigger GCD calculation anew. */ + lrng_gcd_set(0); + + for_each_online_cpu(cpu) + atomic_set(per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); +} + +static u32 lrng_irq_avail_pool_size(void) +{ + u32 max_size = 0, max_pool = lrng_get_digestsize(); + int cpu; + + if (!lrng_irq_continuous_compression) + max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); + + for_each_online_cpu(cpu) { + if (lrng_irq_pool_online(cpu)) + max_size += max_pool; + } + + return max_size; +} + +/* Return entropy of unused IRQs present in all per-CPU pools. */ +static u32 lrng_irq_avail_entropy(u32 __unused) +{ + u32 digestsize_irqs, irq = 0; + int cpu; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) + return 0; + + /* Obtain the cap of maximum numbers of IRQs we count */ + digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize(), + lrng_irq_entropy_bits); + if (!lrng_irq_continuous_compression) { + /* Cap to max. number of IRQs the array can hold */ + digestsize_irqs = min_t(u32, digestsize_irqs, + LRNG_DATA_NUM_VALUES); + } + + for_each_online_cpu(cpu) { + if (!lrng_irq_pool_online(cpu)) + continue; + irq += min_t(u32, digestsize_irqs, + atomic_read_u32(per_cpu_ptr(&lrng_irq_array_irqs, + cpu))); + } + + /* Consider oversampling rate */ + return lrng_reduce_by_osr(lrng_data_to_entropy(irq, + lrng_irq_entropy_bits)); +} + +/* + * Trigger a switch of the hash implementation for the per-CPU pool. + * + * For each per-CPU pool, obtain the message digest with the old hash + * implementation, initialize the per-CPU pool again with the new hash + * implementation and inject the message digest into the new state. + * + * Assumption: the caller must guarantee that the new_cb is available during the + * entire operation (e.g. it must hold the lock against pointer updating). + */ +static int +lrng_irq_switch_hash(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + u8 digest[LRNG_MAX_DIGESTSIZE]; + u32 digestsize_irqs, found_irqs; + int ret = 0, cpu; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + for_each_online_cpu(cpu) { + struct shash_desc *pcpu_shash; + + /* + * Only switch the per-CPU pools for the current node because + * the hash_cb only applies NUMA-node-wide. + */ + if (cpu_to_node(cpu) != node || !lrng_irq_pool_online(cpu)) + continue; + + pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, + cpu); + + digestsize_irqs = old_cb->hash_digestsize(pcpu_shash); + digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3, + lrng_irq_entropy_bits); + + if (pcpu_shash->tfm == new_hash) + continue; + + /* Get the per-CPU pool hash with old digest ... */ + ret = old_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash with the new digest ... */ + new_cb->hash_init(pcpu_shash, new_hash) ?: + /* + * ... feed the old hash into the new state. We may feed + * uninitialized memory into the new state, but this is + * considered no issue and even good as we have some more + * uncertainty here. + */ + new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); + if (ret) + goto out; + + /* + * In case the new digest is larger than the old one, cap + * the available entropy to the old message digest used to + * process the existing data. + */ + found_irqs = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); + found_irqs = min_t(u32, found_irqs, digestsize_irqs); + atomic_add_return_relaxed(found_irqs, + per_cpu_ptr(&lrng_irq_array_irqs, cpu)); + + pr_debug("Re-initialize per-CPU interrupt entropy pool for CPU %d on NUMA node %d with hash %s\n", + cpu, node, new_cb->hash_name()); + } + +out: + memzero_explicit(digest, sizeof(digest)); + return ret; +} + +/* + * When reading the per-CPU message digest, make sure we use the crypto + * callbacks defined for the NUMA node the per-CPU pool is defined for because + * the LRNG crypto switch support is only atomic per NUMA node. + */ +static u32 +lrng_irq_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, + void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) +{ + struct shash_desc *pcpu_shash = + (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, cpu); + spinlock_t *lock = per_cpu_ptr(&lrng_irq_lock, cpu); + unsigned long flags; + u32 digestsize_irqs, found_irqs; + + /* Lock guarding against reading / writing to per-CPU pool */ + spin_lock_irqsave(lock, flags); + + *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); + digestsize_irqs = lrng_entropy_to_data(*digestsize << 3, + lrng_irq_entropy_bits); + + /* Obtain entropy statement like for the entropy pool */ + found_irqs = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); + /* Cap to maximum amount of data we can hold in hash */ + found_irqs = min_t(u32, found_irqs, digestsize_irqs); + + /* Cap to maximum amount of data we can hold in array */ + if (!lrng_irq_continuous_compression) + found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES); + + /* Store all not-yet compressed data in data array into hash, ... */ + if (pcpu_hash_cb->hash_update(pcpu_shash, + (u8 *)per_cpu_ptr(lrng_irq_array, cpu), + LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: + /* ... get the per-CPU pool digest, ... */ + pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash, ... */ + pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: + /* ... feed the old hash into the new state. */ + pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) + found_irqs = 0; + + spin_unlock_irqrestore(lock, flags); + return found_irqs; +} + +/* + * Hash all per-CPU pools and return the digest to be used as seed data for + * seeding a DRNG. The caller must guarantee backtracking resistance. + * The function will only copy as much data as entropy is available into the + * caller-provided output buffer. + * + * This function handles the translation from the number of received interrupts + * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS + * which defines how many interrupts must be received to obtain 256 bits of + * entropy. With this value, the function lrng_data_to_entropy converts a given + * data size (received interrupts, requested amount of data, etc.) into an + * entropy statement. lrng_entropy_to_data does the reverse. + * + * @eb: entropy buffer to store entropy + * @requested_bits: Requested amount of entropy + * @fully_seeded: indicator whether LRNG is fully seeded + */ +static void lrng_irq_pool_hash(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *drng = lrng_drng_init_instance(); + u8 digest[LRNG_MAX_DIGESTSIZE]; + unsigned long flags, flags2; + u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs, + returned_ent_bits; + int ret, cpu; + void *hash; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) { + eb->e_bits[lrng_int_es_irq] = 0; + return; + } + + /* Lock guarding replacement of per-NUMA hash */ + read_lock_irqsave(&drng->hash_lock, flags); + + hash_cb = drng->hash_cb; + hash = drng->hash; + + /* The hash state of filled with all per-CPU pool hashes. */ + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto err; + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); + requested_irqs = lrng_entropy_to_data(requested_bits + + lrng_compress_osr(), + lrng_irq_entropy_bits); + + /* + * Harvest entropy from each per-CPU hash state - even though we may + * have collected sufficient entropy, we will hash all per-CPU pools. + */ + for_each_online_cpu(cpu) { + struct lrng_drng *pcpu_drng = drng; + u32 digestsize, pcpu_unused_irqs = 0; + int node = cpu_to_node(cpu); + + /* If pool is not online, then no entropy is present. */ + if (!lrng_irq_pool_online(cpu)) + continue; + + if (lrng_drng && lrng_drng[node]) + pcpu_drng = lrng_drng[node]; + + if (pcpu_drng == drng) { + found_irqs = lrng_irq_pool_hash_one(hash_cb, hash, + cpu, digest, + &digestsize); + } else { + read_lock_irqsave(&pcpu_drng->hash_lock, flags2); + found_irqs = + lrng_irq_pool_hash_one(pcpu_drng->hash_cb, + pcpu_drng->hash, cpu, + digest, &digestsize); + read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); + } + + /* Inject the digest into the state of all per-CPU pools */ + ret = hash_cb->hash_update(shash, digest, digestsize); + if (ret) + goto err; + + collected_irqs += found_irqs; + if (collected_irqs > requested_irqs) { + pcpu_unused_irqs = collected_irqs - requested_irqs; + atomic_add_return_relaxed(pcpu_unused_irqs, + per_cpu_ptr(&lrng_irq_array_irqs, cpu)); + collected_irqs = requested_irqs; + } + pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n", + found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs); + } + + ret = hash_cb->hash_final(shash, digest); + if (ret) + goto err; + + collected_ent_bits = lrng_data_to_entropy(collected_irqs, + lrng_irq_entropy_bits); + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n", + returned_ent_bits, collected_ent_bits); + + /* + * Truncate to available entropy as implicitly allowed by SP800-90B + * section 3.1.5.1.1 table 1 which awards truncated hashes full + * entropy. + * + * During boot time, we read requested_bits data with + * returned_ent_bits entropy. In case our conservative entropy + * estimate underestimates the available entropy we can transport as + * much available entropy as possible. + */ + memcpy(eb->e[lrng_int_es_irq], digest, + fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); + eb->e_bits[lrng_int_es_irq] = returned_ent_bits; + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(digest, sizeof(digest)); + return; + +err: + eb->e_bits[lrng_int_es_irq] = 0; + goto out; +} + +/* Compress the lrng_irq_array array into lrng_irq_pool */ +static void lrng_irq_array_compress(void) +{ + struct shash_desc *shash = + (struct shash_desc *)this_cpu_ptr(lrng_irq_pool); + struct lrng_drng *drng = lrng_drng_node_instance(); + const struct lrng_hash_cb *hash_cb; + spinlock_t *lock = this_cpu_ptr(&lrng_irq_lock); + unsigned long flags, flags2; + void *hash; + bool init = false; + + read_lock_irqsave(&drng->hash_lock, flags); + hash_cb = drng->hash_cb; + hash = drng->hash; + + if (unlikely(!this_cpu_read(lrng_irq_lock_init))) { + init = true; + spin_lock_init(lock); + this_cpu_write(lrng_irq_lock_init, true); + pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", + raw_smp_processor_id(), numa_node_id(), + hash_cb->hash_name()); + } + + spin_lock_irqsave(lock, flags2); + + if (unlikely(init) && hash_cb->hash_init(shash, hash)) { + this_cpu_write(lrng_irq_lock_init, false); + pr_warn("Initialization of hash failed\n"); + } else if (lrng_irq_continuous_compression) { + /* Add entire per-CPU data array content into entropy pool. */ + if (hash_cb->hash_update(shash, + (u8 *)this_cpu_ptr(lrng_irq_array), + LRNG_DATA_ARRAY_SIZE * sizeof(u32))) + pr_warn_ratelimited("Hashing of entropy data failed\n"); + } + + spin_unlock_irqrestore(lock, flags2); + read_unlock_irqrestore(&drng->hash_lock, flags); +} + +/* Compress data array into hash */ +static void lrng_irq_array_to_hash(u32 ptr) +{ + u32 *array = this_cpu_ptr(lrng_irq_array); + + /* + * During boot time the hash operation is triggered more often than + * during regular operation. + */ + if (unlikely(!lrng_state_fully_seeded())) { + if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK)) + return; + } else if (ptr < LRNG_DATA_WORD_MASK) { + return; + } + + if (lrng_raw_array_entropy_store(*array)) { + u32 i; + + /* + * If we fed even a part of the array to external analysis, we + * mark that the entire array and the per-CPU pool to have no + * entropy. This is due to the non-IID property of the data as + * we do not fully know whether the existing dependencies + * diminish the entropy beyond to what we expect it has. + */ + atomic_set(this_cpu_ptr(&lrng_irq_array_irqs), 0); + + for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++) + lrng_raw_array_entropy_store(*(array + i)); + } else { + lrng_irq_array_compress(); + /* Ping pool handler about received entropy */ + if (lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) + lrng_es_add_entropy(); + } +} + +/* + * Concatenate full 32 bit word at the end of time array even when current + * ptr is not aligned to sizeof(data). + */ +static void _lrng_irq_array_add_u32(u32 data) +{ + /* Increment pointer by number of slots taken for input value */ + u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_irq_array_ptr, + LRNG_DATA_SLOTS_PER_UINT); + unsigned int pre_array; + + /* + * This function injects a unit into the array - guarantee that + * array unit size is equal to data type of input data. + */ + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3)); + + /* + * The following logic requires at least two units holding + * the data as otherwise the pointer would immediately wrap when + * injection an u32 word. + */ + BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT); + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_irq_array[pre_array], ~(0xffffffff & ~mask)); + this_cpu_or(lrng_irq_array[pre_array], data & ~mask); + + /* Invoke compression as we just filled data array completely */ + if (unlikely(pre_ptr > ptr)) + lrng_irq_array_to_hash(LRNG_DATA_WORD_MASK); + + /* LSB of data go into current unit */ + this_cpu_write(lrng_irq_array[lrng_data_idx2array(ptr)], + data & mask); + + if (likely(pre_ptr <= ptr)) + lrng_irq_array_to_hash(ptr); +} + +/* Concatenate a 32-bit word at the end of the per-CPU array */ +void lrng_irq_array_add_u32(u32 data) +{ + /* + * Disregard entropy-less data without continuous compression to + * avoid it overwriting data with entropy when array ptr wraps. + */ + if (lrng_irq_continuous_compression) + _lrng_irq_array_add_u32(data); +} + +/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ +static void lrng_irq_array_add_slot(u32 data) +{ + /* Get slot */ + u32 ptr = this_cpu_inc_return(lrng_irq_array_ptr) & + LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS); + /* Ensure consistency of values */ + BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != + sizeof(lrng_irq_array[0]) << 3); + + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_irq_array[array], + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot))); + /* Store data into slot */ + this_cpu_or(lrng_irq_array[array], lrng_data_slot_val(data, slot)); + + lrng_irq_array_to_hash(ptr); +} + +static void +lrng_time_process_common(u32 time, void(*add_time)(u32 data)) +{ + enum lrng_health_res health_test; + + if (lrng_raw_hires_entropy_store(time)) + return; + + health_test = lrng_health_test(time, lrng_int_es_irq); + if (health_test > lrng_health_fail_use) + return; + + if (health_test == lrng_health_pass) + atomic_inc_return(this_cpu_ptr(&lrng_irq_array_irqs)); + + add_time(time); +} + +/* + * Batching up of entropy in per-CPU array before injecting into entropy pool. + */ +static void lrng_time_process(void) +{ + u32 now_time = random_get_entropy(); + + if (unlikely(!lrng_gcd_tested())) { + /* When GCD is unknown, we process the full time stamp */ + lrng_time_process_common(now_time, _lrng_irq_array_add_u32); + lrng_gcd_add_value(now_time); + } else { + /* GCD is known and applied */ + lrng_time_process_common((now_time / lrng_gcd_get()) & + LRNG_DATA_SLOTSIZE_MASK, + lrng_irq_array_add_slot); + } + + lrng_perf_time(now_time); +} + +/* Hot code path - Callback for interrupt handler */ +void add_interrupt_randomness(int irq) +{ + if (lrng_highres_timer()) { + lrng_time_process(); + } else { + struct pt_regs *regs = get_irq_regs(); + static atomic_t reg_idx = ATOMIC_INIT(0); + u64 ip; + u32 tmp; + + if (regs) { + u32 *ptr = (u32 *)regs; + int reg_ptr = atomic_add_return_relaxed(1, ®_idx); + size_t n = (sizeof(struct pt_regs) / sizeof(u32)); + + ip = instruction_pointer(regs); + tmp = *(ptr + (reg_ptr % n)); + tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp; + _lrng_irq_array_add_u32(tmp); + } else { + ip = _RET_IP_; + } + + lrng_time_process(); + + /* + * The XOR operation combining the different values is not + * considered to destroy entropy since the entirety of all + * processed values delivers the entropy (and not each + * value separately of the other values). + */ + tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies; + tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq; + tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip; + tmp ^= ip >> 32; + _lrng_irq_array_add_u32(tmp); + } +} +EXPORT_SYMBOL(add_interrupt_randomness); + +static void lrng_irq_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n" + " per-CPU interrupt collection size: %u\n" + " Standards compliance: %s\n" + " High-resolution timer: %s\n" + " Continuous compression: %s\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_irq_avail_entropy(0), + LRNG_DATA_NUM_VALUES, + lrng_sp80090b_compliant(lrng_int_es_irq) ? "SP800-90B " : "", + lrng_highres_timer() ? "true" : "false", + lrng_irq_continuous_compression ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_irq = { + .name = "IRQ", + .get_ent = lrng_irq_pool_hash, + .curr_entropy = lrng_irq_avail_entropy, + .max_entropy = lrng_irq_avail_pool_size, + .state = lrng_irq_es_state, + .reset = lrng_irq_reset, + .switch_hash = lrng_irq_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_irq.h b/drivers/char/lrng/lrng_es_irq.h new file mode 100644 index 0000000000000..2cd746611cf0c --- /dev/null +++ b/drivers/char/lrng/lrng_es_irq.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_IRQ_H +#define _LRNG_ES_IRQ_H + +#include + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_IRQ +void lrng_irq_es_init(bool highres_timer); +void lrng_irq_array_add_u32(u32 data); + +extern struct lrng_es_cb lrng_es_irq; + +#else /* CONFIG_LRNG_IRQ */ +static inline void lrng_irq_es_init(bool highres_timer) { } +static inline void lrng_irq_array_add_u32(u32 data) { } +#endif /* CONFIG_LRNG_IRQ */ + +#endif /* _LRNG_ES_IRQ_H */ diff --git a/drivers/char/lrng/lrng_es_jent.c b/drivers/char/lrng/lrng_es_jent.c new file mode 100644 index 0000000000000..dbb2fce591f08 --- /dev/null +++ b/drivers/char/lrng/lrng_es_jent.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: Jitter RNG + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_es_aux.h" +#include "lrng_es_jent.h" + +/* + * Estimated entropy of data is a 16th of LRNG_DRNG_SECURITY_STRENGTH_BITS. + * Albeit a full entropy assessment is provided for the noise source indicating + * that it provides high entropy rates and considering that it deactivates + * when it detects insufficient hardware, the chosen under estimation of + * entropy is considered to be acceptable to all reviewers. + */ +static u32 jent_entropy = CONFIG_LRNG_JENT_ENTROPY_RATE; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(jent_entropy, uint, 0644); +MODULE_PARM_DESC(jent_entropy, "Entropy in bits of 256 data bits from Jitter RNG noise source"); +#endif + +static bool lrng_jent_initialized = false; +static struct rand_data *lrng_jent_state; + +static int __init lrng_jent_initialize(void) +{ + /* Initialize the Jitter RNG after the clocksources are initialized. */ + if (jent_entropy_init() || + (lrng_jent_state = jent_entropy_collector_alloc(1, 0)) == NULL) { + jent_entropy = 0; + pr_info("Jitter RNG unusable on current system\n"); + return 0; + } + lrng_jent_initialized = true; + pr_debug("Jitter RNG working on current system\n"); + + /* + * In FIPS mode, the Jitter RNG is defined to have full of entropy + * unless a different value has been specified at the command line + * (i.e. the user overrides the default), and the default value is + * larger than zero (if it is zero, it is assumed that an RBG2(P) or + * RBG2(NP) construction is attempted that intends to exclude the + * Jitter RNG). + */ + if (fips_enabled && + CONFIG_LRNG_JENT_ENTROPY_RATE > 0 && + jent_entropy == CONFIG_LRNG_JENT_ENTROPY_RATE) + jent_entropy = LRNG_DRNG_SECURITY_STRENGTH_BITS; + + lrng_drng_force_reseed(); + if (jent_entropy) + lrng_es_add_entropy(); + + return 0; +} +device_initcall(lrng_jent_initialize); + +static u32 lrng_jent_entropylevel(u32 requested_bits) +{ + return lrng_fast_noise_entropylevel(lrng_jent_initialized ? + jent_entropy : 0, requested_bits); +} + +static u32 lrng_jent_poolsize(void) +{ + return lrng_jent_entropylevel(lrng_security_strength()); +} + +/* + * lrng_get_jent() - Get Jitter RNG entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_jent_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + int ret; + u32 ent_bits = lrng_jent_entropylevel(requested_bits); + unsigned long flags; + static DEFINE_SPINLOCK(lrng_jent_lock); + + spin_lock_irqsave(&lrng_jent_lock, flags); + + if (!lrng_jent_initialized) { + spin_unlock_irqrestore(&lrng_jent_lock, flags); + goto err; + } + + ret = jent_read_entropy(lrng_jent_state, eb->e[lrng_ext_es_jitter], + requested_bits >> 3); + spin_unlock_irqrestore(&lrng_jent_lock, flags); + + if (ret) { + pr_debug("Jitter RNG failed with %d\n", ret); + goto err; + } + + pr_debug("obtained %u bits of entropy from Jitter RNG noise source\n", + ent_bits); + + eb->e_bits[lrng_ext_es_jitter] = ent_bits; + return; + +err: + eb->e_bits[lrng_ext_es_jitter] = 0; +} + +static void lrng_jent_es_state(unsigned char *buf, size_t buflen) +{ + snprintf(buf, buflen, + " Available entropy: %u\n" + " Enabled: %s\n", + lrng_jent_poolsize(), + lrng_jent_initialized ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_jent = { + .name = "JitterRNG", + .get_ent = lrng_jent_get, + .curr_entropy = lrng_jent_entropylevel, + .max_entropy = lrng_jent_poolsize, + .state = lrng_jent_es_state, + .reset = NULL, + .switch_hash = NULL, +}; diff --git a/drivers/char/lrng/lrng_es_jent.h b/drivers/char/lrng/lrng_es_jent.h new file mode 100644 index 0000000000000..32882d4bdf99b --- /dev/null +++ b/drivers/char/lrng/lrng_es_jent.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_JENT_H +#define _LRNG_ES_JENT_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_JENT + +extern struct lrng_es_cb lrng_es_jent; + +#endif /* CONFIG_LRNG_JENT */ + +#endif /* _LRNG_ES_JENT_H */ diff --git a/drivers/char/lrng/lrng_es_krng.c b/drivers/char/lrng/lrng_es_krng.c new file mode 100644 index 0000000000000..0ef88e6c8d562 --- /dev/null +++ b/drivers/char/lrng/lrng_es_krng.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Fast Entropy Source: Linux kernel RNG (random.c) + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_krng.h" + +static u32 krng_entropy = CONFIG_LRNG_KERNEL_RNG_ENTROPY_RATE; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(krng_entropy, uint, 0644); +MODULE_PARM_DESC(krng_entropy, "Entropy in bits of 256 data bits from the kernel RNG noise source"); +#endif + +static atomic_t lrng_krng_initial_rate = ATOMIC_INIT(0); + +static struct notifier_block lrng_krng_ready = { + .notifier_call = NULL, +}; + +static u32 lrng_krng_fips_entropylevel(u32 entropylevel) +{ + return fips_enabled ? 0 : entropylevel; +} + +static int lrng_krng_adjust_entropy(struct notifier_block *nb, + unsigned long action, void *data) +{ + u32 entropylevel; + + krng_entropy = atomic_read_u32(&lrng_krng_initial_rate); + + entropylevel = lrng_krng_fips_entropylevel(krng_entropy); + pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", + entropylevel); + lrng_drng_force_reseed(); + if (entropylevel) + lrng_es_add_entropy(); + return 0; +} + +static u32 lrng_krng_entropylevel(u32 requested_bits) +{ + if (lrng_krng_ready.notifier_call == NULL) { + int err; + + lrng_krng_ready.notifier_call = lrng_krng_adjust_entropy; + + err = register_random_ready_notifier(&lrng_krng_ready); + switch (err) { + case 0: + atomic_set(&lrng_krng_initial_rate, krng_entropy); + krng_entropy = 0; + pr_debug("Kernel RNG is not yet seeded, setting entropy rate to 0 bits of entropy\n"); + break; + + case -EALREADY: + pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", + lrng_krng_fips_entropylevel(krng_entropy)); + break; + default: + lrng_krng_ready.notifier_call = NULL; + return 0; + } + } + + return lrng_fast_noise_entropylevel( + lrng_krng_fips_entropylevel(krng_entropy), requested_bits); +} + +static u32 lrng_krng_poolsize(void) +{ + return lrng_krng_entropylevel(lrng_security_strength()); +} + +/* + * lrng_krng_get() - Get kernel RNG entropy + * + * @eb: entropy buffer to store entropy + * @requested_bits: requested entropy in bits + */ +static void lrng_krng_get(struct entropy_buf *eb, u32 requested_bits, + bool __unused) +{ + u32 ent_bits = lrng_krng_entropylevel(requested_bits); + + get_random_bytes(eb->e[lrng_ext_es_krng], requested_bits >> 3); + + pr_debug("obtained %u bits of entropy from kernel RNG noise source\n", + ent_bits); + + eb->e_bits[lrng_ext_es_krng] = ent_bits; +} + +static void lrng_krng_es_state(unsigned char *buf, size_t buflen) +{ + snprintf(buf, buflen, + " Available entropy: %u\n" + " Entropy Rate per 256 data bits: %u\n", + lrng_krng_poolsize(), + lrng_krng_entropylevel(256)); +} + +struct lrng_es_cb lrng_es_krng = { + .name = "KernelRNG", + .get_ent = lrng_krng_get, + .curr_entropy = lrng_krng_entropylevel, + .max_entropy = lrng_krng_poolsize, + .state = lrng_krng_es_state, + .reset = NULL, + .switch_hash = NULL, +}; diff --git a/drivers/char/lrng/lrng_es_krng.h b/drivers/char/lrng/lrng_es_krng.h new file mode 100644 index 0000000000000..cf982b9eea053 --- /dev/null +++ b/drivers/char/lrng/lrng_es_krng.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_RANDOM_H +#define _LRNG_ES_RANDOM_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_KERNEL_RNG + +extern struct lrng_es_cb lrng_es_krng; + +#endif /* CONFIG_LRNG_KERNEL_RNG */ + +#endif /* _LRNG_ES_RANDOM_H */ diff --git a/drivers/char/lrng/lrng_es_mgr.c b/drivers/char/lrng/lrng_es_mgr.c new file mode 100644 index 0000000000000..5bec39d9ef912 --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Entropy sources management + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_cpu.h" +#include "lrng_es_irq.h" +#include "lrng_es_jent.h" +#include "lrng_es_krng.h" +#include "lrng_es_mgr.h" +#include "lrng_es_sched.h" +#include "lrng_interface_dev_common.h" +#include "lrng_interface_random_kernel.h" + +struct lrng_state { + bool can_invalidate; /* Can invalidate batched entropy? */ + bool perform_seedwork; /* Can seed work be performed? */ + bool lrng_operational; /* Is DRNG operational? */ + bool lrng_fully_seeded; /* Is DRNG fully seeded? */ + bool lrng_min_seeded; /* Is DRNG minimally seeded? */ + bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */ + + /* + * To ensure that external entropy providers cannot dominate the + * internal noise sources but yet cannot be dominated by internal + * noise sources, the following booleans are intended to allow + * external to provide seed once when a DRNG reseed occurs. This + * triggering of external noise source is performed even when the + * entropy pool has sufficient entropy. + */ + + atomic_t boot_entropy_thresh; /* Reseed threshold */ + atomic_t reseed_in_progress; /* Flag for on executing reseed */ + struct work_struct lrng_seed_work; /* (re)seed work queue */ +}; + +static struct lrng_state lrng_state = { + false, false, false, false, false, false, + .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS), + .reseed_in_progress = ATOMIC_INIT(0), +}; + +/* + * If the entropy count falls under this number of bits, then we + * should wake up processes which are selecting or polling on write + * access to /dev/random. + */ +u32 lrng_write_wakeup_bits = (LRNG_WRITE_WAKEUP_ENTROPY << 3); + +/* + * The entries must be in the same order as defined by enum lrng_internal_es and + * enum lrng_external_es + */ +struct lrng_es_cb *lrng_es[] = { +#ifdef CONFIG_LRNG_IRQ + &lrng_es_irq, +#endif +#ifdef CONFIG_LRNG_SCHED + &lrng_es_sched, +#endif +#ifdef CONFIG_LRNG_JENT + &lrng_es_jent, +#endif +#ifdef CONFIG_LRNG_CPU + &lrng_es_cpu, +#endif +#ifdef CONFIG_LRNG_KERNEL_RNG + &lrng_es_krng, +#endif + &lrng_es_aux +}; + +/********************************** Helper ***********************************/ + +void lrng_debug_report_seedlevel(const char *name) +{ +#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM + static void *previous = NULL; + void *caller = (void *) _RET_IP_; + + if (READ_ONCE(previous) == caller) + return; + + if (!lrng_state_min_seeded()) + pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n", + caller, name, lrng_avail_entropy()); + + WRITE_ONCE(previous, caller); +#endif +} + +/* + * Reading of the LRNG pool is only allowed by one caller. The reading is + * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken, + * the reseeding operation is in progress. The caller is not intended to wait + * but continue with its other operation. + */ +int lrng_pool_trylock(void) +{ + return atomic_cmpxchg(&lrng_state.reseed_in_progress, 0, 1); +} + +void lrng_pool_unlock(void) +{ + atomic_set(&lrng_state.reseed_in_progress, 0); +} + +/* Set new entropy threshold for reseeding during boot */ +void lrng_set_entropy_thresh(u32 new_entropy_bits) +{ + atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits); +} + +/* + * Reset LRNG state - the entropy counters are reset, but the data that may + * or may not have entropy remains in the pools as this data will not hurt. + */ +void lrng_reset_state(void) +{ + u32 i; + + for_each_lrng_es(i) { + if (lrng_es[i]->reset) + lrng_es[i]->reset(); + } + lrng_state.lrng_operational = false; + lrng_state.lrng_fully_seeded = false; + lrng_state.lrng_min_seeded = false; + lrng_state.all_online_numa_node_seeded = false; + pr_debug("reset LRNG\n"); +} + +/* Set flag that all DRNGs are fully seeded */ +void lrng_pool_all_numa_nodes_seeded(bool set) +{ + lrng_state.all_online_numa_node_seeded = set; +} + +/* Return boolean whether LRNG reached minimally seed level */ +bool lrng_state_min_seeded(void) +{ + return lrng_state.lrng_min_seeded; +} + +/* Return boolean whether LRNG reached fully seed level */ +bool lrng_state_fully_seeded(void) +{ + return lrng_state.lrng_fully_seeded; +} + +/* Return boolean whether LRNG is considered fully operational */ +bool lrng_state_operational(void) +{ + return lrng_state.lrng_operational; +} + +static void lrng_init_wakeup(void) +{ + wake_up_all(&lrng_init_wait); + lrng_init_wakeup_dev(); +} + +bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy) +{ + return (collected_entropy >= lrng_get_seed_entropy_osr(fully_seeded)); +} + +u32 lrng_entropy_rate_eb(struct entropy_buf *eb) +{ + u32 i, collected_entropy = 0; + + for_each_lrng_es(i) + collected_entropy += eb->e_bits[i]; + + return collected_entropy; +} + +/* Mark one DRNG as not fully seeded */ +void lrng_unset_fully_seeded(struct lrng_drng *drng) +{ + drng->fully_seeded = false; + lrng_pool_all_numa_nodes_seeded(false); + + /* + * The init DRNG instance must always be fully seeded as this instance + * is the fall-back if any of the per-NUMA node DRNG instances is + * insufficiently seeded. Thus, we mark the entire LRNG as + * non-operational if the initial DRNG becomes not fully seeded. + */ + if (drng == lrng_drng_init_instance() && lrng_state_operational()) { + pr_debug("LRNG set to non-operational\n"); + lrng_state.lrng_operational = false; + lrng_state.lrng_fully_seeded = false; + + /* If sufficient entropy is available, reseed now. */ + lrng_es_add_entropy(); + } +} + +/* Policy to enable LRNG operational mode */ +static void lrng_set_operational(void) +{ + /* + * LRNG is operational if the initial DRNG is fully seeded. This state + * can only occur if either the external entropy sources provided + * sufficient entropy, or the SP800-90B startup test completed for + * the internal ES to supply also entropy data. + */ + if (lrng_state.lrng_fully_seeded) { + lrng_state.lrng_operational = true; + lrng_init_wakeup(); + pr_info("LRNG fully operational\n"); + } +} + +static u32 lrng_avail_entropy_thresh(void) +{ + u32 ent_thresh = lrng_security_strength(); + + /* + * Apply oversampling during initialization according to SP800-90C as + * we request a larger buffer from the ES. + */ + if (lrng_sp80090c_compliant() && + !lrng_state.all_online_numa_node_seeded) + ent_thresh += LRNG_SEED_BUFFER_INIT_ADD_BITS; + + return ent_thresh; +} + +/* Available entropy in the entire LRNG considering all entropy sources */ +u32 lrng_avail_entropy(void) +{ + u32 i, ent = 0, ent_thresh = lrng_avail_entropy_thresh(); + + BUILD_BUG_ON(ARRAY_SIZE(lrng_es) != lrng_ext_es_last); + for_each_lrng_es(i) + ent += lrng_es[i]->curr_entropy(ent_thresh); + return ent; +} + +u32 lrng_avail_entropy_aux(void) +{ + u32 ent_thresh = lrng_avail_entropy_thresh(); + + return lrng_es[lrng_ext_es_aux]->curr_entropy(ent_thresh); +} + +/* + * lrng_init_ops() - Set seed stages of LRNG + * + * Set the slow noise source reseed trigger threshold. The initial threshold + * is set to the minimum data size that can be read from the pool: a word. Upon + * reaching this value, the next seed threshold of 128 bits is set followed + * by 256 bits. + * + * @eb: buffer containing the size of entropy currently injected into DRNG - if + * NULL, the function obtains the available entropy from the ES. + */ +void lrng_init_ops(struct entropy_buf *eb) +{ + struct lrng_state *state = &lrng_state; + u32 i, requested_bits, seed_bits = 0; + + if (state->lrng_operational) + return; + + requested_bits = lrng_get_seed_entropy_osr( + state->all_online_numa_node_seeded); + + if (eb) { + for_each_lrng_es(i) + seed_bits += eb->e_bits[i]; + } else { + u32 ent_thresh = lrng_avail_entropy_thresh(); + + for_each_lrng_es(i) + seed_bits += lrng_es[i]->curr_entropy(ent_thresh); + } + + /* DRNG is seeded with full security strength */ + if (state->lrng_fully_seeded) { + lrng_set_operational(); + lrng_set_entropy_thresh(requested_bits); + } else if (lrng_fully_seeded(state->all_online_numa_node_seeded, + seed_bits)) { + if (state->can_invalidate) + invalidate_batched_entropy(); + + state->lrng_fully_seeded = true; + lrng_set_operational(); + state->lrng_min_seeded = true; + pr_info("LRNG fully seeded with %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(requested_bits); + } else if (!state->lrng_min_seeded) { + + /* DRNG is seeded with at least 128 bits of entropy */ + if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) { + if (state->can_invalidate) + invalidate_batched_entropy(); + + state->lrng_min_seeded = true; + pr_info("LRNG minimally seeded with %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(requested_bits); + lrng_init_wakeup(); + + /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */ + } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) { + pr_info("LRNG initial entropy level %u bits of entropy\n", + seed_bits); + lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS); + } + } +} + +int __init lrng_rand_initialize(void) +{ + struct seed { + ktime_t time; + unsigned long data[((LRNG_MAX_DIGESTSIZE + + sizeof(unsigned long) - 1) / + sizeof(unsigned long))]; + struct new_utsname utsname; + } seed __aligned(LRNG_KCAPI_ALIGN); + unsigned int i; + + seed.time = ktime_get_real(); + + for (i = 0; i < ARRAY_SIZE(seed.data); i++) { +#ifdef CONFIG_LRNG_RANDOM_IF + if (!arch_get_random_seed_long_early(&(seed.data[i])) && + !arch_get_random_long_early(&seed.data[i])) +#else + if (!arch_get_random_seed_long(&(seed.data[i])) && + !arch_get_random_long(&seed.data[i])) +#endif + seed.data[i] = random_get_entropy(); + } + memcpy(&seed.utsname, utsname(), sizeof(*(utsname()))); + + lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0); + memzero_explicit(&seed, sizeof(seed)); + + /* Initialize the seed work queue */ + INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work); + lrng_state.perform_seedwork = true; + + invalidate_batched_entropy(); + + lrng_state.can_invalidate = true; + + return 0; +} + +#ifndef CONFIG_LRNG_RANDOM_IF +early_initcall(lrng_rand_initialize); +#endif + +/* Interface requesting a reseed of the DRNG */ +void lrng_es_add_entropy(void) +{ + /* + * Once all DRNGs are fully seeded, the system-triggered arrival of + * entropy will not cause any reseeding any more. + */ + if (likely(lrng_state.all_online_numa_node_seeded)) + return; + + /* Only trigger the DRNG reseed if we have collected entropy. */ + if (lrng_avail_entropy() < + atomic_read_u32(&lrng_state.boot_entropy_thresh)) + return; + + /* Ensure that the seeding only occurs once at any given time. */ + if (lrng_pool_trylock()) + return; + + /* Seed the DRNG with any available noise. */ + if (lrng_state.perform_seedwork) + schedule_work(&lrng_state.lrng_seed_work); + else + lrng_drng_seed_work(NULL); +} + +/* Fill the seed buffer with data from the noise sources */ +void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits) +{ + struct lrng_state *state = &lrng_state; + u32 i, req_ent = lrng_sp80090c_compliant() ? + lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS; + + /* Guarantee that requested bits is a multiple of bytes */ + BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8); + + /* always reseed the DRNG with the current time stamp */ + eb->now = random_get_entropy(); + + /* + * Require at least 128 bits of entropy for any reseed. If the LRNG is + * operated SP800-90C compliant we want to comply with SP800-90A section + * 9.2 mandating that DRNG is reseeded with the security strength. + */ + if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) { + for_each_lrng_es(i) + eb->e_bits[i] = 0; + + goto wakeup; + } + + /* Concatenate the output of the entropy sources. */ + for_each_lrng_es(i) { + lrng_es[i]->get_ent(eb, requested_bits, + state->lrng_fully_seeded); + } + + /* allow external entropy provider to provide seed */ + lrng_state_exseed_allow_all(); + +wakeup: + lrng_writer_wakeup(); +} diff --git a/drivers/char/lrng/lrng_es_mgr.h b/drivers/char/lrng/lrng_es_mgr.h new file mode 100644 index 0000000000000..877698dbadffb --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_MGR_H +#define _LRNG_ES_MGR_H + +#include "lrng_es_mgr_cb.h" + +/*************************** General LRNG parameter ***************************/ + +#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ + +/* Helper to concatenate a macro with an integer type */ +#define LRNG_PASTER(x, y) x ## y +#define LRNG_UINT32_C(x) LRNG_PASTER(x, U) + +/************************* Entropy sources management *************************/ + +extern struct lrng_es_cb *lrng_es[]; + +#define for_each_lrng_es(ctr) \ + for ((ctr) = 0; (ctr) < lrng_ext_es_last; (ctr)++) + +bool lrng_state_min_seeded(void); +void lrng_debug_report_seedlevel(const char *name); +int lrng_rand_initialize(void); +bool lrng_state_operational(void); + +extern u32 lrng_write_wakeup_bits; +void lrng_set_entropy_thresh(u32 new); +u32 lrng_avail_entropy(void); +u32 lrng_avail_entropy_aux(void); +void lrng_reset_state(void); + +bool lrng_state_fully_seeded(void); + +int lrng_pool_trylock(void); +void lrng_pool_unlock(void); +void lrng_pool_all_numa_nodes_seeded(bool set); + +bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy); +u32 lrng_entropy_rate_eb(struct entropy_buf *eb); +void lrng_unset_fully_seeded(struct lrng_drng *drng); +void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits); +void lrng_init_ops(struct entropy_buf *eb); + +#endif /* _LRNG_ES_MGR_H */ diff --git a/drivers/char/lrng/lrng_es_mgr_cb.h b/drivers/char/lrng/lrng_es_mgr_cb.h new file mode 100644 index 0000000000000..08b24e1b7766e --- /dev/null +++ b/drivers/char/lrng/lrng_es_mgr_cb.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + * + * Definition of an entropy source. + */ + +#ifndef _LRNG_ES_MGR_CB_H +#define _LRNG_ES_MGR_CB_H + +#include + +#include "lrng_definitions.h" +#include "lrng_drng_mgr.h" + +enum lrng_internal_es { +#ifdef CONFIG_LRNG_IRQ + lrng_int_es_irq, /* IRQ-based entropy source */ +#endif +#ifdef CONFIG_LRNG_SCHED + lrng_int_es_sched, /* Scheduler entropy source */ +#endif + lrng_int_es_last, /* MUST be the last entry */ +}; + +enum lrng_external_es { + lrng_ext_link = lrng_int_es_last - 1, /* Link entry */ +#ifdef CONFIG_LRNG_JENT + lrng_ext_es_jitter, /* Jitter RNG */ +#endif +#ifdef CONFIG_LRNG_CPU + lrng_ext_es_cpu, /* CPU-based, e.g. RDSEED */ +#endif +#ifdef CONFIG_LRNG_KERNEL_RNG + lrng_ext_es_krng, /* random.c */ +#endif + lrng_ext_es_aux, /* MUST BE LAST ES! */ + lrng_ext_es_last /* MUST be the last entry */ +}; + +struct entropy_buf { + u8 e[lrng_ext_es_last][LRNG_DRNG_INIT_SEED_SIZE_BYTES]; + u32 now, e_bits[lrng_ext_es_last]; +}; + +/* + * struct lrng_es_cb - callback defining an entropy source + * @name: Name of the entropy source. + * @get_ent: Fetch entropy into the entropy_buf. The ES shall only deliver + * data if its internal initialization is complete, including any + * SP800-90B startup testing or similar. + * @curr_entropy: Return amount of currently available entropy. + * @max_entropy: Maximum amount of entropy the entropy source is able to + * maintain. + * @state: Buffer with human-readable ES state. + * @reset: Reset entropy source (drop all entropy and reinitialize). + * This callback may be NULL. + * @switch_hash: callback to switch from an old hash callback definition to + * a new one. This callback may be NULL. + */ +struct lrng_es_cb { + const char *name; + void (*get_ent)(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded); + u32 (*curr_entropy)(u32 requested_bits); + u32 (*max_entropy)(void); + void (*state)(unsigned char *buf, size_t buflen); + void (*reset)(void); + int (*switch_hash)(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb); +}; + +/* Allow entropy sources to tell the ES manager that new entropy is there */ +void lrng_es_add_entropy(void); + +/* Cap to maximum entropy that can ever be generated with given hash */ +#define lrng_cap_requested(__digestsize_bits, __requested_bits) \ + do { \ + if (__digestsize_bits < __requested_bits) { \ + pr_debug("Cannot satisfy requested entropy %u due to insufficient hash size %u\n",\ + __requested_bits, __digestsize_bits); \ + __requested_bits = __digestsize_bits; \ + } \ + } while (0) + +#endif /* _LRNG_ES_MGR_CB_H */ diff --git a/drivers/char/lrng/lrng_es_sched.c b/drivers/char/lrng/lrng_es_sched.c new file mode 100644 index 0000000000000..c3abaa7ff3108 --- /dev/null +++ b/drivers/char/lrng/lrng_es_sched.c @@ -0,0 +1,562 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Scheduler-based data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_sched.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" +#include "lrng_numa.h" +#include "lrng_testing.h" + +/* + * Number of scheduler-based context switches to be recorded to assume that + * DRNG security strength bits of entropy are received. + * Note: a value below the DRNG security strength should not be defined as this + * may imply the DRNG can never be fully seeded in case other noise + * sources are unavailable. + */ +#define LRNG_SCHED_ENTROPY_BITS \ + LRNG_UINT32_C(CONFIG_LRNG_SCHED_ENTROPY_RATE) + +/* Number of events required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ +static u32 lrng_sched_entropy_bits = LRNG_SCHED_ENTROPY_BITS; + +static u32 sched_entropy __read_mostly = LRNG_SCHED_ENTROPY_BITS; +#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG +module_param(sched_entropy, uint, 0444); +MODULE_PARM_DESC(sched_entropy, + "How many scheduler-based context switches must be collected for obtaining 256 bits of entropy\n"); +#endif + +/* Per-CPU array holding concatenated entropy events */ +static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_sched_array) + __aligned(LRNG_KCAPI_ALIGN); +static DEFINE_PER_CPU(u32, lrng_sched_array_ptr) = 0; +static DEFINE_PER_CPU(atomic_t, lrng_sched_array_events) = ATOMIC_INIT(0); + +/* + * Per-CPU entropy pool with compressed entropy event + * + * The per-CPU entropy pool is defined as the hash state. New data is simply + * inserted into the entropy pool by performing a hash update operation. + * To read the entropy pool, a hash final must be invoked. However, before + * the entropy pool is released again after a hash final, the hash init must + * be performed. + */ +static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_sched_pool) + __aligned(LRNG_KCAPI_ALIGN); +/* + * Lock to allow other CPUs to read the pool - as this is only done during + * reseed which is infrequent, this lock is hardly contended. + */ +static DEFINE_PER_CPU(spinlock_t, lrng_sched_lock); +static DEFINE_PER_CPU(bool, lrng_sched_lock_init) = false; + +static bool lrng_sched_pool_online(int cpu) +{ + return per_cpu(lrng_sched_lock_init, cpu); +} + +static void __init lrng_sched_check_compression_state(void) +{ + /* One pool should hold sufficient entropy for disabled compression */ + u32 max_ent = min_t(u32, lrng_get_digestsize(), + lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, + lrng_sched_entropy_bits)); + if (max_ent < lrng_security_strength()) { + pr_devel("Scheduler entropy source will never provide %u bits of entropy required for fully seeding the DRNG all by itself\n", + lrng_security_strength()); + } +} + +void __init lrng_sched_es_init(bool highres_timer) +{ + /* Set a minimum number of scheduler events that must be collected */ + sched_entropy = max_t(u32, LRNG_SCHED_ENTROPY_BITS, sched_entropy); + + if (highres_timer) { + lrng_sched_entropy_bits = sched_entropy; + } else { + u32 new_entropy = sched_entropy * LRNG_ES_OVERSAMPLING_FACTOR; + + lrng_sched_entropy_bits = (sched_entropy < new_entropy) ? + new_entropy : sched_entropy; + pr_warn("operating without high-resolution timer and applying oversampling factor %u\n", + LRNG_ES_OVERSAMPLING_FACTOR); + } + + lrng_sched_check_compression_state(); +} + +static u32 lrng_sched_avail_pool_size(void) +{ + u32 max_pool = lrng_get_digestsize(), + max_size = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); + int cpu; + + for_each_online_cpu(cpu) + max_size += max_pool; + + return max_size; +} + +/* Return entropy of unused scheduler events present in all per-CPU pools. */ +static u32 lrng_sched_avail_entropy(u32 __unused) +{ + u32 digestsize_events, events = 0; + int cpu; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) + return 0; + + /* Obtain the cap of maximum numbers of scheduler events we count */ + digestsize_events = lrng_entropy_to_data(lrng_get_digestsize(), + lrng_sched_entropy_bits); + /* Cap to max. number of scheduler events the array can hold */ + digestsize_events = min_t(u32, digestsize_events, LRNG_DATA_NUM_VALUES); + + for_each_online_cpu(cpu) { + events += min_t(u32, digestsize_events, + atomic_read_u32(per_cpu_ptr(&lrng_sched_array_events, + cpu))); + } + + /* Consider oversampling rate */ + return lrng_reduce_by_osr( + lrng_data_to_entropy(events, lrng_sched_entropy_bits)); +} + +/* + * Reset all per-CPU pools - reset entropy estimator but leave the pool data + * that may or may not have entropy unchanged. + */ +static void lrng_sched_reset(void) +{ + int cpu; + + /* Trigger GCD calculation anew. */ + lrng_gcd_set(0); + + for_each_online_cpu(cpu) + atomic_set(per_cpu_ptr(&lrng_sched_array_events, cpu), 0); +} + +/* + * Trigger a switch of the hash implementation for the per-CPU pool. + * + * For each per-CPU pool, obtain the message digest with the old hash + * implementation, initialize the per-CPU pool again with the new hash + * implementation and inject the message digest into the new state. + * + * Assumption: the caller must guarantee that the new_cb is available during the + * entire operation (e.g. it must hold the lock against pointer updating). + */ +static int +lrng_sched_switch_hash(struct lrng_drng *drng, int node, + const struct lrng_hash_cb *new_cb, void *new_hash, + const struct lrng_hash_cb *old_cb) +{ + u8 digest[LRNG_MAX_DIGESTSIZE]; + u32 digestsize_events, found_events; + int ret = 0, cpu; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) + return -EOPNOTSUPP; + + for_each_online_cpu(cpu) { + struct shash_desc *pcpu_shash; + + /* + * Only switch the per-CPU pools for the current node because + * the hash_cb only applies NUMA-node-wide. + */ + if (cpu_to_node(cpu) != node || !lrng_sched_pool_online(cpu)) + continue; + + pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, + cpu); + + digestsize_events = old_cb->hash_digestsize(pcpu_shash); + digestsize_events = lrng_entropy_to_data(digestsize_events << 3, + lrng_sched_entropy_bits); + + if (pcpu_shash->tfm == new_hash) + continue; + + /* Get the per-CPU pool hash with old digest ... */ + ret = old_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash with the new digest ... */ + new_cb->hash_init(pcpu_shash, new_hash) ?: + /* + * ... feed the old hash into the new state. We may feed + * uninitialized memory into the new state, but this is + * considered no issue and even good as we have some more + * uncertainty here. + */ + new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); + if (ret) + goto out; + + /* + * In case the new digest is larger than the old one, cap + * the available entropy to the old message digest used to + * process the existing data. + */ + found_events = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_sched_array_events, cpu), 0); + found_events = min_t(u32, found_events, digestsize_events); + atomic_add_return_relaxed(found_events, + per_cpu_ptr(&lrng_sched_array_events, cpu)); + + pr_debug("Re-initialize per-CPU scheduler entropy pool for CPU %d on NUMA node %d with hash %s\n", + cpu, node, new_cb->hash_name()); + } + +out: + memzero_explicit(digest, sizeof(digest)); + return ret; +} + +static u32 +lrng_sched_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, + void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) +{ + struct shash_desc *pcpu_shash = + (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, cpu); + spinlock_t *lock = per_cpu_ptr(&lrng_sched_lock, cpu); + unsigned long flags; + u32 digestsize_events, found_events; + + if (unlikely(!per_cpu(lrng_sched_lock_init, cpu))) { + if (pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash)) { + pr_warn("Initialization of hash failed\n"); + return 0; + } + spin_lock_init(lock); + per_cpu(lrng_sched_lock_init, cpu) = true; + pr_debug("Initializing per-CPU scheduler entropy pool for CPU %d with hash %s\n", + raw_smp_processor_id(), pcpu_hash_cb->hash_name()); + } + + /* Lock guarding against reading / writing to per-CPU pool */ + spin_lock_irqsave(lock, flags); + + *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); + digestsize_events = lrng_entropy_to_data(*digestsize << 3, + lrng_sched_entropy_bits); + + /* Obtain entropy statement like for the entropy pool */ + found_events = atomic_xchg_relaxed( + per_cpu_ptr(&lrng_sched_array_events, cpu), 0); + /* Cap to maximum amount of data we can hold in hash */ + found_events = min_t(u32, found_events, digestsize_events); + + /* Cap to maximum amount of data we can hold in array */ + found_events = min_t(u32, found_events, LRNG_DATA_NUM_VALUES); + + /* Store all not-yet compressed data in data array into hash, ... */ + if (pcpu_hash_cb->hash_update(pcpu_shash, + (u8 *)per_cpu_ptr(lrng_sched_array, cpu), + LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: + /* ... get the per-CPU pool digest, ... */ + pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: + /* ... re-initialize the hash, ... */ + pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: + /* ... feed the old hash into the new state. */ + pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) + found_events = 0; + + spin_unlock_irqrestore(lock, flags); + return found_events; +} + +/* + * Hash all per-CPU arrays and return the digest to be used as seed data for + * seeding a DRNG. The caller must guarantee backtracking resistance. + * The function will only copy as much data as entropy is available into the + * caller-provided output buffer. + * + * This function handles the translation from the number of received scheduler + * events into an entropy statement. The conversion depends on + * LRNG_SCHED_ENTROPY_BITS which defines how many scheduler events must be + * received to obtain 256 bits of entropy. With this value, the function + * lrng_data_to_entropy converts a given data size (received scheduler events, + * requested amount of data, etc.) into an entropy statement. + * lrng_entropy_to_data does the reverse. + * + * @eb: entropy buffer to store entropy + * @requested_bits: Requested amount of entropy + * @fully_seeded: indicator whether LRNG is fully seeded + */ +static void lrng_sched_pool_hash(struct entropy_buf *eb, u32 requested_bits, + bool fully_seeded) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb; + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *drng = lrng_drng_init_instance(); + u8 digest[LRNG_MAX_DIGESTSIZE]; + unsigned long flags, flags2; + u32 found_events, collected_events = 0, collected_ent_bits, + requested_events, returned_ent_bits; + int ret, cpu; + void *hash; + + /* Only deliver entropy when SP800-90B self test is completed */ + if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) { + eb->e_bits[lrng_int_es_sched] = 0; + return; + } + + /* Lock guarding replacement of per-NUMA hash */ + read_lock_irqsave(&drng->hash_lock, flags); + + hash_cb = drng->hash_cb; + hash = drng->hash; + + /* The hash state of filled with all per-CPU pool hashes. */ + ret = hash_cb->hash_init(shash, hash); + if (ret) + goto err; + + /* Cap to maximum entropy that can ever be generated with given hash */ + lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); + requested_events = lrng_entropy_to_data(requested_bits + + lrng_compress_osr(), + lrng_sched_entropy_bits); + + /* + * Harvest entropy from each per-CPU hash state - even though we may + * have collected sufficient entropy, we will hash all per-CPU pools. + */ + for_each_online_cpu(cpu) { + struct lrng_drng *pcpu_drng = drng; + u32 digestsize, unused_events = 0; + int node = cpu_to_node(cpu); + + if (lrng_drng && lrng_drng[node]) + pcpu_drng = lrng_drng[node]; + + if (pcpu_drng == drng) { + found_events = lrng_sched_pool_hash_one(hash_cb, hash, + cpu, digest, + &digestsize); + } else { + read_lock_irqsave(&pcpu_drng->hash_lock, flags2); + found_events = + lrng_sched_pool_hash_one(pcpu_drng->hash_cb, + pcpu_drng->hash, cpu, + digest, &digestsize); + read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); + } + + /* Store all not-yet compressed data in data array into hash */ + ret = hash_cb->hash_update(shash, digest, digestsize); + if (ret) + goto err; + + collected_events += found_events; + if (collected_events > requested_events) { + unused_events = collected_events - requested_events; + atomic_add_return_relaxed(unused_events, + per_cpu_ptr(&lrng_sched_array_events, cpu)); + collected_events = requested_events; + } + pr_debug("%u scheduler-based events used from entropy array of CPU %d, %u scheduler-based events remain unused\n", + found_events - unused_events, cpu, unused_events); + } + + ret = hash_cb->hash_final(shash, digest); + if (ret) + goto err; + + collected_ent_bits = lrng_data_to_entropy(collected_events, + lrng_sched_entropy_bits); + /* Apply oversampling: discount requested oversampling rate */ + returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); + + pr_debug("obtained %u bits by collecting %u bits of entropy from scheduler-based noise source\n", + returned_ent_bits, collected_ent_bits); + + /* + * Truncate to available entropy as implicitly allowed by SP800-90B + * section 3.1.5.1.1 table 1 which awards truncated hashes full + * entropy. + * + * During boot time, we read requested_bits data with + * returned_ent_bits entropy. In case our conservative entropy + * estimate underestimates the available entropy we can transport as + * much available entropy as possible. + */ + memcpy(eb->e[lrng_int_es_sched], digest, + fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); + eb->e_bits[lrng_int_es_sched] = returned_ent_bits; + +out: + hash_cb->hash_desc_zero(shash); + read_unlock_irqrestore(&drng->hash_lock, flags); + memzero_explicit(digest, sizeof(digest)); + return; + +err: + eb->e_bits[lrng_int_es_sched] = 0; + goto out; +} + +/* + * Concatenate full 32 bit word at the end of time array even when current + * ptr is not aligned to sizeof(data). + */ +static void lrng_sched_array_add_u32(u32 data) +{ + /* Increment pointer by number of slots taken for input value */ + u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_sched_array_ptr, + LRNG_DATA_SLOTS_PER_UINT); + unsigned int pre_array; + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_sched_array[pre_array], ~(0xffffffff & ~mask)); + this_cpu_or(lrng_sched_array[pre_array], data & ~mask); + + /* + * Continuous compression is not allowed for scheduler noise source, + * so do not call lrng_sched_array_to_hash here. + */ + + /* LSB of data go into current unit */ + this_cpu_write(lrng_sched_array[lrng_data_idx2array(ptr)], + data & mask); +} + +/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ +static void lrng_sched_array_add_slot(u32 data) +{ + /* Get slot */ + u32 ptr = this_cpu_inc_return(lrng_sched_array_ptr) & + LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + /* zeroization of slot to ensure the following OR adds the data */ + this_cpu_and(lrng_sched_array[array], + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot))); + /* Store data into slot */ + this_cpu_or(lrng_sched_array[array], lrng_data_slot_val(data, slot)); + + /* + * Continuous compression is not allowed for scheduler noise source, + * so do not call lrng_sched_array_to_hash here. + */ +} + +static void +lrng_time_process_common(u32 time, void(*add_time)(u32 data)) +{ + enum lrng_health_res health_test; + + if (lrng_raw_sched_hires_entropy_store(time)) + return; + + health_test = lrng_health_test(time, lrng_int_es_sched); + if (health_test > lrng_health_fail_use) + return; + + if (health_test == lrng_health_pass) + atomic_inc_return(this_cpu_ptr(&lrng_sched_array_events)); + + add_time(time); + + /* + * We cannot call lrng_es_add_entropy() as this would call a schedule + * operation that is not permissible in scheduler context. + * As the scheduler ES provides a high bandwidth of entropy, we assume + * that other reseed triggers happen to pick up the scheduler ES + * entropy in due time. + */ +} + +/* Batching up of entropy in per-CPU array */ +static void lrng_sched_time_process(void) +{ + u32 now_time = random_get_entropy(); + + if (unlikely(!lrng_gcd_tested())) { + /* When GCD is unknown, we process the full time stamp */ + lrng_time_process_common(now_time, lrng_sched_array_add_u32); + lrng_gcd_add_value(now_time); + } else { + /* GCD is known and applied */ + lrng_time_process_common((now_time / lrng_gcd_get()) & + LRNG_DATA_SLOTSIZE_MASK, + lrng_sched_array_add_slot); + } + + lrng_sched_perf_time(now_time); +} + +void add_sched_randomness(const struct task_struct *p, int cpu) +{ + if (lrng_highres_timer()) { + lrng_sched_time_process(); + } else { + u32 tmp = cpu; + + tmp ^= lrng_raw_sched_pid_entropy_store(p->pid) ? + 0 : (u32)p->pid; + tmp ^= lrng_raw_sched_starttime_entropy_store(p->start_time) ? + 0 : (u32)p->start_time; + tmp ^= lrng_raw_sched_nvcsw_entropy_store(p->nvcsw) ? + 0 : (u32)p->nvcsw; + + lrng_sched_time_process(); + lrng_sched_array_add_u32(tmp); + } +} +EXPORT_SYMBOL(add_sched_randomness); + +static void lrng_sched_es_state(unsigned char *buf, size_t buflen) +{ + const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + + /* Assume the lrng_drng_init lock is taken by caller */ + snprintf(buf, buflen, + " Hash for operating entropy pool: %s\n" + " Available entropy: %u\n" + " per-CPU scheduler event collection size: %u\n" + " Standards compliance: %s\n" + " High-resolution timer: %s\n", + lrng_drng_init->hash_cb->hash_name(), + lrng_sched_avail_entropy(0), + LRNG_DATA_NUM_VALUES, + lrng_sp80090b_compliant(lrng_int_es_sched) ? "SP800-90B " : "", + lrng_highres_timer() ? "true" : "false"); +} + +struct lrng_es_cb lrng_es_sched = { + .name = "Scheduler", + .get_ent = lrng_sched_pool_hash, + .curr_entropy = lrng_sched_avail_entropy, + .max_entropy = lrng_sched_avail_pool_size, + .state = lrng_sched_es_state, + .reset = lrng_sched_reset, + .switch_hash = lrng_sched_switch_hash, +}; diff --git a/drivers/char/lrng/lrng_es_sched.h b/drivers/char/lrng/lrng_es_sched.h new file mode 100644 index 0000000000000..f1e596dd89d9f --- /dev/null +++ b/drivers/char/lrng/lrng_es_sched.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_SCHED_H +#define _LRNG_ES_SCHED_H + +#include "lrng_es_mgr_cb.h" + +#ifdef CONFIG_LRNG_SCHED +void lrng_sched_es_init(bool highres_timer); + +extern struct lrng_es_cb lrng_es_sched; + +#else /* CONFIG_LRNG_SCHED */ +static inline void lrng_sched_es_init(bool highres_timer) { } +#endif /* CONFIG_LRNG_SCHED */ + +#endif /* _LRNG_ES_SCHED_H */ diff --git a/drivers/char/lrng/lrng_es_timer_common.c b/drivers/char/lrng/lrng_es_timer_common.c new file mode 100644 index 0000000000000..70f3ff074fe6e --- /dev/null +++ b/drivers/char/lrng/lrng_es_timer_common.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Slow Entropy Source: Interrupt data collection + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_irq.h" +#include "lrng_es_sched.h" +#include "lrng_es_timer_common.h" +#include "lrng_health.h" + +/* Is high-resolution timer present? */ +static bool lrng_highres_timer_val = false; + +/* Number of time stamps analyzed to calculate a GCD */ +#define LRNG_GCD_WINDOW_SIZE 100 +static u32 lrng_gcd_history[LRNG_GCD_WINDOW_SIZE]; +static atomic_t lrng_gcd_history_ptr = ATOMIC_INIT(-1); + +/* The common divisor for all timestamps */ +static u32 lrng_gcd_timer = 0; + +bool lrng_gcd_tested(void) +{ + return (lrng_gcd_timer != 0); +} + +u32 lrng_gcd_get(void) +{ + return lrng_gcd_timer; +} + +/* Set the GCD for use in IRQ ES - if 0, the GCD calculation is restarted. */ +void lrng_gcd_set(u32 running_gcd) +{ + lrng_gcd_timer = running_gcd; + /* Ensure that update to global variable lrng_gcd_timer is visible */ + mb(); +} + +static void lrng_gcd_set_check(u32 running_gcd) +{ + if (!lrng_gcd_tested()) { + lrng_gcd_set(running_gcd); + pr_debug("Setting GCD to %u\n", running_gcd); + } +} + +u32 lrng_gcd_analyze(u32 *history, size_t nelem) +{ + u32 running_gcd = 0; + size_t i; + + /* Now perform the analysis on the accumulated time data. */ + for (i = 0; i < nelem; i++) { + /* + * NOTE: this would be the place to add more analysis on the + * appropriateness of the timer like checking the presence + * of sufficient variations in the timer. + */ + + /* + * This calculates the gcd of all the time values. that is + * gcd(time_1, time_2, ..., time_nelem) + * + * Some timers increment by a fixed (non-1) amount each step. + * This code checks for such increments, and allows the library + * to output the number of such changes have occurred. + */ + running_gcd = (u32)gcd(history[i], running_gcd); + + /* Zeroize data */ + history[i] = 0; + } + + return running_gcd; +} + +void lrng_gcd_add_value(u32 time) +{ + u32 ptr = (u32)atomic_inc_return_relaxed(&lrng_gcd_history_ptr); + + if (ptr < LRNG_GCD_WINDOW_SIZE) { + lrng_gcd_history[ptr] = time; + } else if (ptr == LRNG_GCD_WINDOW_SIZE) { + u32 gcd = lrng_gcd_analyze(lrng_gcd_history, + LRNG_GCD_WINDOW_SIZE); + + if (!gcd) + gcd = 1; + + /* + * Ensure that we have variations in the time stamp below the + * given value. This is just a safety measure to prevent the GCD + * becoming too large. + */ + if (gcd >= 1000) { + pr_warn("calculated GCD is larger than expected: %u\n", + gcd); + gcd = 1000; + } + + /* Adjust all deltas by the observed (small) common factor. */ + lrng_gcd_set_check(gcd); + atomic_set(&lrng_gcd_history_ptr, 0); + } +} + +/* Return boolean whether LRNG identified presence of high-resolution timer */ +bool lrng_highres_timer(void) +{ + return lrng_highres_timer_val; +} + +static int __init lrng_init_time_source(void) +{ + if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) || + (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) { + /* + * As the highres timer is identified here, previous interrupts + * obtained during boot time are treated like a lowres-timer + * would have been present. + */ + lrng_highres_timer_val = true; + } else { + lrng_health_disable(); + lrng_highres_timer_val = false; + } + + lrng_irq_es_init(lrng_highres_timer_val); + lrng_sched_es_init(lrng_highres_timer_val); + + /* Ensure that changes to global variables are visible */ + mb(); + + return 0; +} +core_initcall(lrng_init_time_source); diff --git a/drivers/char/lrng/lrng_es_timer_common.h b/drivers/char/lrng/lrng_es_timer_common.h new file mode 100644 index 0000000000000..b45b9f683dea1 --- /dev/null +++ b/drivers/char/lrng/lrng_es_timer_common.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG Slow Noise Source: Time stamp array handling + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_ES_TIMER_COMMON_H +#define _LRNG_ES_TIMER_COMMON_H + +bool lrng_gcd_tested(void); +void lrng_gcd_set(u32 running_gcd); +u32 lrng_gcd_get(void); +u32 lrng_gcd_analyze(u32 *history, size_t nelem); +void lrng_gcd_add_value(u32 time); +bool lrng_highres_timer(void); + +/* + * To limit the impact on the interrupt handling, the LRNG concatenates + * entropic LSB parts of the time stamps in a per-CPU array and only + * injects them into the entropy pool when the array is full. + */ + +/* Store multiple integers in one u32 */ +#define LRNG_DATA_SLOTSIZE_BITS (8) +#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1) +#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */ +#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \ + LRNG_DATA_SLOTSIZE_BITS) + +/* + * Number of time values to store in the array - in small environments + * only one atomic_t variable per CPU is used. + */ +#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE) +/* Mask of LSB of time stamp to store */ +#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1) + +#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1) +#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \ + LRNG_DATA_SLOTS_PER_UINT) + +/* Starting bit index of slot */ +static inline unsigned int lrng_data_slot2bitindex(unsigned int slot) +{ + return (LRNG_DATA_SLOTSIZE_BITS * slot); +} + +/* Convert index into the array index */ +static inline unsigned int lrng_data_idx2array(unsigned int idx) +{ + return idx / LRNG_DATA_SLOTS_PER_UINT; +} + +/* Convert index into the slot of a given array index */ +static inline unsigned int lrng_data_idx2slot(unsigned int idx) +{ + return idx & LRNG_DATA_SLOTS_MASK; +} + +/* Convert value into slot value */ +static inline unsigned int lrng_data_slot_val(unsigned int val, + unsigned int slot) +{ + return val << lrng_data_slot2bitindex(slot); +} + +/* + * Return the pointers for the previous and current units to inject a u32 into. + * Also return the mask which the u32 word is to be processed. + */ +static inline void lrng_data_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask) +{ + /* ptr to previous unit */ + *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK; + *ptr &= LRNG_DATA_WORD_MASK; + + /* mask to split data into the two parts for the two units */ + *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) * + LRNG_DATA_SLOTSIZE_BITS)) - 1; +} + +#endif /* _LRNG_ES_TIMER_COMMON_H */ \ No newline at end of file diff --git a/drivers/char/lrng/lrng_hash_kcapi.c b/drivers/char/lrng/lrng_hash_kcapi.c new file mode 100644 index 0000000000000..13e62db9b6c8e --- /dev/null +++ b/drivers/char/lrng/lrng_hash_kcapi.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for providing the hash primitive using the kernel crypto API. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + + +static char *lrng_hash_name = "sha512"; + +/* The parameter must be r/o in sysfs as otherwise races appear. */ +module_param(lrng_hash_name, charp, 0444); +MODULE_PARM_DESC(lrng_hash_name, "Kernel crypto API hash name"); + +struct lrng_hash_info { + struct crypto_shash *tfm; +}; + +static const char *lrng_kcapi_hash_name(void) +{ + return lrng_hash_name; +} + +static void _lrng_kcapi_hash_free(struct lrng_hash_info *lrng_hash) +{ + struct crypto_shash *tfm = lrng_hash->tfm; + + crypto_free_shash(tfm); + kfree(lrng_hash); +} + +static void *lrng_kcapi_hash_alloc(const char *name) +{ + struct lrng_hash_info *lrng_hash; + struct crypto_shash *tfm; + int ret; + + if (!name) { + pr_err("Hash name missing\n"); + return ERR_PTR(-EINVAL); + } + + tfm = crypto_alloc_shash(name, 0, 0); + if (IS_ERR(tfm)) { + pr_err("could not allocate hash %s\n", name); + return ERR_CAST(tfm); + } + + ret = sizeof(struct lrng_hash_info); + lrng_hash = kmalloc(ret, GFP_KERNEL); + if (!lrng_hash) { + crypto_free_shash(tfm); + return ERR_PTR(-ENOMEM); + } + + lrng_hash->tfm = tfm; + + pr_info("Hash %s allocated\n", name); + + return lrng_hash; +} + +static void *lrng_kcapi_hash_name_alloc(void) +{ + return lrng_kcapi_hash_alloc(lrng_kcapi_hash_name()); +} + +static u32 lrng_kcapi_hash_digestsize(void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + struct crypto_shash *tfm = lrng_hash->tfm; + + return crypto_shash_digestsize(tfm); +} + +static void lrng_kcapi_hash_dealloc(void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + + _lrng_kcapi_hash_free(lrng_hash); + pr_info("Hash deallocated\n"); +} + +static int lrng_kcapi_hash_init(struct shash_desc *shash, void *hash) +{ + struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; + struct crypto_shash *tfm = lrng_hash->tfm; + + shash->tfm = tfm; + return crypto_shash_init(shash); +} + +static int lrng_kcapi_hash_update(struct shash_desc *shash, const u8 *inbuf, + u32 inbuflen) +{ + return crypto_shash_update(shash, inbuf, inbuflen); +} + +static int lrng_kcapi_hash_final(struct shash_desc *shash, u8 *digest) +{ + return crypto_shash_final(shash, digest); +} + +static void lrng_kcapi_hash_zero(struct shash_desc *shash) +{ + shash_desc_zero(shash); +} + +static const struct lrng_hash_cb lrng_kcapi_hash_cb = { + .hash_name = lrng_kcapi_hash_name, + .hash_alloc = lrng_kcapi_hash_name_alloc, + .hash_dealloc = lrng_kcapi_hash_dealloc, + .hash_digestsize = lrng_kcapi_hash_digestsize, + .hash_init = lrng_kcapi_hash_init, + .hash_update = lrng_kcapi_hash_update, + .hash_final = lrng_kcapi_hash_final, + .hash_desc_zero = lrng_kcapi_hash_zero, +}; + +static int __init lrng_kcapi_init(void) +{ + return lrng_set_hash_cb(&lrng_kcapi_hash_cb); +} + +static void __exit lrng_kcapi_exit(void) +{ + lrng_set_hash_cb(NULL); +} + +late_initcall(lrng_kcapi_init); +module_exit(lrng_kcapi_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager - Kernel crypto API hash backend"); diff --git a/drivers/char/lrng/lrng_health.c b/drivers/char/lrng/lrng_health.c new file mode 100644 index 0000000000000..a60c8378eea5d --- /dev/null +++ b/drivers/char/lrng/lrng_health.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Entropy Source and DRNG Manager (LRNG) Health Testing + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_health.h" + +/* Stuck Test */ +struct lrng_stuck_test { + u32 last_time; /* Stuck test: time of previous IRQ */ + u32 last_delta; /* Stuck test: delta of previous IRQ */ + u32 last_delta2; /* Stuck test: 2. time derivation of prev IRQ */ +}; + +/* Repetition Count Test */ +struct lrng_rct { + atomic_t rct_count; /* Number of stuck values */ +}; + +/* Adaptive Proportion Test */ +struct lrng_apt { + /* Data window size */ +#define LRNG_APT_WINDOW_SIZE 512 + /* LSB of time stamp to process */ +#define LRNG_APT_LSB 16 +#define LRNG_APT_WORD_MASK (LRNG_APT_LSB - 1) + atomic_t apt_count; /* APT counter */ + atomic_t apt_base; /* APT base reference */ + + atomic_t apt_trigger; + bool apt_base_set; /* Is APT base set? */ +}; + +/* Health data collected for one entropy source */ +struct lrng_health_es_state { + struct lrng_rct rct; + struct lrng_apt apt; + + /* SP800-90B startup health tests */ +#define LRNG_SP80090B_STARTUP_SAMPLES 1024 +#define LRNG_SP80090B_STARTUP_BLOCKS ((LRNG_SP80090B_STARTUP_SAMPLES + \ + LRNG_APT_WINDOW_SIZE - 1) / \ + LRNG_APT_WINDOW_SIZE) + bool sp80090b_startup_done; + atomic_t sp80090b_startup_blocks; +}; + +#define LRNG_HEALTH_ES_INIT(x) \ + x.rct.rct_count = ATOMIC_INIT(0), \ + x.apt.apt_count = ATOMIC_INIT(0), \ + x.apt.apt_base = ATOMIC_INIT(-1), \ + x.apt.apt_trigger = ATOMIC_INIT(LRNG_APT_WINDOW_SIZE), \ + x.apt.apt_base_set = false, \ + x.sp80090b_startup_blocks = ATOMIC_INIT(LRNG_SP80090B_STARTUP_BLOCKS), \ + x.sp80090b_startup_done = false, + +/* The health test code must operate lock-less */ +struct lrng_health { + bool health_test_enabled; + struct lrng_health_es_state es_state[lrng_int_es_last]; +}; + +static struct lrng_health lrng_health = { + .health_test_enabled = true, + +#ifdef CONFIG_LRNG_IRQ + LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_irq]) +#endif +#ifdef CONFIG_LRNG_SCHED + LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_sched]) +#endif +}; + +static DEFINE_PER_CPU(struct lrng_stuck_test[lrng_int_es_last], + lrng_stuck_test_array); + +static bool lrng_sp80090b_health_requested(void) +{ + /* Health tests are only requested in FIPS mode */ + return fips_enabled; +} + +static bool lrng_sp80090b_health_enabled(void) +{ + struct lrng_health *health = &lrng_health; + + return lrng_sp80090b_health_requested() && health->health_test_enabled; +} + +/*************************************************************************** + * SP800-90B Compliance + * + * If the Linux-RNG is booted into FIPS mode, the following interfaces + * provide an SP800-90B compliant noise source: + * + * * /dev/random + * * getrandom(2) + * * get_random_bytes_full + * + * All other interfaces, including /dev/urandom or get_random_bytes without + * the add_random_ready_callback cannot claim to use an SP800-90B compliant + * noise source. + ***************************************************************************/ + +/* + * Perform SP800-90B startup testing + */ +static void lrng_sp80090b_startup(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (!es_state->sp80090b_startup_done && + atomic_dec_and_test(&es_state->sp80090b_startup_blocks)) { + es_state->sp80090b_startup_done = true; + pr_info("SP800-90B startup health tests for internal entropy source %u completed\n", + es); + lrng_drng_force_reseed(); + + /* + * We cannot call lrng_es_add_entropy() as this may cause a + * schedule operation while in scheduler context for the + * scheduler ES. + */ + } +} + +/* + * Handle failure of SP800-90B startup testing + */ +static void lrng_sp80090b_startup_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + /* Reset of LRNG and its entropy - NOTE: we are in atomic context */ + lrng_reset(); + + /* + * Reset the SP800-90B startup test. + * + * NOTE SP800-90B section 4.3 bullet 4 does not specify what + * exactly is to be done in case of failure! Thus, we do what + * makes sense, i.e. restarting the health test and thus gating + * the output function of /dev/random and getrandom(2). + */ + atomic_set(&es_state->sp80090b_startup_blocks, + LRNG_SP80090B_STARTUP_BLOCKS); +} + +/* + * Handle failure of SP800-90B runtime testing + */ +static void lrng_sp80090b_runtime_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + lrng_sp80090b_startup_failure(health, es); + es_state->sp80090b_startup_done = false; +} + +static void lrng_sp80090b_failure(struct lrng_health *health, + enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (es_state->sp80090b_startup_done) { + pr_err("SP800-90B runtime health test failure for internal entropy source %u - invalidating all existing entropy and initiate SP800-90B startup\n", es); + lrng_sp80090b_runtime_failure(health, es); + } else { + pr_err("SP800-90B startup test failure for internal entropy source %u - resetting\n", es); + lrng_sp80090b_startup_failure(health, es); + } +} + +bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + struct lrng_health_es_state *es_state = &health->es_state[es]; + + if (!lrng_sp80090b_health_enabled()) + return true; + + return es_state->sp80090b_startup_done; +} + +bool lrng_sp80090b_compliant(enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + struct lrng_health_es_state *es_state = &health->es_state[es]; + + return lrng_sp80090b_health_enabled() && + es_state->sp80090b_startup_done; +} + +/*************************************************************************** + * Adaptive Proportion Test + * + * This test complies with SP800-90B section 4.4.2. + ***************************************************************************/ + +/* + * Reset the APT counter + * + * @health [in] Reference to health state + */ +static void lrng_apt_reset(struct lrng_apt *apt, unsigned int time_masked) +{ + /* Reset APT */ + atomic_set(&apt->apt_count, 0); + atomic_set(&apt->apt_base, time_masked); +} + +static void lrng_apt_restart(struct lrng_apt *apt) +{ + atomic_set(&apt->apt_trigger, LRNG_APT_WINDOW_SIZE); +} + +/* + * Insert a new entropy event into APT + * + * This function does is void as it does not decide about the fate of a time + * stamp. An APT failure can only happen at the same time of a stuck test + * failure. Thus, the stuck failure will already decide how the time stamp + * is handled. + * + * @health [in] Reference to health state + * @now_time [in] Time stamp to process + */ +static void lrng_apt_insert(struct lrng_health *health, + unsigned int now_time, enum lrng_internal_es es) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + struct lrng_apt *apt = &es_state->apt; + + if (!lrng_sp80090b_health_requested()) + return; + + now_time &= LRNG_APT_WORD_MASK; + + /* Initialization of APT */ + if (!apt->apt_base_set) { + atomic_set(&apt->apt_base, now_time); + apt->apt_base_set = true; + return; + } + + if (now_time == (unsigned int)atomic_read(&apt->apt_base)) { + u32 apt_val = (u32)atomic_inc_return_relaxed(&apt->apt_count); + + if (apt_val >= CONFIG_LRNG_APT_CUTOFF) + lrng_sp80090b_failure(health, es); + } + + if (atomic_dec_and_test(&apt->apt_trigger)) { + lrng_apt_restart(apt); + lrng_apt_reset(apt, now_time); + lrng_sp80090b_startup(health, es); + } +} + +/*************************************************************************** + * Repetition Count Test + * + * The LRNG uses an enhanced version of the Repetition Count Test + * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical + * back-to-back values, the input to the RCT is the counting of the stuck + * values while filling the entropy pool. + * + * The RCT is applied with an alpha of 2^-30 compliant to FIPS 140-2 IG 9.8. + * + * During the counting operation, the LRNG always calculates the RCT + * cut-off value of C. If that value exceeds the allowed cut-off value, + * the LRNG will invalidate all entropy for the entropy pool which implies + * that no data can be extracted from the entropy pool unless new entropy + * is received. + ***************************************************************************/ + +/* + * Hot code path - Insert data for Repetition Count Test + * + * @health: Reference to health information + * @stuck: Decision of stuck test + */ +static void lrng_rct(struct lrng_health *health, enum lrng_internal_es es, + int stuck) +{ + struct lrng_health_es_state *es_state = &health->es_state[es]; + struct lrng_rct *rct = &es_state->rct; + + if (!lrng_sp80090b_health_requested()) + return; + + if (stuck) { + u32 rct_count = atomic_add_return_relaxed(1, &rct->rct_count); + + /* + * The cutoff value is based on the following consideration: + * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. + * In addition, we imply an entropy value H of 1 bit as this + * is the minimum entropy required to provide full entropy. + * + * Note, rct_count (which equals to value B in the + * pseudo code of SP800-90B section 4.4.1) starts with zero. + * Hence we need to subtract one from the cutoff value as + * calculated following SP800-90B. + */ + if (rct_count >= CONFIG_LRNG_RCT_CUTOFF) { + atomic_set(&rct->rct_count, 0); + + /* + * APT must start anew as we consider all previously + * recorded data to contain no entropy. + */ + lrng_apt_restart(&es_state->apt); + + lrng_sp80090b_failure(health, es); + } + } else { + atomic_set(&rct->rct_count, 0); + } +} + +/*************************************************************************** + * Stuck Test + * + * Checking the: + * 1st derivative of the event occurrence (time delta) + * 2nd derivative of the event occurrence (delta of time deltas) + * 3rd derivative of the event occurrence (delta of delta of time deltas) + * + * All values must always be non-zero. The stuck test is only valid disabled if + * high-resolution time stamps are identified after initialization. + ***************************************************************************/ + +static u32 lrng_delta(u32 prev, u32 next) +{ + /* + * Note that this (unsigned) subtraction does yield the correct value + * in the wraparound-case, i.e. when next < prev. + */ + return (next - prev); +} + +/* + * Hot code path + * + * @health: Reference to health information + * @now: Event time + * @return: 0 event occurrence not stuck (good time stamp) + * != 0 event occurrence stuck (reject time stamp) + */ +static int lrng_irq_stuck(enum lrng_internal_es es, u32 now_time) +{ + struct lrng_stuck_test *stuck = this_cpu_ptr(lrng_stuck_test_array); + u32 delta = lrng_delta(stuck[es].last_time, now_time); + u32 delta2 = lrng_delta(stuck[es].last_delta, delta); + u32 delta3 = lrng_delta(stuck[es].last_delta2, delta2); + + stuck[es].last_time = now_time; + stuck[es].last_delta = delta; + stuck[es].last_delta2 = delta2; + + if (!delta || !delta2 || !delta3) + return 1; + + return 0; +} + +/*************************************************************************** + * Health test interfaces + ***************************************************************************/ + +/* + * Disable all health tests + */ +void lrng_health_disable(void) +{ + struct lrng_health *health = &lrng_health; + + health->health_test_enabled = false; + + if (lrng_sp80090b_health_requested()) + pr_warn("SP800-90B compliance requested but the Linux RNG is NOT SP800-90B compliant\n"); +} + +/* + * Hot code path - Perform health test on time stamp received from an event + * + * @now_time Time stamp + */ +enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es) +{ + struct lrng_health *health = &lrng_health; + int stuck; + + if (!health->health_test_enabled) + return lrng_health_pass; + + lrng_apt_insert(health, now_time, es); + + stuck = lrng_irq_stuck(es, now_time); + lrng_rct(health, es, stuck); + if (stuck) { + /* SP800-90B disallows using a failing health test time stamp */ + return lrng_sp80090b_health_requested() ? + lrng_health_fail_drop : lrng_health_fail_use; + } + + return lrng_health_pass; +} diff --git a/drivers/char/lrng/lrng_health.h b/drivers/char/lrng/lrng_health.h new file mode 100644 index 0000000000000..4f9f5033fc307 --- /dev/null +++ b/drivers/char/lrng/lrng_health.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_HEALTH_H +#define _LRNG_HEALTH_H + +#include "lrng_es_mgr.h" + +enum lrng_health_res { + lrng_health_pass, /* Health test passes on time stamp */ + lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ + lrng_health_fail_drop /* Time stamp unhealthy, drop it */ +}; + +#ifdef CONFIG_LRNG_HEALTH_TESTS +bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es); +bool lrng_sp80090b_compliant(enum lrng_internal_es es); + +enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es); +void lrng_health_disable(void); +#else /* CONFIG_LRNG_HEALTH_TESTS */ +static inline bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) +{ + return true; +} + +static inline bool lrng_sp80090b_compliant(enum lrng_internal_es es) +{ + return false; +} + +static inline enum lrng_health_res +lrng_health_test(u32 now_time, enum lrng_internal_es es) +{ + return lrng_health_pass; +} +static inline void lrng_health_disable(void) { } +#endif /* CONFIG_LRNG_HEALTH_TESTS */ + +#endif /* _LRNG_HEALTH_H */ diff --git a/drivers/char/lrng/lrng_interface_aux.c b/drivers/char/lrng/lrng_interface_aux.c new file mode 100644 index 0000000000000..0eb49bd6b48f6 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_aux.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG auxiliary interfaces + * + * Copyright (C) 2022 Stephan Mueller + * Copyright (C) 2017 Jason A. Donenfeld . All + * Rights Reserved. + * Copyright (C) 2016 Jason Cooper + */ + +#include +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_interface_random_kernel.h" + +struct batched_entropy { + union { + u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)]; + u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)]; + }; + unsigned int position; + spinlock_t batch_lock; +}; + +/* + * Get a random word for internal kernel use only. The quality of the random + * number is as good as /dev/urandom, but there is no backtrack protection, + * with the goal of being quite fast and not depleting entropy. + */ +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { + .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), +}; + +u64 get_random_u64(void) +{ + u64 ret; + unsigned long flags; + struct batched_entropy *batch; + + lrng_debug_report_seedlevel("get_random_u64"); + + batch = raw_cpu_ptr(&batched_entropy_u64); + spin_lock_irqsave(&batch->batch_lock, flags); + if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { + lrng_get_random_bytes(batch->entropy_u64, LRNG_DRNG_BLOCKSIZE); + batch->position = 0; + } + ret = batch->entropy_u64[batch->position++]; + spin_unlock_irqrestore(&batch->batch_lock, flags); + return ret; +} +EXPORT_SYMBOL(get_random_u64); + +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { + .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), +}; + +u32 get_random_u32(void) +{ + u32 ret; + unsigned long flags; + struct batched_entropy *batch; + + lrng_debug_report_seedlevel("get_random_u32"); + + batch = raw_cpu_ptr(&batched_entropy_u32); + spin_lock_irqsave(&batch->batch_lock, flags); + if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { + lrng_get_random_bytes(batch->entropy_u32, LRNG_DRNG_BLOCKSIZE); + batch->position = 0; + } + ret = batch->entropy_u32[batch->position++]; + spin_unlock_irqrestore(&batch->batch_lock, flags); + return ret; +} +EXPORT_SYMBOL(get_random_u32); + +#ifdef CONFIG_SMP +/* + * This function is called when the CPU is coming up, with entry + * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP. + */ +int random_prepare_cpu(unsigned int cpu) +{ + /* + * When the cpu comes back online, immediately invalidate all batches, + * so that we serve fresh randomness. + */ + per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0; + per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0; + return 0; +} + +int random_online_cpu(unsigned int cpu) +{ + return 0; +} +#endif + +/* + * It's important to invalidate all potential batched entropy that might + * be stored before the crng is initialized, which we can do lazily by + * simply resetting the counter to zero so that it's re-extracted on the + * next usage. + */ +void invalidate_batched_entropy(void) +{ + int cpu; + unsigned long flags; + + for_each_possible_cpu(cpu) { + struct batched_entropy *batched_entropy; + + batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); + spin_lock_irqsave(&batched_entropy->batch_lock, flags); + batched_entropy->position = 0; + spin_unlock(&batched_entropy->batch_lock); + + batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); + spin_lock(&batched_entropy->batch_lock); + batched_entropy->position = 0; + spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); + } +} diff --git a/drivers/char/lrng/lrng_interface_dev.c b/drivers/char/lrng/lrng_interface_dev.c new file mode 100644 index 0000000000000..e60060d402b30 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG user space device file interface + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include + +#include "lrng_interface_dev_common.h" + +static const struct file_operations lrng_fops = { + .read = lrng_drng_read_block, + .write = lrng_drng_write, + .poll = lrng_random_poll, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +static struct miscdevice lrng_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "lrng", + .nodename = "lrng", + .fops = &lrng_fops, + .mode = 0666 +}; + +static int __init lrng_dev_if_mod_init(void) +{ + return misc_register(&lrng_miscdev); +} +device_initcall(lrng_dev_if_mod_init); diff --git a/drivers/char/lrng/lrng_interface_dev_common.c b/drivers/char/lrng/lrng_interface_dev_common.c new file mode 100644 index 0000000000000..c366b29feb99c --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev_common.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG User and kernel space interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" + +DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait); +static struct fasync_struct *fasync; + +static bool lrng_seed_hw = true; /* Allow HW to provide seed */ +static bool lrng_seed_user = true; /* Allow user space to provide seed */ + +/********************************** Helper ***********************************/ + +static u32 lrng_get_aux_ent(void) +{ + return lrng_es[lrng_ext_es_aux]->curr_entropy(0); +} + +/* Is the DRNG seed level too low? */ +bool lrng_need_entropy(void) +{ + return (lrng_get_aux_ent() < lrng_write_wakeup_bits); +} + +void lrng_writer_wakeup(void) +{ + if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) { + wake_up_interruptible(&lrng_write_wait); + kill_fasync(&fasync, SIGIO, POLL_OUT); + } +} + +void lrng_init_wakeup_dev(void) +{ + kill_fasync(&fasync, SIGIO, POLL_IN); +} + +/* External entropy provider is allowed to provide seed data */ +bool lrng_state_exseed_allow(enum lrng_external_noise_source source) +{ + if (source == lrng_noise_source_hw) + return lrng_seed_hw; + return lrng_seed_user; +} + +/* Enable / disable external entropy provider to furnish seed */ +void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) +{ + /* + * If the LRNG is not yet operational, allow all entropy sources + * to deliver data unconditionally to get fully seeded asap. + */ + if (!lrng_state_operational()) + return; + + if (source == lrng_noise_source_hw) + lrng_seed_hw = type; + else + lrng_seed_user = type; +} + +void lrng_state_exseed_allow_all(void) +{ + lrng_state_exseed_set(lrng_noise_source_hw, true); + lrng_state_exseed_set(lrng_noise_source_user, true); +} + +/************************ LRNG user output interfaces *************************/ + +ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr) +{ + ssize_t ret = 0; + u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN); + u8 *tmp_large = NULL, *tmp = tmpbuf; + u32 tmplen = sizeof(tmpbuf); + + if (nbytes == 0) + return 0; + + /* + * Satisfy large read requests -- as the common case are smaller + * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for + * those by using the stack variable of tmpbuf. + */ + if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) { + tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE); + tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL); + if (!tmp_large) + tmplen = sizeof(tmpbuf); + else + tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN); + } + + while (nbytes) { + u32 todo = min_t(u32, nbytes, tmplen); + int rc = 0; + + /* Reschedule if we received a large request. */ + if ((tmp_large) && need_resched()) { + if (signal_pending(current)) { + if (ret == 0) + ret = -ERESTARTSYS; + break; + } + schedule(); + } + + rc = lrng_drng_get_sleep(tmp, todo, pr); + if (rc <= 0) { + if (rc < 0) + ret = rc; + break; + } + if (copy_to_user(buf, tmp, rc)) { + ret = -EFAULT; + break; + } + + nbytes -= rc; + buf += rc; + ret += rc; + } + + /* Wipe data just returned from memory */ + if (tmp_large) + kfree_sensitive(tmp_large); + else + memzero_explicit(tmpbuf, sizeof(tmpbuf)); + + return ret; +} + +ssize_t lrng_read_common_block(int nonblock, int pr, + char __user *buf, size_t nbytes) +{ + int ret; + + if (nbytes == 0) + return 0; + + ret = lrng_drng_sleep_while_nonoperational(nonblock); + if (ret) + return ret; + + return lrng_read_common(buf, nbytes, !!pr); +} + +ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos) +{ + return lrng_read_common_block(file->f_flags & O_NONBLOCK, + file->f_flags & O_SYNC, buf, nbytes); +} + +__poll_t lrng_random_poll(struct file *file, poll_table *wait) +{ + __poll_t mask; + + poll_wait(file, &lrng_init_wait, wait); + poll_wait(file, &lrng_write_wait, wait); + mask = 0; + if (lrng_state_operational()) + mask |= EPOLLIN | EPOLLRDNORM; + if (lrng_need_entropy() || + lrng_state_exseed_allow(lrng_noise_source_user)) { + lrng_state_exseed_set(lrng_noise_source_user, false); + mask |= EPOLLOUT | EPOLLWRNORM; + } + return mask; +} + +ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, + u32 entropy_bits) +{ + ssize_t ret = 0; + u8 buf[64] __aligned(LRNG_KCAPI_ALIGN); + const char __user *p = buffer; + u32 orig_entropy_bits = entropy_bits; + + if (!lrng_get_available()) { + ret = lrng_drng_initalize(); + if (!ret) + return ret; + } + + count = min_t(size_t, count, INT_MAX); + while (count > 0) { + size_t bytes = min_t(size_t, count, sizeof(buf)); + u32 ent = min_t(u32, bytes<<3, entropy_bits); + + if (copy_from_user(&buf, p, bytes)) + return -EFAULT; + /* Inject data into entropy pool */ + lrng_pool_insert_aux(buf, bytes, ent); + + count -= bytes; + p += bytes; + ret += bytes; + entropy_bits -= ent; + + cond_resched(); + } + + /* Force reseed of DRNG during next data request. */ + if (!orig_entropy_bits) + lrng_drng_force_reseed(); + + return ret; +} + +ssize_t lrng_drng_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + return lrng_drng_write_common(buffer, count, 0); +} + +long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +{ + u32 digestsize_bits; + int size, ent_count_bits; + int __user *p = (int __user *)arg; + + switch (cmd) { + case RNDGETENTCNT: + ent_count_bits = lrng_avail_entropy_aux(); + if (put_user(ent_count_bits, p)) + return -EFAULT; + return 0; + case RNDADDTOENTCNT: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (get_user(ent_count_bits, p)) + return -EFAULT; + ent_count_bits = (int)lrng_get_aux_ent() + ent_count_bits; + if (ent_count_bits < 0) + ent_count_bits = 0; + digestsize_bits = lrng_get_digestsize(); + if (ent_count_bits > digestsize_bits) + ent_count_bits = digestsize_bits; + lrng_pool_set_entropy(ent_count_bits); + return 0; + case RNDADDENTROPY: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (get_user(ent_count_bits, p++)) + return -EFAULT; + if (ent_count_bits < 0) + return -EINVAL; + if (get_user(size, p++)) + return -EFAULT; + if (size < 0) + return -EINVAL; + /* there cannot be more entropy than data */ + ent_count_bits = min(ent_count_bits, size<<3); + return lrng_drng_write_common((const char __user *)p, size, + ent_count_bits); + case RNDZAPENTCNT: + case RNDCLEARPOOL: + /* Clear the entropy pool counter. */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + lrng_pool_set_entropy(0); + return 0; + case RNDRESEEDCRNG: + /* + * We leave the capability check here since it is present + * in the upstream's RNG implementation. Yet, user space + * can trigger a reseed as easy as writing into /dev/random + * or /dev/urandom where no privilege is needed. + */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Force a reseed of all DRNGs */ + lrng_drng_force_reseed(); + return 0; + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(lrng_ioctl); + +int lrng_fasync(int fd, struct file *filp, int on) +{ + return fasync_helper(fd, filp, on, &fasync); +} diff --git a/drivers/char/lrng/lrng_interface_dev_common.h b/drivers/char/lrng/lrng_interface_dev_common.h new file mode 100644 index 0000000000000..8bc289224e97d --- /dev/null +++ b/drivers/char/lrng/lrng_interface_dev_common.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_INTERFACE_DEV_COMMON_H +#define _LRNG_INTERFACE_DEV_COMMON_H + +#include +#include + +/******************* Upstream functions hooked into the LRNG ******************/ +enum lrng_external_noise_source { + lrng_noise_source_hw, + lrng_noise_source_user +}; + +#ifdef CONFIG_LRNG_COMMON_DEV_IF +void lrng_writer_wakeup(void); +void lrng_init_wakeup_dev(void); +void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); +void lrng_state_exseed_allow_all(void); +#else /* CONFIG_LRNG_COMMON_DEV_IF */ +static inline void lrng_writer_wakeup(void) { } +static inline void lrng_init_wakeup_dev(void) { } +static inline void +lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) { } +static inline void lrng_state_exseed_allow_all(void) { } +#endif /* CONFIG_LRNG_COMMON_DEV_IF */ + +/****** Downstream service functions to actual interface implementations ******/ + +bool lrng_state_exseed_allow(enum lrng_external_noise_source source); +int lrng_fasync(int fd, struct file *filp, int on); +long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg); +ssize_t lrng_drng_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos); +ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, + u32 entropy_bits); +__poll_t lrng_random_poll(struct file *file, poll_table *wait); +ssize_t lrng_read_common_block(int nonblock, int pr, + char __user *buf, size_t nbytes); +ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos); +ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr); +bool lrng_need_entropy(void); + +extern struct wait_queue_head lrng_write_wait; + +#endif /* _LRNG_INTERFACE_DEV_COMMON_H */ diff --git a/drivers/char/lrng/lrng_interface_hwrand.c b/drivers/char/lrng/lrng_interface_hwrand.c new file mode 100644 index 0000000000000..e841eea133482 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_hwrand.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG interface with the HW-Random framework + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include + +static int lrng_hwrand_if_random(struct hwrng *rng, void *buf, size_t max, + bool wait) +{ + /* + * lrng_get_random_bytes_full not called as we cannot block. + * + * Note: We should either adjust .quality below depending on + * rng_is_initialized() or block here, but neither is not supported by + * the hw_rand framework. + */ + lrng_get_random_bytes(buf, max); + return (int)max; +} + +static struct hwrng lrng_hwrand = { + .name = "lrng", + .init = NULL, + .cleanup = NULL, + .read = lrng_hwrand_if_random, + + /* + * We set .quality only in case the LRNG does not provide the common + * interfaces or does not use the legacy RNG as entropy source. This + * shall avoid that the LRNG automatically spawns the hw_rand + * framework's hwrng kernel thread to feed data into + * add_hwgenerator_randomness. When the LRNG implements the common + * interfaces, this function feeds the data directly into the LRNG. + * If the LRNG uses the legacy RNG as entropy source, + * add_hwgenerator_randomness is implemented by the legacy RNG, but + * still eventually feeds the data into the LRNG. We should avoid such + * circular loops. + * + * We can specify full entropy here, because the LRNG is designed + * to provide full entropy. + */ +#if !defined(CONFIG_LRNG_RANDOM_IF) && \ + !defined(CONFIG_LRNG_KERNEL_RNG) + .quality = 1024, +#endif +}; + +static int __init lrng_hwrand_if_mod_init(void) +{ + return hwrng_register(&lrng_hwrand); +} + +static void __exit lrng_hwrand_if_mod_exit(void) +{ + hwrng_unregister(&lrng_hwrand); +} + +module_init(lrng_hwrand_if_mod_init); +module_exit(lrng_hwrand_if_mod_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager HW-Random Interface"); diff --git a/drivers/char/lrng/lrng_interface_kcapi.c b/drivers/char/lrng/lrng_interface_kcapi.c new file mode 100644 index 0000000000000..4cb511f8088e0 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_kcapi.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG interface with the RNG framework of the kernel crypto API + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" + +static int lrng_kcapi_if_init(struct crypto_tfm *tfm) +{ + return 0; +} + +static void lrng_kcapi_if_cleanup(struct crypto_tfm *tfm) { } + +static int lrng_kcapi_if_reseed(const u8 *src, unsigned int slen) +{ + int ret; + + if (!slen) + return 0; + + /* Insert caller-provided data without crediting entropy */ + ret = lrng_pool_insert_aux((u8 *)src, slen, 0); + if (ret) + return ret; + + /* Make sure the new data is immediately available to DRNG */ + lrng_drng_force_reseed(); + + return 0; +} + +static int lrng_kcapi_if_random(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *rdata, unsigned int dlen) +{ + int ret = lrng_kcapi_if_reseed(src, slen); + + if (!ret) + lrng_get_random_bytes_full(rdata, dlen); + + return ret; +} + +static int lrng_kcapi_if_reset(struct crypto_rng *tfm, + const u8 *seed, unsigned int slen) +{ + return lrng_kcapi_if_reseed(seed, slen); +} + +static struct rng_alg lrng_alg = { + .generate = lrng_kcapi_if_random, + .seed = lrng_kcapi_if_reset, + .seedsize = 0, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "lrng", + .cra_priority = 500, + .cra_ctxsize = 0, + .cra_module = THIS_MODULE, + .cra_init = lrng_kcapi_if_init, + .cra_exit = lrng_kcapi_if_cleanup, + + } +}; + +#ifdef CONFIG_LRNG_DRNG_ATOMIC +static int lrng_kcapi_if_random_atomic(struct crypto_rng *tfm, + const u8 *src, unsigned int slen, + u8 *rdata, unsigned int dlen) +{ + int ret = lrng_kcapi_if_reseed(src, slen); + + if (!ret) + lrng_get_random_bytes(rdata, dlen); + + return ret; +} + +static struct rng_alg lrng_alg_atomic = { + .generate = lrng_kcapi_if_random_atomic, + .seed = lrng_kcapi_if_reset, + .seedsize = 0, + .base = { + .cra_name = "lrng_atomic", + .cra_driver_name = "lrng_atomic", + .cra_priority = 100, + .cra_ctxsize = 0, + .cra_module = THIS_MODULE, + .cra_init = lrng_kcapi_if_init, + .cra_exit = lrng_kcapi_if_cleanup, + + } +}; +#endif /* CONFIG_LRNG_DRNG_ATOMIC */ + +static int __init lrng_kcapi_if_mod_init(void) +{ + return +#ifdef CONFIG_LRNG_DRNG_ATOMIC + crypto_register_rng(&lrng_alg_atomic) ?: +#endif + crypto_register_rng(&lrng_alg); +} + +static void __exit lrng_kcapi_if_mod_exit(void) +{ + crypto_unregister_rng(&lrng_alg); +#ifdef CONFIG_LRNG_DRNG_ATOMIC + crypto_unregister_rng(&lrng_alg_atomic); +#endif +} + +module_init(lrng_kcapi_if_mod_init); +module_exit(lrng_kcapi_if_mod_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Stephan Mueller "); +MODULE_DESCRIPTION("Entropy Source and DRNG Manager kernel crypto API RNG framework interface"); +MODULE_ALIAS_CRYPTO("lrng"); +MODULE_ALIAS_CRYPTO("lrng_atomic"); +MODULE_ALIAS_CRYPTO("stdrng"); diff --git a/drivers/char/lrng/lrng_interface_random_kernel.c b/drivers/char/lrng/lrng_interface_random_kernel.c new file mode 100644 index 0000000000000..ec6310884865b --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_kernel.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Kernel space interfaces API/ABI compliant to linux/random.h + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include "lrng_es_aux.h" +#include "lrng_es_irq.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" +#include "lrng_interface_random_kernel.h" + +/********************************** Helper ***********************************/ + +int __init random_init(const char *command_line) +{ + int ret = lrng_rand_initialize(); + + lrng_pool_insert_aux(command_line, strlen(command_line), 0); + return ret; +} + +/************************ LRNG kernel input interfaces ************************/ + +/* + * add_hwgenerator_randomness() - Interface for in-kernel drivers of true + * hardware RNGs. + * + * Those devices may produce endless random bits and will be throttled + * when our pool is full. + * + * @buffer: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @count: length of buffer + * @entropy_bits: amount of entropy in buffer (value is in bits) + */ +void add_hwgenerator_randomness(const void *buffer, size_t count, + size_t entropy_bits) +{ + /* + * Suspend writing if we are fully loaded with entropy. + * We'll be woken up again once below lrng_write_wakeup_thresh, + * or when the calling thread is about to terminate. + */ + wait_event_interruptible(lrng_write_wait, + lrng_need_entropy() || + lrng_state_exseed_allow(lrng_noise_source_hw) || + kthread_should_stop()); + lrng_state_exseed_set(lrng_noise_source_hw, false); + lrng_pool_insert_aux(buffer, count, entropy_bits); +} +EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); + +/* + * add_bootloader_randomness() - Handle random seed passed by bootloader. + * + * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise + * it would be regarded as device data. + * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. + * + * @buf: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @size: length of buffer + */ +void add_bootloader_randomness(const void *buf, size_t size) +{ + lrng_pool_insert_aux(buf, size, + IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER) ? + size * 8 : 0); +} + +/* + * Callback for HID layer -- use the HID event values to stir the entropy pool + */ +void add_input_randomness(unsigned int type, unsigned int code, + unsigned int value) +{ + static unsigned char last_value; + + /* ignore autorepeat and the like */ + if (value == last_value) + return; + + last_value = value; + + lrng_irq_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value); +} +EXPORT_SYMBOL_GPL(add_input_randomness); + +/* + * add_device_randomness() - Add device- or boot-specific data to the entropy + * pool to help initialize it. + * + * None of this adds any entropy; it is meant to avoid the problem of + * the entropy pool having similar initial state across largely + * identical devices. + * + * @buf: buffer holding the entropic data from HW noise sources to be used to + * insert into entropy pool. + * @size: length of buffer + */ +void add_device_randomness(const void *buf, size_t size) +{ + lrng_pool_insert_aux((u8 *)buf, size, 0); +} +EXPORT_SYMBOL(add_device_randomness); + +#ifdef CONFIG_BLOCK +void rand_initialize_disk(struct gendisk *disk) { } +void add_disk_randomness(struct gendisk *disk) { } +EXPORT_SYMBOL(add_disk_randomness); +#endif + +#ifndef CONFIG_LRNG_IRQ +void add_interrupt_randomness(int irq) { } +EXPORT_SYMBOL(add_interrupt_randomness); +#endif + +#if IS_ENABLED(CONFIG_VMGENID) +static BLOCKING_NOTIFIER_HEAD(lrng_vmfork_chain); + +/* + * Handle a new unique VM ID, which is unique, not secret, so we + * don't credit it, but we do immediately force a reseed after so + * that it's used by the crng posthaste. + */ +void add_vmfork_randomness(const void *unique_vm_id, size_t size) +{ + add_device_randomness(unique_vm_id, size); + if (lrng_state_operational()) + lrng_drng_force_reseed(); + blocking_notifier_call_chain(&lrng_vmfork_chain, 0, NULL); +} +#if IS_MODULE(CONFIG_VMGENID) +EXPORT_SYMBOL_GPL(add_vmfork_randomness); +#endif + +int register_random_vmfork_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&lrng_vmfork_chain, nb); +} +EXPORT_SYMBOL_GPL(register_random_vmfork_notifier); + +int unregister_random_vmfork_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&lrng_vmfork_chain, nb); +} +EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier); +#endif + +/*********************** LRNG kernel output interfaces ************************/ + +/* + * get_random_bytes() - Provider of cryptographic strong random numbers for + * kernel-internal usage. + * + * This function is appropriate for all in-kernel use cases. However, + * it will always use the ChaCha20 DRNG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +void get_random_bytes(void *buf, size_t nbytes) +{ + lrng_get_random_bytes(buf, nbytes); +} +EXPORT_SYMBOL(get_random_bytes); + +/* + * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus + * guaranteed to supply cryptographically secure random numbers. + * + * This applies to: the /dev/urandom device, the get_random_bytes function, + * and the get_random_{u32,u64,int,long} family of functions. Using any of + * these functions without first calling this function forfeits the guarantee + * of security. + * + * Return: + * * 0 if the LRNG has been seeded. + * * -ERESTARTSYS if the function was interrupted by a signal. + */ +int wait_for_random_bytes(void) +{ + return lrng_drng_sleep_while_non_min_seeded(); +} +EXPORT_SYMBOL(wait_for_random_bytes); + +/* + * Returns whether or not the LRNG has been seeded. + * + * Returns: true if the urandom pool has been seeded. + * false if the urandom pool has not been seeded. + */ +bool rng_is_initialized(void) +{ + return lrng_state_operational(); +} +EXPORT_SYMBOL(rng_is_initialized); diff --git a/drivers/char/lrng/lrng_interface_random_kernel.h b/drivers/char/lrng/lrng_interface_random_kernel.h new file mode 100644 index 0000000000000..f4d09d1fbb140 --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_kernel.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_INTERFACE_RANDOM_H +#define _LRNG_INTERFACE_RANDOM_H + +#ifdef CONFIG_LRNG_RANDOM_IF +void invalidate_batched_entropy(void); +#else /* CONFIG_LRNG_RANDOM_IF */ +static inline void invalidate_batched_entropy(void) { } +#endif /* CONFIG_LRNG_RANDOM_IF */ + +#endif /* _LRNG_INTERFACE_RANDOM_H */ diff --git a/drivers/char/lrng/lrng_interface_random_user.c b/drivers/char/lrng/lrng_interface_random_user.c new file mode 100644 index 0000000000000..2d49ebb83539b --- /dev/null +++ b/drivers/char/lrng/lrng_interface_random_user.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG Common user space interfaces compliant to random(4), random(7) and + * getrandom(2) man pages. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" + +static ssize_t lrng_drng_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + if (!lrng_state_min_seeded()) + pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n", + current->comm, nbytes); + else if (!lrng_state_operational()) + pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n", + current->comm, nbytes); + + return lrng_read_common(buf, nbytes, false); +} + +const struct file_operations random_fops = { + .read = lrng_drng_read_block, + .write = lrng_drng_write, + .poll = lrng_random_poll, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +const struct file_operations urandom_fops = { + .read = lrng_drng_read, + .write = lrng_drng_write, + .unlocked_ioctl = lrng_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = lrng_fasync, + .llseek = noop_llseek, +}; + +SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, + unsigned int, flags) +{ + if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE)) + return -EINVAL; + + /* + * Requesting insecure and blocking randomness at the same time makes + * no sense. + */ + if ((flags & + (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) + return -EINVAL; + + if (count > INT_MAX) + count = INT_MAX; + + if (flags & GRND_INSECURE) + return lrng_drng_read(NULL, buf, count, NULL); + + return lrng_read_common_block(flags & GRND_NONBLOCK, + flags & GRND_RANDOM, buf, count); +} diff --git a/drivers/char/lrng/lrng_numa.c b/drivers/char/lrng/lrng_numa.c new file mode 100644 index 0000000000000..d74dd8df28437 --- /dev/null +++ b/drivers/char/lrng/lrng_numa.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG NUMA support + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_irq.h" +#include "lrng_es_mgr.h" +#include "lrng_numa.h" +#include "lrng_proc.h" + +static struct lrng_drng **lrng_drng __read_mostly = NULL; + +struct lrng_drng **lrng_drng_instances(void) +{ + /* counterpart to cmpxchg_release in _lrng_drngs_numa_alloc */ + return READ_ONCE(lrng_drng); +} + +/* Allocate the data structures for the per-NUMA node DRNGs */ +static void _lrng_drngs_numa_alloc(struct work_struct *work) +{ + struct lrng_drng **drngs; + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + u32 node; + bool init_drng_used = false; + + mutex_lock(&lrng_crypto_cb_update); + + /* per-NUMA-node DRNGs are already present */ + if (lrng_drng) + goto unlock; + + /* Make sure the initial DRNG is initialized and its drng_cb is set */ + if (lrng_drng_initalize()) + goto err; + + drngs = kcalloc(nr_node_ids, sizeof(void *), GFP_KERNEL|__GFP_NOFAIL); + for_each_online_node(node) { + struct lrng_drng *drng; + + if (!init_drng_used) { + drngs[node] = lrng_drng_init; + init_drng_used = true; + continue; + } + + drng = kmalloc_node(sizeof(struct lrng_drng), + GFP_KERNEL|__GFP_NOFAIL, node); + memset(drng, 0, sizeof(lrng_drng)); + + if (lrng_drng_alloc_common(drng, lrng_drng_init->drng_cb)) { + kfree(drng); + goto err; + } + + drng->hash_cb = lrng_drng_init->hash_cb; + drng->hash = lrng_drng_init->hash_cb->hash_alloc(); + if (IS_ERR(drng->hash)) { + lrng_drng_init->drng_cb->drng_dealloc(drng->drng); + kfree(drng); + goto err; + } + + mutex_init(&drng->lock); + rwlock_init(&drng->hash_lock); + + /* + * No reseeding of NUMA DRNGs from previous DRNGs as this + * would complicate the code. Let it simply reseed. + */ + drngs[node] = drng; + + lrng_pool_inc_numa_node(); + pr_info("DRNG and entropy pool read hash for NUMA node %d allocated\n", + node); + } + + /* counterpart to READ_ONCE in lrng_drng_instances */ + if (!cmpxchg_release(&lrng_drng, NULL, drngs)) { + lrng_pool_all_numa_nodes_seeded(false); + goto unlock; + } + +err: + for_each_online_node(node) { + struct lrng_drng *drng = drngs[node]; + + if (drng == lrng_drng_init) + continue; + + if (drng) { + drng->hash_cb->hash_dealloc(drng->hash); + drng->drng_cb->drng_dealloc(drng->drng); + kfree(drng); + } + } + kfree(drngs); + +unlock: + mutex_unlock(&lrng_crypto_cb_update); +} + +static DECLARE_WORK(lrng_drngs_numa_alloc_work, _lrng_drngs_numa_alloc); + +static void lrng_drngs_numa_alloc(void) +{ + schedule_work(&lrng_drngs_numa_alloc_work); +} + +static int __init lrng_numa_init(void) +{ + lrng_drngs_numa_alloc(); + return 0; +} + +late_initcall(lrng_numa_init); diff --git a/drivers/char/lrng/lrng_numa.h b/drivers/char/lrng/lrng_numa.h new file mode 100644 index 0000000000000..dc8dff9816eeb --- /dev/null +++ b/drivers/char/lrng/lrng_numa.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_NUMA_H +#define _LRNG_NUMA_H + +#ifdef CONFIG_NUMA +struct lrng_drng **lrng_drng_instances(void); +#else /* CONFIG_NUMA */ +static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } +#endif /* CONFIG_NUMA */ + +#endif /* _LRNG_NUMA_H */ diff --git a/drivers/char/lrng/lrng_proc.c b/drivers/char/lrng/lrng_proc.c new file mode 100644 index 0000000000000..92329cc6ef28c --- /dev/null +++ b/drivers/char/lrng/lrng_proc.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG proc interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_proc.h" + +/* Number of online DRNGs */ +static u32 numa_drngs = 1; + +void lrng_pool_inc_numa_node(void) +{ + numa_drngs++; +} + +static int lrng_proc_type_show(struct seq_file *m, void *v) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + unsigned char buf[270]; + u32 i; + + mutex_lock(&lrng_drng_init->lock); + snprintf(buf, sizeof(buf), + "DRNG name: %s\n" + "LRNG security strength in bits: %d\n" + "Number of DRNG instances: %u\n" + "Standards compliance: %s%s\n" + "LRNG minimally seeded: %s\n" + "LRNG fully seeded: %s\n" + "LRNG entropy level: %u\n", + lrng_drng_init->drng_cb->drng_name(), + lrng_security_strength(), + numa_drngs, + lrng_sp80090c_compliant() ? "SP800-90C " : "", + lrng_ntg1_compliant() ? "NTG.1 " : "", + lrng_state_min_seeded() ? "true" : "false", + lrng_state_fully_seeded() ? "true" : "false", + lrng_avail_entropy()); + seq_write(m, buf, strlen(buf)); + + for_each_lrng_es(i) { + snprintf(buf, sizeof(buf), + "Entropy Source %u properties:\n" + " Name: %s\n", + i, lrng_es[i]->name); + seq_write(m, buf, strlen(buf)); + + buf[0] = '\0'; + lrng_es[i]->state(buf, sizeof(buf)); + seq_write(m, buf, strlen(buf)); + } + + mutex_unlock(&lrng_drng_init->lock); + + return 0; +} + +static int __init lrng_proc_type_init(void) +{ + proc_create_single("lrng_type", 0444, NULL, &lrng_proc_type_show); + return 0; +} + +module_init(lrng_proc_type_init); diff --git a/drivers/char/lrng/lrng_proc.h b/drivers/char/lrng/lrng_proc.h new file mode 100644 index 0000000000000..c653274f19547 --- /dev/null +++ b/drivers/char/lrng/lrng_proc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_PROC_H +#define _LRNG_PROC_H + +#ifdef CONFIG_SYSCTL +void lrng_pool_inc_numa_node(void); +#else +static inline void lrng_pool_inc_numa_node(void) { } +#endif + +#endif /* _LRNG_PROC_H */ diff --git a/drivers/char/lrng/lrng_selftest.c b/drivers/char/lrng/lrng_selftest.c new file mode 100644 index 0000000000000..15f1e4a2a719a --- /dev/null +++ b/drivers/char/lrng/lrng_selftest.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG power-on and on-demand self-test + * + * Copyright (C) 2022, Stephan Mueller + */ + +/* + * In addition to the self-tests below, the following LRNG components + * are covered with self-tests during regular operation: + * + * * power-on self-test: SP800-90A DRBG provided by the Linux kernel crypto API + * * power-on self-test: PRNG provided by the Linux kernel crypto API + * * runtime test: Raw noise source data testing including SP800-90B compliant + * tests when enabling CONFIG_LRNG_HEALTH_TESTS + * + * Additional developer tests present with LRNG code: + * * SP800-90B APT and RCT test enforcement validation when enabling + * CONFIG_LRNG_APT_BROKEN or CONFIG_LRNG_RCT_BROKEN. + * * Collection of raw entropy from the interrupt noise source when enabling + * CONFIG_LRNG_TESTING and pulling the data from the kernel with the provided + * interface. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_drng_chacha20.h" +#include "lrng_sha.h" + +#define LRNG_SELFTEST_PASSED 0 +#define LRNG_SEFLTEST_ERROR_TIME (1 << 0) +#define LRNG_SEFLTEST_ERROR_CHACHA20 (1 << 1) +#define LRNG_SEFLTEST_ERROR_HASH (1 << 2) +#define LRNG_SEFLTEST_ERROR_GCD (1 << 3) +#define LRNG_SELFTEST_NOT_EXECUTED 0xffffffff + +#ifdef CONFIG_LRNG_TIMER_COMMON + +#include "lrng_es_timer_common.h" + +static u32 lrng_data_selftest_ptr = 0; +static u32 lrng_data_selftest[LRNG_DATA_ARRAY_SIZE]; + +static void lrng_data_process_selftest_insert(u32 time) +{ + u32 ptr = lrng_data_selftest_ptr++ & LRNG_DATA_WORD_MASK; + unsigned int array = lrng_data_idx2array(ptr); + unsigned int slot = lrng_data_idx2slot(ptr); + + /* zeroization of slot to ensure the following OR adds the data */ + lrng_data_selftest[array] &= + ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, + slot)); + lrng_data_selftest[array] |= + lrng_data_slot_val(time & LRNG_DATA_SLOTSIZE_MASK, slot); +} + +static void lrng_data_process_selftest_u32(u32 data) +{ + u32 pre_ptr, ptr, mask; + unsigned int pre_array; + + /* Increment pointer by number of slots taken for input value */ + lrng_data_selftest_ptr += LRNG_DATA_SLOTS_PER_UINT; + + /* ptr to current unit */ + ptr = lrng_data_selftest_ptr; + + lrng_data_split_u32(&ptr, &pre_ptr, &mask); + + /* MSB of data go into previous unit */ + pre_array = lrng_data_idx2array(pre_ptr); + /* zeroization of slot to ensure the following OR adds the data */ + lrng_data_selftest[pre_array] &= ~(0xffffffff & ~mask); + lrng_data_selftest[pre_array] |= data & ~mask; + + /* LSB of data go into current unit */ + lrng_data_selftest[lrng_data_idx2array(ptr)] = data & mask; +} + +static unsigned int lrng_data_process_selftest(void) +{ + u32 time; + u32 idx_zero_compare = (0 << 0) | (1 << 8) | (2 << 16) | (3 << 24); + u32 idx_one_compare = (4 << 0) | (5 << 8) | (6 << 16) | (7 << 24); + u32 idx_last_compare = + (((LRNG_DATA_NUM_VALUES - 4) & LRNG_DATA_SLOTSIZE_MASK) << 0) | + (((LRNG_DATA_NUM_VALUES - 3) & LRNG_DATA_SLOTSIZE_MASK) << 8) | + (((LRNG_DATA_NUM_VALUES - 2) & LRNG_DATA_SLOTSIZE_MASK) << 16) | + (((LRNG_DATA_NUM_VALUES - 1) & LRNG_DATA_SLOTSIZE_MASK) << 24); + + (void)idx_one_compare; + + /* "poison" the array to verify the operation of the zeroization */ + lrng_data_selftest[0] = 0xffffffff; + lrng_data_selftest[1] = 0xffffffff; + + lrng_data_process_selftest_insert(0); + /* + * Note, when using lrng_data_process_u32() on unaligned ptr, + * the first slots will go into next word, and the last slots go + * into the previous word. + */ + lrng_data_process_selftest_u32((4 << 0) | (1 << 8) | (2 << 16) | + (3 << 24)); + lrng_data_process_selftest_insert(5); + lrng_data_process_selftest_insert(6); + lrng_data_process_selftest_insert(7); + + if ((lrng_data_selftest[0] != idx_zero_compare) || + (lrng_data_selftest[1] != idx_one_compare)) + goto err; + + /* Reset for next test */ + lrng_data_selftest[0] = 0; + lrng_data_selftest[1] = 0; + lrng_data_selftest_ptr = 0; + + for (time = 0; time < LRNG_DATA_NUM_VALUES; time++) + lrng_data_process_selftest_insert(time); + + if ((lrng_data_selftest[0] != idx_zero_compare) || + (lrng_data_selftest[1] != idx_one_compare) || + (lrng_data_selftest[LRNG_DATA_ARRAY_SIZE - 1] != idx_last_compare)) + goto err; + + return LRNG_SELFTEST_PASSED; + +err: + pr_err("LRNG data array self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_TIME; +} + +static unsigned int lrng_gcd_selftest(void) +{ + u32 history[10]; + unsigned int i; + +#define LRNG_GCD_SELFTEST 3 + for (i = 0; i < ARRAY_SIZE(history); i++) + history[i] = i * LRNG_GCD_SELFTEST; + + if (lrng_gcd_analyze(history, ARRAY_SIZE(history)) == LRNG_GCD_SELFTEST) + return LRNG_SELFTEST_PASSED; + + pr_err("LRNG GCD self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_GCD; +} + +#else /* CONFIG_LRNG_TIMER_COMMON */ + +static unsigned int lrng_data_process_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +static unsigned int lrng_gcd_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +#endif /* CONFIG_LRNG_TIMER_COMMON */ + +/* The test vectors are taken from crypto/testmgr.h */ +static unsigned int lrng_hash_selftest(void) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; + static const u8 lrng_hash_selftest_result[] = +#ifdef CONFIG_CRYPTO_LIB_SHA256 + { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }; +#else /* CONFIG_CRYPTO_LIB_SHA256 */ + { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, + 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d }; +#endif /* CONFIG_CRYPTO_LIB_SHA256 */ + static const u8 hash_input[] = { 0x61, 0x62, 0x63 }; /* "abc" */ + u8 digest[sizeof(lrng_hash_selftest_result)] __aligned(sizeof(u32)); + + if (sizeof(digest) != hash_cb->hash_digestsize(NULL)) + return LRNG_SEFLTEST_ERROR_HASH; + + if (!hash_cb->hash_init(shash, NULL) && + !hash_cb->hash_update(shash, hash_input, + sizeof(hash_input)) && + !hash_cb->hash_final(shash, digest) && + !memcmp(digest, lrng_hash_selftest_result, sizeof(digest))) + return 0; + + pr_err("LRNG %s Hash self-test FAILED\n", hash_cb->hash_name()); + return LRNG_SEFLTEST_ERROR_HASH; +} + +#ifdef CONFIG_LRNG_DRNG_CHACHA20 + +static void lrng_selftest_bswap32(u32 *ptr, u32 words) +{ + u32 i; + + /* Byte-swap data which is an LE representation */ + for (i = 0; i < words; i++) { + __le32 *p = (__le32 *)ptr; + + *p = cpu_to_le32(*ptr); + ptr++; + } +} + +/* + * The test vectors were generated using the ChaCha20 DRNG from + * https://www.chronox.de/chacha20.html + */ +static unsigned int lrng_chacha20_drng_selftest(void) +{ + const struct lrng_drng_cb *drng_cb = &lrng_cc20_drng_cb; + u8 seed[CHACHA_KEY_SIZE * 2] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + }; + struct chacha20_block chacha20; + int ret; + u8 outbuf[CHACHA_KEY_SIZE * 2] __aligned(sizeof(u32)); + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * and pulling one half ChaCha20 DRNG block. + */ + static const u8 expected_halfblock[CHACHA_KEY_SIZE] = { + 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, + 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, + 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, + 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7 }; + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * followed by a reseed with two keyblocks + * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + * 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + * 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + * 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + * 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f + * and pulling one ChaCha20 DRNG block. + */ + static const u8 expected_oneblock[CHACHA_KEY_SIZE * 2] = { + 0xe3, 0xb0, 0x8a, 0xcc, 0x34, 0xc3, 0x17, 0x0e, + 0xc3, 0xd8, 0xc3, 0x40, 0xe7, 0x73, 0xe9, 0x0d, + 0xd1, 0x62, 0xa3, 0x5d, 0x7d, 0xf2, 0xf1, 0x4a, + 0x24, 0x42, 0xb7, 0x1e, 0xb0, 0x05, 0x17, 0x07, + 0xb9, 0x35, 0x10, 0x69, 0x8b, 0x46, 0xfb, 0x51, + 0xe9, 0x91, 0x3f, 0x46, 0xf2, 0x4d, 0xea, 0xd0, + 0x81, 0xc1, 0x1b, 0xa9, 0x5d, 0x52, 0x91, 0x5f, + 0xcd, 0xdc, 0xc6, 0xd6, 0xc3, 0x7c, 0x50, 0x23 }; + + /* + * Expected result when ChaCha20 DRNG state is zero: + * * constants are set to "expand 32-byte k" + * * remaining state is 0 + * followed by a reseed with one key block plus one byte + * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + * 0x20 + * and pulling less than one ChaCha20 DRNG block. + */ + static const u8 expected_block_nonalinged[CHACHA_KEY_SIZE + 4] = { + 0x9c, 0xfc, 0x5e, 0x31, 0x21, 0x62, 0x11, 0x85, + 0xd3, 0x77, 0xd3, 0x69, 0x0f, 0xa8, 0x16, 0x55, + 0xb4, 0x4c, 0xf6, 0x52, 0xf3, 0xa8, 0x37, 0x99, + 0x38, 0x76, 0xa0, 0x66, 0xec, 0xbb, 0xce, 0xa9, + 0x9c, 0x95, 0xa1, 0xfd }; + + BUILD_BUG_ON(sizeof(seed) % sizeof(u32)); + + memset(&chacha20, 0, sizeof(chacha20)); + lrng_cc20_init_rfc7539(&chacha20); + lrng_selftest_bswap32((u32 *)seed, sizeof(seed) / sizeof(u32)); + + /* Generate with zero state */ + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_halfblock)); + if (ret != sizeof(expected_halfblock)) + goto err; + if (memcmp(outbuf, expected_halfblock, sizeof(expected_halfblock))) + goto err; + + /* Clear state of DRNG */ + memset(&chacha20.key.u[0], 0, 48); + + /* Reseed with 2 key blocks */ + ret = drng_cb->drng_seed(&chacha20, seed, sizeof(expected_oneblock)); + if (ret < 0) + goto err; + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_oneblock)); + if (ret != sizeof(expected_oneblock)) + goto err; + if (memcmp(outbuf, expected_oneblock, sizeof(expected_oneblock))) + goto err; + + /* Clear state of DRNG */ + memset(&chacha20.key.u[0], 0, 48); + + /* Reseed with 1 key block and one byte */ + ret = drng_cb->drng_seed(&chacha20, seed, + sizeof(expected_block_nonalinged)); + if (ret < 0) + goto err; + ret = drng_cb->drng_generate(&chacha20, outbuf, + sizeof(expected_block_nonalinged)); + if (ret != sizeof(expected_block_nonalinged)) + goto err; + if (memcmp(outbuf, expected_block_nonalinged, + sizeof(expected_block_nonalinged))) + goto err; + + return LRNG_SELFTEST_PASSED; + +err: + pr_err("LRNG ChaCha20 DRNG self-test FAILED\n"); + return LRNG_SEFLTEST_ERROR_CHACHA20; +} + +#else /* CONFIG_LRNG_DRNG_CHACHA20 */ + +static unsigned int lrng_chacha20_drng_selftest(void) +{ + return LRNG_SELFTEST_PASSED; +} + +#endif /* CONFIG_LRNG_DRNG_CHACHA20 */ + +static unsigned int lrng_selftest_status = LRNG_SELFTEST_NOT_EXECUTED; + +static int lrng_selftest(void) +{ + unsigned int ret = lrng_data_process_selftest(); + + ret |= lrng_chacha20_drng_selftest(); + ret |= lrng_hash_selftest(); + ret |= lrng_gcd_selftest(); + + if (ret) { + if (IS_ENABLED(CONFIG_LRNG_SELFTEST_PANIC)) + panic("LRNG self-tests failed: %u\n", ret); + } else { + pr_info("LRNG self-tests passed\n"); + } + + lrng_selftest_status = ret; + + if (lrng_selftest_status) + return -EFAULT; + return 0; +} + +#ifdef CONFIG_SYSFS +/* Re-perform self-test when any value is written to the sysfs file. */ +static int lrng_selftest_sysfs_set(const char *val, + const struct kernel_param *kp) +{ + return lrng_selftest(); +} + +static const struct kernel_param_ops lrng_selftest_sysfs = { + .set = lrng_selftest_sysfs_set, + .get = param_get_uint, +}; +module_param_cb(selftest_status, &lrng_selftest_sysfs, &lrng_selftest_status, + 0644); +#endif /* CONFIG_SYSFS */ + +static int __init lrng_selftest_init(void) +{ + return lrng_selftest(); +} + +module_init(lrng_selftest_init); diff --git a/drivers/char/lrng/lrng_sha.h b/drivers/char/lrng/lrng_sha.h new file mode 100644 index 0000000000000..d2f134f547737 --- /dev/null +++ b/drivers/char/lrng/lrng_sha.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * LRNG SHA definition usable in atomic contexts right from the start of the + * kernel. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_SHA_H +#define _LRNG_SHA_H + +extern const struct lrng_hash_cb lrng_sha_hash_cb; + +#endif /* _LRNG_SHA_H */ diff --git a/drivers/char/lrng/lrng_sha1.c b/drivers/char/lrng/lrng_sha1.c new file mode 100644 index 0000000000000..9cbc7a6fee49a --- /dev/null +++ b/drivers/char/lrng/lrng_sha1.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the SHA-1 implementation that can be used + * without the kernel crypto API available including during early boot and in + * atomic contexts. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "lrng_sha.h" + +/* + * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always + * compiled and present in the kernel. + */ +static u32 lrng_sha1_hash_digestsize(void *hash) +{ + return SHA1_DIGEST_SIZE; +} + +static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src, + int blocks) +{ + u32 temp[SHA1_WORKSPACE_WORDS]; + + while (blocks--) { + sha1_transform(sctx->state, src, temp); + src += SHA1_BLOCK_SIZE; + } + memzero_explicit(temp, sizeof(temp)); +} + +static int lrng_sha1_hash_init(struct shash_desc *shash, void *hash) +{ + /* + * We do not need a TFM - we only need sufficient space for + * struct sha1_state on the stack. + */ + sha1_base_init(shash); + return 0; +} + +static int lrng_sha1_hash_update(struct shash_desc *shash, + const u8 *inbuf, u32 inbuflen) +{ + return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn); +} + +static int lrng_sha1_hash_final(struct shash_desc *shash, u8 *digest) +{ + return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?: + sha1_base_finish(shash, digest); +} + +static const char *lrng_sha1_hash_name(void) +{ + return "SHA-1"; +} + +static void lrng_sha1_hash_desc_zero(struct shash_desc *shash) +{ + memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state)); +} + +static void *lrng_sha1_hash_alloc(void) +{ + pr_info("Hash %s allocated\n", lrng_sha1_hash_name()); + return NULL; +} + +static void lrng_sha1_hash_dealloc(void *hash) { } + +const struct lrng_hash_cb lrng_sha_hash_cb = { + .hash_name = lrng_sha1_hash_name, + .hash_alloc = lrng_sha1_hash_alloc, + .hash_dealloc = lrng_sha1_hash_dealloc, + .hash_digestsize = lrng_sha1_hash_digestsize, + .hash_init = lrng_sha1_hash_init, + .hash_update = lrng_sha1_hash_update, + .hash_final = lrng_sha1_hash_final, + .hash_desc_zero = lrng_sha1_hash_desc_zero, +}; diff --git a/drivers/char/lrng/lrng_sha256.c b/drivers/char/lrng/lrng_sha256.c new file mode 100644 index 0000000000000..50705351a71cc --- /dev/null +++ b/drivers/char/lrng/lrng_sha256.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Backend for the LRNG providing the SHA-256 implementation that can be used + * without the kernel crypto API available including during early boot and in + * atomic contexts. + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "lrng_sha.h" + +static u32 lrng_sha256_hash_digestsize(void *hash) +{ + return SHA256_DIGEST_SIZE; +} + +static int lrng_sha256_hash_init(struct shash_desc *shash, void *hash) +{ + /* + * We do not need a TFM - we only need sufficient space for + * struct sha256_state on the stack. + */ + sha256_init(shash_desc_ctx(shash)); + return 0; +} + +static int lrng_sha256_hash_update(struct shash_desc *shash, + const u8 *inbuf, u32 inbuflen) +{ + sha256_update(shash_desc_ctx(shash), inbuf, inbuflen); + return 0; +} + +static int lrng_sha256_hash_final(struct shash_desc *shash, u8 *digest) +{ + sha256_final(shash_desc_ctx(shash), digest); + return 0; +} + +static const char *lrng_sha256_hash_name(void) +{ + return "SHA-256"; +} + +static void lrng_sha256_hash_desc_zero(struct shash_desc *shash) +{ + memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state)); +} + +static void *lrng_sha256_hash_alloc(void) +{ + pr_info("Hash %s allocated\n", lrng_sha256_hash_name()); + return NULL; +} + +static void lrng_sha256_hash_dealloc(void *hash) { } + +const struct lrng_hash_cb lrng_sha_hash_cb = { + .hash_name = lrng_sha256_hash_name, + .hash_alloc = lrng_sha256_hash_alloc, + .hash_dealloc = lrng_sha256_hash_dealloc, + .hash_digestsize = lrng_sha256_hash_digestsize, + .hash_init = lrng_sha256_hash_init, + .hash_update = lrng_sha256_hash_update, + .hash_final = lrng_sha256_hash_final, + .hash_desc_zero = lrng_sha256_hash_desc_zero, +}; diff --git a/drivers/char/lrng/lrng_switch.c b/drivers/char/lrng/lrng_switch.c new file mode 100644 index 0000000000000..aae75594926f0 --- /dev/null +++ b/drivers/char/lrng/lrng_switch.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG DRNG switching support + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lrng_es_aux.h" +#include "lrng_es_mgr.h" +#include "lrng_interface_dev_common.h" +#include "lrng_numa.h" + +static int __maybe_unused +lrng_hash_switch(struct lrng_drng *drng_store, const void *cb, int node) +{ + const struct lrng_hash_cb *new_cb = (const struct lrng_hash_cb *)cb; + const struct lrng_hash_cb *old_cb = drng_store->hash_cb; + unsigned long flags; + u32 i; + void *new_hash, *old_hash; + int ret; + + if (node == -1) + return 0; + + new_hash = new_cb->hash_alloc(); + old_hash = drng_store->hash; + + if (IS_ERR(new_hash)) { + pr_warn("could not allocate new LRNG pool hash (%ld)\n", + PTR_ERR(new_hash)); + return PTR_ERR(new_hash); + } + + if (new_cb->hash_digestsize(new_hash) > LRNG_MAX_DIGESTSIZE) { + pr_warn("digest size of newly requested hash too large\n"); + new_cb->hash_dealloc(new_hash); + return -EINVAL; + } + + write_lock_irqsave(&drng_store->hash_lock, flags); + + /* Trigger the switch for each entropy source */ + for_each_lrng_es(i) { + if (!lrng_es[i]->switch_hash) + continue; + ret = lrng_es[i]->switch_hash(drng_store, node, new_cb, + new_hash, old_cb); + if (ret) { + u32 j; + + /* Revert all already executed operations */ + for (j = 0; j < i; j++) { + if (!lrng_es[j]->switch_hash) + continue; + WARN_ON(lrng_es[j]->switch_hash(drng_store, + node, old_cb, + old_hash, + new_cb)); + } + goto err; + } + } + + drng_store->hash = new_hash; + drng_store->hash_cb = new_cb; + old_cb->hash_dealloc(old_hash); + pr_info("Conditioning function allocated for DRNG for NUMA node %d\n", + node); + +err: + write_unlock_irqrestore(&drng_store->hash_lock, flags); + return ret; +} + +static int __maybe_unused +lrng_drng_switch(struct lrng_drng *drng_store, const void *cb, int node) +{ + const struct lrng_drng_cb *new_cb = (const struct lrng_drng_cb *)cb; + const struct lrng_drng_cb *old_cb = drng_store->drng_cb; + int ret; + u8 seed[LRNG_DRNG_SECURITY_STRENGTH_BYTES]; + void *new_drng = new_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); + void *old_drng = drng_store->drng; + u32 current_security_strength; + bool reset_drng = !lrng_get_available(); + + if (IS_ERR(new_drng)) { + pr_warn("could not allocate new DRNG for NUMA node %d (%ld)\n", + node, PTR_ERR(new_drng)); + return PTR_ERR(new_drng); + } + + current_security_strength = lrng_security_strength(); + mutex_lock(&drng_store->lock); + + /* + * Pull from existing DRNG to seed new DRNG regardless of seed status + * of old DRNG -- the entropy state for the DRNG is left unchanged which + * implies that als the new DRNG is reseeded when deemed necessary. This + * seeding of the new DRNG shall only ensure that the new DRNG has the + * same entropy as the old DRNG. + */ + ret = old_cb->drng_generate(old_drng, seed, sizeof(seed)); + mutex_unlock(&drng_store->lock); + + if (ret < 0) { + reset_drng = true; + pr_warn("getting random data from DRNG failed for NUMA node %d (%d)\n", + node, ret); + } else { + /* seed new DRNG with data */ + ret = new_cb->drng_seed(new_drng, seed, ret); + memzero_explicit(seed, sizeof(seed)); + if (ret < 0) { + reset_drng = true; + pr_warn("seeding of new DRNG failed for NUMA node %d (%d)\n", + node, ret); + } else { + pr_debug("seeded new DRNG of NUMA node %d instance from old DRNG instance\n", + node); + } + } + + mutex_lock(&drng_store->lock); + + if (reset_drng) + lrng_drng_reset(drng_store); + + drng_store->drng = new_drng; + drng_store->drng_cb = new_cb; + + /* Reseed if previous LRNG security strength was insufficient */ + if (current_security_strength < lrng_security_strength()) + drng_store->force_reseed = true; + + /* Force oversampling seeding as we initialize DRNG */ + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) + lrng_unset_fully_seeded(drng_store); + + if (lrng_state_min_seeded()) + lrng_set_entropy_thresh(lrng_get_seed_entropy_osr( + drng_store->fully_seeded)); + + old_cb->drng_dealloc(old_drng); + + pr_info("DRNG of NUMA node %d switched\n", node); + + mutex_unlock(&drng_store->lock); + return ret; +} + +/* + * Switch the existing DRNG and hash instances with new using the new crypto + * callbacks. The caller must hold the lrng_crypto_cb_update lock. + */ +static int lrng_switch(const void *cb, + int(*switcher)(struct lrng_drng *drng_store, + const void *cb, int node)) +{ + struct lrng_drng **lrng_drng = lrng_drng_instances(); + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + struct lrng_drng *lrng_drng_pr = lrng_drng_pr_instance(); + int ret = 0; + + if (lrng_drng) { + u32 node; + + for_each_online_node(node) { + if (lrng_drng[node]) + ret |= switcher(lrng_drng[node], cb, node); + } + } else { + ret |= switcher(lrng_drng_init, cb, 0); + } + + ret |= switcher(lrng_drng_pr, cb, -1); + + return ret; +} + +/* + * lrng_set_drng_cb - Register new cryptographic callback functions for DRNG + * The registering implies that all old DRNG states are replaced with new + * DRNG states. + * + * drng_cb: Callback functions to be registered -- if NULL, use the default + * callbacks defined at compile time. + * + * Return: + * * 0 on success + * * < 0 on error + */ +int lrng_set_drng_cb(const struct lrng_drng_cb *drng_cb) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH_DRNG)) + return -EOPNOTSUPP; + + if (!drng_cb) + drng_cb = lrng_default_drng_cb; + + mutex_lock(&lrng_crypto_cb_update); + + /* + * If a callback other than the default is set, allow it only to be + * set back to the default callback. This ensures that multiple + * different callbacks can be registered at the same time. If a + * callback different from the current callback and the default + * callback shall be set, the current callback must be deregistered + * (e.g. the kernel module providing it must be unloaded) and the new + * implementation can be registered. + */ + if ((drng_cb != lrng_default_drng_cb) && + (lrng_drng_init->drng_cb != lrng_default_drng_cb)) { + pr_warn("disallow setting new DRNG callbacks, unload the old callbacks first!\n"); + ret = -EINVAL; + goto out; + } + + ret = lrng_switch(drng_cb, lrng_drng_switch); + /* The switch may imply new entropy due to larger DRNG sec strength. */ + if (!ret) + lrng_es_add_entropy(); + +out: + mutex_unlock(&lrng_crypto_cb_update); + return ret; +} +EXPORT_SYMBOL(lrng_set_drng_cb); + +/* + * lrng_set_hash_cb - Register new cryptographic callback functions for hash + * The registering implies that all old hash states are replaced with new + * hash states. + * + * @hash_cb: Callback functions to be registered -- if NULL, use the default + * callbacks defined at compile time. + * + * Return: + * * 0 on success + * * < 0 on error + */ +int lrng_set_hash_cb(const struct lrng_hash_cb *hash_cb) +{ + struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); + int ret; + + if (!IS_ENABLED(CONFIG_LRNG_SWITCH_HASH)) + return -EOPNOTSUPP; + + if (!hash_cb) + hash_cb = lrng_default_hash_cb; + + mutex_lock(&lrng_crypto_cb_update); + + /* Comment from lrng_set_drng_cb applies. */ + if ((hash_cb != lrng_default_hash_cb) && + (lrng_drng_init->hash_cb != lrng_default_hash_cb)) { + pr_warn("disallow setting new hash callbacks, unload the old callbacks first!\n"); + ret = -EINVAL; + goto out; + } + + ret = lrng_switch(hash_cb, lrng_hash_switch); + /* + * The switch may imply new entropy due to larger digest size. But + * it may also offer more room in the aux pool which means we ping + * any waiting entropy providers. + */ + if (!ret) { + lrng_es_add_entropy(); + lrng_writer_wakeup(); + } + +out: + mutex_unlock(&lrng_crypto_cb_update); + return ret; +} +EXPORT_SYMBOL(lrng_set_hash_cb); diff --git a/drivers/char/lrng/lrng_sysctl.c b/drivers/char/lrng/lrng_sysctl.c new file mode 100644 index 0000000000000..ecdd96a842b41 --- /dev/null +++ b/drivers/char/lrng/lrng_sysctl.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG sysctl interfaces + * + * Copyright (C) 2022, Stephan Mueller + */ + +#include +#include +#include +#include + +#include "lrng_drng_mgr.h" +#include "lrng_es_mgr.h" +#include "lrng_sysctl.h" + +/* + * This function is used to return both the bootid UUID, and random + * UUID. The difference is in whether table->data is NULL; if it is, + * then a new UUID is generated and returned to the user. + * + * If the user accesses this via the proc interface, the UUID will be + * returned as an ASCII string in the standard UUID format; if via the + * sysctl system call, as 16 bytes of binary data. + */ +static int lrng_sysctl_do_uuid(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + unsigned char buf[64], tmp_uuid[16], *uuid; + + uuid = table->data; + if (!uuid) { + uuid = tmp_uuid; + generate_random_uuid(uuid); + } else { + static DEFINE_SPINLOCK(bootid_spinlock); + + spin_lock(&bootid_spinlock); + if (!uuid[8]) + generate_random_uuid(uuid); + spin_unlock(&bootid_spinlock); + } + + sprintf(buf, "%pU", uuid); + + fake_table.data = buf; + fake_table.maxlen = sizeof(buf); + + return proc_dostring(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_sysctl_do_entropy(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + int entropy_count = lrng_avail_entropy_aux(); + + fake_table.data = &entropy_count; + fake_table.maxlen = sizeof(entropy_count); + + return proc_dointvec(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_sysctl_do_poolsize(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table fake_table; + u32 entropy_count = lrng_es[lrng_ext_es_aux]->max_entropy(); + + fake_table.data = &entropy_count; + fake_table.maxlen = sizeof(entropy_count); + + return proc_dointvec(&fake_table, write, buffer, lenp, ppos); +} + +static int lrng_min_write_thresh; +static int lrng_max_write_thresh = (LRNG_WRITE_WAKEUP_ENTROPY << 3); +static char lrng_sysctl_bootid[16]; +static int lrng_drng_reseed_max_min; + +void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) +{ + lrng_max_write_thresh = (int)new_digestsize; + /* Ensure that changes to the global variable are visible */ + mb(); +} + +static struct ctl_table random_table[] = { + { + .procname = "poolsize", + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = lrng_sysctl_do_poolsize, + }, + { + .procname = "entropy_avail", + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = lrng_sysctl_do_entropy, + }, + { + .procname = "write_wakeup_threshold", + .data = &lrng_write_wakeup_bits, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &lrng_min_write_thresh, + .extra2 = &lrng_max_write_thresh, + }, + { + .procname = "boot_id", + .data = &lrng_sysctl_bootid, + .maxlen = 16, + .mode = 0444, + .proc_handler = lrng_sysctl_do_uuid, + }, + { + .procname = "uuid", + .maxlen = 16, + .mode = 0444, + .proc_handler = lrng_sysctl_do_uuid, + }, + { + .procname = "urandom_min_reseed_secs", + .data = &lrng_drng_reseed_max_time, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + .extra1 = &lrng_drng_reseed_max_min, + }, + { } +}; + +static int __init random_sysctls_init(void) +{ + register_sysctl_init("kernel/random", random_table); + return 0; +} +device_initcall(random_sysctls_init); diff --git a/drivers/char/lrng/lrng_sysctl.h b/drivers/char/lrng/lrng_sysctl.h new file mode 100644 index 0000000000000..4b487e5077ed9 --- /dev/null +++ b/drivers/char/lrng/lrng_sysctl.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_SYSCTL_H +#define _LRNG_SYSCTL_H + +#ifdef CONFIG_LRNG_SYSCTL +void lrng_sysctl_update_max_write_thresh(u32 new_digestsize); +#else +static inline void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) { } +#endif + +#endif /* _LRNG_SYSCTL_H */ diff --git a/drivers/char/lrng/lrng_testing.c b/drivers/char/lrng/lrng_testing.c new file mode 100644 index 0000000000000..101140085d818 --- /dev/null +++ b/drivers/char/lrng/lrng_testing.c @@ -0,0 +1,901 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * LRNG testing interfaces to obtain raw entropy + * + * Copyright (C) 2022, Stephan Mueller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lrng_definitions.h" +#include "lrng_drng_chacha20.h" +#include "lrng_sha.h" +#include "lrng_testing.h" + +#if defined(CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_PID_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY) || \ + defined(CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY) || \ + defined(CONFIG_LRNG_SCHED_PERF) +#define LRNG_TESTING_USE_BUSYLOOP +#endif + +#ifdef CONFIG_LRNG_TESTING_RECORDING + +#define LRNG_TESTING_RINGBUFFER_SIZE 1024 +#define LRNG_TESTING_RINGBUFFER_MASK (LRNG_TESTING_RINGBUFFER_SIZE - 1) + +struct lrng_testing { + u32 lrng_testing_rb[LRNG_TESTING_RINGBUFFER_SIZE]; + u32 rb_reader; + atomic_t rb_writer; + atomic_t lrng_testing_enabled; + spinlock_t lock; + wait_queue_head_t read_wait; +}; + +/*************************** Generic Data Handling ****************************/ + +/* + * boot variable: + * 0 ==> No boot test, gathering of runtime data allowed + * 1 ==> Boot test enabled and ready for collecting data, gathering runtime + * data is disabled + * 2 ==> Boot test completed and disabled, gathering of runtime data is + * disabled + */ + +static void lrng_testing_reset(struct lrng_testing *data) +{ + unsigned long flags; + + spin_lock_irqsave(&data->lock, flags); + data->rb_reader = 0; + atomic_set(&data->rb_writer, 0); + spin_unlock_irqrestore(&data->lock, flags); +} + +static void lrng_testing_init(struct lrng_testing *data, u32 boot) +{ + /* + * The boot time testing implies we have a running test. If the + * caller wants to clear it, he has to unset the boot_test flag + * at runtime via sysfs to enable regular runtime testing + */ + if (boot) + return; + + lrng_testing_reset(data); + atomic_set(&data->lrng_testing_enabled, 1); + pr_warn("Enabling data collection\n"); +} + +static void lrng_testing_fini(struct lrng_testing *data, u32 boot) +{ + /* If we have boot data, we do not reset yet to allow data to be read */ + if (boot) + return; + + atomic_set(&data->lrng_testing_enabled, 0); + lrng_testing_reset(data); + pr_warn("Disabling data collection\n"); +} + +static bool lrng_testing_store(struct lrng_testing *data, u32 value, + u32 *boot) +{ + unsigned long flags; + + if (!atomic_read(&data->lrng_testing_enabled) && (*boot != 1)) + return false; + + spin_lock_irqsave(&data->lock, flags); + + /* + * Disable entropy testing for boot time testing after ring buffer + * is filled. + */ + if (*boot) { + if (((u32)atomic_read(&data->rb_writer)) > + LRNG_TESTING_RINGBUFFER_SIZE) { + *boot = 2; + pr_warn_once("One time data collection test disabled\n"); + spin_unlock_irqrestore(&data->lock, flags); + return false; + } + + if (atomic_read(&data->rb_writer) == 1) + pr_warn("One time data collection test enabled\n"); + } + + data->lrng_testing_rb[((u32)atomic_read(&data->rb_writer)) & + LRNG_TESTING_RINGBUFFER_MASK] = value; + atomic_inc(&data->rb_writer); + + spin_unlock_irqrestore(&data->lock, flags); + +#ifndef LRNG_TESTING_USE_BUSYLOOP + if (wq_has_sleeper(&data->read_wait)) + wake_up_interruptible(&data->read_wait); +#endif + + return true; +} + +static bool lrng_testing_have_data(struct lrng_testing *data) +{ + return ((((u32)atomic_read(&data->rb_writer)) & + LRNG_TESTING_RINGBUFFER_MASK) != + (data->rb_reader & LRNG_TESTING_RINGBUFFER_MASK)); +} + +static int lrng_testing_reader(struct lrng_testing *data, u32 *boot, + u8 *outbuf, u32 outbuflen) +{ + unsigned long flags; + int collected_data = 0; + + lrng_testing_init(data, *boot); + + while (outbuflen) { + u32 writer = (u32)atomic_read(&data->rb_writer); + + spin_lock_irqsave(&data->lock, flags); + + /* We have no data or reached the writer. */ + if (!writer || (writer == data->rb_reader)) { + + spin_unlock_irqrestore(&data->lock, flags); + + /* + * Now we gathered all boot data, enable regular data + * collection. + */ + if (*boot) { + *boot = 0; + goto out; + } + +#ifdef LRNG_TESTING_USE_BUSYLOOP + while (!lrng_testing_have_data(data)) + ; +#else + wait_event_interruptible(data->read_wait, + lrng_testing_have_data(data)); +#endif + if (signal_pending(current)) { + collected_data = -ERESTARTSYS; + goto out; + } + + continue; + } + + /* We copy out word-wise */ + if (outbuflen < sizeof(u32)) { + spin_unlock_irqrestore(&data->lock, flags); + goto out; + } + + memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], + sizeof(u32)); + data->rb_reader++; + + spin_unlock_irqrestore(&data->lock, flags); + + outbuf += sizeof(u32); + outbuflen -= sizeof(u32); + collected_data += sizeof(u32); + } + +out: + lrng_testing_fini(data, *boot); + return collected_data; +} + +static int lrng_testing_extract_user(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos, + int (*reader)(u8 *outbuf, u32 outbuflen)) +{ + u8 *tmp, *tmp_aligned; + int ret = 0, large_request = (nbytes > 256); + + if (!nbytes) + return 0; + + /* + * The intention of this interface is for collecting at least + * 1000 samples due to the SP800-90B requirements. So, we make no + * effort in avoiding allocating more memory that actually needed + * by the user. Hence, we allocate sufficient memory to always hold + * that amount of data. + */ + tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); + + while (nbytes) { + int i; + + if (large_request && need_resched()) { + if (signal_pending(current)) { + if (ret == 0) + ret = -ERESTARTSYS; + break; + } + schedule(); + } + + i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); + i = reader(tmp_aligned, i); + if (i <= 0) { + if (i < 0) + ret = i; + break; + } + if (copy_to_user(buf, tmp_aligned, i)) { + ret = -EFAULT; + break; + } + + nbytes -= i; + buf += i; + ret += i; + } + + kfree_sensitive(tmp); + + if (ret > 0) + *ppos += ret; + + return ret; +} + +#endif /* CONFIG_LRNG_TESTING_RECORDING */ + +/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY + +static u32 boot_raw_hires_test = 0; +module_param(boot_raw_hires_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); + +static struct lrng_testing lrng_raw_hires = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) +}; + +bool lrng_raw_hires_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); +} + +static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_hires_entropy_reader); +} + +static const struct file_operations lrng_raw_hires_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_hires_read, +}; + +#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + +/********************* Raw Jiffies Entropy Data Handling **********************/ + +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY + +static u32 boot_raw_jiffies_test = 0; +module_param(boot_raw_jiffies_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); + +static struct lrng_testing lrng_raw_jiffies = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) +}; + +bool lrng_raw_jiffies_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_jiffies, value, + &boot_raw_jiffies_test); +} + +static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_jiffies_entropy_reader); +} + +static const struct file_operations lrng_raw_jiffies_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_jiffies_read, +}; + +#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + +/************************** Raw IRQ Data Handling ****************************/ + +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY + +static u32 boot_raw_irq_test = 0; +module_param(boot_raw_irq_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); + +static struct lrng_testing lrng_raw_irq = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) +}; + +bool lrng_raw_irq_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); +} + +static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, + outbuflen); +} + +static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_irq_entropy_reader); +} + +static const struct file_operations lrng_raw_irq_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_irq_read, +}; + +#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + +/************************ Raw _RET_IP_ Data Handling **************************/ + +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY + +static u32 boot_raw_retip_test = 0; +module_param(boot_raw_retip_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); + +static struct lrng_testing lrng_raw_retip = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) +}; + +bool lrng_raw_retip_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); +} + +static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_retip_entropy_reader); +} + +static const struct file_operations lrng_raw_retip_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_retip_read, +}; + +#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + +/********************** Raw IRQ register Data Handling ************************/ + +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY + +static u32 boot_raw_regs_test = 0; +module_param(boot_raw_regs_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); + +static struct lrng_testing lrng_raw_regs = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) +}; + +bool lrng_raw_regs_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); +} + +static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_regs_entropy_reader); +} + +static const struct file_operations lrng_raw_regs_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_regs_read, +}; + +#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + +/********************** Raw Entropy Array Data Handling ***********************/ + +#ifdef CONFIG_LRNG_RAW_ARRAY + +static u32 boot_raw_array = 0; +module_param(boot_raw_array, uint, 0644); +MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); + +static struct lrng_testing lrng_raw_array = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) +}; + +bool lrng_raw_array_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); +} + +static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, + outbuflen); +} + +static ssize_t lrng_raw_array_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_array_entropy_reader); +} + +static const struct file_operations lrng_raw_array_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_array_read, +}; + +#endif /* CONFIG_LRNG_RAW_ARRAY */ + +/******************** Interrupt Performance Data Handling *********************/ + +#ifdef CONFIG_LRNG_IRQ_PERF + +static u32 boot_irq_perf = 0; +module_param(boot_irq_perf, uint, 0644); +MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); + +static struct lrng_testing lrng_irq_perf = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) +}; + +bool lrng_perf_time(u32 start) +{ + return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, + &boot_irq_perf); +} + +static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, + outbuflen); +} + +static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_irq_perf_reader); +} + +static const struct file_operations lrng_irq_perf_fops = { + .owner = THIS_MODULE, + .read = lrng_irq_perf_read, +}; + +#endif /* CONFIG_LRNG_IRQ_PERF */ + +/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ + +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY + +static u32 boot_raw_sched_hires_test = 0; +module_param(boot_raw_sched_hires_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); + +static struct lrng_testing lrng_raw_sched_hires = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), + .read_wait = + __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) +}; + +bool lrng_raw_sched_hires_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_hires, value, + &boot_raw_sched_hires_test); +} + +static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_hires, + &boot_raw_sched_hires_test, + outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_hires_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_hires_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_hires_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ + +/******************** Interrupt Performance Data Handling *********************/ + +#ifdef CONFIG_LRNG_SCHED_PERF + +static u32 boot_sched_perf = 0; +module_param(boot_sched_perf, uint, 0644); +MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); + +static struct lrng_testing lrng_sched_perf = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) +}; + +bool lrng_sched_perf_time(u32 start) +{ + return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, + &boot_sched_perf); +} + +static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, + outbuflen); +} + +static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_sched_perf_reader); +} + +static const struct file_operations lrng_sched_perf_fops = { + .owner = THIS_MODULE, + .read = lrng_sched_perf_read, +}; + +#endif /* CONFIG_LRNG_SCHED_PERF */ + +/*************** Raw Scheduler task_struct->pid Data Handling *****************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY + +static u32 boot_raw_sched_pid_test = 0; +module_param(boot_raw_sched_pid_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_pid = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) +}; + +bool lrng_raw_sched_pid_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_pid, value, + &boot_raw_sched_pid_test); +} + +static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_pid, + &boot_raw_sched_pid_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_pid_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_pid_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_pid_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ + + +/*********** Raw Scheduler task_struct->start_time Data Handling **************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY + +static u32 boot_raw_sched_starttime_test = 0; +module_param(boot_raw_sched_starttime_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_starttime = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) +}; + +bool lrng_raw_sched_starttime_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_starttime, value, + &boot_raw_sched_starttime_test); +} + +static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_starttime, + &boot_raw_sched_starttime_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_starttime_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_starttime_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_starttime_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ + +/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ + +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY + +static u32 boot_raw_sched_nvcsw_test = 0; +module_param(boot_raw_sched_nvcsw_test, uint, 0644); +MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); + +static struct lrng_testing lrng_raw_sched_nvcsw = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), + .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), + .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) +}; + +bool lrng_raw_sched_nvcsw_entropy_store(u32 value) +{ + return lrng_testing_store(&lrng_raw_sched_nvcsw, value, + &boot_raw_sched_nvcsw_test); +} + +static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) +{ + return lrng_testing_reader(&lrng_raw_sched_nvcsw, + &boot_raw_sched_nvcsw_test, outbuf, outbuflen); +} + +static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, + lrng_raw_sched_nvcsw_entropy_reader); +} + +static const struct file_operations lrng_raw_sched_nvcsw_fops = { + .owner = THIS_MODULE, + .read = lrng_raw_sched_nvcsw_read, +}; + +#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ + +/*********************************** ACVT ************************************/ + +#ifdef CONFIG_LRNG_ACVT_HASH + +/* maximum amount of data to be hashed as defined by ACVP */ +#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) + +/* + * As we use static variables to store the data, it is clear that the + * test interface is only able to handle single threaded testing. This is + * considered to be sufficient for testing. If multi-threaded use of the + * ACVT test interface would be performed, the caller would get garbage + * but the kernel operation is unaffected by this. + */ +static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] + __aligned(LRNG_KCAPI_ALIGN); +static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); +static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; + +static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + if (nbytes > LRNG_ACVT_MAX_SHA_MSG) + return -EINVAL; + + atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); + + return simple_write_to_buffer(lrng_acvt_hash_data, + LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); +} + +static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + SHASH_DESC_ON_STACK(shash, NULL); + const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; + ssize_t ret; + + if (count > LRNG_ATOMIC_DIGEST_SIZE) + return -EINVAL; + + ret = hash_cb->hash_init(shash, NULL) ?: + hash_cb->hash_update(shash, lrng_acvt_hash_data, + atomic_read_u32(&lrng_acvt_hash_data_size)) ?: + hash_cb->hash_final(shash, lrng_acvt_hash_digest); + if (ret) + return ret; + + return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, + sizeof(lrng_acvt_hash_digest)); +} + +static const struct file_operations lrng_acvt_hash_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .llseek = default_llseek, + .read = lrng_acvt_hash_read, + .write = lrng_acvt_hash_write, +}; + +#endif /* CONFIG_LRNG_ACVT_DRNG */ + +/************************************************************************** + * Debugfs interface + **************************************************************************/ + +static int __init lrng_raw_init(void) +{ + struct dentry *lrng_raw_debugfs_root; + + lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_hires", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_hires_fops); +#endif +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_jiffies_fops); +#endif +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY + debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, + NULL, &lrng_raw_irq_fops); +#endif +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY + debugfs_create_file_unsafe("lrng_raw_retip", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_retip_fops); +#endif +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY + debugfs_create_file_unsafe("lrng_raw_regs", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_regs_fops); +#endif +#ifdef CONFIG_LRNG_RAW_ARRAY + debugfs_create_file_unsafe("lrng_raw_array", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_array_fops); +#endif +#ifdef CONFIG_LRNG_IRQ_PERF + debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, + NULL, &lrng_irq_perf_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, + lrng_raw_debugfs_root, + NULL, &lrng_raw_sched_hires_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_pid_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_starttime_fops); +#endif +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY + debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_raw_sched_nvcsw_fops); +#endif +#ifdef CONFIG_LRNG_SCHED_PERF + debugfs_create_file_unsafe("lrng_sched_perf", 0400, + lrng_raw_debugfs_root, NULL, + &lrng_sched_perf_fops); +#endif +#ifdef CONFIG_LRNG_ACVT_HASH + debugfs_create_file_unsafe("lrng_acvt_hash", 0600, + lrng_raw_debugfs_root, NULL, + &lrng_acvt_hash_fops); +#endif + + return 0; +} + +module_init(lrng_raw_init); diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h new file mode 100644 index 0000000000000..672a7fca45956 --- /dev/null +++ b/drivers/char/lrng/lrng_testing.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_TESTING_H +#define _LRNG_TESTING_H + +#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY +bool lrng_raw_hires_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ +static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY +bool lrng_raw_jiffies_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ +static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY +bool lrng_raw_irq_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ +static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY +bool lrng_raw_retip_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ +static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY +bool lrng_raw_regs_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ +static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_ARRAY +bool lrng_raw_array_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_ARRAY */ +static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_ARRAY */ + +#ifdef CONFIG_LRNG_IRQ_PERF +bool lrng_perf_time(u32 start); +#else /* CONFIG_LRNG_IRQ_PERF */ +static inline bool lrng_perf_time(u32 start) { return false; } +#endif /*CONFIG_LRNG_IRQ_PERF */ + +#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY +bool lrng_raw_sched_hires_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ +static inline bool +lrng_raw_sched_hires_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY +bool lrng_raw_sched_pid_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ +static inline bool +lrng_raw_sched_pid_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY +bool lrng_raw_sched_starttime_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ +static inline bool +lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ + +#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY +bool lrng_raw_sched_nvcsw_entropy_store(u32 value); +#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ +static inline bool +lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } +#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ + +#ifdef CONFIG_LRNG_SCHED_PERF +bool lrng_sched_perf_time(u32 start); +#else /* CONFIG_LRNG_SCHED_PERF */ +static inline bool lrng_sched_perf_time(u32 start) { return false; } +#endif /*CONFIG_LRNG_SCHED_PERF */ + +#endif /* _LRNG_TESTING_H */ diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index af5ad51d3eef8..b12ae9bdebf43 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -283,4 +283,11 @@ enum drbg_prefixes { DRBG_PREFIX3 }; +extern int drbg_alloc_state(struct drbg_state *drbg); +extern void drbg_dealloc_state(struct drbg_state *drbg); +extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, + bool *pr); +extern const struct drbg_core drbg_cores[]; +extern unsigned short drbg_sec_strength(drbg_flag_t flags); + #endif /* _DRBG_H */ diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h similarity index 100% rename from crypto/jitterentropy.h rename to include/crypto/internal/jitterentropy.h diff --git a/include/linux/lrng.h b/include/linux/lrng.h new file mode 100644 index 0000000000000..8ec50ac7d83c5 --- /dev/null +++ b/include/linux/lrng.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (C) 2022, Stephan Mueller + */ + +#ifndef _LRNG_H +#define _LRNG_H + +#include +#include +#include +#include + +/* + * struct lrng_drng_cb - cryptographic callback functions defining a DRNG + * @drng_name Name of DRNG + * @drng_alloc: Allocate DRNG -- the provided integer should be used for + * sanity checks. + * return: allocated data structure or PTR_ERR on error + * @drng_dealloc: Deallocate DRNG + * @drng_seed: Seed the DRNG with data of arbitrary length drng: is + * pointer to data structure allocated with drng_alloc + * return: >= 0 on success, < 0 on error + * @drng_generate: Generate random numbers from the DRNG with arbitrary + * length + */ +struct lrng_drng_cb { + const char *(*drng_name)(void); + void *(*drng_alloc)(u32 sec_strength); + void (*drng_dealloc)(void *drng); + int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); + int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); +}; + +/* + * struct lrng_hash_cb - cryptographic callback functions defining a hash + * @hash_name Name of Hash used for reading entropy pool arbitrary + * length + * @hash_alloc: Allocate the hash for reading the entropy pool + * return: allocated data structure (NULL is success too) + * or ERR_PTR on error + * @hash_dealloc: Deallocate Hash + * @hash_digestsize: Return the digestsize for the used hash to read out + * entropy pool + * hash: is pointer to data structure allocated with + * hash_alloc + * return: size of digest of hash in bytes + * @hash_init: Initialize hash + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_update: Update hash operation + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_final Final hash operation + * hash: is pointer to data structure allocated with + * hash_alloc + * return: 0 on success, < 0 on error + * @hash_desc_zero Zeroization of hash state buffer + * + * Assumptions: + * + * 1. Hash operation will not sleep + * 2. The hash' volatile state information is provided with *shash by caller. + */ +struct lrng_hash_cb { + const char *(*hash_name)(void); + void *(*hash_alloc)(void); + void (*hash_dealloc)(void *hash); + u32 (*hash_digestsize)(void *hash); + int (*hash_init)(struct shash_desc *shash, void *hash); + int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, + u32 inbuflen); + int (*hash_final)(struct shash_desc *shash, u8 *digest); + void (*hash_desc_zero)(struct shash_desc *shash); +}; + +/* Register cryptographic backend */ +#ifdef CONFIG_LRNG_SWITCH +int lrng_set_drng_cb(const struct lrng_drng_cb *cb); +int lrng_set_hash_cb(const struct lrng_hash_cb *cb); +#else /* CONFIG_LRNG_SWITCH */ +static inline int +lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } +static inline int +lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } +#endif /* CONFIG_LRNG_SWITCH */ + +/* Callback to feed events to the scheduler entropy source */ +#ifdef CONFIG_LRNG_SCHED +extern void add_sched_randomness(const struct task_struct *p, int cpu); +#else +static inline void +add_sched_randomness(const struct task_struct *p, int cpu) { } +#endif + +/* + * lrng_get_random_bytes() - Provider of cryptographic strong random numbers + * for kernel-internal usage. + * + * This function is appropriate for in-kernel use cases operating in atomic + * contexts. It will always use the ChaCha20 DRNG and it may be the case that + * it is not fully seeded when being used. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG_DRNG_ATOMIC +void lrng_get_random_bytes(void *buf, int nbytes); +#endif + +/* + * lrng_get_random_bytes_full() - Provider of cryptographic strong + * random numbers for kernel-internal usage from a fully initialized LRNG. + * + * This function will always return random numbers from a fully seeded and + * fully initialized LRNG. + * + * This function is appropriate only for non-atomic use cases as this + * function may sleep. It provides access to the full functionality of LRNG + * including the switchable DRNG support, that may support other DRNGs such + * as the SP800-90A DRBG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG +void lrng_get_random_bytes_full(void *buf, int nbytes); +#endif + +/* + * lrng_get_random_bytes_min() - Provider of cryptographic strong + * random numbers for kernel-internal usage from at least a minimally seeded + * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C + * oversampling applied in FIPS mode is not applied yet). + * + * This function is appropriate only for non-atomic use cases as this + * function may sleep. It provides access to the full functionality of LRNG + * including the switchable DRNG support, that may support other DRNGs such + * as the SP800-90A DRBG. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + */ +#ifdef CONFIG_LRNG +void lrng_get_random_bytes_min(void *buf, int nbytes); +#endif + +/* + * lrng_get_random_bytes_pr() - Provider of cryptographic strong + * random numbers for kernel-internal usage from a fully initialized LRNG and + * requiring a reseed from the entropy sources before. + * + * This function will always return random numbers from a fully seeded and + * fully initialized LRNG. + * + * This function is appropriate only for non-atomic use cases as this + * function may sleep. It provides access to the full functionality of LRNG + * including the switchable DRNG support, that may support other DRNGs such + * as the SP800-90A DRBG. + * + * This call only returns no more data than entropy was pulled from the + * entropy sources. Thus, it is likely that this call returns less data + * than requested by the caller. Also, the caller MUST be prepared that this + * call returns 0 bytes, i.e. it did not generate data. + * + * @buf: buffer to store the random bytes + * @nbytes: size of the buffer + * + * @return: positive number indicates amount of generated bytes, < 0 on error + */ +#ifdef CONFIG_LRNG +int lrng_get_random_bytes_pr(void *buf, int nbytes); +#endif + +#endif /* _LRNG_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 5555e49c4e120..f60c0c26fdf0b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6,6 +6,7 @@ * * Copyright (C) 1991-2002 Linus Torvalds */ +#include #include #include #include @@ -3574,6 +3575,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) { struct rq *rq; + add_sched_randomness(p, cpu); + if (!schedstat_enabled()) return; -- 2.37.1 From ddc9a7e6058c656e8ea91e2537dc38a27ddd6e73 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:34:44 +0200 Subject: [PATCH 12/16] MG-LRU-le9 Signed-off-by: Peter Jung --- Documentation/admin-guide/mm/index.rst | 1 + Documentation/admin-guide/mm/multigen_lru.rst | 156 + Documentation/admin-guide/sysctl/vm.rst | 66 + Documentation/vm/index.rst | 1 + Documentation/vm/multigen_lru.rst | 159 + arch/Kconfig | 8 + arch/arm64/include/asm/pgtable.h | 15 +- arch/x86/Kconfig | 1 + arch/x86/include/asm/pgtable.h | 9 +- arch/x86/mm/pgtable.c | 5 +- fs/exec.c | 2 + fs/fuse/dev.c | 3 +- include/linux/cgroup.h | 15 +- include/linux/memcontrol.h | 36 + include/linux/mm.h | 11 + include/linux/mm_inline.h | 231 +- include/linux/mm_types.h | 77 + include/linux/mmzone.h | 212 ++ include/linux/nodemask.h | 1 + include/linux/page-flags-layout.h | 16 +- include/linux/page-flags.h | 4 +- include/linux/pgtable.h | 17 +- include/linux/sched.h | 4 + include/linux/swap.h | 4 + kernel/bounds.c | 7 + kernel/cgroup/cgroup-internal.h | 1 - kernel/exit.c | 1 + kernel/fork.c | 9 + kernel/sched/core.c | 1 + kernel/sysctl.c | 21 + mm/Kconfig | 89 + mm/huge_memory.c | 3 +- mm/internal.h | 1 + mm/memcontrol.c | 28 + mm/memory.c | 39 +- mm/mm_init.c | 6 +- mm/mmzone.c | 2 + mm/rmap.c | 6 + mm/swap.c | 52 +- mm/vmscan.c | 3218 ++++++++++++++++- mm/workingset.c | 110 +- 41 files changed, 4403 insertions(+), 245 deletions(-) create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst create mode 100644 Documentation/vm/multigen_lru.rst diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst index c21b5823f1261..2cf5bae620367 100644 --- a/Documentation/admin-guide/mm/index.rst +++ b/Documentation/admin-guide/mm/index.rst @@ -32,6 +32,7 @@ the Linux memory management. idle_page_tracking ksm memory-hotplug + multigen_lru nommu-mmap numa_memory_policy numaperf diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst new file mode 100644 index 0000000000000..6355f2b5019d3 --- /dev/null +++ b/Documentation/admin-guide/mm/multigen_lru.rst @@ -0,0 +1,156 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +Multi-Gen LRU +============= +The multi-gen LRU is an alternative LRU implementation that optimizes +page reclaim and improves performance under memory pressure. Page +reclaim decides the kernel's caching policy and ability to overcommit +memory. It directly impacts the kswapd CPU usage and RAM efficiency. + +Quick start +=========== +Build the kernel with the following configurations. + +* ``CONFIG_LRU_GEN=y`` +* ``CONFIG_LRU_GEN_ENABLED=y`` + +All set! + +Runtime options +=============== +``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the +following subsections. + +Kill switch +----------- +``enabled`` accepts different values to enable or disable the +following components. Its default value depends on +``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled +unless some of them have unforeseen side effects. Writing to +``enabled`` has no effect when a component is not supported by the +hardware, and valid values will be accepted even when the main switch +is off. + +====== =============================================================== +Values Components +====== =============================================================== +0x0001 The main switch for the multi-gen LRU. +0x0002 Clearing the accessed bit in leaf page table entries in large + batches, when MMU sets it (e.g., on x86). This behavior can + theoretically worsen lock contention (mmap_lock). If it is + disabled, the multi-gen LRU will suffer a minor performance + degradation for workloads that contiguously map hot pages, + whose accessed bits can be otherwise cleared by fewer larger + batches. +0x0004 Clearing the accessed bit in non-leaf page table entries as + well, when MMU sets it (e.g., on x86). This behavior was not + verified on x86 varieties other than Intel and AMD. If it is + disabled, the multi-gen LRU will suffer a negligible + performance degradation. +[yYnN] Apply to all the components above. +====== =============================================================== + +E.g., +:: + + echo y >/sys/kernel/mm/lru_gen/enabled + cat /sys/kernel/mm/lru_gen/enabled + 0x0007 + echo 5 >/sys/kernel/mm/lru_gen/enabled + cat /sys/kernel/mm/lru_gen/enabled + 0x0005 + +Thrashing prevention +-------------------- +Personal computers are more sensitive to thrashing because it can +cause janks (lags when rendering UI) and negatively impact user +experience. The multi-gen LRU offers thrashing prevention to the +majority of laptop and desktop users who do not have ``oomd``. + +Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of +``N`` milliseconds from getting evicted. The OOM killer is triggered +if this working set cannot be kept in memory. In other words, this +option works as an adjustable pressure relief valve, and when open, it +terminates applications that are hopefully not being used. + +Based on the average human detectable lag (~100ms), ``N=1000`` usually +eliminates intolerable janks due to thrashing. Larger values like +``N=3000`` make janks less noticeable at the risk of premature OOM +kills. + +The default value ``0`` means disabled. + +Experimental features +===================== +``/sys/kernel/debug/lru_gen`` accepts commands described in the +following subsections. Multiple command lines are supported, so does +concatenation with delimiters ``,`` and ``;``. + +``/sys/kernel/debug/lru_gen_full`` provides additional stats for +debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from +evicted generations in this file. + +Working set estimation +---------------------- +Working set estimation measures how much memory an application needs +in a given time interval, and it is usually done with little impact on +the performance of the application. E.g., data centers want to +optimize job scheduling (bin packing) to improve memory utilizations. +When a new job comes in, the job scheduler needs to find out whether +each server it manages can allocate a certain amount of memory for +this new job before it can pick a candidate. To do so, the job +scheduler needs to estimate the working sets of the existing jobs. + +When it is read, ``lru_gen`` returns a histogram of numbers of pages +accessed over different time intervals for each memcg and node. +``MAX_NR_GENS`` decides the number of bins for each histogram. The +histograms are noncumulative. +:: + + memcg memcg_id memcg_path + node node_id + min_gen_nr age_in_ms nr_anon_pages nr_file_pages + ... + max_gen_nr age_in_ms nr_anon_pages nr_file_pages + +Each bin contains an estimated number of pages that have been accessed +within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages +and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of +the former is the largest and that of the latter is the smallest. + +Users can write ``+ memcg_id node_id max_gen_nr +[can_swap [force_scan]]`` to ``lru_gen`` to create a new generation +``max_gen_nr+1``. ``can_swap`` defaults to the swap setting and, if it +is set to ``1``, it forces the scan of anon pages when swap is off, +and vice versa. ``force_scan`` defaults to ``1`` and, if it is set to +``0``, it employs heuristics to reduce the overhead, which is likely +to reduce the coverage as well. + +A typical use case is that a job scheduler writes to ``lru_gen`` at a +certain time interval to create new generations, and it ranks the +servers it manages based on the sizes of their cold pages defined by +this time interval. + +Proactive reclaim +----------------- +Proactive reclaim induces page reclaim when there is no memory +pressure. It usually targets cold pages only. E.g., when a new job +comes in, the job scheduler wants to proactively reclaim cold pages on +the server it selected to improve the chance of successfully landing +this new job. + +Users can write ``- memcg_id node_id min_gen_nr [swappiness +[nr_to_reclaim]]`` to ``lru_gen`` to evict generations less than or +equal to ``min_gen_nr``. Note that ``min_gen_nr`` should be less than +``max_gen_nr-1`` as ``max_gen_nr`` and ``max_gen_nr-1`` are not fully +aged and therefore cannot be evicted. ``swappiness`` overrides the +default value in ``/proc/sys/vm/swappiness``. ``nr_to_reclaim`` limits +the number of pages to evict. + +A typical use case is that a job scheduler writes to ``lru_gen`` +before it tries to land a new job on a server. If it fails to +materialize enough cold pages because of the overestimation, it +retries on the next server according to the ranking result obtained +from the working set estimation step. This less forceful approach +limits the impacts on the existing jobs. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 5c9aa171a0d34..6b581e64a0850 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -25,6 +25,9 @@ files can be found in mm/swap.c. Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes +- anon_min_kbytes +- clean_low_kbytes +- clean_min_kbytes - compact_memory - compaction_proactiveness - compact_unevictable_allowed @@ -106,6 +109,61 @@ On x86_64 this is about 128MB. Changing this takes effect whenever an application requests memory. +anon_min_kbytes +=============== + +This knob provides *hard* protection of anonymous pages. The anonymous pages +on the current node won't be reclaimed under any conditions when their amount +is below vm.anon_min_kbytes. + +This knob may be used to prevent excessive swap thrashing when anonymous +memory is low (for example, when memory is going to be overfilled by +compressed data of zram module). + +Setting this value too high (close to MemTotal) can result in inability to +swap and can lead to early OOM under memory pressure. + +The default value is defined by CONFIG_ANON_MIN_KBYTES. + + +clean_low_kbytes +================ + +This knob provides *best-effort* protection of clean file pages. The file pages +on the current node won't be reclaimed under memory pressure when the amount of +clean file pages is below vm.clean_low_kbytes *unless* we threaten to OOM. + +Protection of clean file pages using this knob may be used when swapping is +still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory pressure. + +Setting it to a high value may result in a early eviction of anonymous pages +into the swap space by attempting to hold the protected amount of clean file +pages in memory. + +The default value is defined by CONFIG_CLEAN_LOW_KBYTES. + + +clean_min_kbytes +================ + +This knob provides *hard* protection of clean file pages. The file pages on the +current node won't be reclaimed under memory pressure when the amount of clean +file pages is below vm.clean_min_kbytes. + +Hard protection of clean file pages using this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free swap space; + - improve performance in disk cache-bound tasks under memory pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + +Setting it to a high value may result in a early out-of-memory condition due to +the inability to reclaim the protected amount of clean file pages when other +types of pages cannot be reclaimed. + +The default value is defined by CONFIG_CLEAN_MIN_KBYTES. + + compact_memory ============== @@ -912,6 +970,14 @@ be 133 (x + 2x = 200, 2x = 133.33). At 0, the kernel will not initiate swap until the amount of free and file-backed pages is less than the high watermark in a zone. +This knob has no effect if the amount of clean file pages on the current +node is below vm.clean_low_kbytes or vm.clean_min_kbytes. In this case, +only anonymous pages can be reclaimed. + +If the number of anonymous pages on the current node is below +vm.anon_min_kbytes, then only file pages can be reclaimed with +any vm.swappiness value. + unprivileged_userfaultfd ======================== diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst index 575ccd40e30cf..4aa12b8be278d 100644 --- a/Documentation/vm/index.rst +++ b/Documentation/vm/index.rst @@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. ksm memory-model mmu_notifier + multigen_lru numa overcommit-accounting page_migration diff --git a/Documentation/vm/multigen_lru.rst b/Documentation/vm/multigen_lru.rst new file mode 100644 index 0000000000000..d7062c6a89464 --- /dev/null +++ b/Documentation/vm/multigen_lru.rst @@ -0,0 +1,159 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +Multi-Gen LRU +============= +The multi-gen LRU is an alternative LRU implementation that optimizes +page reclaim and improves performance under memory pressure. Page +reclaim decides the kernel's caching policy and ability to overcommit +memory. It directly impacts the kswapd CPU usage and RAM efficiency. + +Design overview +=============== +Objectives +---------- +The design objectives are: + +* Good representation of access recency +* Try to profit from spatial locality +* Fast paths to make obvious choices +* Simple self-correcting heuristics + +The representation of access recency is at the core of all LRU +implementations. In the multi-gen LRU, each generation represents a +group of pages with similar access recency. Generations establish a +(time-based) common frame of reference and therefore help make better +choices, e.g., between different memcgs on a computer or different +computers in a data center (for job scheduling). + +Exploiting spatial locality improves efficiency when gathering the +accessed bit. A rmap walk targets a single page and does not try to +profit from discovering a young PTE. A page table walk can sweep all +the young PTEs in an address space, but the address space can be too +sparse to make a profit. The key is to optimize both methods and use +them in combination. + +Fast paths reduce code complexity and runtime overhead. Unmapped pages +do not require TLB flushes; clean pages do not require writeback. +These facts are only helpful when other conditions, e.g., access +recency, are similar. With generations as a common frame of reference, +additional factors stand out. But obvious choices might not be good +choices; thus self-correction is necessary. + +The benefits of simple self-correcting heuristics are self-evident. +Again, with generations as a common frame of reference, this becomes +attainable. Specifically, pages in the same generation can be +categorized based on additional factors, and a feedback loop can +statistically compare the refault percentages across those categories +and infer which of them are better choices. + +Assumptions +----------- +The protection of hot pages and the selection of cold pages are based +on page access channels and patterns. There are two access channels: + +* Accesses through page tables +* Accesses through file descriptors + +The protection of the former channel is by design stronger because: + +1. The uncertainty in determining the access patterns of the former + channel is higher due to the approximation of the accessed bit. +2. The cost of evicting the former channel is higher due to the TLB + flushes required and the likelihood of encountering the dirty bit. +3. The penalty of underprotecting the former channel is higher because + applications usually do not prepare themselves for major page + faults like they do for blocked I/O. E.g., GUI applications + commonly use dedicated I/O threads to avoid blocking rendering + threads. + +There are also two access patterns: + +* Accesses exhibiting temporal locality +* Accesses not exhibiting temporal locality + +For the reasons listed above, the former channel is assumed to follow +the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is +present, and the latter channel is assumed to follow the latter +pattern unless outlying refaults have been observed. + +Workflow overview +================= +Evictable pages are divided into multiple generations for each +``lruvec``. The youngest generation number is stored in +``lrugen->max_seq`` for both anon and file types as they are aged on +an equal footing. The oldest generation numbers are stored in +``lrugen->min_seq[]`` separately for anon and file types as clean file +pages can be evicted regardless of swap constraints. These three +variables are monotonically increasing. + +Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` +bits in order to fit into the gen counter in ``folio->flags``. Each +truncated generation number is an index to ``lrugen->lists[]``. The +sliding window technique is used to track at least ``MIN_NR_GENS`` and +at most ``MAX_NR_GENS`` generations. The gen counter stores a value +within ``[1, MAX_NR_GENS]`` while a page is on one of +``lrugen->lists[]``; otherwise it stores zero. + +Each generation is divided into multiple tiers. A page accessed ``N`` +times through file descriptors is in tier ``order_base_2(N)``. Unlike +generations, tiers do not have dedicated ``lrugen->lists[]``. In +contrast to moving across generations, which requires the LRU lock, +moving across tiers only involves atomic operations on +``folio->flags`` and therefore has a negligible cost. A feedback loop +modeled after the PID controller monitors refaults over all the tiers +from anon and file types and decides which tiers from which types to +evict or protect. + +There are two conceptually independent procedures: the aging and the +eviction. They form a closed-loop system, i.e., the page reclaim. + +Aging +----- +The aging produces young generations. Given an ``lruvec``, it +increments ``max_seq`` when ``max_seq-min_seq+1`` approaches +``MIN_NR_GENS``. The aging promotes hot pages to the youngest +generation when it finds them accessed through page tables; the +demotion of cold pages happens consequently when it increments +``max_seq``. The aging uses page table walks and rmap walks to find +young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` +and calls ``walk_page_range()`` with each ``mm_struct`` on this list +to scan PTEs, and after each iteration, it increments ``max_seq``. For +the latter, when the eviction walks the rmap and finds a young PTE, +the aging scans the adjacent PTEs. For both, on finding a young PTE, +the aging clears the accessed bit and updates the gen counter of the +page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. + +Eviction +-------- +The eviction consumes old generations. Given an ``lruvec``, it +increments ``min_seq`` when ``lrugen->lists[]`` indexed by +``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to +evict from, it first compares ``min_seq[]`` to select the older type. +If both types are equally old, it selects the one whose first tier has +a lower refault percentage. The first tier contains single-use +unmapped clean pages, which are the best bet. The eviction sorts a +page according to its gen counter if the aging has found this page +accessed through page tables and updated its gen counter. It also +moves a page to the next generation, i.e., ``min_seq+1``, if this page +was accessed multiple times through file descriptors and the feedback +loop has detected outlying refaults from the tier this page is in. To +this end, the feedback loop uses the first tier as the baseline, for +the reason stated earlier. + +Summary +------- +The multi-gen LRU can be disassembled into the following parts: + +* Generations +* Rmap walks +* Page table walks +* Bloom filters +* PID controller + +The aging and the eviction form a producer-consumer model; +specifically, the latter drives the former by the sliding window over +generations. Within the aging, rmap walks drive page table walks by +inserting hot densely populated page tables to the Bloom filters. +Within the eviction, the PID controller uses refaults as the feedback +to select types to evict and tiers to protect. diff --git a/arch/Kconfig b/arch/Kconfig index 71b9272acb28b..3b06abf999440 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1410,6 +1410,14 @@ config DYNAMIC_SIGFRAME config HAVE_ARCH_NODE_DEV_GROUP bool +config ARCH_HAS_NONLEAF_PMD_YOUNG + bool + help + Architectures that select this option are capable of setting the + accessed bit in non-leaf PMD entries when using them as part of linear + address translations. Page table walkers that clear the accessed bit + may use this capability to reduce their search space. + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0b6632f183644..c46399c0500c0 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1066,24 +1066,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, * page after fork() + CoW for pfn mappings. We don't always have a * hardware-managed access flag on arm64. */ -static inline bool arch_faults_on_old_pte(void) -{ - /* The register read below requires a stable CPU to make any sense */ - cant_migrate(); - - return !cpu_has_hw_af(); -} -#define arch_faults_on_old_pte arch_faults_on_old_pte +#define arch_has_hw_pte_young cpu_has_hw_af /* * Experimentally, it's cheap to set the access flag in hardware and we * benefit from prefaulting mappings as 'old' to start with. */ -static inline bool arch_wants_old_prefaulted_pte(void) -{ - return !arch_faults_on_old_pte(); -} -#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte +#define arch_wants_old_prefaulted_pte cpu_has_hw_af static inline bool pud_sect_supported(void) { diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 52a7f91527fe0..83661b6c7b140 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -85,6 +85,7 @@ config X86 select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_PTE_DEVMAP if X86_64 select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 select ARCH_HAS_COPY_MC if X86_64 select ARCH_HAS_SET_MEMORY diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 44e2d6f1dbaa8..5059799bebe36 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { - return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; + return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != + (_KERNPG_TABLE & ~_PAGE_ACCESSED); } static inline unsigned long pages_to_mb(unsigned long npg) @@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) return boot_cpu_has_bug(X86_BUG_L1TF); } -#define arch_faults_on_old_pte arch_faults_on_old_pte -static inline bool arch_faults_on_old_pte(void) +#define arch_has_hw_pte_young arch_has_hw_pte_young +static inline bool arch_has_hw_pte_young(void) { - return false; + return true; } #ifdef CONFIG_PAGE_TABLE_CHECK diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index a932d7712d851..8525f2876fb40 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { @@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, return ret; } +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE int pudp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pud_t *pudp) { diff --git a/fs/exec.c b/fs/exec.c index 778123259e424..39e9bbf864427 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1015,6 +1015,7 @@ static int exec_mmap(struct mm_struct *mm) active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; + lru_gen_add_mm(mm); /* * This prevents preemption while active_mm is being loaded and * it and mm are being updated, which could cause problems for @@ -1030,6 +1031,7 @@ static int exec_mmap(struct mm_struct *mm) tsk->mm->vmacache_seqnum = 0; vmacache_flush(tsk); task_unlock(tsk); + lru_gen_use_mm(mm); if (old_mm) { mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 0e537e580dc1e..5d36015071d22 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -777,7 +777,8 @@ static int fuse_check_page(struct page *page) 1 << PG_active | 1 << PG_workingset | 1 << PG_reclaim | - 1 << PG_waiters))) { + 1 << PG_waiters | + LRU_GEN_MASK | LRU_REFS_MASK))) { dump_page(page, "fuse: trying to steal weird page"); return 1; } diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 0d1ada8968d75..1bc0cabf993ff 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) css_put(&cgrp->self); } +extern struct mutex cgroup_mutex; + +static inline void cgroup_lock(void) +{ + mutex_lock(&cgroup_mutex); +} + +static inline void cgroup_unlock(void) +{ + mutex_unlock(&cgroup_mutex); +} + /** * task_css_set_check - obtain a task's css_set with extra access conditions * @task: the task to obtain css_set for @@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) * as locks used during the cgroup_subsys::attach() methods. */ #ifdef CONFIG_PROVE_RCU -extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; #define task_css_set_check(task, __c) \ rcu_dereference_check((task)->cgroups, \ @@ -708,6 +719,8 @@ struct cgroup; static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } static inline void css_get(struct cgroup_subsys_state *css) {} static inline void css_put(struct cgroup_subsys_state *css) {} +static inline void cgroup_lock(void) {} +static inline void cgroup_unlock(void) {} static inline int cgroup_attach_task_all(struct task_struct *from, struct task_struct *t) { return 0; } static inline int cgroupstats_build(struct cgroupstats *stats, diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9ecead1042b9c..eca62345fdd55 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -350,6 +350,11 @@ struct mem_cgroup { struct deferred_split deferred_split_queue; #endif +#ifdef CONFIG_LRU_GEN + /* per-memcg mm_struct list */ + struct lru_gen_mm_list mm_list; +#endif + struct mem_cgroup_per_node *nodeinfo[]; }; @@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() * * For a kmem folio a caller should hold an rcu read lock to protect memcg * associated with a kmem folio from being released. @@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() * * For a kmem page a caller should hold an rcu read lock to protect memcg * associated with a kmem page from being released. @@ -950,6 +957,23 @@ void unlock_page_memcg(struct page *page); void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); +/* try to stablize folio_memcg() for all the pages in a memcg */ +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) +{ + rcu_read_lock(); + + if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) + return true; + + rcu_read_unlock(); + return false; +} + +static inline void mem_cgroup_unlock_pages(void) +{ + rcu_read_unlock(); +} + /* idx can be of type enum memcg_stat_item or node_stat_item */ static inline void mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) @@ -1401,6 +1425,18 @@ static inline void folio_memcg_unlock(struct folio *folio) { } +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) +{ + /* to match folio_memcg_rcu() */ + rcu_read_lock(); + return true; +} + +static inline void mem_cgroup_unlock_pages(void) +{ + rcu_read_unlock(); +} + static inline void mem_cgroup_handle_over_high(void) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index 7898e29bcfb54..425755adc4559 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -196,6 +196,10 @@ static inline void __mm_zero_struct_page(struct page *page) extern int sysctl_max_map_count; +extern unsigned long sysctl_anon_min_kbytes; +extern unsigned long sysctl_clean_low_kbytes; +extern unsigned long sysctl_clean_min_kbytes; + extern unsigned long sysctl_user_reserve_kbytes; extern unsigned long sysctl_admin_reserve_kbytes; @@ -1060,6 +1064,8 @@ vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) +#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) +#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) /* * Define the bit shifts to access each section. For non-existent @@ -1525,6 +1531,11 @@ static inline unsigned long folio_pfn(struct folio *folio) return page_to_pfn(&folio->page); } +static inline struct folio *pfn_folio(unsigned long pfn) +{ + return page_folio(pfn_to_page(pfn)); +} + static inline atomic_t *folio_pincount_ptr(struct folio *folio) { return &folio_page(folio, 1)->compound_pincount; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 7b25b53c474a7..4949eda9a9a2a 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) return folio_is_file_lru(page_folio(page)); } -static __always_inline void update_lru_size(struct lruvec *lruvec, +static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) { struct pglist_data *pgdat = lruvec_pgdat(lruvec); + lockdep_assert_held(&lruvec->lru_lock); + WARN_ON_ONCE(nr_pages != (int)nr_pages); + __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); __mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LRU_BASE + lru, nr_pages); +} + +static __always_inline void update_lru_size(struct lruvec *lruvec, + enum lru_list lru, enum zone_type zid, + long nr_pages) +{ + __update_lru_size(lruvec, lru, zid, nr_pages); #ifdef CONFIG_MEMCG mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); #endif @@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) return lru; } +#ifdef CONFIG_LRU_GEN + +#ifdef CONFIG_LRU_GEN_ENABLED +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); +} +#else +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); +} +#endif + +static inline bool lru_gen_in_fault(void) +{ + return current->in_lru_fault; +} + +static inline int lru_gen_from_seq(unsigned long seq) +{ + return seq % MAX_NR_GENS; +} + +static inline int lru_hist_from_seq(unsigned long seq) +{ + return seq % NR_HIST_GENS; +} + +static inline int lru_tier_from_refs(int refs) +{ + VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); + + /* see the comment in folio_lru_refs() */ + return order_base_2(refs + 1); +} + +static inline int folio_lru_refs(struct folio *folio) +{ + unsigned long flags = READ_ONCE(folio->flags); + bool workingset = flags & BIT(PG_workingset); + + /* + * Return the number of accesses beyond PG_referenced, i.e., N-1 if the + * total number of accesses is N>1, since N=0,1 both map to the first + * tier. lru_tier_from_refs() will account for this off-by-one. Also see + * the comment on MAX_NR_TIERS. + */ + return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; +} + +static inline int folio_lru_gen(struct folio *folio) +{ + unsigned long flags = READ_ONCE(folio->flags); + + return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + +static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) +{ + unsigned long max_seq = lruvec->lrugen.max_seq; + + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + + /* see the comment on MIN_NR_GENS */ + return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); +} + +static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, + int old_gen, int new_gen) +{ + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + enum lru_list lru = type * LRU_INACTIVE_FILE; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); + + if (old_gen >= 0) + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + if (new_gen >= 0) + WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], + lrugen->nr_pages[new_gen][type][zone] + delta); + + /* addition */ + if (old_gen < 0) { + if (lru_gen_is_active(lruvec, new_gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, delta); + return; + } + + /* deletion */ + if (new_gen < 0) { + if (lru_gen_is_active(lruvec, old_gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, -delta); + return; + } + + /* promotion */ + if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { + __update_lru_size(lruvec, lru, zone, -delta); + __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); + } + + /* demotion requires isolation, e.g., lru_deactivate_fn() */ + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); +} + +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + unsigned long seq; + unsigned long flags; + int gen = folio_lru_gen(folio); + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); + + if (folio_test_unevictable(folio) || !lrugen->enabled) + return false; + /* + * There are three common cases for this page: + * 1. If it's hot, e.g., freshly faulted in or previously hot and + * migrated, add it to the youngest generation. + * 2. If it's cold but can't be evicted immediately, i.e., an anon page + * not in swapcache or a dirty page pending writeback, add it to the + * second oldest generation. + * 3. Everything else (clean, cold) is added to the oldest generation. + */ + if (folio_test_active(folio)) + seq = lrugen->max_seq; + else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || + (folio_test_reclaim(folio) && + (folio_test_dirty(folio) || folio_test_writeback(folio)))) + seq = lrugen->min_seq[type] + 1; + else + seq = lrugen->min_seq[type]; + + gen = lru_gen_from_seq(seq); + flags = (gen + 1UL) << LRU_GEN_PGOFF; + /* see the comment on MIN_NR_GENS about PG_active */ + set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); + + lru_gen_update_size(lruvec, folio, -1, gen); + /* for folio_rotate_reclaimable() */ + if (reclaiming) + list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); + else + list_add(&folio->lru, &lrugen->lists[gen][type][zone]); + + return true; +} + +static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + unsigned long flags; + int gen = folio_lru_gen(folio); + + if (gen < 0) + return false; + + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + + /* for folio_migrate_flags() */ + flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; + flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); + gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + + lru_gen_update_size(lruvec, folio, gen, -1); + list_del(&folio->lru); + + return true; +} + +#else /* !CONFIG_LRU_GEN */ + +static inline bool lru_gen_enabled(void) +{ + return false; +} + +static inline bool lru_gen_in_fault(void) +{ + return false; +} + +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + return false; +} + +static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + return false; +} + +#endif /* CONFIG_LRU_GEN */ + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_add_folio(lruvec, folio, false)) + return; + update_lru_size(lruvec, lru, folio_zonenum(folio), folio_nr_pages(folio)); if (lru != LRU_UNEVICTABLE) @@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_add_folio(lruvec, folio, true)) + return; + update_lru_size(lruvec, lru, folio_zonenum(folio), folio_nr_pages(folio)); /* This is not expected to be used on LRU_UNEVICTABLE */ @@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_del_folio(lruvec, folio, false)) + return; + if (lru != LRU_UNEVICTABLE) list_del(&folio->lru); update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index e6321ec7621d5..194bb1bb2f360 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -3,6 +3,7 @@ #define _LINUX_MM_TYPES_H #include +#include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include @@ -672,6 +674,22 @@ struct mm_struct { */ unsigned long ksm_merging_pages; #endif +#ifdef CONFIG_LRU_GEN + struct { + /* this mm_struct is on lru_gen_mm_list */ + struct list_head list; + /* + * Set when switching to this mm_struct, as a hint of + * whether it has been used since the last time per-node + * page table walkers cleared the corresponding bits. + */ + unsigned long bitmap; +#ifdef CONFIG_MEMCG + /* points to the memcg of "owner" above */ + struct mem_cgroup *memcg; +#endif + } lru_gen; +#endif /* CONFIG_LRU_GEN */ } __randomize_layout; /* @@ -698,6 +716,65 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) return (struct cpumask *)&mm->cpu_bitmap; } +#ifdef CONFIG_LRU_GEN + +struct lru_gen_mm_list { + /* mm_struct list for page table walkers */ + struct list_head fifo; + /* protects the list above */ + spinlock_t lock; +}; + +void lru_gen_add_mm(struct mm_struct *mm); +void lru_gen_del_mm(struct mm_struct *mm); +#ifdef CONFIG_MEMCG +void lru_gen_migrate_mm(struct mm_struct *mm); +#endif + +static inline void lru_gen_init_mm(struct mm_struct *mm) +{ + INIT_LIST_HEAD(&mm->lru_gen.list); + mm->lru_gen.bitmap = 0; +#ifdef CONFIG_MEMCG + mm->lru_gen.memcg = NULL; +#endif +} + +static inline void lru_gen_use_mm(struct mm_struct *mm) +{ + /* unlikely but not a bug when racing with lru_gen_migrate_mm() */ + VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); + + if (!(current->flags & PF_KTHREAD)) + WRITE_ONCE(mm->lru_gen.bitmap, -1); +} + +#else /* !CONFIG_LRU_GEN */ + +static inline void lru_gen_add_mm(struct mm_struct *mm) +{ +} + +static inline void lru_gen_del_mm(struct mm_struct *mm) +{ +} + +#ifdef CONFIG_MEMCG +static inline void lru_gen_migrate_mm(struct mm_struct *mm) +{ +} +#endif + +static inline void lru_gen_init_mm(struct mm_struct *mm) +{ +} + +static inline void lru_gen_use_mm(struct mm_struct *mm) +{ +} + +#endif /* CONFIG_LRU_GEN */ + struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 299259cfe4627..961da7953b025 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -314,6 +314,207 @@ enum lruvec_flags { */ }; +#endif /* !__GENERATING_BOUNDS_H */ + +/* + * Evictable pages are divided into multiple generations. The youngest and the + * oldest generation numbers, max_seq and min_seq, are monotonically increasing. + * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An + * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the + * corresponding generation. The gen counter in folio->flags stores gen+1 while + * a page is on one of lrugen->lists[]. Otherwise it stores 0. + * + * A page is added to the youngest generation on faulting. The aging needs to + * check the accessed bit at least twice before handing this page over to the + * eviction. The first check takes care of the accessed bit set on the initial + * fault; the second check makes sure this page hasn't been used since then. + * This process, AKA second chance, requires a minimum of two generations, + * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive + * LRU, e.g., /proc/vmstat, these two generations are considered active; the + * rest of generations, if they exist, are considered inactive. See + * lru_gen_is_active(). + * + * PG_active is always cleared while a page is on one of lrugen->lists[] so that + * the aging needs not to worry about it. And it's set again when a page + * considered active is isolated for non-reclaiming purposes, e.g., migration. + * See lru_gen_add_folio() and lru_gen_del_folio(). + * + * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits + * in folio->flags. + */ +#define MIN_NR_GENS 2U +#define MAX_NR_GENS 4U + +/* + * Each generation is divided into multiple tiers. A page accessed N times + * through file descriptors is in tier order_base_2(N). A page in the first tier + * (N=0,1) is marked by PG_referenced unless it was faulted in through page + * tables or read ahead. A page in any other tier (N>1) is marked by + * PG_referenced and PG_workingset. This implies a minimum of two tiers is + * supported without using additional bits in folio->flags. + * + * In contrast to moving across generations which requires the LRU lock, moving + * across tiers only involves atomic operations on folio->flags and therefore + * has a negligible cost in the buffered access path. In the eviction path, + * comparisons of refaulted/(evicted+protected) from the first tier and the + * rest infer whether pages accessed multiple times through file descriptors + * are statistically hot and thus worth protecting. + * + * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in + * folio->flags. + */ +#define MAX_NR_TIERS 4U + +#ifndef __GENERATING_BOUNDS_H + +struct lruvec; +struct page_vma_mapped_walk; + +#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) +#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) + +#ifdef CONFIG_LRU_GEN + +enum { + LRU_GEN_ANON, + LRU_GEN_FILE, +}; + +enum { + LRU_GEN_CORE, + LRU_GEN_MM_WALK, + LRU_GEN_NONLEAF_YOUNG, + NR_LRU_GEN_CAPS +}; + +#define MIN_LRU_BATCH BITS_PER_LONG +#define MAX_LRU_BATCH (MIN_LRU_BATCH * 128) + +/* whether to keep historical stats from evicted generations */ +#ifdef CONFIG_LRU_GEN_STATS +#define NR_HIST_GENS MAX_NR_GENS +#else +#define NR_HIST_GENS 1U +#endif + +/* + * The youngest generation number is stored in max_seq for both anon and file + * types as they are aged on an equal footing. The oldest generation numbers are + * stored in min_seq[] separately for anon and file types as clean file pages + * can be evicted regardless of swap constraints. + * + * Normally anon and file min_seq are in sync. But if swapping is constrained, + * e.g., out of swap space, file min_seq is allowed to advance and leave anon + * min_seq behind. + * + * The number of pages in each generation is eventually consistent and therefore + * can be transiently negative when reset_batch_size() is pending. + */ +struct lru_gen_struct { + /* the aging increments the youngest generation number */ + unsigned long max_seq; + /* the eviction increments the oldest generation numbers */ + unsigned long min_seq[ANON_AND_FILE]; + /* the birth time of each generation in jiffies */ + unsigned long timestamps[MAX_NR_GENS]; + /* the multi-gen LRU lists, lazily sorted on eviction */ + struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* the multi-gen LRU sizes, eventually consistent */ + long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* the exponential moving average of refaulted */ + unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; + /* the exponential moving average of evicted+protected */ + unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; + /* the first tier doesn't need protection, hence the minus one */ + unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; + /* can be modified without holding the LRU lock */ + atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; + atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; + /* whether the multi-gen LRU is enabled */ + bool enabled; +}; + +enum { + MM_LEAF_TOTAL, /* total leaf entries */ + MM_LEAF_OLD, /* old leaf entries */ + MM_LEAF_YOUNG, /* young leaf entries */ + MM_NONLEAF_TOTAL, /* total non-leaf entries */ + MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ + MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ + NR_MM_STATS +}; + +/* double-buffering Bloom filters */ +#define NR_BLOOM_FILTERS 2 + +struct lru_gen_mm_state { + /* set to max_seq after each iteration */ + unsigned long seq; + /* where the current iteration continues (inclusive) */ + struct list_head *head; + /* where the last iteration ended (exclusive) */ + struct list_head *tail; + /* to wait for the last page table walker to finish */ + struct wait_queue_head wait; + /* Bloom filters flip after each iteration */ + unsigned long *filters[NR_BLOOM_FILTERS]; + /* the mm stats for debugging */ + unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; + /* the number of concurrent page table walkers */ + int nr_walkers; +}; + +struct lru_gen_mm_walk { + /* the lruvec under reclaim */ + struct lruvec *lruvec; + /* unstable max_seq from lru_gen_struct */ + unsigned long max_seq; + /* the next address within an mm to scan */ + unsigned long next_addr; + /* to batch promoted pages */ + int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* to batch the mm stats */ + int mm_stats[NR_MM_STATS]; + /* total batched items */ + int batched; + bool can_swap; + bool force_scan; +}; + +void lru_gen_init_lruvec(struct lruvec *lruvec); +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); + +#ifdef CONFIG_MEMCG +void lru_gen_init_memcg(struct mem_cgroup *memcg); +void lru_gen_exit_memcg(struct mem_cgroup *memcg); +#endif + +#else /* !CONFIG_LRU_GEN */ + +static inline void lru_gen_init_lruvec(struct lruvec *lruvec) +{ +} + +static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +{ +} + +#ifdef CONFIG_MEMCG +static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) +{ +} + +static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) +{ +} +#endif + +#endif /* CONFIG_LRU_GEN */ + struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ @@ -331,6 +532,12 @@ struct lruvec { unsigned long refaults[ANON_AND_FILE]; /* Various lruvec state flags (enum lruvec_flags) */ unsigned long flags; +#ifdef CONFIG_LRU_GEN + /* evictable pages divided into generations */ + struct lru_gen_struct lrugen; + /* to concurrently iterate lru_gen_mm_list */ + struct lru_gen_mm_state mm_state; +#endif #ifdef CONFIG_MEMCG struct pglist_data *pgdat; #endif @@ -927,6 +1134,11 @@ typedef struct pglist_data { unsigned long flags; +#ifdef CONFIG_LRU_GEN + /* kswap mm walk data */ + struct lru_gen_mm_walk mm_walk; +#endif + ZONE_PADDING(_pad2_) /* Per-node vmstats */ diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 0f233b76c9cec..292ec0ce0d63d 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -485,6 +485,7 @@ static inline int num_node_state(enum node_states state) #define first_online_node 0 #define first_memory_node 0 #define next_online_node(nid) (MAX_NUMNODES) +#define next_memory_node(nid) (MAX_NUMNODES) #define nr_node_ids 1U #define nr_online_nodes 1U diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index ef1e3e736e148..7d79818dc0651 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -55,7 +55,8 @@ #define SECTIONS_WIDTH 0 #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ + <= BITS_PER_LONG - NR_PAGEFLAGS #define NODES_WIDTH NODES_SHIFT #elif defined(CONFIG_SPARSEMEM_VMEMMAP) #error "Vmemmap: No space for nodes field in page flags" @@ -89,8 +90,8 @@ #define LAST_CPUPID_SHIFT 0 #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ - <= BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT #else #define LAST_CPUPID_WIDTH 0 @@ -100,10 +101,15 @@ #define LAST_CPUPID_NOT_IN_PAGE_FLAGS #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ - > BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS #error "Not enough bits in page flags" #endif +/* see the comment on MAX_NR_TIERS */ +#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ + ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ + NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) + #endif #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index e66f7aa3191df..8d466d7248523 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1059,7 +1059,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) 1UL << PG_private | 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ 1UL << PG_slab | 1UL << PG_active | \ - 1UL << PG_unevictable | __PG_MLOCKED) + 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) /* * Flags checked when a page is prepped for return by the page allocator. @@ -1070,7 +1070,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) * alloc-free cycle to prevent from reusing the page. */ #define PAGE_FLAGS_CHECK_AT_PREP \ - (PAGEFLAGS_MASK & ~__PG_HWPOISON) + ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) #define PAGE_FLAGS_PRIVATE \ (1UL << PG_private | 1UL << PG_private_2) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 3cdc16cfd8673..9c57c5cc49c27 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, #endif #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) @@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, BUILD_BUG(); return 0; } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ #endif #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH @@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif +#ifndef arch_has_hw_pte_young +/* + * Return whether the accessed bit is supported on the local CPU. + * + * This stub assumes accessing through an old PTE triggers a page fault. + * Architectures that automatically set the access bit should overwrite it. + */ +static inline bool arch_has_hw_pte_young(void) +{ + return false; +} +#endif + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, diff --git a/include/linux/sched.h b/include/linux/sched.h index 0b3052d3a0511..4bc88e979b2f4 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -913,6 +913,10 @@ struct task_struct { #ifdef CONFIG_MEMCG unsigned in_user_fault:1; #endif +#ifdef CONFIG_LRU_GEN + /* whether the LRU algorithm may apply to this access */ + unsigned in_lru_fault:1; +#endif #ifdef CONFIG_COMPAT_BRK unsigned brk_randomized:1; #endif diff --git a/include/linux/swap.h b/include/linux/swap.h index 0c0fed1b348f2..b66cbc7ea93cf 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -162,6 +162,10 @@ union swap_header { */ struct reclaim_state { unsigned long reclaimed_slab; +#ifdef CONFIG_LRU_GEN + /* per-thread mm walk data */ + struct lru_gen_mm_walk *mm_walk; +#endif }; #ifdef __KERNEL__ diff --git a/kernel/bounds.c b/kernel/bounds.c index 9795d75b09b23..b529182e8b04f 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -22,6 +22,13 @@ int main(void) DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); #endif DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); +#ifdef CONFIG_LRU_GEN + DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); + DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); +#else + DEFINE(LRU_GEN_WIDTH, 0); + DEFINE(__LRU_REFS_WIDTH, 0); +#endif /* End of constants */ return 0; diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 5da09c74228d0..c966e55cab29d 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -164,7 +164,6 @@ struct cgroup_mgctx { #define DEFINE_CGROUP_MGCTX(name) \ struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) -extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct cgroup_subsys *cgroup_subsys[]; extern struct list_head cgroup_roots; diff --git a/kernel/exit.c b/kernel/exit.c index 64c938ce36fe8..c9ffa2bdb535d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) goto retry; } WRITE_ONCE(mm->owner, c); + lru_gen_migrate_mm(mm); task_unlock(c); put_task_struct(c); } diff --git a/kernel/fork.c b/kernel/fork.c index e5ba00a7e3b27..b0456bd944ff3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1156,6 +1156,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, goto fail_nocontext; mm->user_ns = get_user_ns(user_ns); + lru_gen_init_mm(mm); return mm; fail_nocontext: @@ -1198,6 +1199,7 @@ static inline void __mmput(struct mm_struct *mm) } if (mm->binfmt) module_put(mm->binfmt->module); + lru_gen_del_mm(mm); mmdrop(mm); } @@ -2684,6 +2686,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) get_task_struct(p); } + if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { + /* lock the task to synchronize with memcg migration */ + task_lock(p); + lru_gen_add_mm(p->mm); + task_unlock(p); + } + wake_up_new_task(p); /* forking complete and child started to run, tell ptracer */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f60c0c26fdf0b..98db3c8171602 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5133,6 +5133,7 @@ context_switch(struct rq *rq, struct task_struct *prev, * finish_task_switch()'s mmdrop(). */ switch_mm_irqs_off(prev->active_mm, next->mm, next); + lru_gen_use_mm(next->mm); if (!prev->mm) { // from kernel /* will mmdrop() in finish_task_switch(). */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5e35535807e41..80281b08b55fb 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2379,6 +2379,27 @@ static struct ctl_table vm_table[] = { .extra2 = SYSCTL_ONE, }, #endif + { + .procname = "anon_min_kbytes", + .data = &sysctl_anon_min_kbytes, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "clean_low_kbytes", + .data = &sysctl_clean_low_kbytes, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "clean_min_kbytes", + .data = &sysctl_clean_min_kbytes, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, { .procname = "user_reserve_kbytes", .data = &sysctl_user_reserve_kbytes, diff --git a/mm/Kconfig b/mm/Kconfig index 169e64192e484..fd2c21c64a501 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -441,6 +441,69 @@ config SPARSEMEM_VMEMMAP pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. +config ANON_MIN_KBYTES + int "Default value for vm.anon_min_kbytes" + depends on SYSCTL + range 0 4294967295 + default 0 + help + This option sets the default value for vm.anon_min_kbytes sysctl knob. + + The vm.anon_min_kbytes sysctl knob provides *hard* protection of + anonymous pages. The anonymous pages on the current node won't be + reclaimed under any conditions when their amount is below + vm.anon_min_kbytes. This knob may be used to prevent excessive swap + thrashing when anonymous memory is low (for example, when memory is + going to be overfilled by compressed data of zram module). + + Setting this value too high (close to MemTotal) can result in + inability to swap and can lead to early OOM under memory pressure. + +config CLEAN_LOW_KBYTES + int "Default value for vm.clean_low_kbytes" + depends on SYSCTL + range 0 4294967295 + default 524288 + help + This option sets the default value for vm.clean_low_kbytes sysctl knob. + + The vm.clean_low_kbytes sysctl knob provides *best-effort* + protection of clean file pages. The file pages on the current node + won't be reclaimed under memory pressure when the amount of clean file + pages is below vm.clean_low_kbytes *unless* we threaten to OOM. + Protection of clean file pages using this knob may be used when + swapping is still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory + pressure. + + Setting it to a high value may result in a early eviction of anonymous + pages into the swap space by attempting to hold the protected amount + of clean file pages in memory. + +config CLEAN_MIN_KBYTES + int "Default value for vm.clean_min_kbytes" + depends on SYSCTL + range 0 4294967295 + default 0 + help + This option sets the default value for vm.clean_min_kbytes sysctl knob. + + The vm.clean_min_kbytes sysctl knob provides *hard* protection of + clean file pages. The file pages on the current node won't be + reclaimed under memory pressure when the amount of clean file pages is + below vm.clean_min_kbytes. Hard protection of clean file pages using + this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free + swap space; + - improve performance in disk cache-bound tasks under memory + pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + + Setting it to a high value may result in a early out-of-memory condition + due to the inability to reclaim the protected amount of clean file pages + when other types of pages cannot be reclaimed. + config HAVE_MEMBLOCK_PHYS_MAP bool @@ -1130,6 +1193,32 @@ config PTE_MARKER_UFFD_WP purposes. It is required to enable userfaultfd write protection on file-backed memory types like shmem and hugetlbfs. +# multi-gen LRU { +config LRU_GEN + bool "Multi-Gen LRU" + depends on MMU + # make sure folio->flags has enough spare bits + depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP + help + A high performance LRU implementation to overcommit memory. See + Documentation/admin-guide/mm/multigen_lru.rst for details. + +config LRU_GEN_ENABLED + bool "Enable by default" + depends on LRU_GEN + help + This option enables the multi-gen LRU by default. + +config LRU_GEN_STATS + bool "Full stats for debugging" + depends on LRU_GEN + help + Do not enable this option unless you plan to look at historical stats + from evicted generations for debugging purpose. + + This option has a per-memcg and per-node memory overhead. +# } + source "mm/damon/Kconfig" endmenu diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 834f288b37690..5500583e35b8b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2370,7 +2370,8 @@ static void __split_huge_page_tail(struct page *head, int tail, #ifdef CONFIG_64BIT (1L << PG_arch_2) | #endif - (1L << PG_dirty))); + (1L << PG_dirty) | + LRU_GEN_MASK | LRU_REFS_MASK)); /* ->mapping in first tail page is compound_mapcount */ VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, diff --git a/mm/internal.h b/mm/internal.h index c0f8fbe0445b5..3d070582052ed 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); void folio_rotate_reclaimable(struct folio *folio); bool __folio_end_writeback(struct folio *folio); void deactivate_file_folio(struct folio *folio); +void folio_activate(struct folio *folio); void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, unsigned long floor, unsigned long ceiling); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b6888b0486b67..cefb6523d2d8c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2777,6 +2777,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() */ folio->memcg_data = (unsigned long)memcg; } @@ -5105,6 +5106,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) static void mem_cgroup_free(struct mem_cgroup *memcg) { + lru_gen_exit_memcg(memcg); memcg_wb_domain_exit(memcg); __mem_cgroup_free(memcg); } @@ -5163,6 +5165,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) memcg->deferred_split_queue.split_queue_len = 0; #endif idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); + lru_gen_init_memcg(memcg); return memcg; fail: mem_cgroup_id_remove(memcg); @@ -6130,6 +6133,30 @@ static void mem_cgroup_move_task(void) } #endif +#ifdef CONFIG_LRU_GEN +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + + /* find the first leader if there is any */ + cgroup_taskset_for_each_leader(task, css, tset) + break; + + if (!task) + return; + + task_lock(task); + if (task->mm && task->mm->owner == task) + lru_gen_migrate_mm(task->mm); + task_unlock(task); +} +#else +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ +} +#endif /* CONFIG_LRU_GEN */ + static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) { if (value == PAGE_COUNTER_MAX) @@ -6533,6 +6560,7 @@ struct cgroup_subsys memory_cgrp_subsys = { .css_reset = mem_cgroup_css_reset, .css_rstat_flush = mem_cgroup_css_rstat_flush, .can_attach = mem_cgroup_can_attach, + .attach = mem_cgroup_attach, .cancel_attach = mem_cgroup_cancel_attach, .post_attach = mem_cgroup_move_task, .dfl_cftypes = memory_files, diff --git a/mm/memory.c b/mm/memory.c index 1c6027adc5426..d8c5632ae5548 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = 2; #endif -#ifndef arch_faults_on_old_pte -static inline bool arch_faults_on_old_pte(void) -{ - /* - * Those arches which don't have hw access flag feature need to - * implement their own helper. By default, "true" means pagefault - * will be hit on old pte. - */ - return true; -} -#endif - #ifndef arch_wants_old_prefaulted_pte static inline bool arch_wants_old_prefaulted_pte(void) { @@ -2862,7 +2850,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, * On architectures with software "accessed" bits, we would * take a double page fault, so mark it accessed here. */ - if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { + if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { pte_t entry; vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); @@ -5107,6 +5095,27 @@ static inline void mm_account_fault(struct pt_regs *regs, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } +#ifdef CONFIG_LRU_GEN +static void lru_gen_enter_fault(struct vm_area_struct *vma) +{ + /* the LRU algorithm doesn't apply to sequential or random reads */ + current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); +} + +static void lru_gen_exit_fault(void) +{ + current->in_lru_fault = false; +} +#else +static void lru_gen_enter_fault(struct vm_area_struct *vma) +{ +} + +static void lru_gen_exit_fault(void) +{ +} +#endif /* CONFIG_LRU_GEN */ + /* * By the time we get here, we already hold the mm semaphore * @@ -5138,11 +5147,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, if (flags & FAULT_FLAG_USER) mem_cgroup_enter_user_fault(); + lru_gen_enter_fault(vma); + if (unlikely(is_vm_hugetlb_page(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags); else ret = __handle_mm_fault(vma, address, flags); + lru_gen_exit_fault(); + if (flags & FAULT_FLAG_USER) { mem_cgroup_exit_user_fault(); /* diff --git a/mm/mm_init.c b/mm/mm_init.c index 9ddaf0e1b0ab9..0d7b2bd2454a1 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) shift = 8 * sizeof(unsigned long); width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; + - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", - "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", SECTIONS_WIDTH, NODES_WIDTH, ZONES_WIDTH, LAST_CPUPID_WIDTH, KASAN_TAG_WIDTH, + LRU_GEN_WIDTH, + LRU_REFS_WIDTH, NR_PAGEFLAGS); mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", diff --git a/mm/mmzone.c b/mm/mmzone.c index 0ae7571e35abb..68e1511be12de 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) * Poison its list head, so that any operations on it would crash. */ list_del(&lruvec->lists[LRU_UNEVICTABLE]); + + lru_gen_init_lruvec(lruvec); } #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) diff --git a/mm/rmap.c b/mm/rmap.c index 746c05acad270..301ccb4cf0e69 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -830,6 +830,12 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte) { + if (lru_gen_enabled() && pte_young(*pvmw.pte) && + !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { + lru_gen_look_around(&pvmw); + referenced++; + } + if (ptep_clear_flush_young_notify(vma, address, pvmw.pte)) { /* diff --git a/mm/swap.c b/mm/swap.c index 034bb24879a3b..131fc76242a37 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -342,7 +342,7 @@ static bool need_activate_page_drain(int cpu) return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0; } -static void folio_activate(struct folio *folio) +void folio_activate(struct folio *folio) { if (folio_test_lru(folio) && !folio_test_active(folio) && !folio_test_unevictable(folio)) { @@ -362,7 +362,7 @@ static inline void activate_page_drain(int cpu) { } -static void folio_activate(struct folio *folio) +void folio_activate(struct folio *folio) { struct lruvec *lruvec; @@ -405,6 +405,40 @@ static void __lru_cache_activate_folio(struct folio *folio) local_unlock(&lru_pvecs.lock); } +#ifdef CONFIG_LRU_GEN +static void folio_inc_refs(struct folio *folio) +{ + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + if (folio_test_unevictable(folio)) + return; + + if (!folio_test_referenced(folio)) { + folio_set_referenced(folio); + return; + } + + if (!folio_test_workingset(folio)) { + folio_set_workingset(folio); + return; + } + + /* see the comment on MAX_NR_TIERS */ + do { + new_flags = old_flags & LRU_REFS_MASK; + if (new_flags == LRU_REFS_MASK) + break; + + new_flags += BIT(LRU_REFS_PGOFF); + new_flags |= old_flags & ~LRU_REFS_MASK; + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); +} +#else +static void folio_inc_refs(struct folio *folio) +{ +} +#endif /* CONFIG_LRU_GEN */ + /* * Mark a page as having seen activity. * @@ -417,6 +451,11 @@ static void __lru_cache_activate_folio(struct folio *folio) */ void folio_mark_accessed(struct folio *folio) { + if (lru_gen_enabled()) { + folio_inc_refs(folio); + return; + } + if (!folio_test_referenced(folio)) { folio_set_referenced(folio); } else if (folio_test_unevictable(folio)) { @@ -460,6 +499,11 @@ void folio_add_lru(struct folio *folio) VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio); VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + /* see the comment in lru_gen_add_folio() */ + if (lru_gen_enabled() && !folio_test_unevictable(folio) && + lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) + folio_set_active(folio); + folio_get(folio); local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); @@ -551,7 +595,7 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec) { - if (PageActive(page) && !PageUnevictable(page)) { + if (!PageUnevictable(page) && (PageActive(page) || lru_gen_enabled())) { int nr_pages = thp_nr_pages(page); del_page_from_lru_list(page, lruvec); @@ -666,7 +710,7 @@ void deactivate_file_folio(struct folio *folio) */ void deactivate_page(struct page *page) { - if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { + if (PageLRU(page) && !PageUnevictable(page) && (PageActive(page) || lru_gen_enabled())) { struct pagevec *pvec; local_lock(&lru_pvecs.lock); diff --git a/mm/vmscan.c b/mm/vmscan.c index 82b19744d7b0d..599211074f6fd 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -50,6 +50,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -124,9 +128,25 @@ struct scan_control { /* The file pages on the current node are dangerously low */ unsigned int file_is_tiny:1; + /* The anonymous pages on the current node are below vm.anon_min_kbytes */ + unsigned int anon_below_min:1; + + /* The clean file pages on the current node are below vm.clean_low_kbytes */ + unsigned int clean_below_low:1; + + /* The clean file pages on the current node are below vm.clean_min_kbytes */ + unsigned int clean_below_min:1; + /* Always discard instead of demoting to lower tier memory */ unsigned int no_demotion:1; +#ifdef CONFIG_LRU_GEN + /* help make better choices when multiple memcgs are available */ + unsigned int memcgs_need_aging:1; + unsigned int memcgs_need_swapping:1; + unsigned int memcgs_avoid_swapping:1; +#endif + /* Allocation order */ s8 order; @@ -173,6 +193,10 @@ struct scan_control { #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0) #endif +unsigned long sysctl_anon_min_kbytes __read_mostly = CONFIG_ANON_MIN_KBYTES; +unsigned long sysctl_clean_low_kbytes __read_mostly = CONFIG_CLEAN_LOW_KBYTES; +unsigned long sysctl_clean_min_kbytes __read_mostly = CONFIG_CLEAN_MIN_KBYTES; + /* * From 0 .. 200. Higher means more swappy. */ @@ -1277,9 +1301,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, if (folio_test_swapcache(folio)) { swp_entry_t swap = folio_swap_entry(folio); - mem_cgroup_swapout(folio, swap); + + /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ if (reclaimed && !mapping_exiting(mapping)) shadow = workingset_eviction(folio, target_memcg); + mem_cgroup_swapout(folio, swap); __delete_from_swap_cache(&folio->page, swap, shadow); xa_unlock_irq(&mapping->i_pages); put_swap_page(&folio->page, swap); @@ -1576,6 +1602,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, if (!sc->may_unmap && folio_mapped(folio)) goto keep_locked; + /* folio_update_gen() tried to promote this page? */ + if (lru_gen_enabled() && !ignore_references && + folio_mapped(folio) && folio_test_referenced(folio)) + goto keep_locked; + /* * The number of dirty pages determines if a node is marked * reclaim_congested. kswapd will stall and start writing @@ -2674,6 +2705,162 @@ enum scan_balance { SCAN_FILE, }; +static void prepare_workingset_protection(pg_data_t *pgdat, struct scan_control *sc) +{ + /* + * Check the number of anonymous pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_anon_min_kbytes) { + unsigned long reclaimable_anon; + + reclaimable_anon = + node_page_state(pgdat, NR_ACTIVE_ANON) + + node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_ISOLATED_ANON); + reclaimable_anon <<= (PAGE_SHIFT - 10); + + sc->anon_below_min = reclaimable_anon < sysctl_anon_min_kbytes; + } else + sc->anon_below_min = 0; + + /* + * Check the number of clean file pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_clean_low_kbytes || sysctl_clean_min_kbytes) { + unsigned long reclaimable_file, dirty, clean; + + reclaimable_file = + node_page_state(pgdat, NR_ACTIVE_FILE) + + node_page_state(pgdat, NR_INACTIVE_FILE) + + node_page_state(pgdat, NR_ISOLATED_FILE); + dirty = node_page_state(pgdat, NR_FILE_DIRTY); + /* + * node_page_state() sum can go out of sync since + * all the values are not read at once. + */ + if (likely(reclaimable_file > dirty)) + clean = (reclaimable_file - dirty) << (PAGE_SHIFT - 10); + else + clean = 0; + + sc->clean_below_low = clean < sysctl_clean_low_kbytes; + sc->clean_below_min = clean < sysctl_clean_min_kbytes; + } else { + sc->clean_below_low = 0; + sc->clean_below_min = 0; + } +} + +static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) +{ + unsigned long file; + struct lruvec *target_lruvec; + + if (lru_gen_enabled()) + return; + + target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + + /* + * Flush the memory cgroup stats, so that we read accurate per-memcg + * lruvec stats for heuristics. + */ + mem_cgroup_flush_stats(); + + /* + * Determine the scan balance between anon and file LRUs. + */ + spin_lock_irq(&target_lruvec->lru_lock); + sc->anon_cost = target_lruvec->anon_cost; + sc->file_cost = target_lruvec->file_cost; + spin_unlock_irq(&target_lruvec->lru_lock); + + /* + * Target desirable inactive:active list ratios for the anon + * and file LRU lists. + */ + if (!sc->force_deactivate) { + unsigned long refaults; + + refaults = lruvec_page_state(target_lruvec, + WORKINGSET_ACTIVATE_ANON); + if (refaults != target_lruvec->refaults[0] || + inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) + sc->may_deactivate |= DEACTIVATE_ANON; + else + sc->may_deactivate &= ~DEACTIVATE_ANON; + + /* + * When refaults are being observed, it means a new + * workingset is being established. Deactivate to get + * rid of any stale active pages quickly. + */ + refaults = lruvec_page_state(target_lruvec, + WORKINGSET_ACTIVATE_FILE); + if (refaults != target_lruvec->refaults[1] || + inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) + sc->may_deactivate |= DEACTIVATE_FILE; + else + sc->may_deactivate &= ~DEACTIVATE_FILE; + } else + sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; + + /* + * If we have plenty of inactive file pages that aren't + * thrashing, try to reclaim those first before touching + * anonymous pages. + */ + file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); + if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) + sc->cache_trim_mode = 1; + else + sc->cache_trim_mode = 0; + + /* + * Prevent the reclaimer from falling into the cache trap: as + * cache pages start out inactive, every cache fault will tip + * the scan balance towards the file LRU. And as the file LRU + * shrinks, so does the window for rotation from references. + * This means we have a runaway feedback loop where a tiny + * thrashing file LRU becomes infinitely more attractive than + * anon pages. Try to detect this based on file LRU size. + */ + if (!cgroup_reclaim(sc)) { + unsigned long total_high_wmark = 0; + unsigned long free, anon; + int z; + + free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); + file = node_page_state(pgdat, NR_ACTIVE_FILE) + + node_page_state(pgdat, NR_INACTIVE_FILE); + + for (z = 0; z < MAX_NR_ZONES; z++) { + struct zone *zone = &pgdat->node_zones[z]; + + if (!managed_zone(zone)) + continue; + + total_high_wmark += high_wmark_pages(zone); + } + + /* + * Consider anon: if that's low too, this isn't a + * runaway file reclaim problem, but rather just + * extreme pressure. Reclaim as per usual then. + */ + anon = node_page_state(pgdat, NR_INACTIVE_ANON); + + sc->file_is_tiny = + file + free <= total_high_wmark && + !(sc->may_deactivate & DEACTIVATE_ANON) && + anon >> sc->priority; + } + + prepare_workingset_protection(pgdat, sc); +} + /* * Determine how aggressively the anon and file LRU lists should be * scanned. @@ -2730,6 +2917,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } + /* + * Force-scan anon if clean file pages is under vm.clean_low_kbytes + * or vm.clean_min_kbytes. + */ + if (sc->clean_below_low || sc->clean_below_min) { + scan_balance = SCAN_ANON; + goto out; + } + /* * If there is enough inactive page cache, we do not reclaim * anything from the anonymous working right now. @@ -2874,6 +3070,25 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, BUG(); } + /* + * Hard protection of the working set. + */ + if (file) { + /* + * Don't reclaim file pages when the amount of + * clean file pages is below vm.clean_min_kbytes. + */ + if (sc->clean_below_min) + scan = 0; + } else { + /* + * Don't reclaim anonymous pages when their + * amount is below vm.anon_min_kbytes. + */ + if (sc->anon_below_min) + scan = 0; + } + nr[lru] = scan; } } @@ -2893,113 +3108,2818 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, return can_demote(pgdat->node_id, sc); } -static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -{ - unsigned long nr[NR_LRU_LISTS]; - unsigned long targets[NR_LRU_LISTS]; - unsigned long nr_to_scan; - enum lru_list lru; - unsigned long nr_reclaimed = 0; - unsigned long nr_to_reclaim = sc->nr_to_reclaim; - struct blk_plug plug; - bool scan_adjusted; +#ifdef CONFIG_LRU_GEN - get_scan_count(lruvec, sc, nr); +#ifdef CONFIG_LRU_GEN_ENABLED +DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) +#else +DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) +#endif - /* Record the original scan target for proportional adjustments later */ - memcpy(targets, nr, sizeof(nr)); +/****************************************************************************** + * shorthand helpers + ******************************************************************************/ - /* - * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal - * event that can occur when there is little memory pressure e.g. - * multiple streaming readers/writers. Hence, we do not abort scanning - * when the requested number of pages are reclaimed when scanning at - * DEF_PRIORITY on the assumption that the fact we are direct - * reclaiming implies that kswapd is not keeping up and it is best to - * do a batch of work at once. For memcg reclaim one check is made to - * abort proportional reclaim if either the file or anon lru has already - * dropped to zero at the first pass. - */ - scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && - sc->priority == DEF_PRIORITY); +#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) - blk_start_plug(&plug); - while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || - nr[LRU_INACTIVE_FILE]) { - unsigned long nr_anon, nr_file, percentage; - unsigned long nr_scanned; +#define DEFINE_MAX_SEQ(lruvec) \ + unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) - for_each_evictable_lru(lru) { - if (nr[lru]) { - nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); - nr[lru] -= nr_to_scan; +#define DEFINE_MIN_SEQ(lruvec) \ + unsigned long min_seq[ANON_AND_FILE] = { \ + READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ + READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ + } - nr_reclaimed += shrink_list(lru, nr_to_scan, - lruvec, sc); - } - } +#define for_each_gen_type_zone(gen, type, zone) \ + for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ + for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ + for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) - cond_resched(); +static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) +{ + struct pglist_data *pgdat = NODE_DATA(nid); - if (nr_reclaimed < nr_to_reclaim || scan_adjusted) - continue; +#ifdef CONFIG_MEMCG + if (memcg) { + struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; - /* - * For kswapd and memcg, reclaim at least the number of pages - * requested. Ensure that the anon and file LRUs are scanned - * proportionally what was requested by get_scan_count(). We - * stop reclaiming one LRU and reduce the amount scanning - * proportional to the original scan target. - */ - nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; - nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; + /* for hotadd_new_pgdat() */ + if (!lruvec->pgdat) + lruvec->pgdat = pgdat; - /* - * It's just vindictive to attack the larger once the smaller - * has gone to zero. And given the way we stop scanning the - * smaller below, this makes sure that we only make one nudge - * towards proportionality once we've got nr_to_reclaim. - */ - if (!nr_file || !nr_anon) - break; + return lruvec; + } +#endif + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); - if (nr_file > nr_anon) { - unsigned long scan_target = targets[LRU_INACTIVE_ANON] + - targets[LRU_ACTIVE_ANON] + 1; - lru = LRU_BASE; - percentage = nr_anon * 100 / scan_target; - } else { - unsigned long scan_target = targets[LRU_INACTIVE_FILE] + - targets[LRU_ACTIVE_FILE] + 1; - lru = LRU_FILE; - percentage = nr_file * 100 / scan_target; - } + return pgdat ? &pgdat->__lruvec : NULL; +} - /* Stop scanning the smaller of the LRU */ - nr[lru] = 0; - nr[lru + LRU_ACTIVE] = 0; +static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) +{ + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct pglist_data *pgdat = lruvec_pgdat(lruvec); - /* - * Recalculate the other LRU scan count based on its original - * scan target and the percentage scanning already complete - */ - lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; - nr_scanned = targets[lru] - nr[lru]; - nr[lru] = targets[lru] * (100 - percentage) / 100; - nr[lru] -= min(nr[lru], nr_scanned); + if (!can_demote(pgdat->node_id, sc) && + mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) + return 0; - lru += LRU_ACTIVE; - nr_scanned = targets[lru] - nr[lru]; - nr[lru] = targets[lru] * (100 - percentage) / 100; - nr[lru] -= min(nr[lru], nr_scanned); + return mem_cgroup_swappiness(memcg); +} - scan_adjusted = true; - } - blk_finish_plug(&plug); - sc->nr_reclaimed += nr_reclaimed; +static int get_nr_gens(struct lruvec *lruvec, int type) +{ + return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; +} - /* - * Even if we did not try to evict anon pages at all, we want to +static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) +{ + /* see the comment on lru_gen_struct */ + return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && + get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && + get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; +} + +/****************************************************************************** + * mm_struct list + ******************************************************************************/ + +static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) +{ + static struct lru_gen_mm_list mm_list = { + .fifo = LIST_HEAD_INIT(mm_list.fifo), + .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), + }; + +#ifdef CONFIG_MEMCG + if (memcg) + return &memcg->mm_list; +#endif + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); + + return &mm_list; +} + +void lru_gen_add_mm(struct mm_struct *mm) +{ + int nid; + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + + VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); +#ifdef CONFIG_MEMCG + VM_WARN_ON_ONCE(mm->lru_gen.memcg); + mm->lru_gen.memcg = memcg; +#endif + spin_lock(&mm_list->lock); + + for_each_node_state(nid, N_MEMORY) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + /* the first addition since the last iteration */ + if (lruvec->mm_state.tail == &mm_list->fifo) + lruvec->mm_state.tail = &mm->lru_gen.list; + } + + list_add_tail(&mm->lru_gen.list, &mm_list->fifo); + + spin_unlock(&mm_list->lock); +} + +void lru_gen_del_mm(struct mm_struct *mm) +{ + int nid; + struct lru_gen_mm_list *mm_list; + struct mem_cgroup *memcg = NULL; + + if (list_empty(&mm->lru_gen.list)) + return; + +#ifdef CONFIG_MEMCG + memcg = mm->lru_gen.memcg; +#endif + mm_list = get_mm_list(memcg); + + spin_lock(&mm_list->lock); + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + /* where the last iteration ended (exclusive) */ + if (lruvec->mm_state.tail == &mm->lru_gen.list) + lruvec->mm_state.tail = lruvec->mm_state.tail->next; + + /* where the current iteration continues (inclusive) */ + if (lruvec->mm_state.head != &mm->lru_gen.list) + continue; + + lruvec->mm_state.head = lruvec->mm_state.head->next; + /* the deletion ends the current iteration */ + if (lruvec->mm_state.head == &mm_list->fifo) + WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); + } + + list_del_init(&mm->lru_gen.list); + + spin_unlock(&mm_list->lock); + +#ifdef CONFIG_MEMCG + mem_cgroup_put(mm->lru_gen.memcg); + mm->lru_gen.memcg = NULL; +#endif +} + +#ifdef CONFIG_MEMCG +void lru_gen_migrate_mm(struct mm_struct *mm) +{ + struct mem_cgroup *memcg; + + lockdep_assert_held(&mm->owner->alloc_lock); + + /* for mm_update_next_owner() */ + if (mem_cgroup_disabled()) + return; + + rcu_read_lock(); + memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); + rcu_read_unlock(); + if (memcg == mm->lru_gen.memcg) + return; + + VM_WARN_ON_ONCE(!mm->lru_gen.memcg); + VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); + + lru_gen_del_mm(mm); + lru_gen_add_mm(mm); +} +#endif + +/* + * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when + * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of + * bits in a bitmap, k is the number of hash functions and n is the number of + * inserted items. + * + * Page table walkers use one of the two filters to reduce their search space. + * To get rid of non-leaf entries that no longer have enough leaf entries, the + * aging uses the double-buffering technique to flip to the other filter each + * time it produces a new generation. For non-leaf entries that have enough + * leaf entries, the aging carries them over to the next generation in + * walk_pmd_range(); the eviction also report them when walking the rmap + * in lru_gen_look_around(). + * + * For future optimizations: + * 1. It's not necessary to keep both filters all the time. The spare one can be + * freed after the RCU grace period and reallocated if needed again. + * 2. And when reallocating, it's worth scaling its size according to the number + * of inserted entries in the other filter, to reduce the memory overhead on + * small systems and false positives on large systems. + * 3. Jenkins' hash function is an alternative to Knuth's. + */ +#define BLOOM_FILTER_SHIFT 15 + +static inline int filter_gen_from_seq(unsigned long seq) +{ + return seq % NR_BLOOM_FILTERS; +} + +static void get_item_key(void *item, int *key) +{ + u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); + + BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); + + key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); + key[1] = hash >> BLOOM_FILTER_SHIFT; +} + +static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) +{ + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = lruvec->mm_state.filters[gen]; + if (filter) { + bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); + return; + } + + filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), + __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + WRITE_ONCE(lruvec->mm_state.filters[gen], filter); +} + +static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ + int key[2]; + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = READ_ONCE(lruvec->mm_state.filters[gen]); + if (!filter) + return; + + get_item_key(item, key); + + if (!test_bit(key[0], filter)) + set_bit(key[0], filter); + if (!test_bit(key[1], filter)) + set_bit(key[1], filter); +} + +static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ + int key[2]; + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = READ_ONCE(lruvec->mm_state.filters[gen]); + if (!filter) + return true; + + get_item_key(item, key); + + return test_bit(key[0], filter) && test_bit(key[1], filter); +} + +static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) +{ + int i; + int hist; + + lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); + + if (walk) { + hist = lru_hist_from_seq(walk->max_seq); + + for (i = 0; i < NR_MM_STATS; i++) { + WRITE_ONCE(lruvec->mm_state.stats[hist][i], + lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); + walk->mm_stats[i] = 0; + } + } + + if (NR_HIST_GENS > 1 && last) { + hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); + + for (i = 0; i < NR_MM_STATS; i++) + WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); + } +} + +static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) +{ + int type; + unsigned long size = 0; + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); + + if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) + return true; + + clear_bit(key, &mm->lru_gen.bitmap); + + for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { + size += type ? get_mm_counter(mm, MM_FILEPAGES) : + get_mm_counter(mm, MM_ANONPAGES) + + get_mm_counter(mm, MM_SHMEMPAGES); + } + + if (size < MIN_LRU_BATCH) + return true; + + if (test_bit(MMF_OOM_REAP_QUEUED, &mm->flags)) + return true; + + return !mmget_not_zero(mm); +} + +static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, + struct mm_struct **iter) +{ + bool first = false; + bool last = true; + struct mm_struct *mm = NULL; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + + /* + * There are four interesting cases for this page table walker: + * 1. It tries to start a new iteration of mm_list with a stale max_seq; + * there is nothing left to do. + * 2. It's the first of the current generation, and it needs to reset + * the Bloom filter for the next generation. + * 3. It reaches the end of mm_list, and it needs to increment + * mm_state->seq; the iteration is done. + * 4. It's the last of the current generation, and it needs to reset the + * mm stats counters for the next generation. + */ + spin_lock(&mm_list->lock); + + VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); + VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); + VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); + + if (walk->max_seq <= mm_state->seq) { + if (!*iter) + last = false; + goto done; + } + + if (!mm_state->nr_walkers) { + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + + mm_state->head = mm_list->fifo.next; + first = true; + } + + while (!mm && mm_state->head != &mm_list->fifo) { + mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); + + mm_state->head = mm_state->head->next; + + /* force scan for those added after the last iteration */ + if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { + mm_state->tail = mm_state->head; + walk->force_scan = true; + } + + if (should_skip_mm(mm, walk)) + mm = NULL; + } + + if (mm_state->head == &mm_list->fifo) + WRITE_ONCE(mm_state->seq, mm_state->seq + 1); +done: + if (*iter && !mm) + mm_state->nr_walkers--; + if (!*iter && mm) + mm_state->nr_walkers++; + + if (mm_state->nr_walkers) + last = false; + + if (*iter || last) + reset_mm_stats(lruvec, walk, last); + + spin_unlock(&mm_list->lock); + + if (mm && first) + reset_bloom_filter(lruvec, walk->max_seq + 1); + + if (*iter) + mmput_async(*iter); + + *iter = mm; + + return last; +} + +static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) +{ + bool success = false; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + + spin_lock(&mm_list->lock); + + VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); + + if (max_seq > mm_state->seq && !mm_state->nr_walkers) { + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + + WRITE_ONCE(mm_state->seq, mm_state->seq + 1); + reset_mm_stats(lruvec, NULL, true); + success = true; + } + + spin_unlock(&mm_list->lock); + + return success; +} + +/****************************************************************************** + * refault feedback loop + ******************************************************************************/ + +/* + * A feedback loop based on Proportional-Integral-Derivative (PID) controller. + * + * The P term is refaulted/(evicted+protected) from a tier in the generation + * currently being evicted; the I term is the exponential moving average of the + * P term over the generations previously evicted, using the smoothing factor + * 1/2; the D term isn't supported. + * + * The setpoint (SP) is always the first tier of one type; the process variable + * (PV) is either any tier of the other type or any other tier of the same + * type. + * + * The error is the difference between the SP and the PV; the correction is to + * turn off protection when SP>PV or turn on protection when SPlrugen; + int hist = lru_hist_from_seq(lrugen->min_seq[type]); + + pos->refaulted = lrugen->avg_refaulted[type][tier] + + atomic_long_read(&lrugen->refaulted[hist][type][tier]); + pos->total = lrugen->avg_total[type][tier] + + atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + pos->total += lrugen->protected[hist][type][tier - 1]; + pos->gain = gain; +} + +static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) +{ + int hist, tier; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; + unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; + + lockdep_assert_held(&lruvec->lru_lock); + + if (!carryover && !clear) + return; + + hist = lru_hist_from_seq(seq); + + for (tier = 0; tier < MAX_NR_TIERS; tier++) { + if (carryover) { + unsigned long sum; + + sum = lrugen->avg_refaulted[type][tier] + + atomic_long_read(&lrugen->refaulted[hist][type][tier]); + WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); + + sum = lrugen->avg_total[type][tier] + + atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + sum += lrugen->protected[hist][type][tier - 1]; + WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); + } + + if (clear) { + atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); + atomic_long_set(&lrugen->evicted[hist][type][tier], 0); + if (tier) + WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); + } + } +} + +static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) +{ + /* + * Return true if the PV has a limited number of refaults or a lower + * refaulted/total than the SP. + */ + return pv->refaulted < MIN_LRU_BATCH || + pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= + (sp->refaulted + 1) * pv->total * pv->gain; +} + +/****************************************************************************** + * the aging + ******************************************************************************/ + +/* promote pages accessed through page tables */ +static int folio_update_gen(struct folio *folio, int gen) +{ + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(!rcu_read_lock_held()); + + do { + /* lru_gen_del_folio() has isolated this page? */ + if (!(old_flags & LRU_GEN_MASK)) { + /* for shrink_page_list() */ + new_flags = old_flags | BIT(PG_referenced); + continue; + } + + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); + new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + + return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + int type = folio_is_file_lru(folio); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); + + do { + new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + /* folio_update_gen() has promoted this page? */ + if (new_gen >= 0 && new_gen != old_gen) + return new_gen; + + new_gen = (old_gen + 1) % MAX_NR_GENS; + + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); + new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; + /* for folio_end_writeback() */ + if (reclaiming) + new_flags |= BIT(PG_reclaim); + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + + lru_gen_update_size(lruvec, folio, old_gen, new_gen); + + return new_gen; +} + +static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, + int old_gen, int new_gen) +{ + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + + VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); + + walk->batched++; + + walk->nr_pages[old_gen][type][zone] -= delta; + walk->nr_pages[new_gen][type][zone] += delta; +} + +static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) +{ + int gen, type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + walk->batched = 0; + + for_each_gen_type_zone(gen, type, zone) { + enum lru_list lru = type * LRU_INACTIVE_FILE; + int delta = walk->nr_pages[gen][type][zone]; + + if (!delta) + continue; + + walk->nr_pages[gen][type][zone] = 0; + WRITE_ONCE(lrugen->nr_pages[gen][type][zone], + lrugen->nr_pages[gen][type][zone] + delta); + + if (lru_gen_is_active(lruvec, gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, delta); + } +} + +static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) +{ + struct address_space *mapping; + struct vm_area_struct *vma = args->vma; + struct lru_gen_mm_walk *walk = args->private; + + if (!vma_is_accessible(vma)) + return true; + + if (is_vm_hugetlb_page(vma)) + return true; + + if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) + return true; + + if (vma == get_gate_vma(vma->vm_mm)) + return true; + + if (vma_is_anonymous(vma)) + return !walk->can_swap; + + if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) + return true; + + mapping = vma->vm_file->f_mapping; + if (mapping_unevictable(mapping)) + return true; + + if (shmem_mapping(mapping)) + return !walk->can_swap; + + /* to exclude special mappings like dax, etc. */ + return !mapping->a_ops->read_folio; +} + +/* + * Some userspace memory allocators map many single-page VMAs. Instead of + * returning back to the PGD table for each of such VMAs, finish an entire PMD + * table to reduce zigzags and improve cache performance. + */ +static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, + unsigned long *vm_start, unsigned long *vm_end) +{ + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; + + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); + + while (args->vma) { + if (start >= args->vma->vm_end) { + args->vma = args->vma->vm_next; + continue; + } + + if (end && end <= args->vma->vm_start) + return false; + + if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { + args->vma = args->vma->vm_next; + continue; + } + + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; + + return true; + } + + return false; +} + +static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pte_pfn(pte); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pte_present(pte) || is_zero_pfn(pfn)) + return -1; + + if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} + +static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pmd_pfn(pmd); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) + return -1; + + if (WARN_ON_ONCE(pmd_devmap(pmd))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} + +static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, + struct pglist_data *pgdat, bool can_swap) +{ + struct folio *folio; + + /* try to avoid unnecessary memory loads */ + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + return NULL; + + folio = pfn_folio(pfn); + if (folio_nid(folio) != pgdat->node_id) + return NULL; + + if (folio_memcg_rcu(folio) != memcg) + return NULL; + + /* file VMAs can contain anon pages from COW */ + if (!folio_is_file_lru(folio) && !can_swap) + return NULL; + + return folio; +} + +static bool suitable_to_scan(int total, int young) +{ + int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); + + /* suitable if the average number of young PTEs per cacheline is >=1 */ + return young * n >= total; +} + +static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pte_t *pte; + spinlock_t *ptl; + unsigned long addr; + int total = 0; + int young = 0; + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + + VM_WARN_ON_ONCE(pmd_leaf(*pmd)); + + ptl = pte_lockptr(args->mm, pmd); + if (!spin_trylock(ptl)) + return false; + + arch_enter_lazy_mmu_mode(); + + pte = pte_offset_map(pmd, start & PMD_MASK); +restart: + for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { + unsigned long pfn; + struct folio *folio; + + total++; + walk->mm_stats[MM_LEAF_TOTAL]++; + + pfn = get_pte_pfn(pte[i], args->vma, addr); + if (pfn == -1) + continue; + + if (!pte_young(pte[i])) { + walk->mm_stats[MM_LEAF_OLD]++; + continue; + } + + folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); + if (!folio) + continue; + + if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) + continue; + + young++; + walk->mm_stats[MM_LEAF_YOUNG]++; + + if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen >= 0 && old_gen != new_gen) + update_batch_size(walk, folio, old_gen, new_gen); + } + + if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + goto restart; + + pte_unmap(pte); + + arch_leave_lazy_mmu_mode(); + spin_unlock(ptl); + + return suitable_to_scan(total, young); +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) +{ + int i; + pmd_t *pmd; + spinlock_t *ptl; + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + + VM_WARN_ON_ONCE(pud_leaf(*pud)); + + /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ + if (*start == -1) { + *start = next; + return; + } + + i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); + if (i && i <= MIN_LRU_BATCH) { + __set_bit(i - 1, bitmap); + return; + } + + pmd = pmd_offset(pud, *start); + + ptl = pmd_lockptr(args->mm, pmd); + if (!spin_trylock(ptl)) + goto done; + + arch_enter_lazy_mmu_mode(); + + do { + unsigned long pfn; + struct folio *folio; + unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; + + pfn = get_pmd_pfn(pmd[i], vma, addr); + if (pfn == -1) + goto next; + + if (!pmd_trans_huge(pmd[i])) { + if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && + get_cap(LRU_GEN_NONLEAF_YOUNG)) + pmdp_test_and_clear_young(vma, addr, pmd + i); + goto next; + } + + folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); + if (!folio) + goto next; + + if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) + goto next; + + walk->mm_stats[MM_LEAF_YOUNG]++; + + if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen >= 0 && old_gen != new_gen) + update_batch_size(walk, folio, old_gen, new_gen); +next: + i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; + } while (i <= MIN_LRU_BATCH); + + arch_leave_lazy_mmu_mode(); + spin_unlock(ptl); +done: + *start = -1; + bitmap_zero(bitmap, MIN_LRU_BATCH); +} +#else +static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) +{ +} +#endif + +static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pmd_t *pmd; + unsigned long next; + unsigned long addr; + struct vm_area_struct *vma; + unsigned long pos = -1; + struct lru_gen_mm_walk *walk = args->private; + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; + + VM_WARN_ON_ONCE(pud_leaf(*pud)); + + /* + * Finish an entire PMD in two passes: the first only reaches to PTE + * tables to avoid taking the PMD lock; the second, if necessary, takes + * the PMD lock to clear the accessed bit in PMD entries. + */ + pmd = pmd_offset(pud, start & PUD_MASK); +restart: + /* walk_pte_range() may call get_next_vma() */ + vma = args->vma; + for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + pmd_t val = pmd_read_atomic(pmd + i); + + /* for pmd_read_atomic() */ + barrier(); + + next = pmd_addr_end(addr, end); + + if (!pmd_present(val) || is_huge_zero_pmd(val)) { + walk->mm_stats[MM_LEAF_TOTAL]++; + continue; + } + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_trans_huge(val)) { + unsigned long pfn = pmd_pfn(val); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + + walk->mm_stats[MM_LEAF_TOTAL]++; + + if (!pmd_young(val)) { + walk->mm_stats[MM_LEAF_OLD]++; + continue; + } + + /* try to avoid unnecessary memory loads */ + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + continue; + + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); + continue; + } +#endif + walk->mm_stats[MM_NONLEAF_TOTAL]++; + +#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG + if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { + if (!pmd_young(val)) + continue; + + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); + } +#endif + if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) + continue; + + walk->mm_stats[MM_NONLEAF_FOUND]++; + + if (!walk_pte_range(&val, addr, next, args)) + continue; + + walk->mm_stats[MM_NONLEAF_ADDED]++; + + /* carry over to the next generation */ + update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); + } + + walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); + + if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + goto restart; +} + +static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pud_t *pud; + unsigned long addr; + unsigned long next; + struct lru_gen_mm_walk *walk = args->private; + + VM_WARN_ON_ONCE(p4d_leaf(*p4d)); + + pud = pud_offset(p4d, start & P4D_MASK); +restart: + for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + pud_t val = READ_ONCE(pud[i]); + + next = pud_addr_end(addr, end); + + if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) + continue; + + walk_pmd_range(&val, addr, next, args); + + if (walk->batched >= MAX_LRU_BATCH) { + end = (addr | ~PUD_MASK) + 1; + goto done; + } + } + + if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + goto restart; + + end = round_up(end, P4D_SIZE); +done: + if (!end || !args->vma) + return 1; + + walk->next_addr = max(end, args->vma->vm_start); + + return -EAGAIN; +} + +static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) +{ + static const struct mm_walk_ops mm_walk_ops = { + .test_walk = should_skip_vma, + .p4d_entry = walk_pud_range, + }; + + int err; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + walk->next_addr = FIRST_USER_ADDRESS; + + do { + err = -EBUSY; + + /* folio_update_gen() requires stable folio_memcg() */ + if (!mem_cgroup_trylock_pages(memcg)) + break; + + /* the caller might be holding the lock for write */ + if (mmap_read_trylock(mm)) { + err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); + + mmap_read_unlock(mm); + } + + mem_cgroup_unlock_pages(); + + if (walk->batched) { + spin_lock_irq(&lruvec->lru_lock); + reset_batch_size(lruvec, walk); + spin_unlock_irq(&lruvec->lru_lock); + } + + cond_resched(); + } while (err == -EAGAIN); +} + +static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) +{ + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + + if (pgdat && current_is_kswapd()) { + VM_WARN_ON_ONCE(walk); + + walk = &pgdat->mm_walk; + } else if (!pgdat && !walk) { + VM_WARN_ON_ONCE(current_is_kswapd()); + + walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + } + + current->reclaim_state->mm_walk = walk; + + return walk; +} + +static void clear_mm_walk(void) +{ + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + + VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); + VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + + current->reclaim_state->mm_walk = NULL; + + if (!current_is_kswapd()) + kfree(walk); +} + +static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) +{ + int zone; + int remaining = MAX_LRU_BATCH; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + + if (type == LRU_GEN_ANON && !can_swap) + goto done; + + /* prevent cold/hot inversion if force_scan is true */ + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *head = &lrugen->lists[old_gen][type][zone]; + + while (!list_empty(head)) { + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + new_gen = folio_inc_gen(lruvec, folio, false); + list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); + + if (!--remaining) + return false; + } + } +done: + reset_ctrl_pos(lruvec, type, true); + WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + + return true; +} + +static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) +{ + int gen, type, zone; + bool success = false; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + DEFINE_MIN_SEQ(lruvec); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + + /* find the oldest populated generation */ + for (type = !can_swap; type < ANON_AND_FILE; type++) { + while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { + gen = lru_gen_from_seq(min_seq[type]); + + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + if (!list_empty(&lrugen->lists[gen][type][zone])) + goto next; + } + + min_seq[type]++; + } +next: + ; + } + + /* see the comment on lru_gen_struct */ + if (can_swap) { + min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); + min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); + } + + for (type = !can_swap; type < ANON_AND_FILE; type++) { + if (min_seq[type] == lrugen->min_seq[type]) + continue; + + reset_ctrl_pos(lruvec, type, true); + WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); + success = true; + } + + return success; +} + +static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) +{ + int prev, next; + int type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + spin_lock_irq(&lruvec->lru_lock); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + + for (type = 0; type < ANON_AND_FILE; type++) { + if (get_nr_gens(lruvec, type) != MAX_NR_GENS) + continue; + + VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); + + while (!inc_min_seq(lruvec, type, can_swap)) { + spin_unlock_irq(&lruvec->lru_lock); + cond_resched(); + spin_lock_irq(&lruvec->lru_lock); + } + } + + /* + * Update the active/inactive LRU sizes for compatibility. Both sides of + * the current max_seq need to be covered, since max_seq+1 can overlap + * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do + * overlap, cold/hot inversion happens. + */ + prev = lru_gen_from_seq(lrugen->max_seq - 1); + next = lru_gen_from_seq(lrugen->max_seq + 1); + + for (type = 0; type < ANON_AND_FILE; type++) { + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + enum lru_list lru = type * LRU_INACTIVE_FILE; + long delta = lrugen->nr_pages[prev][type][zone] - + lrugen->nr_pages[next][type][zone]; + + if (!delta) + continue; + + __update_lru_size(lruvec, lru, zone, delta); + __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); + } + } + + for (type = 0; type < ANON_AND_FILE; type++) + reset_ctrl_pos(lruvec, type, false); + + WRITE_ONCE(lrugen->timestamps[next], jiffies); + /* make sure preceding modifications appear */ + smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); + + spin_unlock_irq(&lruvec->lru_lock); +} + +static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, + struct scan_control *sc, bool can_swap, bool force_scan) +{ + bool success; + struct lru_gen_mm_walk *walk; + struct mm_struct *mm = NULL; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); + + /* see the comment in iterate_mm_list() */ + if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { + success = false; + goto done; + } + + /* + * If the hardware doesn't automatically set the accessed bit, fallback + * to lru_gen_look_around(), which only clears the accessed bit in a + * handful of PTEs. Spreading the work out over a period of time usually + * is less efficient, but it avoids bursty page faults. + */ + if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { + success = iterate_mm_list_nowalk(lruvec, max_seq); + goto done; + } + + walk = set_mm_walk(NULL); + if (!walk) { + success = iterate_mm_list_nowalk(lruvec, max_seq); + goto done; + } + + walk->lruvec = lruvec; + walk->max_seq = max_seq; + walk->can_swap = can_swap; + walk->force_scan = force_scan; + + do { + success = iterate_mm_list(lruvec, walk, &mm); + if (mm) + walk_mm(lruvec, mm, walk); + + cond_resched(); + } while (mm); +done: + if (!success) { + if (!current_is_kswapd() && !sc->priority) + wait_event_killable(lruvec->mm_state.wait, + max_seq < READ_ONCE(lrugen->max_seq)); + + return max_seq < READ_ONCE(lrugen->max_seq); + } + + VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); + + inc_max_seq(lruvec, can_swap, force_scan); + /* either this sees any waiters or they will see updated max_seq */ + if (wq_has_sleeper(&lruvec->mm_state.wait)) + wake_up_all(&lruvec->mm_state.wait); + + wakeup_flusher_threads(WB_REASON_VMSCAN); + + return true; +} + +static unsigned long get_nr_evictable(struct lruvec *lruvec, unsigned long max_seq, + unsigned long *min_seq, bool can_swap, bool *need_aging) +{ + int gen, type, zone; + unsigned long old = 0; + unsigned long young = 0; + unsigned long total = 0; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + for (type = !can_swap; type < ANON_AND_FILE; type++) { + unsigned long seq; + + for (seq = min_seq[type]; seq <= max_seq; seq++) { + unsigned long size = 0; + + gen = lru_gen_from_seq(seq); + + for (zone = 0; zone < MAX_NR_ZONES; zone++) + size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + + total += size; + if (seq == max_seq) + young += size; + if (seq + MIN_NR_GENS == max_seq) + old += size; + } + } + + /* + * The aging tries to be lazy to reduce the overhead. On the other hand, + * the eviction stalls when the number of generations reaches + * MIN_NR_GENS. So ideally, there should be MIN_NR_GENS+1 generations, + * hence the first two if's. + * + * Also it's ideal to spread pages out evenly, meaning 1/(MIN_NR_GENS+1) + * of the total number of pages for each generation. A reasonable range + * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The + * eviction cares about the lower bound of cold pages, whereas the aging + * cares about the upper bound of hot pages. + */ + if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) + *need_aging = true; + else if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) + *need_aging = false; + else if (young * MIN_NR_GENS > total) + *need_aging = true; + else if (old * (MIN_NR_GENS + 2) < total) + *need_aging = true; + else + *need_aging = false; + + return total; +} + +static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) +{ + bool need_aging; + unsigned long nr_to_scan; + int swappiness = get_swappiness(lruvec, sc); + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + VM_WARN_ON_ONCE(sc->memcg_low_reclaim); + + mem_cgroup_calculate_protection(NULL, memcg); + + if (mem_cgroup_below_min(memcg)) + return false; + + nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, swappiness, &need_aging); + if (!nr_to_scan) + return false; + + nr_to_scan >>= mem_cgroup_online(memcg) ? sc->priority : 0; + + if (min_ttl) { + int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); + unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + + if (time_is_after_jiffies(birth + min_ttl)) + return false; + + /* the size is likely too small to be helpful */ + if (!nr_to_scan && sc->priority != DEF_PRIORITY) + return false; + } + + if (nr_to_scan && need_aging) + try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); + + return true; +} + +/* to protect the working set of the last N jiffies */ +static unsigned long lru_gen_min_ttl __read_mostly = HZ; + +static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ + struct mem_cgroup *memcg; + bool success = false; + unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); + + VM_WARN_ON_ONCE(!current_is_kswapd()); + + /* + * To reduce the chance of going into the aging path or swapping, which + * can be costly, optimistically skip them unless their corresponding + * flags were cleared in the eviction path. This improves the overall + * performance when multiple memcgs are available. + */ + if (!sc->memcgs_need_aging) { + sc->memcgs_need_aging = true; + sc->memcgs_avoid_swapping = !sc->memcgs_need_swapping; + sc->memcgs_need_swapping = true; + return; + } + + sc->memcgs_need_swapping = true; + sc->memcgs_avoid_swapping = true; + + set_mm_walk(pgdat); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + + if (age_lruvec(lruvec, sc, min_ttl)) + success = true; + + cond_resched(); + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + + clear_mm_walk(); + + /* + * The main goal is to OOM kill if every generation from all memcgs is + * younger than min_ttl. However, another theoretical possibility is all + * memcgs are either below min or empty. + */ + if (!success && !sc->order && mutex_trylock(&oom_lock)) { + struct oom_control oc = { + .gfp_mask = sc->gfp_mask, + }; + + out_of_memory(&oc); + + mutex_unlock(&oom_lock); + } +} + +/* + * This function exploits spatial locality when shrink_page_list() walks the + * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If + * the scan was done cacheline efficiently, it adds the PMD entry pointing to + * the PTE table to the Bloom filter. This forms a feedback loop between the + * eviction and the aging. + */ +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +{ + int i; + pte_t *pte; + unsigned long start; + unsigned long end; + unsigned long addr; + struct lru_gen_mm_walk *walk; + int young = 0; + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; + struct folio *folio = pfn_folio(pvmw->pfn); + struct mem_cgroup *memcg = folio_memcg(folio); + struct pglist_data *pgdat = folio_pgdat(folio); + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + DEFINE_MAX_SEQ(lruvec); + int old_gen, new_gen = lru_gen_from_seq(max_seq); + + lockdep_assert_held(pvmw->ptl); + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + + if (spin_is_contended(pvmw->ptl)) + return; + + start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); + end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; + + if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { + if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) + end = start + MIN_LRU_BATCH * PAGE_SIZE; + else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) + start = end - MIN_LRU_BATCH * PAGE_SIZE; + else { + start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; + end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; + } + } + + pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; + walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; + + rcu_read_lock(); + arch_enter_lazy_mmu_mode(); + + for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { + unsigned long pfn; + + pfn = get_pte_pfn(pte[i], pvmw->vma, addr); + if (pfn == -1) + continue; + + if (!pte_young(pte[i])) + continue; + + folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); + if (!folio) + continue; + + if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) + continue; + + young++; + + if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_lru_gen(folio); + if (old_gen < 0) + folio_set_referenced(folio); + else if (old_gen != new_gen) + __set_bit(i, bitmap); + } + + arch_leave_lazy_mmu_mode(); + rcu_read_unlock(); + + /* feedback from rmap walkers to page table walkers */ + if (suitable_to_scan(i, young)) + update_bloom_filter(lruvec, max_seq, pvmw->pmd); + + if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { + folio = pfn_folio(pte_pfn(pte[i])); + folio_activate(folio); + } + return; + } + + /* folio_update_gen() requires stable folio_memcg() */ + if (!mem_cgroup_trylock_pages(memcg)) + return; + + if (!walk) { + spin_lock_irq(&lruvec->lru_lock); + new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); + } + + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { + folio = pfn_folio(pte_pfn(pte[i])); + if (folio_memcg_rcu(folio) != memcg) + continue; + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen < 0 || old_gen == new_gen) + continue; + + if (walk) + update_batch_size(walk, folio, old_gen, new_gen); + else + lru_gen_update_size(lruvec, folio, old_gen, new_gen); + } + + if (!walk) + spin_unlock_irq(&lruvec->lru_lock); + + mem_cgroup_unlock_pages(); +} + +/****************************************************************************** + * the eviction + ******************************************************************************/ + +static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) +{ + bool success; + int gen = folio_lru_gen(folio); + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + int refs = folio_lru_refs(folio); + int tier = lru_tier_from_refs(refs); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); + + /* unevictable */ + if (!folio_evictable(folio)) { + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + folio_set_unevictable(folio); + lruvec_add_folio(lruvec, folio); + __count_vm_events(UNEVICTABLE_PGCULLED, delta); + return true; + } + + /* dirty lazyfree */ + if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + folio_set_swapbacked(folio); + lruvec_add_folio_tail(lruvec, folio); + return true; + } + + /* promoted */ + if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { + list_move(&folio->lru, &lrugen->lists[gen][type][zone]); + return true; + } + + /* protected */ + if (tier > tier_idx) { + int hist = lru_hist_from_seq(lrugen->min_seq[type]); + + gen = folio_inc_gen(lruvec, folio, false); + list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); + + WRITE_ONCE(lrugen->protected[hist][type][tier - 1], + lrugen->protected[hist][type][tier - 1] + delta); + __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); + return true; + } + + /* waiting for writeback */ + if (folio_test_locked(folio) || folio_test_writeback(folio) || + (type == LRU_GEN_FILE && folio_test_dirty(folio))) { + gen = folio_inc_gen(lruvec, folio, true); + list_move(&folio->lru, &lrugen->lists[gen][type][zone]); + return true; + } + + return false; +} + +static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) +{ + bool success; + + /* unmapping inhibited */ + if (!sc->may_unmap && folio_mapped(folio)) + return false; + + /* swapping inhibited */ + if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && + (folio_test_dirty(folio) || + (folio_test_anon(folio) && !folio_test_swapcache(folio)))) + return false; + + /* raced with release_pages() */ + if (!folio_try_get(folio)) + return false; + + /* raced with another isolation */ + if (!folio_test_clear_lru(folio)) { + folio_put(folio); + return false; + } + + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + + /* for shrink_page_list() */ + folio_clear_reclaim(folio); + folio_clear_referenced(folio); + + return true; +} + +static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, + int type, int tier, struct list_head *list) +{ + int gen, zone; + enum vm_event_item item; + int sorted = 0; + int scanned = 0; + int isolated = 0; + int remaining = MAX_LRU_BATCH; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + VM_WARN_ON_ONCE(!list_empty(list)); + + if (get_nr_gens(lruvec, type) == MIN_NR_GENS) + return 0; + + gen = lru_gen_from_seq(lrugen->min_seq[type]); + + for (zone = sc->reclaim_idx; zone >= 0; zone--) { + LIST_HEAD(moved); + int skipped = 0; + struct list_head *head = &lrugen->lists[gen][type][zone]; + + while (!list_empty(head)) { + struct folio *folio = lru_to_folio(head); + int delta = folio_nr_pages(folio); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + scanned += delta; + + if (sort_folio(lruvec, folio, tier)) + sorted += delta; + else if (isolate_folio(lruvec, folio, sc)) { + list_add(&folio->lru, list); + isolated += delta; + } else { + list_move(&folio->lru, &moved); + skipped += delta; + } + + if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) + break; + } + + if (skipped) { + list_splice(&moved, head); + __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); + } + + if (!remaining || isolated >= MIN_LRU_BATCH) + break; + } + + item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; + if (!cgroup_reclaim(sc)) { + __count_vm_events(item, isolated); + __count_vm_events(PGREFILL, sorted); + } + __count_memcg_events(memcg, item, isolated); + __count_memcg_events(memcg, PGREFILL, sorted); + __count_vm_events(PGSCAN_ANON + type, isolated); + + /* + * There might not be eligible pages due to reclaim_idx, may_unmap and + * may_writepage. Check the remaining to prevent livelock if it's not + * making progress. + */ + return isolated || !remaining ? scanned : 0; +} + +static int get_tier_idx(struct lruvec *lruvec, int type) +{ + int tier; + struct ctrl_pos sp, pv; + + /* + * To leave a margin for fluctuations, use a larger gain factor (1:2). + * This value is chosen because any other tier would have at least twice + * as many refaults as the first tier. + */ + read_ctrl_pos(lruvec, type, 0, 1, &sp); + for (tier = 1; tier < MAX_NR_TIERS; tier++) { + read_ctrl_pos(lruvec, type, tier, 2, &pv); + if (!positive_ctrl_err(&sp, &pv)) + break; + } + + return tier - 1; +} + +static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) +{ + int type, tier; + struct ctrl_pos sp, pv; + int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; + + /* + * Compare the first tier of anon with that of file to determine which + * type to scan. Also need to compare other tiers of the selected type + * with the first tier of the other type to determine the last tier (of + * the selected type) to evict. + */ + read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); + read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); + type = positive_ctrl_err(&sp, &pv); + + read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); + for (tier = 1; tier < MAX_NR_TIERS; tier++) { + read_ctrl_pos(lruvec, type, tier, gain[type], &pv); + if (!positive_ctrl_err(&sp, &pv)) + break; + } + + *tier_idx = tier - 1; + + return type; +} + +static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, + int *type_scanned, struct list_head *list) +{ + int i; + int type; + int scanned; + int tier = -1; + DEFINE_MIN_SEQ(lruvec); + + /* + * Try to make the obvious choice first. When anon and file are both + * available from the same generation, interpret swappiness 1 as file + * first and 200 as anon first. + */ + if (!swappiness) + type = LRU_GEN_FILE; + else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) + type = LRU_GEN_ANON; + else if (swappiness == 1) + type = LRU_GEN_FILE; + else if (swappiness == 200) + type = LRU_GEN_ANON; + else + type = get_type_to_scan(lruvec, swappiness, &tier); + + for (i = !swappiness; i < ANON_AND_FILE; i++) { + if (tier < 0) + tier = get_tier_idx(lruvec, type); + + scanned = scan_folios(lruvec, sc, type, tier, list); + if (scanned) + break; + + type = !type; + tier = -1; + } + + *type_scanned = type; + + return scanned; +} + +static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, + bool *need_swapping) +{ + int type; + int scanned; + int reclaimed; + LIST_HEAD(list); + struct folio *folio; + enum vm_event_item item; + struct reclaim_stat stat; + struct lru_gen_mm_walk *walk; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + + spin_lock_irq(&lruvec->lru_lock); + + scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); + + scanned += try_to_inc_min_seq(lruvec, swappiness); + + if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) + scanned = 0; + + spin_unlock_irq(&lruvec->lru_lock); + + if (list_empty(&list)) + return scanned; + + reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); + + /* + * To avoid livelock, don't add rejected pages back to the same lists + * they were isolated from. See lru_gen_add_folio(). + */ + list_for_each_entry(folio, &list, lru) { + if (folio_test_reclaim(folio) && + (folio_test_dirty(folio) || folio_test_writeback(folio))) + folio_clear_active(folio); + else + folio_set_active(folio); + } + + spin_lock_irq(&lruvec->lru_lock); + + move_pages_to_lru(lruvec, &list); + + walk = current->reclaim_state->mm_walk; + if (walk && walk->batched) + reset_batch_size(lruvec, walk); + + item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; + if (!cgroup_reclaim(sc)) + __count_vm_events(item, reclaimed); + __count_memcg_events(memcg, item, reclaimed); + __count_vm_events(PGSTEAL_ANON + type, reclaimed); + + spin_unlock_irq(&lruvec->lru_lock); + + mem_cgroup_uncharge_list(&list); + free_unref_page_list(&list); + + sc->nr_reclaimed += reclaimed; + + if (type == LRU_GEN_ANON && need_swapping) + *need_swapping = true; + + return scanned; +} + +static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, + bool can_swap, unsigned long reclaimed, bool *need_aging) +{ + int priority; + unsigned long nr_to_scan; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (mem_cgroup_below_min(memcg) || + (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) + return 0; + + nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, can_swap, need_aging); + if (!nr_to_scan) + return 0; + + /* adjust priority if memcg is offline or the target is met */ + if (!mem_cgroup_online(memcg)) + priority = 0; + else if (sc->nr_reclaimed - reclaimed >= sc->nr_to_reclaim) + priority = DEF_PRIORITY; + else + priority = sc->priority; + + nr_to_scan >>= priority; + if (!nr_to_scan) + return 0; + + if (!*need_aging) + return nr_to_scan; + + /* skip the aging path at the default priority */ + if (priority == DEF_PRIORITY) + return nr_to_scan; + + /* leave the work to lru_gen_age_node() */ + if (current_is_kswapd()) + return 0; + + if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) + return nr_to_scan; + + return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +} + +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ + struct blk_plug plug; + bool need_aging = false; + bool need_swapping = false; + unsigned long scanned = 0; + unsigned long reclaimed = sc->nr_reclaimed; + + lru_add_drain(); + + blk_start_plug(&plug); + + set_mm_walk(lruvec_pgdat(lruvec)); + + while (true) { + int delta; + int swappiness; + unsigned long nr_to_scan; + + if (sc->may_swap) + swappiness = get_swappiness(lruvec, sc); + else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) + swappiness = 1; + else + swappiness = 0; + + nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, reclaimed, &need_aging); + if (!nr_to_scan) + goto done; + + delta = evict_folios(lruvec, sc, swappiness, &need_swapping); + if (!delta) + goto done; + + scanned += delta; + if (scanned >= nr_to_scan) + break; + + if (sc->memcgs_avoid_swapping && swappiness < 200 && need_swapping) + break; + + cond_resched(); + } + + /* see the comment in lru_gen_age_node() */ + if (!need_aging) + sc->memcgs_need_aging = false; + if (!need_swapping) + sc->memcgs_need_swapping = false; +done: + clear_mm_walk(); + + blk_finish_plug(&plug); +} + +/****************************************************************************** + * state change + ******************************************************************************/ + +static bool __maybe_unused state_is_valid(struct lruvec *lruvec) +{ + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + if (lrugen->enabled) { + enum lru_list lru; + + for_each_evictable_lru(lru) { + if (!list_empty(&lruvec->lists[lru])) + return false; + } + } else { + int gen, type, zone; + + for_each_gen_type_zone(gen, type, zone) { + if (!list_empty(&lrugen->lists[gen][type][zone])) + return false; + + /* unlikely but not a bug when reset_batch_size() is pending */ + VM_WARN_ON_ONCE(lrugen->nr_pages[gen][type][zone]); + } + } + + return true; +} + +static bool fill_evictable(struct lruvec *lruvec) +{ + enum lru_list lru; + int remaining = MAX_LRU_BATCH; + + for_each_evictable_lru(lru) { + int type = is_file_lru(lru); + bool active = is_active_lru(lru); + struct list_head *head = &lruvec->lists[lru]; + + while (!list_empty(head)) { + bool success; + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); + + lruvec_del_folio(lruvec, folio); + success = lru_gen_add_folio(lruvec, folio, false); + VM_WARN_ON_ONCE(!success); + + if (!--remaining) + return false; + } + } + + return true; +} + +static bool drain_evictable(struct lruvec *lruvec) +{ + int gen, type, zone; + int remaining = MAX_LRU_BATCH; + + for_each_gen_type_zone(gen, type, zone) { + struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; + + while (!list_empty(head)) { + bool success; + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + success = lru_gen_del_folio(lruvec, folio, false); + VM_WARN_ON_ONCE(!success); + lruvec_add_folio(lruvec, folio); + + if (!--remaining) + return false; + } + } + + return true; +} + +static void lru_gen_change_state(bool enabled) +{ + static DEFINE_MUTEX(state_mutex); + + struct mem_cgroup *memcg; + + cgroup_lock(); + cpus_read_lock(); + get_online_mems(); + mutex_lock(&state_mutex); + + if (enabled == lru_gen_enabled()) + goto unlock; + + if (enabled) + static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); + else + static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + int nid; + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + spin_lock_irq(&lruvec->lru_lock); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + VM_WARN_ON_ONCE(!state_is_valid(lruvec)); + + lruvec->lrugen.enabled = enabled; + + while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { + spin_unlock_irq(&lruvec->lru_lock); + cond_resched(); + spin_lock_irq(&lruvec->lru_lock); + } + + spin_unlock_irq(&lruvec->lru_lock); + } + + cond_resched(); + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); +unlock: + mutex_unlock(&state_mutex); + put_online_mems(); + cpus_read_unlock(); + cgroup_unlock(); +} + +/****************************************************************************** + * sysfs interface + ******************************************************************************/ + +static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) +{ + unsigned int msecs; + + if (kstrtouint(buf, 0, &msecs)) + return -EINVAL; + + WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); + + return len; +} + +static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( + min_ttl_ms, 0644, show_min_ttl, store_min_ttl +); + +static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + unsigned int caps = 0; + + if (get_cap(LRU_GEN_CORE)) + caps |= BIT(LRU_GEN_CORE); + + if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) + caps |= BIT(LRU_GEN_MM_WALK); + + if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) + caps |= BIT(LRU_GEN_NONLEAF_YOUNG); + + return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) +{ + int i; + unsigned int caps; + + if (tolower(*buf) == 'n') + caps = 0; + else if (tolower(*buf) == 'y') + caps = -1; + else if (kstrtouint(buf, 0, &caps)) + return -EINVAL; + + for (i = 0; i < NR_LRU_GEN_CAPS; i++) { + bool enabled = caps & BIT(i); + + if (i == LRU_GEN_CORE) + lru_gen_change_state(enabled); + else if (enabled) + static_branch_enable(&lru_gen_caps[i]); + else + static_branch_disable(&lru_gen_caps[i]); + } + + return len; +} + +static struct kobj_attribute lru_gen_enabled_attr = __ATTR( + enabled, 0644, show_enabled, store_enabled +); + +static struct attribute *lru_gen_attrs[] = { + &lru_gen_min_ttl_attr.attr, + &lru_gen_enabled_attr.attr, + NULL +}; + +static struct attribute_group lru_gen_attr_group = { + .name = "lru_gen", + .attrs = lru_gen_attrs, +}; + +/****************************************************************************** + * debugfs interface + ******************************************************************************/ + +static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) +{ + struct mem_cgroup *memcg; + loff_t nr_to_skip = *pos; + + m->private = kvmalloc(PATH_MAX, GFP_KERNEL); + if (!m->private) + return ERR_PTR(-ENOMEM); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + int nid; + + for_each_node_state(nid, N_MEMORY) { + if (!nr_to_skip--) + return get_lruvec(memcg, nid); + } + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + + return NULL; +} + +static void lru_gen_seq_stop(struct seq_file *m, void *v) +{ + if (!IS_ERR_OR_NULL(v)) + mem_cgroup_iter_break(NULL, lruvec_memcg(v)); + + kvfree(m->private); + m->private = NULL; +} + +static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + int nid = lruvec_pgdat(v)->node_id; + struct mem_cgroup *memcg = lruvec_memcg(v); + + ++*pos; + + nid = next_memory_node(nid); + if (nid == MAX_NUMNODES) { + memcg = mem_cgroup_iter(NULL, memcg, NULL); + if (!memcg) + return NULL; + + nid = first_memory_node; + } + + return get_lruvec(memcg, nid); +} + +static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, + unsigned long max_seq, unsigned long *min_seq, + unsigned long seq) +{ + int i; + int type, tier; + int hist = lru_hist_from_seq(seq); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + for (tier = 0; tier < MAX_NR_TIERS; tier++) { + seq_printf(m, " %10d", tier); + for (type = 0; type < ANON_AND_FILE; type++) { + const char *s = " "; + unsigned long n[3] = {}; + + if (seq == max_seq) { + s = "RT "; + n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); + n[1] = READ_ONCE(lrugen->avg_total[type][tier]); + } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { + s = "rep"; + n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); + n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); + } + + for (i = 0; i < 3; i++) + seq_printf(m, " %10lu%c", n[i], s[i]); + } + seq_putc(m, '\n'); + } + + seq_puts(m, " "); + for (i = 0; i < NR_MM_STATS; i++) { + const char *s = " "; + unsigned long n = 0; + + if (seq == max_seq && NR_HIST_GENS == 1) { + s = "TOYDFA"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } else if (seq != max_seq && NR_HIST_GENS > 1) { + s = "toydfa"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } + + seq_printf(m, " %10lu%c", n, s[i]); + } + seq_putc(m, '\n'); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static int lru_gen_seq_show(struct seq_file *m, void *v) +{ + unsigned long seq; + bool full = !debugfs_real_fops(m->file)->write; + struct lruvec *lruvec = v; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int nid = lruvec_pgdat(lruvec)->node_id; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (nid == first_memory_node) { + const char *path = memcg ? m->private : ""; + +#ifdef CONFIG_MEMCG + if (memcg) + cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); +#endif + seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); + } + + seq_printf(m, " node %5d\n", nid); + + if (!full) + seq = min_seq[LRU_GEN_ANON]; + else if (max_seq >= MAX_NR_GENS) + seq = max_seq - MAX_NR_GENS + 1; + else + seq = 0; + + for (; seq <= max_seq; seq++) { + int type, zone; + int gen = lru_gen_from_seq(seq); + unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + + seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); + + for (type = 0; type < ANON_AND_FILE; type++) { + unsigned long size = 0; + char mark = full && seq < min_seq[type] ? 'x' : ' '; + + for (zone = 0; zone < MAX_NR_ZONES; zone++) + size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + + seq_printf(m, " %10lu%c", size, mark); + } + + seq_putc(m, '\n'); + + if (full) + lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); + } + + return 0; +} + +static const struct seq_operations lru_gen_seq_ops = { + .start = lru_gen_seq_start, + .stop = lru_gen_seq_stop, + .next = lru_gen_seq_next, + .show = lru_gen_seq_show, +}; + +static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, + bool can_swap, bool force_scan) +{ + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (seq < max_seq) + return 0; + + if (seq > max_seq) + return -EINVAL; + + if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) + return -ERANGE; + + try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); + + return 0; +} + +static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, + int swappiness, unsigned long nr_to_reclaim) +{ + DEFINE_MAX_SEQ(lruvec); + + if (seq + MIN_NR_GENS > max_seq) + return -EINVAL; + + sc->nr_reclaimed = 0; + + while (!signal_pending(current)) { + DEFINE_MIN_SEQ(lruvec); + + if (seq < min_seq[!swappiness]) + return 0; + + if (sc->nr_reclaimed >= nr_to_reclaim) + return 0; + + if (!evict_folios(lruvec, sc, swappiness, NULL)) + return 0; + + cond_resched(); + } + + return -EINTR; +} + +static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, + struct scan_control *sc, int swappiness, unsigned long opt) +{ + struct lruvec *lruvec; + int err = -EINVAL; + struct mem_cgroup *memcg = NULL; + + if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) + return -EINVAL; + + if (!mem_cgroup_disabled()) { + rcu_read_lock(); + memcg = mem_cgroup_from_id(memcg_id); +#ifdef CONFIG_MEMCG + if (memcg && !css_tryget(&memcg->css)) + memcg = NULL; +#endif + rcu_read_unlock(); + + if (!memcg) + return -EINVAL; + } + + if (memcg_id != mem_cgroup_id(memcg)) + goto done; + + lruvec = get_lruvec(memcg, nid); + + if (swappiness < 0) + swappiness = get_swappiness(lruvec, sc); + else if (swappiness > 200) + goto done; + + switch (cmd) { + case '+': + err = run_aging(lruvec, seq, sc, swappiness, opt); + break; + case '-': + err = run_eviction(lruvec, seq, sc, swappiness, opt); + break; + } +done: + mem_cgroup_put(memcg); + + return err; +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, + size_t len, loff_t *pos) +{ + void *buf; + char *cur, *next; + unsigned int flags; + struct blk_plug plug; + int err = -EINVAL; + struct scan_control sc = { + .may_writepage = true, + .may_unmap = true, + .may_swap = true, + .reclaim_idx = MAX_NR_ZONES - 1, + .gfp_mask = GFP_KERNEL, + }; + + buf = kvmalloc(len + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, src, len)) { + kvfree(buf); + return -EFAULT; + } + + if (!set_mm_walk(NULL)) { + kvfree(buf); + return -ENOMEM; + } + + set_task_reclaim_state(current, &sc.reclaim_state); + flags = memalloc_noreclaim_save(); + blk_start_plug(&plug); + + next = buf; + next[len] = '\0'; + + while ((cur = strsep(&next, ",;\n"))) { + int n; + int end; + char cmd; + unsigned int memcg_id; + unsigned int nid; + unsigned long seq; + unsigned int swappiness = -1; + unsigned long opt = -1; + + cur = skip_spaces(cur); + if (!*cur) + continue; + + n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, + &seq, &end, &swappiness, &end, &opt, &end); + if (n < 4 || cur[end]) { + err = -EINVAL; + break; + } + + err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); + if (err) + break; + } + + blk_finish_plug(&plug); + memalloc_noreclaim_restore(flags); + set_task_reclaim_state(current, NULL); + + clear_mm_walk(); + kvfree(buf); + + return err ? : len; +} + +static int lru_gen_seq_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &lru_gen_seq_ops); +} + +static const struct file_operations lru_gen_rw_fops = { + .open = lru_gen_seq_open, + .read = seq_read, + .write = lru_gen_seq_write, + .llseek = seq_lseek, + .release = seq_release, +}; + +static const struct file_operations lru_gen_ro_fops = { + .open = lru_gen_seq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +/****************************************************************************** + * initialization + ******************************************************************************/ + +void lru_gen_init_lruvec(struct lruvec *lruvec) +{ + int i; + int gen, type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + lrugen->max_seq = MIN_NR_GENS + 1; + lrugen->enabled = lru_gen_enabled(); + + for (i = 0; i <= MIN_NR_GENS + 1; i++) + lrugen->timestamps[i] = jiffies; + + for_each_gen_type_zone(gen, type, zone) + INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); + + lruvec->mm_state.seq = MIN_NR_GENS; + init_waitqueue_head(&lruvec->mm_state.wait); +} + +#ifdef CONFIG_MEMCG +void lru_gen_init_memcg(struct mem_cgroup *memcg) +{ + INIT_LIST_HEAD(&memcg->mm_list.fifo); + spin_lock_init(&memcg->mm_list.lock); +} + +void lru_gen_exit_memcg(struct mem_cgroup *memcg) +{ + int i; + int nid; + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, + sizeof(lruvec->lrugen.nr_pages))); + + for (i = 0; i < NR_BLOOM_FILTERS; i++) { + bitmap_free(lruvec->mm_state.filters[i]); + lruvec->mm_state.filters[i] = NULL; + } + } +} +#endif + +static int __init init_lru_gen(void) +{ + BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); + BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); + + if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) + pr_err("lru_gen: failed to create sysfs group\n"); + + debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); + debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); + + return 0; +}; +late_initcall(init_lru_gen); + +#else /* !CONFIG_LRU_GEN */ + +static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ +} + +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ +} + +#endif /* CONFIG_LRU_GEN */ + +static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ + unsigned long nr[NR_LRU_LISTS]; + unsigned long targets[NR_LRU_LISTS]; + unsigned long nr_to_scan; + enum lru_list lru; + unsigned long nr_reclaimed = 0; + unsigned long nr_to_reclaim = sc->nr_to_reclaim; + struct blk_plug plug; + bool scan_adjusted; + + if (lru_gen_enabled()) { + lru_gen_shrink_lruvec(lruvec, sc); + return; + } + + get_scan_count(lruvec, sc, nr); + + /* Record the original scan target for proportional adjustments later */ + memcpy(targets, nr, sizeof(nr)); + + /* + * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal + * event that can occur when there is little memory pressure e.g. + * multiple streaming readers/writers. Hence, we do not abort scanning + * when the requested number of pages are reclaimed when scanning at + * DEF_PRIORITY on the assumption that the fact we are direct + * reclaiming implies that kswapd is not keeping up and it is best to + * do a batch of work at once. For memcg reclaim one check is made to + * abort proportional reclaim if either the file or anon lru has already + * dropped to zero at the first pass. + */ + scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && + sc->priority == DEF_PRIORITY); + + blk_start_plug(&plug); + while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || + nr[LRU_INACTIVE_FILE]) { + unsigned long nr_anon, nr_file, percentage; + unsigned long nr_scanned; + + for_each_evictable_lru(lru) { + if (nr[lru]) { + nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); + nr[lru] -= nr_to_scan; + + nr_reclaimed += shrink_list(lru, nr_to_scan, + lruvec, sc); + } + } + + cond_resched(); + + if (nr_reclaimed < nr_to_reclaim || scan_adjusted) + continue; + + /* + * For kswapd and memcg, reclaim at least the number of pages + * requested. Ensure that the anon and file LRUs are scanned + * proportionally what was requested by get_scan_count(). We + * stop reclaiming one LRU and reduce the amount scanning + * proportional to the original scan target. + */ + nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; + nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; + + /* + * It's just vindictive to attack the larger once the smaller + * has gone to zero. And given the way we stop scanning the + * smaller below, this makes sure that we only make one nudge + * towards proportionality once we've got nr_to_reclaim. + */ + if (!nr_file || !nr_anon) + break; + + if (nr_file > nr_anon) { + unsigned long scan_target = targets[LRU_INACTIVE_ANON] + + targets[LRU_ACTIVE_ANON] + 1; + lru = LRU_BASE; + percentage = nr_anon * 100 / scan_target; + } else { + unsigned long scan_target = targets[LRU_INACTIVE_FILE] + + targets[LRU_ACTIVE_FILE] + 1; + lru = LRU_FILE; + percentage = nr_file * 100 / scan_target; + } + + /* Stop scanning the smaller of the LRU */ + nr[lru] = 0; + nr[lru + LRU_ACTIVE] = 0; + + /* + * Recalculate the other LRU scan count based on its original + * scan target and the percentage scanning already complete + */ + lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; + nr_scanned = targets[lru] - nr[lru]; + nr[lru] = targets[lru] * (100 - percentage) / 100; + nr[lru] -= min(nr[lru], nr_scanned); + + lru += LRU_ACTIVE; + nr_scanned = targets[lru] - nr[lru]; + nr[lru] = targets[lru] * (100 - percentage) / 100; + nr[lru] -= min(nr[lru], nr_scanned); + + scan_adjusted = true; + } + blk_finish_plug(&plug); + sc->nr_reclaimed += nr_reclaimed; + + /* + * Even if we did not try to evict anon pages at all, we want to * rebalance the anon lru active/inactive ratio. */ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && @@ -3142,109 +6062,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) unsigned long nr_reclaimed, nr_scanned; struct lruvec *target_lruvec; bool reclaimable = false; - unsigned long file; target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); again: - /* - * Flush the memory cgroup stats, so that we read accurate per-memcg - * lruvec stats for heuristics. - */ - mem_cgroup_flush_stats(); - memset(&sc->nr, 0, sizeof(sc->nr)); nr_reclaimed = sc->nr_reclaimed; nr_scanned = sc->nr_scanned; - /* - * Determine the scan balance between anon and file LRUs. - */ - spin_lock_irq(&target_lruvec->lru_lock); - sc->anon_cost = target_lruvec->anon_cost; - sc->file_cost = target_lruvec->file_cost; - spin_unlock_irq(&target_lruvec->lru_lock); - - /* - * Target desirable inactive:active list ratios for the anon - * and file LRU lists. - */ - if (!sc->force_deactivate) { - unsigned long refaults; - - refaults = lruvec_page_state(target_lruvec, - WORKINGSET_ACTIVATE_ANON); - if (refaults != target_lruvec->refaults[0] || - inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) - sc->may_deactivate |= DEACTIVATE_ANON; - else - sc->may_deactivate &= ~DEACTIVATE_ANON; - - /* - * When refaults are being observed, it means a new - * workingset is being established. Deactivate to get - * rid of any stale active pages quickly. - */ - refaults = lruvec_page_state(target_lruvec, - WORKINGSET_ACTIVATE_FILE); - if (refaults != target_lruvec->refaults[1] || - inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) - sc->may_deactivate |= DEACTIVATE_FILE; - else - sc->may_deactivate &= ~DEACTIVATE_FILE; - } else - sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; - - /* - * If we have plenty of inactive file pages that aren't - * thrashing, try to reclaim those first before touching - * anonymous pages. - */ - file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); - if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) - sc->cache_trim_mode = 1; - else - sc->cache_trim_mode = 0; - - /* - * Prevent the reclaimer from falling into the cache trap: as - * cache pages start out inactive, every cache fault will tip - * the scan balance towards the file LRU. And as the file LRU - * shrinks, so does the window for rotation from references. - * This means we have a runaway feedback loop where a tiny - * thrashing file LRU becomes infinitely more attractive than - * anon pages. Try to detect this based on file LRU size. - */ - if (!cgroup_reclaim(sc)) { - unsigned long total_high_wmark = 0; - unsigned long free, anon; - int z; - - free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); - file = node_page_state(pgdat, NR_ACTIVE_FILE) + - node_page_state(pgdat, NR_INACTIVE_FILE); - - for (z = 0; z < MAX_NR_ZONES; z++) { - struct zone *zone = &pgdat->node_zones[z]; - if (!managed_zone(zone)) - continue; - - total_high_wmark += high_wmark_pages(zone); - } - - /* - * Consider anon: if that's low too, this isn't a - * runaway file reclaim problem, but rather just - * extreme pressure. Reclaim as per usual then. - */ - anon = node_page_state(pgdat, NR_INACTIVE_ANON); - - sc->file_is_tiny = - file + free <= total_high_wmark && - !(sc->may_deactivate & DEACTIVATE_ANON) && - anon >> sc->priority; - } + prepare_scan_count(pgdat, sc); shrink_node_memcgs(pgdat, sc); @@ -3501,6 +6328,9 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) struct lruvec *target_lruvec; unsigned long refaults; + if (lru_gen_enabled()) + return; + target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); target_lruvec->refaults[0] = refaults; @@ -3865,12 +6695,17 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, } #endif -static void age_active_anon(struct pglist_data *pgdat, +static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) { struct mem_cgroup *memcg; struct lruvec *lruvec; + if (lru_gen_enabled()) { + lru_gen_age_node(pgdat, sc); + return; + } + if (!can_age_anon_pages(pgdat, sc)) return; @@ -4190,12 +7025,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) sc.may_swap = !nr_boost_reclaim; /* - * Do some background aging of the anon list, to give - * pages a chance to be referenced before reclaiming. All - * pages are rotated regardless of classzone as this is - * about consistent aging. + * Do some background aging, to give pages a chance to be + * referenced before reclaiming. All pages are rotated + * regardless of classzone as this is about consistent aging. */ - age_active_anon(pgdat, &sc); + kswapd_age_node(pgdat, &sc); /* * If we're getting trouble reclaiming, start doing writepage diff --git a/mm/workingset.c b/mm/workingset.c index 592569a8974c4..84a9e0ab04ad7 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, bool workingset) { - eviction >>= bucket_order; eviction &= EVICTION_MASK; eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; eviction = (eviction << NODES_SHIFT) | pgdat->node_id; @@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, *memcgidp = memcgid; *pgdat = NODE_DATA(nid); - *evictionp = entry << bucket_order; + *evictionp = entry; *workingsetp = workingset; } +#ifdef CONFIG_LRU_GEN + +static void *lru_gen_eviction(struct folio *folio) +{ + int hist; + unsigned long token; + unsigned long min_seq; + struct lruvec *lruvec; + struct lru_gen_struct *lrugen; + int type = folio_is_file_lru(folio); + int delta = folio_nr_pages(folio); + int refs = folio_lru_refs(folio); + int tier = lru_tier_from_refs(refs); + struct mem_cgroup *memcg = folio_memcg(folio); + struct pglist_data *pgdat = folio_pgdat(folio); + + BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); + + lruvec = mem_cgroup_lruvec(memcg, pgdat); + lrugen = &lruvec->lrugen; + min_seq = READ_ONCE(lrugen->min_seq[type]); + token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); + + hist = lru_hist_from_seq(min_seq); + atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); + + return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); +} + +static void lru_gen_refault(struct folio *folio, void *shadow) +{ + int hist, tier, refs; + int memcg_id; + bool workingset; + unsigned long token; + unsigned long min_seq; + struct lruvec *lruvec; + struct lru_gen_struct *lrugen; + struct mem_cgroup *memcg; + struct pglist_data *pgdat; + int type = folio_is_file_lru(folio); + int delta = folio_nr_pages(folio); + + unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); + + if (pgdat != folio_pgdat(folio)) + return; + + rcu_read_lock(); + + memcg = folio_memcg_rcu(folio); + if (memcg_id != mem_cgroup_id(memcg)) + goto unlock; + + lruvec = mem_cgroup_lruvec(memcg, pgdat); + lrugen = &lruvec->lrugen; + + min_seq = READ_ONCE(lrugen->min_seq[type]); + if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) + goto unlock; + + hist = lru_hist_from_seq(min_seq); + /* see the comment in folio_lru_refs() */ + refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; + tier = lru_tier_from_refs(refs); + + atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); + mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); + + /* + * Count the following two cases as stalls: + * 1. For pages accessed through page tables, hotter pages pushed out + * hot pages which refaulted immediately. + * 2. For pages accessed multiple times through file descriptors, + * numbers of accesses might have been out of the range. + */ + if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { + folio_set_workingset(folio); + mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); + } +unlock: + rcu_read_unlock(); +} + +#else /* !CONFIG_LRU_GEN */ + +static void *lru_gen_eviction(struct folio *folio) +{ + return NULL; +} + +static void lru_gen_refault(struct folio *folio, void *shadow) +{ +} + +#endif /* CONFIG_LRU_GEN */ + /** * workingset_age_nonresident - age non-resident entries as LRU ages * @lruvec: the lruvec that was aged @@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (lru_gen_enabled()) + return lru_gen_eviction(folio); + lruvec = mem_cgroup_lruvec(target_memcg, pgdat); /* XXX: target_memcg can be NULL, go through lruvec */ memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); eviction = atomic_long_read(&lruvec->nonresident_age); + eviction >>= bucket_order; workingset_age_nonresident(lruvec, folio_nr_pages(folio)); return pack_shadow(memcgid, pgdat, eviction, folio_test_workingset(folio)); @@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) int memcgid; long nr; + if (lru_gen_enabled()) { + lru_gen_refault(folio, shadow); + return; + } + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); + eviction <<= bucket_order; rcu_read_lock(); /* -- 2.37.1 From 4f1745dd8989a615a3257fc96433db9dc90105e0 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:35:01 +0200 Subject: [PATCH 13/16] misc Signed-off-by: Peter Jung --- Documentation/filesystems/proc.rst | 7 - arch/Kconfig | 3 + arch/x86/Kconfig | 1 + arch/x86/mm/tlb.c | 5 + fs/exec.c | 2 - include/linux/mm.h | 34 +--- include/linux/mm_types_task.h | 49 ++++-- include/linux/sched.h | 3 - kernel/exit.c | 5 - kernel/fork.c | 39 +---- kernel/kthread.c | 1 - kernel/sched/core.c | 4 + mm/madvise.c | 7 +- mm/memory.c | 254 +++++++++++++++++++++++++---- 14 files changed, 281 insertions(+), 133 deletions(-) diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 1bc91fb8c321a..04a0a18da2621 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -224,13 +224,6 @@ memory usage. Its seven fields are explained in Table 1-3. The stat file contains detailed information about the process itself. Its fields are explained in Table 1-4. -(for SMP CONFIG users) - -For making accounting scalable, RSS related information are handled in an -asynchronous manner and the value may not be very precise. To see a precise -snapshot of a moment, you can see /proc//smaps file and scan page table. -It's slow but very precise. - .. table:: Table 1-2: Contents of the status files (as of 4.19) ========================== =================================================== diff --git a/arch/Kconfig b/arch/Kconfig index 3b06abf999440..4477e54462f64 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1403,6 +1403,9 @@ config ARCH_HAS_ELFCORE_COMPAT config ARCH_HAS_PARANOID_L1D_FLUSH bool +config ARCH_PCP_RSS_USE_CPUMASK + bool + config DYNAMIC_SIGFRAME bool diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 83661b6c7b140..9cede651bc74b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -126,6 +126,7 @@ config X86 select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH + select ARCH_PCP_RSS_USE_CPUMASK if X86_64 select BUILDTIME_TABLE_SORT select CLKEVT_I8253 select CLOCKSOURCE_VALIDATE_LAST_CYCLE diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index d9314cc8b81f7..545d916ecab62 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -597,6 +597,11 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, */ cond_mitigation(tsk); + /* + * Flush RSS cache before clear up the bitmask + */ + switch_pcp_rss_cache_no_irq(next); + /* * Stop remote flushes for the previous mm. * Skip kernel threads; we never send init_mm TLB flushing IPIs, diff --git a/fs/exec.c b/fs/exec.c index 39e9bbf864427..1d13114ca0679 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -988,8 +988,6 @@ static int exec_mmap(struct mm_struct *mm) tsk = current; old_mm = current->mm; exec_mm_release(tsk, old_mm); - if (old_mm) - sync_mm_rss(old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); if (ret) diff --git a/include/linux/mm.h b/include/linux/mm.h index 425755adc4559..cd72142b35a20 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2008,23 +2008,14 @@ static inline bool get_user_page_fast_only(unsigned long addr, { return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1; } + +void check_discard_mm(struct mm_struct *mm); + /* * per-process(per-mm_struct) statistics. */ -static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) -{ - long val = atomic_long_read(&mm->rss_stat.count[member]); - -#ifdef SPLIT_RSS_COUNTING - /* - * counter is updated in asynchronous manner and may go to minus. - * But it's never be expected number for users. - */ - if (val < 0) - val = 0; -#endif - return (unsigned long)val; -} +unsigned long get_mm_counter(struct mm_struct *mm, int member); +unsigned long get_mm_rss(struct mm_struct *mm); void mm_trace_rss_stat(struct mm_struct *mm, int member, long count); @@ -2064,13 +2055,6 @@ static inline int mm_counter(struct page *page) return mm_counter_file(page); } -static inline unsigned long get_mm_rss(struct mm_struct *mm) -{ - return get_mm_counter(mm, MM_FILEPAGES) + - get_mm_counter(mm, MM_ANONPAGES) + - get_mm_counter(mm, MM_SHMEMPAGES); -} - static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) { return max(mm->hiwater_rss, get_mm_rss(mm)); @@ -2109,14 +2093,6 @@ static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, *maxrss = hiwater_rss; } -#if defined(SPLIT_RSS_COUNTING) -void sync_mm_rss(struct mm_struct *mm); -#else -static inline void sync_mm_rss(struct mm_struct *mm) -{ -} -#endif - #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL static inline int pte_special(pte_t pte) { diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h index c1bc6731125cb..d5d3fbece174e 100644 --- a/include/linux/mm_types_task.h +++ b/include/linux/mm_types_task.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -43,24 +44,52 @@ struct vmacache { enum { MM_FILEPAGES, /* Resident file mapping pages */ MM_ANONPAGES, /* Resident anonymous pages */ - MM_SWAPENTS, /* Anonymous swap entries */ MM_SHMEMPAGES, /* Resident shared memory pages */ + MM_SWAPENTS, /* Anonymous swap entries */ NR_MM_COUNTERS }; -#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU) -#define SPLIT_RSS_COUNTING -/* per-thread cached information, */ -struct task_rss_stat { - int events; /* for synchronization threshold */ - int count[NR_MM_COUNTERS]; -}; -#endif /* USE_SPLIT_PTE_PTLOCKS */ - struct mm_rss_stat { atomic_long_t count[NR_MM_COUNTERS]; }; +struct mm_rss_cache { + /* + * CPU local only variables, hot path for RSS caching. Readonly for other CPUs. + */ + unsigned long in_use; + long count[NR_MM_COUNTERS]; + + /* Avoid false sharing when other CPUs collect RSS counter */ + struct mm_struct *mm ____cacheline_aligned; + /* Avoid ABA problem and RSS being accounted for wrong mm */ + unsigned long sync_count; +}; + +/* lowest bit of *mm is never used, so use it as a syncing flag */ +#define RSS_CACHE_MM_SYNCING_MASK 1UL + +/* mark the mm as being synced on that cache */ +static __always_inline struct mm_struct *__pcp_rss_mm_mark(struct mm_struct *mm) +{ + unsigned long val = (unsigned long)mm; + + val |= RSS_CACHE_MM_SYNCING_MASK; + + return (struct mm_struct *) val; +} + +static __always_inline struct mm_struct *__pcp_rss_mm_unmark(struct mm_struct *mm) +{ + unsigned long val = (unsigned long)mm; + + val &= ~RSS_CACHE_MM_SYNCING_MASK; + + return (struct mm_struct *) val; +} + +void switch_pcp_rss_cache_no_irq(struct mm_struct *next_mm); + struct page_frag { struct page *page; #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) diff --git a/include/linux/sched.h b/include/linux/sched.h index 4bc88e979b2f4..755fe31f11cbf 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -862,9 +862,6 @@ struct task_struct { /* Per-thread vma caching: */ struct vmacache vmacache; -#ifdef SPLIT_RSS_COUNTING - struct task_rss_stat rss_stat; -#endif int exit_state; int exit_code; int exit_signal; diff --git a/kernel/exit.c b/kernel/exit.c index c9ffa2bdb535d..b127cb896eeb7 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -483,7 +483,6 @@ static void exit_mm(void) exit_mm_release(current, mm); if (!mm) return; - sync_mm_rss(mm); mmap_read_lock(mm); mmgrab(mm); BUG_ON(mm != current->active_mm); @@ -750,10 +749,6 @@ void __noreturn do_exit(long code) io_uring_files_cancel(); exit_signals(tsk); /* sets PF_EXITING */ - - /* sync mm's RSS info before statistics gathering */ - if (tsk->mm) - sync_mm_rss(tsk->mm); acct_update_integrals(tsk); group_dead = atomic_dec_and_test(&tsk->signal->live); if (group_dead) { diff --git a/kernel/fork.c b/kernel/fork.c index b0456bd944ff3..19f9280dfaf83 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -132,15 +132,6 @@ int nr_threads; /* The idle threads do not count.. */ static int max_threads; /* tunable limit on nr_threads */ -#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) - -static const char * const resident_page_types[] = { - NAMED_ARRAY_INDEX(MM_FILEPAGES), - NAMED_ARRAY_INDEX(MM_ANONPAGES), - NAMED_ARRAY_INDEX(MM_SWAPENTS), - NAMED_ARRAY_INDEX(MM_SHMEMPAGES), -}; - DEFINE_PER_CPU(unsigned long, process_counts) = 0; __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ @@ -752,30 +743,6 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) #define mm_free_pgd(mm) #endif /* CONFIG_MMU */ -static void check_mm(struct mm_struct *mm) -{ - int i; - - BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, - "Please make sure 'struct resident_page_types[]' is updated as well"); - - for (i = 0; i < NR_MM_COUNTERS; i++) { - long x = atomic_long_read(&mm->rss_stat.count[i]); - - if (unlikely(x)) - pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", - mm, resident_page_types[i], x); - } - - if (mm_pgtables_bytes(mm)) - pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", - mm_pgtables_bytes(mm)); - -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS - VM_BUG_ON_MM(mm->pmd_huge_pte, mm); -#endif -} - #define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL)) #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) @@ -792,9 +759,9 @@ void __mmdrop(struct mm_struct *mm) mm_free_pgd(mm); destroy_context(mm); mmu_notifier_subscriptions_destroy(mm); - check_mm(mm); put_user_ns(mm->user_ns); mm_pasid_drop(mm); + check_discard_mm(mm); free_mm(mm); } EXPORT_SYMBOL_GPL(__mmdrop); @@ -2155,10 +2122,6 @@ static __latent_entropy struct task_struct *copy_process( p->io_uring = NULL; #endif -#if defined(SPLIT_RSS_COUNTING) - memset(&p->rss_stat, 0, sizeof(p->rss_stat)); -#endif - p->default_timer_slack_ns = current->timer_slack_ns; #ifdef CONFIG_PSI diff --git a/kernel/kthread.c b/kernel/kthread.c index 3c677918d8f2f..6bfbab4e2103c 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -1463,7 +1463,6 @@ void kthread_unuse_mm(struct mm_struct *mm) * clearing tsk->mm. */ smp_mb__after_spinlock(); - sync_mm_rss(mm); local_irq_disable(); tsk->mm = NULL; membarrier_update_current_mm(NULL); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 98db3c8171602..c0b1e38a98638 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5146,6 +5146,10 @@ context_switch(struct rq *rq, struct task_struct *prev, prepare_lock_switch(rq, next, rf); + /* Cache new active_mm */ + if (!IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) + switch_pcp_rss_cache_no_irq(next->active_mm); + /* Here we just switch the register state and the stack. */ switch_to(prev, next, prev); barrier(); diff --git a/mm/madvise.c b/mm/madvise.c index b71963da64641..9aed49db1b0be 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -711,12 +711,9 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, mark_page_lazyfree(page); } out: - if (nr_swap) { - if (current->mm == mm) - sync_mm_rss(mm); - + if (nr_swap) add_mm_counter(mm, MM_SWAPENTS, nr_swap); - } + arch_leave_lazy_mmu_mode(); pte_unmap_unlock(orig_pte, ptl); cond_resched(); diff --git a/mm/memory.c b/mm/memory.c index d8c5632ae5548..ec53af7908a54 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -164,52 +164,245 @@ void mm_trace_rss_stat(struct mm_struct *mm, int member, long count) trace_rss_stat(mm, member, count); } -#if defined(SPLIT_RSS_COUNTING) +static DEFINE_PER_CPU_SHARED_ALIGNED(struct mm_rss_cache, cpu_rss_cache); -void sync_mm_rss(struct mm_struct *mm) +/* + * get_mm_counter and get_mm_rss try to read the RSS cache of each + * CPU that cached target mm. If the cache is flushed while being read, + * skip it. May lead to rare and little bit of accuracy loss, but flushed + * cache will surely be accounted in the next read. + */ +unsigned long get_mm_counter(struct mm_struct *mm, int member) { - int i; + int cpu; + long ret, update, sync_count; + const struct cpumask *mm_mask; - for (i = 0; i < NR_MM_COUNTERS; i++) { - if (current->rss_stat.count[i]) { - add_mm_counter(mm, i, current->rss_stat.count[i]); - current->rss_stat.count[i] = 0; - } + ret = atomic_long_read(&mm->rss_stat.count[member]); + + if (IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) + mm_mask = mm_cpumask(mm); + else + mm_mask = cpu_possible_mask; + + for_each_cpu(cpu, mm_mask) { + if (READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)) != mm) + continue; + sync_count = READ_ONCE(per_cpu(cpu_rss_cache.sync_count, cpu)); + /* see smp_mb in switch_pcp_rss_cache_no_irq */ + smp_rmb(); + + update = READ_ONCE(per_cpu(cpu_rss_cache.count[member], cpu)); + + /* same as above */ + smp_rmb(); + if (READ_ONCE(per_cpu(cpu_rss_cache.sync_count, cpu)) == sync_count && + READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)) == mm) + ret += update; } - current->rss_stat.events = 0; + + if (ret < 0) + ret = 0; + + return ret; } -static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) +/* see comment for get_mm_counter */ +unsigned long get_mm_rss(struct mm_struct *mm) { - struct task_struct *task = current; + int cpu; + long ret, update, sync_count; + const struct cpumask *mm_mask; + + ret = atomic_long_read(&mm->rss_stat.count[MM_FILEPAGES]), + + atomic_long_read(&mm->rss_stat.count[MM_ANONPAGES]), + + atomic_long_read(&mm->rss_stat.count[MM_SHMEMPAGES]); - if (likely(task->mm == mm)) - task->rss_stat.count[member] += val; + if (IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) + mm_mask = mm_cpumask(mm); else - add_mm_counter(mm, member, val); + mm_mask = cpu_possible_mask; + + for_each_cpu(cpu, mm_mask) { + if (READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)) != mm) + continue; + sync_count = READ_ONCE(per_cpu(cpu_rss_cache.sync_count, cpu)); + /* see smp_mb in switch_pcp_rss_cache_no_irq */ + smp_rmb(); + + /* Reads MM_FILEPAGES, MM_ANONPAGES, MM_SHMEMPAGES */ + for (int i = MM_FILEPAGES; i < MM_SWAPENTS; i++) + update += READ_ONCE(per_cpu(cpu_rss_cache.count[i], cpu)); + + /* same as above */ + smp_rmb(); + if (READ_ONCE(per_cpu(cpu_rss_cache.sync_count, cpu)) == sync_count && + READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)) == mm) + ret += update; + } + + if (ret < 0) + ret = 0; + + return ret; } -#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) -#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) -/* sync counter once per 64 page faults */ -#define TASK_RSS_EVENTS_THRESH (64) -static void check_sync_rss_stat(struct task_struct *task) +/* flush the rss cache of current CPU with IRQ disabled, and switch to new mm */ +void switch_pcp_rss_cache_no_irq(struct mm_struct *next_mm) { - if (unlikely(task != current)) + long count; + struct mm_struct *cpu_mm; + + cpu_mm = this_cpu_read(cpu_rss_cache.mm); + if (cpu_mm == next_mm) + return; + + /* + * `in_use` counter is hold with preempt disabled, if non-zero, this would be a + * interrupt switching the mm, just ignore it. + */ + if (this_cpu_read(cpu_rss_cache.in_use)) return; - if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) - sync_mm_rss(task->mm); + + if (cpu_mm == NULL) + goto commit_done; + + /* Arch will take care of cache invalidation */ + if (!IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) { + /* Race with check_discard_rss_cache */ + if (cpu_mm != cmpxchg(this_cpu_ptr(&cpu_rss_cache.mm), cpu_mm, + __pcp_rss_mm_mark(cpu_mm))) + goto commit_done; + } + + for (int i = 0; i < NR_MM_COUNTERS; i++) { + count = this_cpu_read(cpu_rss_cache.count[i]); + if (count) + add_mm_counter(cpu_mm, i, count); + } + +commit_done: + for (int i = 0; i < NR_MM_COUNTERS; i++) + this_cpu_write(cpu_rss_cache.count[i], 0); + + /* + * For remote reading in get_mm_{rss,counter}, + * ensure new mm and sync counter have zero'ed counters + */ + smp_wmb(); + this_cpu_write(cpu_rss_cache.mm, next_mm); + this_cpu_inc(cpu_rss_cache.sync_count); } -#else /* SPLIT_RSS_COUNTING */ -#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) -#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) +static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) +{ + /* + * Disable preempt so task is pinned, and the mm is pinned on this CPU + * since caller must be holding a reference. + */ + preempt_disable(); + this_cpu_inc(cpu_rss_cache.in_use); + + if (likely(mm == this_cpu_read(cpu_rss_cache.mm))) { + this_cpu_add(cpu_rss_cache.count[member], val); + this_cpu_dec(cpu_rss_cache.in_use); + /* Avoid the resched checking oveahead for fast path */ + preempt_enable_no_resched(); + } else { + this_cpu_dec(cpu_rss_cache.in_use); + preempt_enable_no_resched(); + add_mm_counter(mm, member, val); + } +} -static void check_sync_rss_stat(struct task_struct *task) +#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) +#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) + +#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) +static const char * const resident_page_types[] = { + NAMED_ARRAY_INDEX(MM_FILEPAGES), + NAMED_ARRAY_INDEX(MM_ANONPAGES), + NAMED_ARRAY_INDEX(MM_SWAPENTS), + NAMED_ARRAY_INDEX(MM_SHMEMPAGES), +}; + +static void check_discard_rss_cache(struct mm_struct *mm) { + int cpu; + long cached_count[NR_MM_COUNTERS] = { 0 }; + struct mm_struct *cpu_mm; + + /* Arch will take care of cache invalidation */ + if (!IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) { + /* Invalidate the RSS cache on every CPU */ + for_each_possible_cpu(cpu) { + cpu_mm = READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)); + if (__pcp_rss_mm_unmark(cpu_mm) != mm) + continue; + + /* + * If not being flusehd, try read-in the counter and mark it NULL, + * once cache's mm is set NULL, counter are considered invalided. + */ + if (cpu_mm != __pcp_rss_mm_mark(cpu_mm)) { + long count[NR_MM_COUNTERS]; + + for (int i = 0; i < NR_MM_COUNTERS; i++) + count[i] = READ_ONCE(per_cpu(cpu_rss_cache.count[i], cpu)); + + /* + * If successfully set to NULL, the owner CPU is not flushing it, + * counters are uncommitted and untouched during this period, since + * a dying mm won't be accouted anymore. + */ + cpu_mm = cmpxchg(&per_cpu(cpu_rss_cache.mm, cpu), mm, NULL); + if (cpu_mm == mm) { + for (int i = 0; i < NR_MM_COUNTERS; i++) + cached_count[i] += count[i]; + continue; + } + } + + /* + * It's being flushed, just busy wait as the critial section + * is really short. + */ + do { + cpu_relax(); + cpu_mm = READ_ONCE(per_cpu(cpu_rss_cache.mm, cpu)); + } while (cpu_mm == __pcp_rss_mm_mark(mm)); + } + } + + for (int i = 0; i < NR_MM_COUNTERS; i++) { + long val = atomic_long_read(&mm->rss_stat.count[i]); + + if (!IS_ENABLED(CONFIG_ARCH_PCP_RSS_USE_CPUMASK)) { + val += cached_count[i]; + } + + if (unlikely(val)) { + pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", + mm, resident_page_types[i], val); + } + } } -#endif /* SPLIT_RSS_COUNTING */ +void check_discard_mm(struct mm_struct *mm) +{ + BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, + "Please make sure 'struct resident_page_types[]' is updated as well"); + + if (mm_pgtables_bytes(mm)) + pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", + mm_pgtables_bytes(mm)); + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS + VM_BUG_ON_MM(mm->pmd_huge_pte, mm); +#endif + + check_discard_rss_cache(mm); +} /* * Note: this doesn't free the actual pages themselves. That @@ -490,8 +683,6 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) { int i; - if (current->mm == mm) - sync_mm_rss(mm); for (i = 0; i < NR_MM_COUNTERS; i++) if (rss[i]) add_mm_counter(mm, i, rss[i]); @@ -4240,7 +4431,7 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) if (write) entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); - add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR); + add_mm_counter_fast(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR); page_add_file_rmap(page, vma, true); /* @@ -5132,9 +5323,6 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, count_vm_event(PGFAULT); count_memcg_event_mm(vma->vm_mm, PGFAULT); - /* do counter updates before entering really critical section. */ - check_sync_rss_stat(current); - if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, flags & FAULT_FLAG_INSTRUCTION, flags & FAULT_FLAG_REMOTE)) -- 2.37.1 From bda11a4c1d2b4729d7646a30b312db1427764951 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:35:13 +0200 Subject: [PATCH 14/16] perf Signed-off-by: Peter Jung --- arch/x86/events/amd/uncore.c | 146 ++++++++++++++++++++++++------ arch/x86/events/intel/core.c | 7 +- arch/x86/events/intel/ds.c | 129 +++++++++++++++++--------- arch/x86/events/perf_event.h | 17 +++- arch/x86/include/asm/perf_event.h | 16 ++++ include/linux/perf_event.h | 2 + include/uapi/linux/perf_event.h | 5 +- kernel/events/core.c | 21 ++++- kernel/events/ring_buffer.c | 5 +- 9 files changed, 270 insertions(+), 78 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 0d04414b97d2d..d568afc705d2e 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -21,7 +21,6 @@ #define NUM_COUNTERS_NB 4 #define NUM_COUNTERS_L2 4 #define NUM_COUNTERS_L3 6 -#define MAX_COUNTERS 6 #define RDPMC_BASE_NB 6 #define RDPMC_BASE_LLC 10 @@ -31,6 +30,7 @@ #undef pr_fmt #define pr_fmt(fmt) "amd_uncore: " fmt +static int pmu_version; static int num_counters_llc; static int num_counters_nb; static bool l3_mask; @@ -46,7 +46,7 @@ struct amd_uncore { u32 msr_base; cpumask_t *active_mask; struct pmu *pmu; - struct perf_event *events[MAX_COUNTERS]; + struct perf_event **events; struct hlist_node node; }; @@ -158,6 +158,16 @@ static int amd_uncore_add(struct perf_event *event, int flags) hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + /* + * The first four DF counters are accessible via RDPMC index 6 to 9 + * followed by the L3 counters from index 10 to 15. For processors + * with more than four DF counters, the DF RDPMC assignments become + * discontiguous as the additional counters are accessible starting + * from index 16. + */ + if (is_nb_event(event) && hwc->idx >= NUM_COUNTERS_NB) + hwc->event_base_rdpmc += NUM_COUNTERS_L3; + if (flags & PERF_EF_START) amd_uncore_start(event, PERF_EF_RELOAD); @@ -209,10 +219,14 @@ static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore *uncore; struct hw_perf_event *hwc = &event->hw; + u64 event_mask = AMD64_RAW_EVENT_MASK_NB; if (event->attr.type != event->pmu->type) return -ENOENT; + if (pmu_version >= 2 && is_nb_event(event)) + event_mask = AMD64_PERFMON_V2_RAW_EVENT_MASK_NB; + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, @@ -221,7 +235,7 @@ static int amd_uncore_event_init(struct perf_event *event) * out. So we do not support sampling and per-thread events via * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts: */ - hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB; + hwc->config = event->attr.config & event_mask; hwc->idx = -1; if (event->cpu < 0) @@ -247,6 +261,19 @@ static int amd_uncore_event_init(struct perf_event *event) return 0; } +static umode_t +amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ? + attr->mode : 0; +} + +static umode_t +amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0; +} + static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, struct device_attribute *attr, char *buf) @@ -287,8 +314,10 @@ static struct device_attribute format_attr_##_var = \ DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */ +DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */ DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */ -DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); +DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15"); +DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */ DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */ DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */ @@ -297,20 +326,33 @@ DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ +/* Common DF and NB attributes */ static struct attribute *amd_uncore_df_format_attr[] = { - &format_attr_event12.attr, /* event14 if F17h+ */ - &format_attr_umask.attr, + &format_attr_event12.attr, /* event */ + &format_attr_umask8.attr, /* umask */ NULL, }; +/* Common L2 and L3 attributes */ static struct attribute *amd_uncore_l3_format_attr[] = { - &format_attr_event12.attr, /* event8 if F17h+ */ - &format_attr_umask.attr, - NULL, /* slicemask if F17h, coreid if F19h */ - NULL, /* threadmask8 if F17h, enallslices if F19h */ - NULL, /* enallcores if F19h */ - NULL, /* sliceid if F19h */ - NULL, /* threadmask2 if F19h */ + &format_attr_event12.attr, /* event */ + &format_attr_umask8.attr, /* umask */ + NULL, /* threadmask */ + NULL, +}; + +/* F17h unique L3 attributes */ +static struct attribute *amd_f17h_uncore_l3_format_attr[] = { + &format_attr_slicemask.attr, /* slicemask */ + NULL, +}; + +/* F19h unique L3 attributes */ +static struct attribute *amd_f19h_uncore_l3_format_attr[] = { + &format_attr_coreid.attr, /* coreid */ + &format_attr_enallslices.attr, /* enallslices */ + &format_attr_enallcores.attr, /* enallcores */ + &format_attr_sliceid.attr, /* sliceid */ NULL, }; @@ -324,6 +366,18 @@ static struct attribute_group amd_uncore_l3_format_group = { .attrs = amd_uncore_l3_format_attr, }; +static struct attribute_group amd_f17h_uncore_l3_format_group = { + .name = "format", + .attrs = amd_f17h_uncore_l3_format_attr, + .is_visible = amd_f17h_uncore_is_visible, +}; + +static struct attribute_group amd_f19h_uncore_l3_format_group = { + .name = "format", + .attrs = amd_f19h_uncore_l3_format_attr, + .is_visible = amd_f19h_uncore_is_visible, +}; + static const struct attribute_group *amd_uncore_df_attr_groups[] = { &amd_uncore_attr_group, &amd_uncore_df_format_group, @@ -336,6 +390,12 @@ static const struct attribute_group *amd_uncore_l3_attr_groups[] = { NULL, }; +static const struct attribute_group *amd_uncore_l3_attr_update[] = { + &amd_f17h_uncore_l3_format_group, + &amd_f19h_uncore_l3_format_group, + NULL, +}; + static struct pmu amd_nb_pmu = { .task_ctx_nr = perf_invalid_context, .attr_groups = amd_uncore_df_attr_groups, @@ -353,6 +413,7 @@ static struct pmu amd_nb_pmu = { static struct pmu amd_llc_pmu = { .task_ctx_nr = perf_invalid_context, .attr_groups = amd_uncore_l3_attr_groups, + .attr_update = amd_uncore_l3_attr_update, .name = "amd_l2", .event_init = amd_uncore_event_init, .add = amd_uncore_add, @@ -370,11 +431,19 @@ static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) cpu_to_node(cpu)); } +static inline struct perf_event ** +amd_uncore_events_alloc(unsigned int num, unsigned int cpu) +{ + return kzalloc_node(sizeof(struct perf_event *) * num, GFP_KERNEL, + cpu_to_node(cpu)); +} + static int amd_uncore_cpu_up_prepare(unsigned int cpu) { - struct amd_uncore *uncore_nb = NULL, *uncore_llc; + struct amd_uncore *uncore_nb = NULL, *uncore_llc = NULL; if (amd_uncore_nb) { + *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; uncore_nb = amd_uncore_alloc(cpu); if (!uncore_nb) goto fail; @@ -384,11 +453,15 @@ static int amd_uncore_cpu_up_prepare(unsigned int cpu) uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL; uncore_nb->active_mask = &amd_nb_active_mask; uncore_nb->pmu = &amd_nb_pmu; + uncore_nb->events = amd_uncore_events_alloc(num_counters_nb, cpu); + if (!uncore_nb->events) + goto fail; uncore_nb->id = -1; *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb; } if (amd_uncore_llc) { + *per_cpu_ptr(amd_uncore_llc, cpu) = NULL; uncore_llc = amd_uncore_alloc(cpu); if (!uncore_llc) goto fail; @@ -398,6 +471,9 @@ static int amd_uncore_cpu_up_prepare(unsigned int cpu) uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL; uncore_llc->active_mask = &amd_llc_active_mask; uncore_llc->pmu = &amd_llc_pmu; + uncore_llc->events = amd_uncore_events_alloc(num_counters_llc, cpu); + if (!uncore_llc->events) + goto fail; uncore_llc->id = -1; *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc; } @@ -405,9 +481,16 @@ static int amd_uncore_cpu_up_prepare(unsigned int cpu) return 0; fail: - if (amd_uncore_nb) - *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; - kfree(uncore_nb); + if (uncore_nb) { + kfree(uncore_nb->events); + kfree(uncore_nb); + } + + if (uncore_llc) { + kfree(uncore_llc->events); + kfree(uncore_llc); + } + return -ENOMEM; } @@ -540,8 +623,11 @@ static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) if (cpu == uncore->cpu) cpumask_clear_cpu(cpu, uncore->active_mask); - if (!--uncore->refcnt) + if (!--uncore->refcnt) { + kfree(uncore->events); kfree(uncore); + } + *per_cpu_ptr(uncores, cpu) = NULL; } @@ -560,6 +646,7 @@ static int __init amd_uncore_init(void) { struct attribute **df_attr = amd_uncore_df_format_attr; struct attribute **l3_attr = amd_uncore_l3_format_attr; + union cpuid_0x80000022_ebx ebx; int ret = -ENODEV; if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && @@ -569,6 +656,9 @@ static int __init amd_uncore_init(void) if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) return -ENODEV; + if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) + pmu_version = 2; + num_counters_nb = NUM_COUNTERS_NB; num_counters_llc = NUM_COUNTERS_L2; if (boot_cpu_data.x86 >= 0x17) { @@ -585,8 +675,12 @@ static int __init amd_uncore_init(void) } if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { - if (boot_cpu_data.x86 >= 0x17) + if (pmu_version >= 2) { + *df_attr++ = &format_attr_event14v2.attr; + *df_attr++ = &format_attr_umask12.attr; + } else if (boot_cpu_data.x86 >= 0x17) { *df_attr = &format_attr_event14.attr; + } amd_uncore_nb = alloc_percpu(struct amd_uncore *); if (!amd_uncore_nb) { @@ -597,6 +691,11 @@ static int __init amd_uncore_init(void) if (ret) goto fail_nb; + if (pmu_version >= 2) { + ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); + num_counters_nb = ebx.split.num_df_pmc; + } + pr_info("%d %s %s counters detected\n", num_counters_nb, boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", amd_nb_pmu.name); @@ -607,16 +706,11 @@ static int __init amd_uncore_init(void) if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { if (boot_cpu_data.x86 >= 0x19) { *l3_attr++ = &format_attr_event8.attr; - *l3_attr++ = &format_attr_umask.attr; - *l3_attr++ = &format_attr_coreid.attr; - *l3_attr++ = &format_attr_enallslices.attr; - *l3_attr++ = &format_attr_enallcores.attr; - *l3_attr++ = &format_attr_sliceid.attr; + *l3_attr++ = &format_attr_umask8.attr; *l3_attr++ = &format_attr_threadmask2.attr; } else if (boot_cpu_data.x86 >= 0x17) { *l3_attr++ = &format_attr_event8.attr; - *l3_attr++ = &format_attr_umask.attr; - *l3_attr++ = &format_attr_slicemask.attr; + *l3_attr++ = &format_attr_umask8.attr; *l3_attr++ = &format_attr_threadmask8.attr; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 45024abd929f0..bd8b988576097 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4141,6 +4141,8 @@ tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx, { struct event_constraint *c; + c = intel_get_event_constraints(cpuc, idx, event); + /* * :ppp means to do reduced skid PEBS, * which is available on PMC0 and fixed counter 0. @@ -4153,8 +4155,6 @@ tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx, return &counter0_constraint; } - c = intel_get_event_constraints(cpuc, idx, event); - return c; } @@ -6241,7 +6241,8 @@ __init int intel_pmu_init(void) x86_pmu.flags |= PMU_FL_INSTR_LATENCY; x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX; x86_pmu.lbr_pt_coexist = true; - intel_pmu_pebs_data_source_skl(false); + intel_pmu_pebs_data_source_adl(); + x86_pmu.pebs_latency_data = adl_latency_data_small; x86_pmu.num_topdown_events = 8; x86_pmu.update_topdown_event = adl_update_topdown_event; x86_pmu.set_topdown_event_period = adl_set_topdown_event_period; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 376cc3d66094c..ba60427caa6d3 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -94,15 +94,40 @@ void __init intel_pmu_pebs_data_source_nhm(void) pebs_data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM); } -void __init intel_pmu_pebs_data_source_skl(bool pmem) +static void __init __intel_pmu_pebs_data_source_skl(bool pmem, u64 *data_source) { u64 pmem_or_l4 = pmem ? LEVEL(PMEM) : LEVEL(L4); - pebs_data_source[0x08] = OP_LH | pmem_or_l4 | P(SNOOP, HIT); - pebs_data_source[0x09] = OP_LH | pmem_or_l4 | REM | P(SNOOP, HIT); - pebs_data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE); - pebs_data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD); - pebs_data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM); + data_source[0x08] = OP_LH | pmem_or_l4 | P(SNOOP, HIT); + data_source[0x09] = OP_LH | pmem_or_l4 | REM | P(SNOOP, HIT); + data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE); + data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD); + data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM); +} + +void __init intel_pmu_pebs_data_source_skl(bool pmem) +{ + __intel_pmu_pebs_data_source_skl(pmem, pebs_data_source); +} + +static void __init intel_pmu_pebs_data_source_grt(u64 *data_source) +{ + data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT); + data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM); + data_source[0x08] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD); +} + +void __init intel_pmu_pebs_data_source_adl(void) +{ + u64 *data_source; + + data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX].pebs_data_source; + memcpy(data_source, pebs_data_source, sizeof(pebs_data_source)); + __intel_pmu_pebs_data_source_skl(false, data_source); + + data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source; + memcpy(data_source, pebs_data_source, sizeof(pebs_data_source)); + intel_pmu_pebs_data_source_grt(data_source); } static u64 precise_store_data(u64 status) @@ -171,7 +196,50 @@ static u64 precise_datala_hsw(struct perf_event *event, u64 status) return dse.val; } -static u64 load_latency_data(u64 status) +static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock) +{ + /* + * TLB access + * 0 = did not miss 2nd level TLB + * 1 = missed 2nd level TLB + */ + if (tlb) + *val |= P(TLB, MISS) | P(TLB, L2); + else + *val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2); + + /* locked prefix */ + if (lock) + *val |= P(LOCK, LOCKED); +} + +/* Retrieve the latency data for e-core of ADL */ +u64 adl_latency_data_small(struct perf_event *event, u64 status) +{ + union intel_x86_pebs_dse dse; + u64 val; + + WARN_ON_ONCE(hybrid_pmu(event->pmu)->cpu_type == hybrid_big); + + dse.val = status; + + val = hybrid_var(event->pmu, pebs_data_source)[dse.ld_dse]; + + /* + * For the atom core on ADL, + * bit 4: lock, bit 5: TLB access. + */ + pebs_set_tlb_lock(&val, dse.ld_locked, dse.ld_stlb_miss); + + if (dse.ld_data_blk) + val |= P(BLK, DATA); + else + val |= P(BLK, NA); + + return val; +} + +static u64 load_latency_data(struct perf_event *event, u64 status) { union intel_x86_pebs_dse dse; u64 val; @@ -181,7 +249,7 @@ static u64 load_latency_data(u64 status) /* * use the mapping table for bit 0-3 */ - val = pebs_data_source[dse.ld_dse]; + val = hybrid_var(event->pmu, pebs_data_source)[dse.ld_dse]; /* * Nehalem models do not support TLB, Lock infos @@ -190,21 +258,8 @@ static u64 load_latency_data(u64 status) val |= P(TLB, NA) | P(LOCK, NA); return val; } - /* - * bit 4: TLB access - * 0 = did not miss 2nd level TLB - * 1 = missed 2nd level TLB - */ - if (dse.ld_stlb_miss) - val |= P(TLB, MISS) | P(TLB, L2); - else - val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2); - /* - * bit 5: locked prefix - */ - if (dse.ld_locked) - val |= P(LOCK, LOCKED); + pebs_set_tlb_lock(&val, dse.ld_stlb_miss, dse.ld_locked); /* * Ice Lake and earlier models do not support block infos. @@ -233,7 +288,7 @@ static u64 load_latency_data(u64 status) return val; } -static u64 store_latency_data(u64 status) +static u64 store_latency_data(struct perf_event *event, u64 status) { union intel_x86_pebs_dse dse; u64 val; @@ -243,23 +298,9 @@ static u64 store_latency_data(u64 status) /* * use the mapping table for bit 0-3 */ - val = pebs_data_source[dse.st_lat_dse]; + val = hybrid_var(event->pmu, pebs_data_source)[dse.st_lat_dse]; - /* - * bit 4: TLB access - * 0 = did not miss 2nd level TLB - * 1 = missed 2nd level TLB - */ - if (dse.st_lat_stlb_miss) - val |= P(TLB, MISS) | P(TLB, L2); - else - val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2); - - /* - * bit 5: locked prefix - */ - if (dse.st_lat_locked) - val |= P(LOCK, LOCKED); + pebs_set_tlb_lock(&val, dse.st_lat_stlb_miss, dse.st_lat_locked); val |= P(BLK, NA); @@ -781,8 +822,8 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_PLD_CONSTRAINT(0x5d0, 0xf), - INTEL_PSD_CONSTRAINT(0x6d0, 0xf), + INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xf), + INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf), EVENT_CONSTRAINT_END }; @@ -1443,9 +1484,11 @@ static u64 get_data_src(struct perf_event *event, u64 aux) bool fst = fl & (PERF_X86_EVENT_PEBS_ST | PERF_X86_EVENT_PEBS_HSW_PREC); if (fl & PERF_X86_EVENT_PEBS_LDLAT) - val = load_latency_data(aux); + val = load_latency_data(event, aux); else if (fl & PERF_X86_EVENT_PEBS_STLAT) - val = store_latency_data(aux); + val = store_latency_data(event, aux); + else if (fl & PERF_X86_EVENT_PEBS_LAT_HYBRID) + val = x86_pmu.pebs_latency_data(event, aux); else if (fst && (fl & PERF_X86_EVENT_PEBS_HSW_PREC)) val = precise_datala_hsw(event, aux); else if (fst) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 21a5482bcf845..ca2f8bfe6ff10 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -84,6 +84,7 @@ static inline bool constraint_match(struct event_constraint *c, u64 ecode) #define PERF_X86_EVENT_TOPDOWN 0x04000 /* Count Topdown slots/metrics events */ #define PERF_X86_EVENT_PEBS_STLAT 0x08000 /* st+stlat data address sampling */ #define PERF_X86_EVENT_AMD_BRS 0x10000 /* AMD Branch Sampling */ +#define PERF_X86_EVENT_PEBS_LAT_HYBRID 0x20000 /* ld and st lat for hybrid */ static inline bool is_topdown_count(struct perf_event *event) { @@ -136,7 +137,8 @@ struct amd_nb { PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \ PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \ PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \ - PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE) + PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \ + PERF_SAMPLE_WEIGHT_TYPE) #define PEBS_GP_REGS \ ((1ULL << PERF_REG_X86_AX) | \ @@ -460,6 +462,10 @@ struct cpu_hw_events { __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST) +#define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \ + __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ + HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID) + /* Event constraint, but match on all event flags too. */ #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \ EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS) @@ -638,6 +644,8 @@ enum { x86_lbr_exclusive_max, }; +#define PERF_PEBS_DATA_SOURCE_MAX 0x10 + struct x86_hybrid_pmu { struct pmu pmu; const char *name; @@ -665,6 +673,8 @@ struct x86_hybrid_pmu { unsigned int late_ack :1, mid_ack :1, enabled_ack :1; + + u64 pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX]; }; static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu) @@ -825,6 +835,7 @@ struct x86_pmu { void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); struct event_constraint *pebs_constraints; void (*pebs_aliases)(struct perf_event *event); + u64 (*pebs_latency_data)(struct perf_event *event, u64 status); unsigned long large_pebs_flags; u64 rtm_abort_event; @@ -1392,6 +1403,8 @@ void intel_pmu_disable_bts(void); int intel_pmu_drain_bts_buffer(void); +u64 adl_latency_data_small(struct perf_event *event, u64 status); + extern struct event_constraint intel_core2_pebs_event_constraints[]; extern struct event_constraint intel_atom_pebs_event_constraints[]; @@ -1499,6 +1512,8 @@ void intel_pmu_pebs_data_source_nhm(void); void intel_pmu_pebs_data_source_skl(bool pmem); +void intel_pmu_pebs_data_source_adl(void); + int intel_pmu_setup_lbr_filter(struct perf_event *event); void intel_pt_interrupt(void); diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 409725e86f42c..34348ae41cdbd 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -89,6 +89,19 @@ #define AMD64_RAW_EVENT_MASK_NB \ (AMD64_EVENTSEL_EVENT | \ ARCH_PERFMON_EVENTSEL_UMASK) + +#define AMD64_PERFMON_V2_EVENTSEL_EVENT_NB \ + (AMD64_EVENTSEL_EVENT | \ + GENMASK_ULL(37, 36)) + +#define AMD64_PERFMON_V2_EVENTSEL_UMASK_NB \ + (ARCH_PERFMON_EVENTSEL_UMASK | \ + GENMASK_ULL(27, 24)) + +#define AMD64_PERFMON_V2_RAW_EVENT_MASK_NB \ + (AMD64_PERFMON_V2_EVENTSEL_EVENT_NB | \ + AMD64_PERFMON_V2_EVENTSEL_UMASK_NB) + #define AMD64_NUM_COUNTERS 4 #define AMD64_NUM_COUNTERS_CORE 6 #define AMD64_NUM_COUNTERS_NB 4 @@ -194,6 +207,9 @@ union cpuid_0x80000022_ebx { struct { /* Number of Core Performance Counters */ unsigned int num_core_pmc:4; + unsigned int reserved:6; + /* Number of Data Fabric Counters */ + unsigned int num_df_pmc:6; } split; unsigned int full; }; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index da759560eec56..ee8b9ecdc03b7 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -759,6 +759,8 @@ struct perf_event { struct pid_namespace *ns; u64 id; + atomic64_t lost_samples; + u64 (*clock)(void); perf_overflow_handler_t overflow_handler; void *overflow_handler_context; diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index d37629dbad72f..0474ee362151f 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -301,6 +301,7 @@ enum { * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING * { u64 id; } && PERF_FORMAT_ID + * { u64 lost; } && PERF_FORMAT_LOST * } && !PERF_FORMAT_GROUP * * { u64 nr; @@ -308,6 +309,7 @@ enum { * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING * { u64 value; * { u64 id; } && PERF_FORMAT_ID + * { u64 lost; } && PERF_FORMAT_LOST * } cntr[nr]; * } && PERF_FORMAT_GROUP * }; @@ -317,8 +319,9 @@ enum perf_event_read_format { PERF_FORMAT_TOTAL_TIME_RUNNING = 1U << 1, PERF_FORMAT_ID = 1U << 2, PERF_FORMAT_GROUP = 1U << 3, + PERF_FORMAT_LOST = 1U << 4, - PERF_FORMAT_MAX = 1U << 4, /* non-ABI */ + PERF_FORMAT_MAX = 1U << 5, /* non-ABI */ }; #define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ diff --git a/kernel/events/core.c b/kernel/events/core.c index d2b354991bf5a..c9d32d4d2e204 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1819,6 +1819,9 @@ static void __perf_event_read_size(struct perf_event *event, int nr_siblings) if (event->attr.read_format & PERF_FORMAT_ID) entry += sizeof(u64); + if (event->attr.read_format & PERF_FORMAT_LOST) + entry += sizeof(u64); + if (event->attr.read_format & PERF_FORMAT_GROUP) { nr += nr_siblings; size += sizeof(u64); @@ -5260,11 +5263,15 @@ static int __perf_read_group_add(struct perf_event *leader, values[n++] += perf_event_count(leader); if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(leader); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&leader->lost_samples); for_each_sibling_event(sub, leader) { values[n++] += perf_event_count(sub); if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(sub); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&sub->lost_samples); } raw_spin_unlock_irqrestore(&ctx->lock, flags); @@ -5321,7 +5328,7 @@ static int perf_read_one(struct perf_event *event, u64 read_format, char __user *buf) { u64 enabled, running; - u64 values[4]; + u64 values[5]; int n = 0; values[n++] = __perf_event_read_value(event, &enabled, &running); @@ -5331,6 +5338,8 @@ static int perf_read_one(struct perf_event *event, values[n++] = running; if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(event); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&event->lost_samples); if (copy_to_user(buf, values, n * sizeof(u64))) return -EFAULT; @@ -6858,7 +6867,7 @@ static void perf_output_read_one(struct perf_output_handle *handle, u64 enabled, u64 running) { u64 read_format = event->attr.read_format; - u64 values[4]; + u64 values[5]; int n = 0; values[n++] = perf_event_count(event); @@ -6872,6 +6881,8 @@ static void perf_output_read_one(struct perf_output_handle *handle, } if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(event); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&event->lost_samples); __output_copy(handle, values, n * sizeof(u64)); } @@ -6882,7 +6893,7 @@ static void perf_output_read_group(struct perf_output_handle *handle, { struct perf_event *leader = event->group_leader, *sub; u64 read_format = event->attr.read_format; - u64 values[5]; + u64 values[6]; int n = 0; values[n++] = 1 + leader->nr_siblings; @@ -6900,6 +6911,8 @@ static void perf_output_read_group(struct perf_output_handle *handle, values[n++] = perf_event_count(leader); if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(leader); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&leader->lost_samples); __output_copy(handle, values, n * sizeof(u64)); @@ -6913,6 +6926,8 @@ static void perf_output_read_group(struct perf_output_handle *handle, values[n++] = perf_event_count(sub); if (read_format & PERF_FORMAT_ID) values[n++] = primary_event_id(sub); + if (read_format & PERF_FORMAT_LOST) + values[n++] = atomic64_read(&sub->lost_samples); __output_copy(handle, values, n * sizeof(u64)); } diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index fb35b926024ca..726132039c388 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -172,8 +172,10 @@ __perf_output_begin(struct perf_output_handle *handle, goto out; if (unlikely(rb->paused)) { - if (rb->nr_pages) + if (rb->nr_pages) { local_inc(&rb->lost); + atomic64_inc(&event->lost_samples); + } goto out; } @@ -254,6 +256,7 @@ __perf_output_begin(struct perf_output_handle *handle, fail: local_inc(&rb->lost); + atomic64_inc(&event->lost_samples); perf_output_put_handle(handle); out: rcu_read_unlock(); -- 2.37.1 From d9200e735c43f50b0854d2b84412b6ece00c88b7 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:35:26 +0200 Subject: [PATCH 15/16] rcu Signed-off-by: Peter Jung --- .../admin-guide/kernel-parameters.txt | 6 + kernel/rcu/Kconfig | 13 + kernel/rcu/tasks.h | 28 +- kernel/rcu/tree.c | 10 +- kernel/rcu/tree.h | 3 +- kernel/rcu/tree_nocb.h | 266 +++++++++++------- kernel/rcu/tree_plugin.h | 33 ++- 7 files changed, 240 insertions(+), 119 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9d96da88c04f1..ee0b0605434a5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3674,6 +3674,9 @@ just as if they had also been called out in the rcu_nocbs= boot parameter. + Note that this argument takes precedence over + the CONFIG_RCU_NOCB_CPU_DEFAULT_ALL option. + noiotrap [SH] Disables trapped I/O port accesses. noirqdebug [X86-32] Disables the code which attempts to detect and @@ -4581,6 +4584,9 @@ no-callback mode from boot but the mode may be toggled at runtime via cpusets. + Note that this argument takes precedence over + the CONFIG_RCU_NOCB_CPU_DEFAULT_ALL option. + rcu_nocb_poll [KNL] Rather than requiring that offloaded CPUs (specified by rcu_nocbs= above) explicitly diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 1c630e573548d..27aab870ae4cf 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -262,6 +262,19 @@ config RCU_NOCB_CPU Say Y here if you need reduced OS jitter, despite added overhead. Say N here if you are unsure. +config RCU_NOCB_CPU_DEFAULT_ALL + bool "Offload RCU callback processing from all CPUs by default" + depends on RCU_NOCB_CPU + default n + help + Use this option to offload callback processing from all CPUs + by default, in the absence of the rcu_nocbs or nohz_full boot + parameter. This also avoids the need to use any boot parameters + to achieve the effect of offloading all CPUs on boot. + + Say Y here if you want offload all CPUs by default on boot. + Say N here if you are unsure. + config TASKS_TRACE_RCU_READ_MB bool "Tasks Trace RCU readers use memory barriers in user and idle" depends on RCU_EXPERT && TASKS_TRACE_RCU diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 3925e32159b5a..19087d45efddc 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -140,6 +140,7 @@ static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY; module_param(rcu_task_ipi_delay, int, 0644); /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ +#define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30) #define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10) static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT; module_param(rcu_task_stall_timeout, int, 0644); @@ -1686,23 +1687,24 @@ struct rcu_tasks_test_desc { struct rcu_head rh; const char *name; bool notrun; + unsigned long runstart; }; static struct rcu_tasks_test_desc tests[] = { { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_RCU), + .notrun = IS_ENABLED(CONFIG_TASKS_RCU), }, { .name = "call_rcu_tasks_rude()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_RUDE_RCU), + .notrun = IS_ENABLED(CONFIG_TASKS_RUDE_RCU), }, { .name = "call_rcu_tasks_trace()", /* If not defined, the test is skipped. */ - .notrun = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU) + .notrun = IS_ENABLED(CONFIG_TASKS_TRACE_RCU) } }; @@ -1713,23 +1715,28 @@ static void test_rcu_tasks_callback(struct rcu_head *rhp) pr_info("Callback from %s invoked.\n", rttd->name); - rttd->notrun = true; + rttd->notrun = false; } static void rcu_tasks_initiate_self_tests(void) { + unsigned long j = jiffies; + pr_info("Running RCU-tasks wait API self tests\n"); #ifdef CONFIG_TASKS_RCU + tests[0].runstart = j; synchronize_rcu_tasks(); call_rcu_tasks(&tests[0].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_RUDE_RCU + tests[1].runstart = j; synchronize_rcu_tasks_rude(); call_rcu_tasks_rude(&tests[1].rh, test_rcu_tasks_callback); #endif #ifdef CONFIG_TASKS_TRACE_RCU + tests[2].runstart = j; synchronize_rcu_tasks_trace(); call_rcu_tasks_trace(&tests[2].rh, test_rcu_tasks_callback); #endif @@ -1739,11 +1746,18 @@ static int rcu_tasks_verify_self_tests(void) { int ret = 0; int i; + unsigned long bst = rcu_task_stall_timeout; + if (bst <= 0 || bst > RCU_TASK_BOOT_STALL_TIMEOUT) + bst = RCU_TASK_BOOT_STALL_TIMEOUT; for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!tests[i].notrun) { // still hanging. - pr_err("%s has been failed.\n", tests[i].name); - ret = -1; + while (tests[i].notrun) { // still hanging. + if (time_after(jiffies, tests[i].runstart + bst)) { + pr_err("%s has failed boot-time tests.\n", tests[i].name); + ret = -1; + break; + } + schedule_timeout_uninterruptible(1); } } diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index fc993202dc35b..aed556a7a8be5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -154,7 +154,11 @@ static void sync_sched_exp_online_cleanup(int cpu); static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp); static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); -/* rcuc/rcub/rcuop kthread realtime priority */ +/* + * rcuc/rcub/rcuop kthread realtime priority. The "rcuop" + * real-time priority(enabling/disabling) is controlled by + * the extra CONFIG_RCU_NOCB_CPU_CB_BOOST configuration. + */ static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; module_param(kthread_prio, int, 0444); @@ -2530,7 +2534,7 @@ static void rcu_do_batch(struct rcu_data *rdp) trace_rcu_batch_end(rcu_state.name, 0, !rcu_segcblist_empty(&rdp->cblist), need_resched(), is_idle_task(current), - rcu_is_callbacks_kthread()); + rcu_is_callbacks_kthread(rdp)); return; } @@ -2608,7 +2612,7 @@ static void rcu_do_batch(struct rcu_data *rdp) rcu_nocb_lock_irqsave(rdp, flags); rdp->n_cbs_invoked += count; trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), - is_idle_task(current), rcu_is_callbacks_kthread()); + is_idle_task(current), rcu_is_callbacks_kthread(rdp)); /* Update counts and requeue any remaining callbacks. */ rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 2ccf5845957df..649ad4f0129b1 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -235,6 +235,7 @@ struct rcu_data { * if rdp_gp. */ struct list_head nocb_entry_rdp; /* rcu_data node in wakeup chain. */ + struct rcu_data *nocb_toggling_rdp; /* rdp queued for (de-)offloading */ /* The following fields are used by CB kthread, hence new cacheline. */ struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp; @@ -425,7 +426,7 @@ static void rcu_flavor_sched_clock_irq(int user); static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); -static bool rcu_is_callbacks_kthread(void); +static bool rcu_is_callbacks_kthread(struct rcu_data *rdp); static void rcu_cpu_kthread_setup(unsigned int cpu); static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp); static bool rcu_preempt_has_tasks(struct rcu_node *rnp); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 46694e13398a3..a8f574d8850d2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -546,52 +546,51 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } } -/* - * Check if we ignore this rdp. - * - * We check that without holding the nocb lock but - * we make sure not to miss a freshly offloaded rdp - * with the current ordering: - * - * rdp_offload_toggle() nocb_gp_enabled_cb() - * ------------------------- ---------------------------- - * WRITE flags LOCK nocb_gp_lock - * LOCK nocb_gp_lock READ/WRITE nocb_gp_sleep - * READ/WRITE nocb_gp_sleep UNLOCK nocb_gp_lock - * UNLOCK nocb_gp_lock READ flags - */ -static inline bool nocb_gp_enabled_cb(struct rcu_data *rdp) -{ - u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_GP; - - return rcu_segcblist_test_flags(&rdp->cblist, flags); -} - -static inline bool nocb_gp_update_state_deoffloading(struct rcu_data *rdp, - bool *needwake_state) +static int nocb_gp_toggle_rdp(struct rcu_data *rdp, + bool *wake_state) { struct rcu_segcblist *cblist = &rdp->cblist; + unsigned long flags; + int ret; - if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { - if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { - rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); - if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) - *needwake_state = true; - } - return false; + rcu_nocb_lock_irqsave(rdp, flags); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && + !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { + /* + * Offloading. Set our flag and notify the offload worker. + * We will handle this rdp until it ever gets de-offloaded. + */ + rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) + *wake_state = true; + ret = 1; + } else if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && + rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { + /* + * De-offloading. Clear our flag and notify the de-offload worker. + * We will ignore this rdp until it ever gets re-offloaded. + */ + rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); + if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) + *wake_state = true; + ret = 0; + } else { + WARN_ON_ONCE(1); + ret = -1; } - /* - * De-offloading. Clear our flag and notify the de-offload worker. - * We will ignore this rdp until it ever gets re-offloaded. - */ - WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); - rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); - if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) - *needwake_state = true; - return true; + rcu_nocb_unlock_irqrestore(rdp, flags); + + return ret; } +static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) +{ + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); + swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, + !READ_ONCE(my_rdp->nocb_gp_sleep)); + trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); +} /* * No-CBs GP kthreads come here to wait for additional callbacks to show up @@ -609,7 +608,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bool needwait_gp = false; // This prevents actual uninitialized use. bool needwake; bool needwake_gp; - struct rcu_data *rdp; + struct rcu_data *rdp, *rdp_toggling = NULL; struct rcu_node *rnp; unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning. bool wasempty = false; @@ -634,19 +633,10 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) * is added to the list, so the skipped-over rcu_data structures * won't be ignored for long. */ - list_for_each_entry_rcu(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp, 1) { - bool needwake_state = false; - - if (!nocb_gp_enabled_cb(rdp)) - continue; + list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); rcu_nocb_lock_irqsave(rdp, flags); - if (nocb_gp_update_state_deoffloading(rdp, &needwake_state)) { - rcu_nocb_unlock_irqrestore(rdp, flags); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); - continue; - } + lockdep_assert_held(&rdp->nocb_lock); bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); if (bypass_ncbs && (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || @@ -656,8 +646,6 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { rcu_nocb_unlock_irqrestore(rdp, flags); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); continue; /* No callbacks here, try next. */ } if (bypass_ncbs) { @@ -705,8 +693,6 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) } if (needwake_gp) rcu_gp_kthread_wake(); - if (needwake_state) - swake_up_one(&rdp->nocb_state_wq); } my_rdp->nocb_gp_bypass = bypass; @@ -723,13 +709,19 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) /* Polling, so trace if first poll in the series. */ if (gotcbs) trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); - schedule_timeout_idle(1); + if (list_empty(&my_rdp->nocb_head_rdp)) { + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + if (!my_rdp->nocb_toggling_rdp) + WRITE_ONCE(my_rdp->nocb_gp_sleep, true); + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + /* Wait for any offloading rdp */ + nocb_gp_sleep(my_rdp, cpu); + } else { + schedule_timeout_idle(1); + } } else if (!needwait_gp) { /* Wait for callbacks to appear. */ - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); - swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq, - !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); + nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); @@ -739,15 +731,49 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) !READ_ONCE(my_rdp->nocb_gp_sleep)); trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); } + if (!rcu_nocb_poll) { raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + // (De-)queue an rdp to/from the group if its nocb state is changing + rdp_toggling = my_rdp->nocb_toggling_rdp; + if (rdp_toggling) + my_rdp->nocb_toggling_rdp = NULL; + if (my_rdp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) { WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); del_timer(&my_rdp->nocb_timer); } WRITE_ONCE(my_rdp->nocb_gp_sleep, true); raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + } else { + rdp_toggling = READ_ONCE(my_rdp->nocb_toggling_rdp); + if (rdp_toggling) { + /* + * Paranoid locking to make sure nocb_toggling_rdp is well + * reset *before* we (re)set SEGCBLIST_KTHREAD_GP or we could + * race with another round of nocb toggling for this rdp. + * Nocb locking should prevent from that already but we stick + * to paranoia, especially in rare path. + */ + raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags); + my_rdp->nocb_toggling_rdp = NULL; + raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags); + } + } + + if (rdp_toggling) { + bool wake_state = false; + int ret; + + ret = nocb_gp_toggle_rdp(rdp_toggling, &wake_state); + if (ret == 1) + list_add_tail(&rdp_toggling->nocb_entry_rdp, &my_rdp->nocb_head_rdp); + else if (ret == 0) + list_del(&rdp_toggling->nocb_entry_rdp); + if (wake_state) + swake_up_one(&rdp_toggling->nocb_state_wq); } + my_rdp->nocb_gp_seq = -1; WARN_ON(signal_pending(current)); } @@ -966,16 +992,15 @@ static int rdp_offload_toggle(struct rcu_data *rdp, swake_up_one(&rdp->nocb_cb_wq); raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); + // Queue this rdp for add/del to/from the list to iterate on rcuog + WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp); if (rdp_gp->nocb_gp_sleep) { rdp_gp->nocb_gp_sleep = false; wake_gp = true; } raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags); - if (wake_gp) - wake_up_process(rdp_gp->nocb_gp_kthread); - - return 0; + return wake_gp; } static long rcu_nocb_rdp_deoffload(void *arg) @@ -983,9 +1008,15 @@ static long rcu_nocb_rdp_deoffload(void *arg) struct rcu_data *rdp = arg; struct rcu_segcblist *cblist = &rdp->cblist; unsigned long flags; - int ret; + int wake_gp; + struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; - WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); + /* + * rcu_nocb_rdp_deoffload() may be called directly if + * rcuog/o[p] spawn failed, because at this time the rdp->cpu + * is not online yet. + */ + WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu)); pr_info("De-offloading %d\n", rdp->cpu); @@ -1009,12 +1040,41 @@ static long rcu_nocb_rdp_deoffload(void *arg) */ rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE); invoke_rcu_core(); - ret = rdp_offload_toggle(rdp, false, flags); - swait_event_exclusive(rdp->nocb_state_wq, - !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | - SEGCBLIST_KTHREAD_GP)); - /* Stop nocb_gp_wait() from iterating over this structure. */ - list_del_rcu(&rdp->nocb_entry_rdp); + wake_gp = rdp_offload_toggle(rdp, false, flags); + + mutex_lock(&rdp_gp->nocb_gp_kthread_mutex); + if (rdp_gp->nocb_gp_kthread) { + if (wake_gp) + wake_up_process(rdp_gp->nocb_gp_kthread); + + /* + * If rcuo[p] kthread spawn failed, directly remove SEGCBLIST_KTHREAD_CB. + * Just wait SEGCBLIST_KTHREAD_GP to be cleared by rcuog. + */ + if (!rdp->nocb_cb_kthread) { + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); + rcu_nocb_unlock_irqrestore(rdp, flags); + } + + swait_event_exclusive(rdp->nocb_state_wq, + !rcu_segcblist_test_flags(cblist, + SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP)); + } else { + /* + * No kthread to clear the flags for us or remove the rdp from the nocb list + * to iterate. Do it here instead. Locking doesn't look stricly necessary + * but we stick to paranoia in this rare path. + */ + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_clear_flags(&rdp->cblist, + SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP); + rcu_nocb_unlock_irqrestore(rdp, flags); + + list_del(&rdp->nocb_entry_rdp); + } + mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); + /* * Lock one last time to acquire latest callback updates from kthreads * so we can later handle callbacks locally without locking. @@ -1035,7 +1095,7 @@ static long rcu_nocb_rdp_deoffload(void *arg) WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); - return ret; + return 0; } int rcu_nocb_cpu_deoffload(int cpu) @@ -1043,8 +1103,8 @@ int rcu_nocb_cpu_deoffload(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int ret = 0; - mutex_lock(&rcu_state.barrier_mutex); cpus_read_lock(); + mutex_lock(&rcu_state.barrier_mutex); if (rcu_rdp_is_offloaded(rdp)) { if (cpu_online(cpu)) { ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); @@ -1055,8 +1115,8 @@ int rcu_nocb_cpu_deoffload(int cpu) ret = -EINVAL; } } - cpus_read_unlock(); mutex_unlock(&rcu_state.barrier_mutex); + cpus_read_unlock(); return ret; } @@ -1067,7 +1127,8 @@ static long rcu_nocb_rdp_offload(void *arg) struct rcu_data *rdp = arg; struct rcu_segcblist *cblist = &rdp->cblist; unsigned long flags; - int ret; + int wake_gp; + struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); /* @@ -1077,17 +1138,10 @@ static long rcu_nocb_rdp_offload(void *arg) if (!rdp->nocb_gp_rdp) return -EINVAL; - pr_info("Offloading %d\n", rdp->cpu); + if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread)) + return -EINVAL; - /* - * Cause future nocb_gp_wait() invocations to iterate over - * structure, resetting ->nocb_gp_sleep and waking up the related - * "rcuog". Since nocb_gp_wait() in turn locks ->nocb_gp_lock - * before setting ->nocb_gp_sleep again, we are guaranteed to - * iterate this newly added structure before "rcuog" goes to - * sleep again. - */ - list_add_tail_rcu(&rdp->nocb_entry_rdp, &rdp->nocb_gp_rdp->nocb_head_rdp); + pr_info("Offloading %d\n", rdp->cpu); /* * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING @@ -1111,7 +1165,9 @@ static long rcu_nocb_rdp_offload(void *arg) * WRITE flags READ callbacks * rcu_nocb_unlock() rcu_nocb_unlock() */ - ret = rdp_offload_toggle(rdp, true, flags); + wake_gp = rdp_offload_toggle(rdp, true, flags); + if (wake_gp) + wake_up_process(rdp_gp->nocb_gp_kthread); swait_event_exclusive(rdp->nocb_state_wq, rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); @@ -1124,7 +1180,7 @@ static long rcu_nocb_rdp_offload(void *arg) rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE); rcu_nocb_unlock_irqrestore(rdp, flags); - return ret; + return 0; } int rcu_nocb_cpu_offload(int cpu) @@ -1132,8 +1188,8 @@ int rcu_nocb_cpu_offload(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int ret = 0; - mutex_lock(&rcu_state.barrier_mutex); cpus_read_lock(); + mutex_lock(&rcu_state.barrier_mutex); if (!rcu_rdp_is_offloaded(rdp)) { if (cpu_online(cpu)) { ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); @@ -1144,8 +1200,8 @@ int rcu_nocb_cpu_offload(int cpu) ret = -EINVAL; } } - cpus_read_unlock(); mutex_unlock(&rcu_state.barrier_mutex); + cpus_read_unlock(); return ret; } @@ -1155,11 +1211,21 @@ void __init rcu_init_nohz(void) { int cpu; bool need_rcu_nocb_mask = false; + bool offload_all = false; struct rcu_data *rdp; +#if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) + if (!rcu_state.nocb_is_setup) { + need_rcu_nocb_mask = true; + offload_all = true; + } +#endif /* #if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) */ + #if defined(CONFIG_NO_HZ_FULL) - if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) + if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) { need_rcu_nocb_mask = true; + offload_all = false; /* NO_HZ_FULL has its own mask. */ + } #endif /* #if defined(CONFIG_NO_HZ_FULL) */ if (need_rcu_nocb_mask) { @@ -1180,6 +1246,9 @@ void __init rcu_init_nohz(void) cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask); #endif /* #if defined(CONFIG_NO_HZ_FULL) */ + if (offload_all) + cpumask_setall(rcu_nocb_mask); + if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); cpumask_and(rcu_nocb_mask, cpu_possible_mask, @@ -1246,7 +1315,7 @@ static void rcu_spawn_cpu_nocb_kthread(int cpu) "rcuog/%d", rdp_gp->cpu); if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) { mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex); - return; + goto end; } WRITE_ONCE(rdp_gp->nocb_gp_kthread, t); if (kthread_prio) @@ -1258,12 +1327,21 @@ static void rcu_spawn_cpu_nocb_kthread(int cpu) t = kthread_run(rcu_nocb_cb_kthread, rdp, "rcuo%c/%d", rcu_state.abbr, cpu); if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__)) - return; + goto end; - if (kthread_prio) + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_CB_BOOST) && kthread_prio) sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); + WRITE_ONCE(rdp->nocb_cb_kthread, t); WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); + return; +end: + mutex_lock(&rcu_state.barrier_mutex); + if (rcu_rdp_is_offloaded(rdp)) { + rcu_nocb_rdp_deoffload(rdp); + cpumask_clear_cpu(cpu, rcu_nocb_mask); + } + mutex_unlock(&rcu_state.barrier_mutex); } /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 271a23bd51409..e59a370eb8b39 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1012,6 +1012,25 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) WRITE_ONCE(rdp->rcuc_activity, jiffies); } +static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) +{ +#ifdef CONFIG_RCU_NOCB_CPU + return rdp->nocb_cb_kthread == current; +#else + return false; +#endif +} + +/* + * Is the current CPU running the RCU-callbacks kthread? + * Caller must have preemption disabled. + */ +static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) +{ + return rdp->rcu_cpu_kthread_task == current || + rcu_is_callbacks_nocb_kthread(rdp); +} + #ifdef CONFIG_RCU_BOOST /* @@ -1151,15 +1170,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) } } -/* - * Is the current CPU running the RCU-callbacks kthread? - * Caller must have preemption disabled. - */ -static bool rcu_is_callbacks_kthread(void) -{ - return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current; -} - #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000) /* @@ -1242,11 +1252,6 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } -static bool rcu_is_callbacks_kthread(void) -{ - return false; -} - static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) { } -- 2.37.1