mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-03-03 18:28:01 +01:00
4725344462
New network transport protocols want NIC drivers to get hardware timestamps of all incoming packets, and possibly all outgoing packets. One example is the upcoming 'Swift congestion control' which is used by TCP transport and is the primary need for timecounter_cyc2time(). This means timecounter_cyc2time() can be called more than 100 million times per second on a busy server. Inlining timecounter_cyc2time() brings a 12% improvement on a UDP receive stress test on a 100Gbit NIC. Note that FDO, LTO, PGO are unable to magically help for this case, presumably because NIC drivers are almost exclusively shipped as modules. Add an unlikely() around the cc_cyc2ns_backwards() case, even if FDO (when used) is able to take care of this optimization. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/ Link: https://patch.msgid.link/20251129095740.3338476-1-edumazet@google.com
65 lines
1.6 KiB
C
65 lines
1.6 KiB
C
// SPDX-License-Identifier: GPL-2.0+
|
|
/*
|
|
* Based on clocksource code. See commit 74d23cc704d1
|
|
*/
|
|
#include <linux/export.h>
|
|
#include <linux/timecounter.h>
|
|
|
|
void timecounter_init(struct timecounter *tc,
|
|
struct cyclecounter *cc,
|
|
u64 start_tstamp)
|
|
{
|
|
tc->cc = cc;
|
|
tc->cycle_last = cc->read(cc);
|
|
tc->nsec = start_tstamp;
|
|
tc->mask = (1ULL << cc->shift) - 1;
|
|
tc->frac = 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(timecounter_init);
|
|
|
|
/**
|
|
* timecounter_read_delta - get nanoseconds since last call of this function
|
|
* @tc: Pointer to time counter
|
|
*
|
|
* When the underlying cycle counter runs over, this will be handled
|
|
* correctly as long as it does not run over more than once between
|
|
* calls.
|
|
*
|
|
* The first call to this function for a new time counter initializes
|
|
* the time tracking and returns an undefined result.
|
|
*/
|
|
static u64 timecounter_read_delta(struct timecounter *tc)
|
|
{
|
|
u64 cycle_now, cycle_delta;
|
|
u64 ns_offset;
|
|
|
|
/* read cycle counter: */
|
|
cycle_now = tc->cc->read(tc->cc);
|
|
|
|
/* calculate the delta since the last timecounter_read_delta(): */
|
|
cycle_delta = (cycle_now - tc->cycle_last) & tc->cc->mask;
|
|
|
|
/* convert to nanoseconds: */
|
|
ns_offset = cyclecounter_cyc2ns(tc->cc, cycle_delta,
|
|
tc->mask, &tc->frac);
|
|
|
|
/* update time stamp of timecounter_read_delta() call: */
|
|
tc->cycle_last = cycle_now;
|
|
|
|
return ns_offset;
|
|
}
|
|
|
|
u64 timecounter_read(struct timecounter *tc)
|
|
{
|
|
u64 nsec;
|
|
|
|
/* increment time by nanoseconds since last call */
|
|
nsec = timecounter_read_delta(tc);
|
|
nsec += tc->nsec;
|
|
tc->nsec = nsec;
|
|
|
|
return nsec;
|
|
}
|
|
EXPORT_SYMBOL_GPL(timecounter_read);
|
|
|