From 820de5c39ef7f6866d2c9e6c7d208bcd2a6e1942 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sat, 21 Jul 2007 04:37:36 -0700 Subject: highres: improve debug output Add some more debug information to the hrtimer and clock events code. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/hrtimer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel/hrtimer.c') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 72d034258ba..065a8978662 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -558,7 +558,8 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, */ static int hrtimer_switch_to_hres(void) { - struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases); + int cpu = smp_processor_id(); + struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); unsigned long flags; if (base->hres_active) @@ -568,6 +569,8 @@ static int hrtimer_switch_to_hres(void) if (tick_init_highres()) { local_irq_restore(flags); + printk(KERN_WARNING "Could not switch to high resolution " + "mode on CPU %d\n", cpu); return 0; } base->hres_active = 1; -- cgit v1.2.3-70-g09d2 From 99bc2fcb283852931fb6bbef40f3df8316b59000 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sat, 21 Jul 2007 04:37:36 -0700 Subject: hrtimer: speedup hrtimer_enqueue Speedup hrtimer_enqueue by evaluating the rbtree insertion result. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Cc: john stultz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/hrtimer.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel/hrtimer.c') diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 065a8978662..eb1ddebd2c0 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -686,6 +686,7 @@ static void enqueue_hrtimer(struct hrtimer *timer, struct rb_node **link = &base->active.rb_node; struct rb_node *parent = NULL; struct hrtimer *entry; + int leftmost = 1; /* * Find the right place in the rbtree: @@ -697,18 +698,19 @@ static void enqueue_hrtimer(struct hrtimer *timer, * We dont care about collisions. Nodes with * the same expiry time stay together. */ - if (timer->expires.tv64 < entry->expires.tv64) + if (timer->expires.tv64 < entry->expires.tv64) { link = &(*link)->rb_left; - else + } else { link = &(*link)->rb_right; + leftmost = 0; + } } /* * Insert the timer to the rbtree and check whether it * replaces the first pending timer */ - if (!base->first || timer->expires.tv64 < - rb_entry(base->first, struct hrtimer, node)->expires.tv64) { + if (leftmost) { /* * Reprogram the clock event device. When the timer is already * expired hrtimer_enqueue_reprogram has either called the -- cgit v1.2.3-70-g09d2 From 2c6b47de17c75d553de3e2fb426d8298d2074585 Mon Sep 17 00:00:00 2001 From: john stultz Date: Tue, 24 Jul 2007 17:47:43 -0700 Subject: Cleanup non-arch xtime uses, use get_seconds() or current_kernel_time(). This avoids use of the kernel-internal "xtime" variable directly outside of the actual time-related functions. Instead, use the helper functions that we already have available to us. This doesn't actually change any behaviour, but this will allow us to fix the fact that "xtime" isn't updated very often with CONFIG_NO_HZ (because much of the realtime information is maintained as separate offsets to 'xtime'), which has caused interfaces that use xtime directly to get a time that is out of sync with the real-time clock by up to a third of a second or so. Signed-off-by: John Stultz Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Linus Torvalds --- drivers/rtc/class.c | 5 +++-- drivers/s390/net/ctcmain.c | 6 +++--- drivers/s390/net/netiucv.c | 4 ++-- include/linux/time.h | 2 +- kernel/acct.c | 2 +- kernel/hrtimer.c | 2 +- kernel/time.c | 16 ---------------- kernel/time/timekeeping.c | 16 ++++++++++++++++ kernel/tsacct.c | 2 +- net/rxrpc/af_rxrpc.c | 2 +- net/rxrpc/ar-connection.c | 4 ++-- net/rxrpc/ar-transport.c | 4 ++-- net/rxrpc/rxkad.c | 2 +- 13 files changed, 34 insertions(+), 33 deletions(-) (limited to 'kernel/hrtimer.c') diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c index 8b3cd31d6a6..10ab3b71ffc 100644 --- a/drivers/rtc/class.c +++ b/drivers/rtc/class.c @@ -46,6 +46,7 @@ static int rtc_suspend(struct device *dev, pm_message_t mesg) { struct rtc_device *rtc = to_rtc_device(dev); struct rtc_time tm; + struct timespec ts = current_kernel_time(); if (strncmp(rtc->dev.bus_id, CONFIG_RTC_HCTOSYS_DEVICE, @@ -57,8 +58,8 @@ static int rtc_suspend(struct device *dev, pm_message_t mesg) /* RTC precision is 1 second; adjust delta for avg 1/2 sec err */ set_normalized_timespec(&delta, - xtime.tv_sec - oldtime, - xtime.tv_nsec - (NSEC_PER_SEC >> 1)); + ts.tv_sec - oldtime, + ts.tv_nsec - (NSEC_PER_SEC >> 1)); return 0; } diff --git a/drivers/s390/net/ctcmain.c b/drivers/s390/net/ctcmain.c index b20fd068173..92e8a37b502 100644 --- a/drivers/s390/net/ctcmain.c +++ b/drivers/s390/net/ctcmain.c @@ -674,7 +674,7 @@ ch_action_txdone(fsm_instance * fi, int event, void *arg) int first = 1; int i; unsigned long duration; - struct timespec done_stamp = xtime; + struct timespec done_stamp = current_kernel_time(); DBF_TEXT(trace, 4, __FUNCTION__); @@ -730,7 +730,7 @@ ch_action_txdone(fsm_instance * fi, int event, void *arg) spin_unlock(&ch->collect_lock); ch->ccw[1].count = ch->trans_skb->len; fsm_addtimer(&ch->timer, CTC_TIMEOUT_5SEC, CH_EVENT_TIMER, ch); - ch->prof.send_stamp = xtime; + ch->prof.send_stamp = current_kernel_time(); rc = ccw_device_start(ch->cdev, &ch->ccw[0], (unsigned long) ch, 0xff, 0); ch->prof.doios_multi++; @@ -2281,7 +2281,7 @@ transmit_skb(struct channel *ch, struct sk_buff *skb) fsm_newstate(ch->fsm, CH_STATE_TX); fsm_addtimer(&ch->timer, CTC_TIMEOUT_5SEC, CH_EVENT_TIMER, ch); spin_lock_irqsave(get_ccwdev_lock(ch->cdev), saveflags); - ch->prof.send_stamp = xtime; + ch->prof.send_stamp = current_kernel_time(); rc = ccw_device_start(ch->cdev, &ch->ccw[ccw_idx], (unsigned long) ch, 0xff, 0); spin_unlock_irqrestore(get_ccwdev_lock(ch->cdev), saveflags); diff --git a/drivers/s390/net/netiucv.c b/drivers/s390/net/netiucv.c index 3d28e1a5bf7..26888947433 100644 --- a/drivers/s390/net/netiucv.c +++ b/drivers/s390/net/netiucv.c @@ -753,7 +753,7 @@ static void conn_action_txdone(fsm_instance *fi, int event, void *arg) header.next = 0; memcpy(skb_put(conn->tx_buff, NETIUCV_HDRLEN), &header, NETIUCV_HDRLEN); - conn->prof.send_stamp = xtime; + conn->prof.send_stamp = current_kernel_time(); txmsg.class = 0; txmsg.tag = 0; rc = iucv_message_send(conn->path, &txmsg, 0, 0, @@ -1185,7 +1185,7 @@ static int netiucv_transmit_skb(struct iucv_connection *conn, memcpy(skb_put(nskb, NETIUCV_HDRLEN), &header, NETIUCV_HDRLEN); fsm_newstate(conn->fsm, CONN_STATE_TX); - conn->prof.send_stamp = xtime; + conn->prof.send_stamp = current_kernel_time(); msg.tag = 1; msg.class = 0; diff --git a/include/linux/time.h b/include/linux/time.h index e6aea5146e5..71181df8b74 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -107,7 +107,7 @@ static inline unsigned long get_seconds(void) struct timespec current_kernel_time(void); #define CURRENT_TIME (current_kernel_time()) -#define CURRENT_TIME_SEC ((struct timespec) { xtime.tv_sec, 0 }) +#define CURRENT_TIME_SEC ((struct timespec) { get_seconds(), 0 }) extern void do_gettimeofday(struct timeval *tv); extern int do_settimeofday(struct timespec *tv); diff --git a/kernel/acct.c b/kernel/acct.c index 70d0d88e555..24f0f8b2ba7 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -468,7 +468,7 @@ static void do_acct_process(struct file *file) } #endif do_div(elapsed, AHZ); - ac.ac_btime = xtime.tv_sec - elapsed; + ac.ac_btime = get_seconds() - elapsed; /* we really need to bite the bullet and change layout */ ac.ac_uid = current->uid; ac.ac_gid = current->gid; diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index eb1ddebd2c0..a7bb05e6cb6 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -144,7 +144,7 @@ static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) #ifdef CONFIG_NO_HZ getnstimeofday(&xts); #else - xts = xtime; + xts = current_kernel_time(); #endif tom = wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); diff --git a/kernel/time.c b/kernel/time.c index 5b81da08bbd..2289a8d6831 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -215,22 +215,6 @@ asmlinkage long sys_adjtimex(struct timex __user *txc_p) return copy_to_user(txc_p, &txc, sizeof(struct timex)) ? -EFAULT : ret; } -inline struct timespec current_kernel_time(void) -{ - struct timespec now; - unsigned long seq; - - do { - seq = read_seqbegin(&xtime_lock); - - now = xtime; - } while (read_seqretry(&xtime_lock, seq)); - - return now; -} - -EXPORT_SYMBOL(current_kernel_time); - /** * current_fs_time - Return FS time * @sb: Superblock. diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 88c81026e00..07a3f1420c2 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -509,3 +509,19 @@ void monotonic_to_bootbased(struct timespec *ts) { ts->tv_sec += total_sleep_time; } + +struct timespec current_kernel_time(void) +{ + struct timespec now; + unsigned long seq; + + do { + seq = read_seqbegin(&xtime_lock); + + now = xtime; + } while (read_seqretry(&xtime_lock, seq)); + + return now; +} + +EXPORT_SYMBOL(current_kernel_time); diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 658f638c402..c122131a122 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -39,7 +39,7 @@ void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk) ac_etime = timespec_to_ns(&ts); do_div(ac_etime, NSEC_PER_USEC); stats->ac_etime = ac_etime; - stats->ac_btime = xtime.tv_sec - ts.tv_sec; + stats->ac_btime = get_seconds() - ts.tv_sec; if (thread_group_leader(tsk)) { stats->ac_exitcode = tsk->exit_code; if (tsk->flags & PF_FORKNOEXEC) diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 16a68df4e36..c58fa0d1be2 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -787,7 +787,7 @@ static int __init af_rxrpc_init(void) BUILD_BUG_ON(sizeof(struct rxrpc_skb_priv) > sizeof(dummy_skb->cb)); - rxrpc_epoch = htonl(xtime.tv_sec); + rxrpc_epoch = htonl(get_seconds()); ret = -ENOMEM; rxrpc_call_jar = kmem_cache_create( diff --git a/net/rxrpc/ar-connection.c b/net/rxrpc/ar-connection.c index 482750efc23..372b24466dc 100644 --- a/net/rxrpc/ar-connection.c +++ b/net/rxrpc/ar-connection.c @@ -791,7 +791,7 @@ void rxrpc_put_connection(struct rxrpc_connection *conn) ASSERTCMP(atomic_read(&conn->usage), >, 0); - conn->put_time = xtime.tv_sec; + conn->put_time = get_seconds(); if (atomic_dec_and_test(&conn->usage)) { _debug("zombie"); rxrpc_queue_delayed_work(&rxrpc_connection_reap, 0); @@ -835,7 +835,7 @@ void rxrpc_connection_reaper(struct work_struct *work) _enter(""); - now = xtime.tv_sec; + now = get_seconds(); earliest = ULONG_MAX; write_lock_bh(&rxrpc_connection_lock); diff --git a/net/rxrpc/ar-transport.c b/net/rxrpc/ar-transport.c index d43d78f1930..bb282a6a19f 100644 --- a/net/rxrpc/ar-transport.c +++ b/net/rxrpc/ar-transport.c @@ -183,7 +183,7 @@ void rxrpc_put_transport(struct rxrpc_transport *trans) ASSERTCMP(atomic_read(&trans->usage), >, 0); - trans->put_time = xtime.tv_sec; + trans->put_time = get_seconds(); if (unlikely(atomic_dec_and_test(&trans->usage))) _debug("zombie"); /* let the reaper determine the timeout to avoid a race with @@ -219,7 +219,7 @@ static void rxrpc_transport_reaper(struct work_struct *work) _enter(""); - now = xtime.tv_sec; + now = get_seconds(); earliest = ULONG_MAX; /* extract all the transports that have been dead too long */ diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 5ec705144e1..ac3cabdca78 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -916,7 +916,7 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn, issue = be32_to_cpu(stamp); } p += 4; - now = xtime.tv_sec; + now = get_seconds(); _debug("KIV ISSUE: %lx [%lx]", issue, now); /* check the ticket is in date */ -- cgit v1.2.3-70-g09d2 From 17c38b7490b3f0300c7812aefdae2ddda7ab4112 Mon Sep 17 00:00:00 2001 From: john stultz Date: Tue, 24 Jul 2007 18:38:34 -0700 Subject: Cache xtime every call to update_wall_time This avoids xtime lag seen with dynticks, because while 'xtime' itself is still not updated often, we keep a 'xtime_cache' variable around that contains the approximate real-time that _is_ updated each time we do a 'update_wall_time()', and is thus never off by more than one tick. IOW, this restores the original semantics for 'xtime' users, as long as you use the proper abstraction functions (ie 'current_kernel_time()' or 'get_seconds()' depending on whether you want a timespec or just the seconds field). [ Updated Patch. As penance for my sins I've also yanked another #ifdef that was added to avoid the xtime lag w/ hrtimers. ] Signed-off-by: John Stultz Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Linus Torvalds --- include/linux/time.h | 6 +----- kernel/hrtimer.c | 4 ---- kernel/time/timekeeping.c | 26 +++++++++++++++++++++++--- 3 files changed, 24 insertions(+), 12 deletions(-) (limited to 'kernel/hrtimer.c') diff --git a/include/linux/time.h b/include/linux/time.h index 71181df8b74..6a5f503b4f1 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -99,11 +99,7 @@ extern int update_persistent_clock(struct timespec now); extern int no_sync_cmos_clock __read_mostly; void timekeeping_init(void); -static inline unsigned long get_seconds(void) -{ - return xtime.tv_sec; -} - +unsigned long get_seconds(void); struct timespec current_kernel_time(void); #define CURRENT_TIME (current_kernel_time()) diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index a7bb05e6cb6..c21ca6bfaa6 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -141,11 +141,7 @@ static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) do { seq = read_seqbegin(&xtime_lock); -#ifdef CONFIG_NO_HZ - getnstimeofday(&xts); -#else xts = current_kernel_time(); -#endif tom = wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 07a3f1420c2..acc417b5a9b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -47,10 +47,22 @@ EXPORT_SYMBOL(xtime_lock); struct timespec xtime __attribute__ ((aligned (16))); struct timespec wall_to_monotonic __attribute__ ((aligned (16))); static unsigned long total_sleep_time; /* seconds */ - EXPORT_SYMBOL(xtime); +#ifdef CONFIG_NO_HZ +static struct timespec xtime_cache __attribute__ ((aligned (16))); +static inline void update_xtime_cache(u64 nsec) +{ + xtime_cache = xtime; + timespec_add_ns(&xtime_cache, nsec); +} +#else +#define xtime_cache xtime +/* We do *not* want to evaluate the argument for this case */ +#define update_xtime_cache(n) do { } while (0) +#endif + static struct clocksource *clock; /* pointer to current clocksource */ @@ -478,6 +490,8 @@ void update_wall_time(void) xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; + update_xtime_cache(cyc2ns(clock, offset)); + /* check to see if there is a new clocksource to use */ change_clocksource(); update_vsyscall(&xtime, clock); @@ -510,6 +524,13 @@ void monotonic_to_bootbased(struct timespec *ts) ts->tv_sec += total_sleep_time; } +unsigned long get_seconds(void) +{ + return xtime_cache.tv_sec; +} +EXPORT_SYMBOL(get_seconds); + + struct timespec current_kernel_time(void) { struct timespec now; @@ -518,10 +539,9 @@ struct timespec current_kernel_time(void) do { seq = read_seqbegin(&xtime_lock); - now = xtime; + now = xtime_cache; } while (read_seqretry(&xtime_lock, seq)); return now; } - EXPORT_SYMBOL(current_kernel_time); -- cgit v1.2.3-70-g09d2 From a272378d1128d1c60a463a315646c86d174ff74c Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Sun, 19 Aug 2007 17:16:05 -0700 Subject: [KTIME]: Introduce ktime_sub_ns and ktime_sub_us First user will be the DCCP transport networking protocol. Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: David S. Miller --- include/linux/ktime.h | 21 +++++++++++++++++++++ kernel/hrtimer.c | 24 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+) (limited to 'kernel/hrtimer.c') diff --git a/include/linux/ktime.h b/include/linux/ktime.h index dae7143644f..a6ddec141f9 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -102,6 +102,13 @@ static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) #define ktime_add_ns(kt, nsval) \ ({ (ktime_t){ .tv64 = (kt).tv64 + (nsval) }; }) +/* + * Subtract a scalar nanosecod from a ktime_t variable + * res = kt - nsval: + */ +#define ktime_sub_ns(kt, nsval) \ + ({ (ktime_t){ .tv64 = (kt).tv64 - (nsval) }; }) + /* convert a timespec to ktime_t format: */ static inline ktime_t timespec_to_ktime(struct timespec ts) { @@ -199,6 +206,15 @@ static inline ktime_t ktime_add(const ktime_t add1, const ktime_t add2) */ extern ktime_t ktime_add_ns(const ktime_t kt, u64 nsec); +/** + * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable + * @kt: minuend + * @nsec: the scalar nsec value to subtract + * + * Returns the subtraction of @nsec from @kt in ktime_t format + */ +extern ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec); + /** * timespec_to_ktime - convert a timespec to ktime_t format * @ts: the timespec variable to convert @@ -289,6 +305,11 @@ static inline ktime_t ktime_add_us(const ktime_t kt, const u64 usec) return ktime_add_ns(kt, usec * 1000); } +static inline ktime_t ktime_sub_us(const ktime_t kt, const u64 usec) +{ + return ktime_sub_ns(kt, usec * 1000); +} + /* * The resolution of the clocks. The resolution value is returned in * the clock_getres() system call to give application programmers an diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index c21ca6bfaa6..dc8a4451d79 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -277,6 +277,30 @@ ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) } EXPORT_SYMBOL_GPL(ktime_add_ns); + +/** + * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable + * @kt: minuend + * @nsec: the scalar nsec value to subtract + * + * Returns the subtraction of @nsec from @kt in ktime_t format + */ +ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec) +{ + ktime_t tmp; + + if (likely(nsec < NSEC_PER_SEC)) { + tmp.tv64 = nsec; + } else { + unsigned long rem = do_div(nsec, NSEC_PER_SEC); + + tmp = ktime_set((long)nsec, rem); + } + + return ktime_sub(kt, tmp); +} + +EXPORT_SYMBOL_GPL(ktime_sub_ns); # endif /* !CONFIG_KTIME_SCALAR */ /* -- cgit v1.2.3-70-g09d2