diff options
Diffstat (limited to 'drivers/idle/intel_idle.c')
| -rw-r--r-- | drivers/idle/intel_idle.c | 299 |
1 files changed, 246 insertions, 53 deletions
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index fa6964d8681..4d140bbbe10 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1,7 +1,7 @@ /* * intel_idle.c - native hardware idle loop for modern Intel processors * - * Copyright (c) 2010, Intel Corporation. + * Copyright (c) 2013, Intel Corporation. * Len Brown <len.brown@intel.com> * * This program is free software; you can redistribute it and/or modify it @@ -123,7 +123,7 @@ static struct cpuidle_state *cpuidle_state_table; * which is also the index into the MWAIT hint array. * Thus C0 is a dummy. */ -static struct cpuidle_state nehalem_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state nehalem_cstates[] = { { .name = "C1-NHM", .desc = "MWAIT 0x00", @@ -156,7 +156,7 @@ static struct cpuidle_state nehalem_cstates[CPUIDLE_STATE_MAX] = { .enter = NULL } }; -static struct cpuidle_state snb_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state snb_cstates[] = { { .name = "C1-SNB", .desc = "MWAIT 0x00", @@ -196,7 +196,54 @@ static struct cpuidle_state snb_cstates[CPUIDLE_STATE_MAX] = { .enter = NULL } }; -static struct cpuidle_state ivb_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state byt_cstates[] = { + { + .name = "C1-BYT", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle }, + { + .name = "C1E-BYT", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 15, + .target_residency = 30, + .enter = &intel_idle }, + { + .name = "C6N-BYT", + .desc = "MWAIT 0x58", + .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 40, + .target_residency = 275, + .enter = &intel_idle }, + { + .name = "C6S-BYT", + .desc = "MWAIT 0x52", + .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 140, + .target_residency = 560, + .enter = &intel_idle }, + { + .name = "C7-BYT", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 1200, + .target_residency = 1500, + .enter = &intel_idle }, + { + .name = "C7S-BYT", + .desc = "MWAIT 0x64", + .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 10000, + .target_residency = 20000, + .enter = &intel_idle }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivb_cstates[] = { { .name = "C1-IVB", .desc = "MWAIT 0x00", @@ -236,7 +283,106 @@ static struct cpuidle_state ivb_cstates[CPUIDLE_STATE_MAX] = { .enter = NULL } }; -static struct cpuidle_state hsw_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state ivt_cstates[] = { + { + .name = "C1-IVT", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle }, + { + .name = "C1E-IVT", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 10, + .target_residency = 80, + .enter = &intel_idle }, + { + .name = "C3-IVT", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 156, + .enter = &intel_idle }, + { + .name = "C6-IVT", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 82, + .target_residency = 300, + .enter = &intel_idle }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivt_cstates_4s[] = { + { + .name = "C1-IVT-4S", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle }, + { + .name = "C1E-IVT-4S", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 10, + .target_residency = 250, + .enter = &intel_idle }, + { + .name = "C3-IVT-4S", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 300, + .enter = &intel_idle }, + { + .name = "C6-IVT-4S", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 84, + .target_residency = 400, + .enter = &intel_idle }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivt_cstates_8s[] = { + { + .name = "C1-IVT-8S", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle }, + { + .name = "C1E-IVT-8S", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 10, + .target_residency = 500, + .enter = &intel_idle }, + { + .name = "C3-IVT-8S", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 600, + .enter = &intel_idle }, + { + .name = "C6-IVT-8S", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 88, + .target_residency = 700, + .enter = &intel_idle }, + { + .enter = NULL } +}; + +static struct cpuidle_state hsw_cstates[] = { { .name = "C1-HSW", .desc = "MWAIT 0x00", @@ -297,7 +443,7 @@ static struct cpuidle_state hsw_cstates[CPUIDLE_STATE_MAX] = { .enter = NULL } }; -static struct cpuidle_state atom_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state atom_cstates[] = { { .name = "C1E-ATM", .desc = "MWAIT 0x00", @@ -329,6 +475,24 @@ static struct cpuidle_state atom_cstates[CPUIDLE_STATE_MAX] = { { .enter = NULL } }; +static struct cpuidle_state avn_cstates[] = { + { + .name = "C1-AVN", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .exit_latency = 2, + .target_residency = 2, + .enter = &intel_idle }, + { + .name = "C6-AVN", + .desc = "MWAIT 0x51", + .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 15, + .target_residency = 45, + .enter = &intel_idle }, + { + .enter = NULL } +}; /** * intel_idle @@ -359,13 +523,7 @@ static int intel_idle(struct cpuidle_device *dev, if (!(lapic_timer_reliable_states & (1 << (cstate)))) clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu); - if (!need_resched()) { - - __monitor((void *)¤t_thread_info()->flags, 0, 0); - smp_mb(); - if (!need_resched()) - __mwait(eax, ecx); - } + mwait_idle_with_hints(eax, ecx); if (!(lapic_timer_reliable_states & (1 << (cstate)))) clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); @@ -390,7 +548,7 @@ static int cpu_hotplug_notify(struct notifier_block *n, int hotcpu = (unsigned long)hcpu; struct cpuidle_device *dev; - switch (action & 0xf) { + switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) @@ -452,16 +610,31 @@ static const struct idle_cpu idle_cpu_snb = { .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_byt = { + .state_table = byt_cstates, + .disable_promotion_to_c1e = true, +}; + static const struct idle_cpu idle_cpu_ivb = { .state_table = ivb_cstates, .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_ivt = { + .state_table = ivt_cstates, + .disable_promotion_to_c1e = true, +}; + static const struct idle_cpu idle_cpu_hsw = { .state_table = hsw_cstates, .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_avn = { + .state_table = avn_cstates, + .disable_promotion_to_c1e = true, +}; + #define ICPU(model, cpu) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } @@ -477,12 +650,15 @@ static const struct x86_cpu_id intel_idle_ids[] = { ICPU(0x2f, idle_cpu_nehalem), ICPU(0x2a, idle_cpu_snb), ICPU(0x2d, idle_cpu_snb), + ICPU(0x36, idle_cpu_atom), + ICPU(0x37, idle_cpu_byt), ICPU(0x3a, idle_cpu_ivb), - ICPU(0x3e, idle_cpu_ivb), + ICPU(0x3e, idle_cpu_ivt), ICPU(0x3c, idle_cpu_hsw), ICPU(0x3f, idle_cpu_hsw), ICPU(0x45, idle_cpu_hsw), ICPU(0x46, idle_cpu_hsw), + ICPU(0x4D, idle_cpu_avn), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); @@ -490,7 +666,7 @@ MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); /* * intel_idle_probe() */ -static int intel_idle_probe(void) +static int __init intel_idle_probe(void) { unsigned int eax, ebx, ecx; const struct x86_cpu_id *id; @@ -554,19 +730,55 @@ static void intel_idle_cpuidle_devices_uninit(void) free_percpu(intel_idle_cpuidle_devices); return; } + +/* + * intel_idle_state_table_update() + * + * Update the default state_table for this CPU-id + * + * Currently used to access tuned IVT multi-socket targets + * Assumption: num_sockets == (max_package_num + 1) + */ +void intel_idle_state_table_update(void) +{ + /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ + if (boot_cpu_data.x86_model == 0x3e) { /* IVT */ + int cpu, package_num, num_sockets = 1; + + for_each_online_cpu(cpu) { + package_num = topology_physical_package_id(cpu); + if (package_num + 1 > num_sockets) { + num_sockets = package_num + 1; + + if (num_sockets > 4) { + cpuidle_state_table = ivt_cstates_8s; + return; + } + } + } + + if (num_sockets > 2) + cpuidle_state_table = ivt_cstates_4s; + /* else, 1 and 2 socket systems use default ivt_cstates */ + } + return; +} + /* * intel_idle_cpuidle_driver_init() * allocate, initialize cpuidle_states */ -static int intel_idle_cpuidle_driver_init(void) +static int __init intel_idle_cpuidle_driver_init(void) { int cstate; struct cpuidle_driver *drv = &intel_idle_driver; + intel_idle_state_table_update(); + drv->state_count = 1; for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { - int num_substates, mwait_hint, mwait_cstate, mwait_substate; + int num_substates, mwait_hint, mwait_cstate; if (cpuidle_state_table[cstate].enter == NULL) break; @@ -579,14 +791,13 @@ static int intel_idle_cpuidle_driver_init(void) mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); - mwait_substate = MWAIT_HINT2SUBSTATE(mwait_hint); - /* does the state exist in CPUID.MWAIT? */ + /* number of sub-states for this state in CPUID.MWAIT */ num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) & MWAIT_SUBSTATE_MASK; - /* if sub-state in table is not enumerated by CPUID */ - if ((mwait_substate + 1) > num_substates) + /* if NO sub-states for this state in CPUID, skip it */ + if (num_substates == 0) continue; if (((mwait_cstate + 1) > 2) && @@ -617,39 +828,10 @@ static int intel_idle_cpuidle_driver_init(void) */ static int intel_idle_cpu_init(int cpu) { - int cstate; struct cpuidle_device *dev; dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); - dev->state_count = 1; - - for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { - int num_substates, mwait_hint, mwait_cstate, mwait_substate; - - if (cpuidle_state_table[cstate].enter == NULL) - continue; - - if (cstate + 1 > max_cstate) { - printk(PREFIX "max_cstate %d reached\n", max_cstate); - break; - } - - mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); - mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); - mwait_substate = MWAIT_HINT2SUBSTATE(mwait_hint); - - /* does the state exist in CPUID.MWAIT? */ - num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) - & MWAIT_SUBSTATE_MASK; - - /* if sub-state in table is not enumerated by CPUID */ - if ((mwait_substate + 1) > num_substates) - continue; - - dev->state_count += 1; - } - dev->cpu = cpu; if (cpuidle_register_device(dev)) { @@ -661,6 +843,9 @@ static int intel_idle_cpu_init(int cpu) if (icpu->auto_demotion_disable_flags) smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); + if (icpu->disable_promotion_to_c1e) + smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); + return 0; } @@ -689,14 +874,19 @@ static int __init intel_idle_init(void) if (intel_idle_cpuidle_devices == NULL) return -ENOMEM; + cpu_notifier_register_begin(); + for_each_online_cpu(i) { retval = intel_idle_cpu_init(i); if (retval) { + cpu_notifier_register_done(); cpuidle_unregister_driver(&intel_idle_driver); return retval; } } - register_cpu_notifier(&cpu_hotplug_notifier); + __register_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return 0; } @@ -706,10 +896,13 @@ static void __exit intel_idle_exit(void) intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); + cpu_notifier_register_begin(); if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) on_each_cpu(__setup_broadcast_timer, (void *)false, 1); - unregister_cpu_notifier(&cpu_hotplug_notifier); + __unregister_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return; } |